From a26a024092de78c7711f67f5bacf3297f1253c63 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 11 May 2020 19:13:10 +0200 Subject: [PATCH 001/119] [wip] Allow file upload from config-panel --- src/yunohost/app.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 2ca931a90..32bb1ece3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -33,6 +33,7 @@ import re import subprocess import glob import urllib.parse +import base64 import tempfile from collections import OrderedDict @@ -1867,6 +1868,7 @@ def app_config_apply(operation_logger, app, args): } args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} + upload_dir = None for tab in config_panel.get("panel", []): tab_id = tab["id"] # this makes things easier to debug on crash for section in tab.get("sections", []): @@ -1878,6 +1880,23 @@ def app_config_apply(operation_logger, app, args): ).upper() if generated_name in args: + # Upload files from API + # A file arg contains a string with "FILENAME:BASE64_CONTENT" + if option["type"] == "file" and msettings.get('interface') == 'api': + if upload_dir is None: + upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') + filename, args[generated_name] = args[generated_name].split(':') + logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) + file_path = os.join(upload_dir, filename) + try: + with open(file_path, 'wb') as f: + f.write(args[generated_name]) + except IOError as e: + raise YunohostError("cannot_write_file", file=file_path, error=str(e)) + except Exception as e: + raise YunohostError("error_writing_file", file=file_path, error=str(e)) + args[generated_name] = file_path + logger.debug( "include into env %s=%s", generated_name, args[generated_name] ) @@ -1899,6 +1918,11 @@ def app_config_apply(operation_logger, app, args): env=env, )[0] + # Delete files uploaded from API + if msettings.get('interface') == 'api': + if upload_dir is not None: + shutil.rmtree(upload_dir) + if return_code != 0: msg = ( "'script/config apply' return value code: %s (considered as an error)" From 4939bbeb2e4b7a0362ee55d955fa33271bcf8c50 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 8 Jun 2020 19:18:22 +0200 Subject: [PATCH 002/119] [fix] Several files with same name --- src/yunohost/app.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 32bb1ece3..ae6accab0 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1882,15 +1882,21 @@ def app_config_apply(operation_logger, app, args): if generated_name in args: # Upload files from API # A file arg contains a string with "FILENAME:BASE64_CONTENT" - if option["type"] == "file" and msettings.get('interface') == 'api': + if 'type' in option and option["type"] == "file" \ + and msettings.get('interface') == 'api': if upload_dir is None: upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') - filename, args[generated_name] = args[generated_name].split(':') + filename = args[generated_name + '[name]'] + content = args[generated_name] logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) - file_path = os.join(upload_dir, filename) + file_path = os.path.join(upload_dir, filename) + i = 2 + while os.path.exists(file_path): + file_path = os.path.join(upload_dir, filename + (".%d" % i)) + i += 1 try: with open(file_path, 'wb') as f: - f.write(args[generated_name]) + f.write(content.decode("base64")) except IOError as e: raise YunohostError("cannot_write_file", file=file_path, error=str(e)) except Exception as e: @@ -1907,7 +1913,7 @@ def app_config_apply(operation_logger, app, args): # for debug purpose for key in args: if key not in env: - logger.warning( + logger.debug( "Ignore key '%s' from arguments because it is not in the config", key ) From 3bc45b5672f332e7cdbe314c756fcf4aac74e11f Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 7 Oct 2020 00:31:20 +0200 Subject: [PATCH 003/119] [enh] Replace os.path.join to improve security --- src/yunohost/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index ae6accab0..f017521d2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1889,10 +1889,14 @@ def app_config_apply(operation_logger, app, args): filename = args[generated_name + '[name]'] content = args[generated_name] logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) - file_path = os.path.join(upload_dir, filename) + + # Filename is given by user of the API. For security reason, we have replaced + # os.path.join to avoid the user to be able to rewrite a file in filesystem + # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" + file_path = os.path.normpath(upload_dir + "/" + filename) i = 2 while os.path.exists(file_path): - file_path = os.path.join(upload_dir, filename + (".%d" % i)) + file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) i += 1 try: with open(file_path, 'wb') as f: From a5508b1db45d2f5ae94578f44b6026fd5b45d017 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 31 May 2021 16:32:19 +0200 Subject: [PATCH 004/119] [fix] Base64 python3 change --- src/yunohost/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index f017521d2..49033d8b4 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1845,7 +1845,7 @@ def app_config_apply(operation_logger, app, args): logger.warning(m18n.n("experimental_feature")) from yunohost.hook import hook_exec - + from base64 import b64decode installed = _is_installed(app) if not installed: raise YunohostValidationError( @@ -1889,8 +1889,8 @@ def app_config_apply(operation_logger, app, args): filename = args[generated_name + '[name]'] content = args[generated_name] logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) - - # Filename is given by user of the API. For security reason, we have replaced + + # Filename is given by user of the API. For security reason, we have replaced # os.path.join to avoid the user to be able to rewrite a file in filesystem # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" file_path = os.path.normpath(upload_dir + "/" + filename) @@ -1900,7 +1900,7 @@ def app_config_apply(operation_logger, app, args): i += 1 try: with open(file_path, 'wb') as f: - f.write(content.decode("base64")) + f.write(b64decode(content)) except IOError as e: raise YunohostError("cannot_write_file", file=file_path, error=str(e)) except Exception as e: From 27ba82bd307ae28268f7e56c1b3a6a40060c62e8 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 1 Jun 2021 00:41:37 +0200 Subject: [PATCH 005/119] [enh] Add configpanel helpers --- data/helpers.d/configpanel | 259 +++++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 data/helpers.d/configpanel diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel new file mode 100644 index 000000000..f648826e4 --- /dev/null +++ b/data/helpers.d/configpanel @@ -0,0 +1,259 @@ +#!/bin/bash + +ynh_lowerdot_to_uppersnake() { + local lowerdot + lowerdot=$(echo "$1" | cut -d= -f1 | sed "s/\./_/g") + echo "${lowerdot^^}" +} + +# Get a value from heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_value_get --file=PATH --key=KEY +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to get +# +# This helpers match several var affectation use case in several languages +# We don't use jq or equivalent to keep comments and blank space in files +# This helpers work line by line, it is not able to work correctly +# if you have several identical keys in your files +# +# Example of line this helpers can managed correctly +# .yml +# title: YunoHost documentation +# email: 'yunohost@yunohost.org' +# .json +# "theme": "colib'ris", +# "port": 8102 +# "some_boolean": false, +# "user": null +# .ini +# some_boolean = On +# action = "Clear" +# port = 20 +# .php +# $user= +# user => 20 +# .py +# USER = 8102 +# user = 'https://donate.local' +# CUSTOM['user'] = 'YunoHost' +# Requires YunoHost version 4.3 or higher. +ynh_value_get() { + # Declare an array to define the options of this helper. + local legacy_args=fk + local -A args_array=( [f]=file= [k]=key= ) + local file + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + + local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + + local first_char="${crazy_value:0:1}" + if [[ "$first_char" == '"' ]] ; then + echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + elif [[ "$first_char" == "'" ]] ; then + echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + else + echo "$crazy_value" + fi +} + +# Set a value into heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_value_set --file=PATH --key=KEY --value=VALUE +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to set +# | arg: -v, --value= - the value to set +# +# Requires YunoHost version 4.3 or higher. +ynh_value_set() { + # Declare an array to define the options of this helper. + local legacy_args=fkv + local -A args_array=( [f]=file= [k]=key= [v]=value=) + local file + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + + local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + local first_char="${crazy_value:0:1}" + if [[ "$first_char" == '"' ]] ; then + value="$(echo "$value" | sed 's/"/\\"/g')" + sed -ri "s%^(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file} + elif [[ "$first_char" == "'" ]] ; then + value="$(echo "$value" | sed "s/'/\\\\'/g")" + sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + else + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then + value="\"$(echo "$value" | sed 's/"/\\"/g')\"" + fi + sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + fi +} + +_ynh_panel_get() { + + # From settings + local params_sources + params_sources=`python3 << EOL +import toml +from collections import OrderedDict +with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f: + file_content = f.read() +loaded_toml = toml.loads(file_content, _dict=OrderedDict) + +for panel_name,panel in loaded_toml.items(): + if isinstance(panel, dict): + for section_name, section in panel.items(): + if isinstance(section, dict): + for name, param in section.items(): + if isinstance(param, dict) and param.get('source', '') == 'settings': + print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings'))) +EOL +` + for param_source in params_sources + do + local _dot_setting=$(echo "$param_source" | cut -d= -f1) + local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)" + local short_setting=$(echo "$_dot_setting" | cut -d. -f3) + local _getter="get__${short_setting}" + local source="$(echo $param_source | cut -d= -f2)" + + # Get value from getter if exists + if type $getter | grep -q '^function$' 2>/dev/null; then + old[$short_setting]="$($getter)" + + # Get value from app settings + elif [[ "$source" == "settings" ]] ; then + old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" + + # Get value from a kind of key/value file + elif [[ "$source" == *":"* ]] ; then + local source_key="$(echo "$source" | cut -d: -f1)" + source_key=${source_key:-$short_setting} + local source_file="$(echo "$source" | cut -d: -f2)" + old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")" + + # Specific case for files (all content of the file is the source) + else + old[$short_setting]="$source" + fi + + done + + +} + +_ynh_panel_apply() { + for short_setting in "${!dot_settings[@]}" + do + local setter="set__${short_setting}" + local source="$sources[$short_setting]" + + # Apply setter if exists + if type $setter | grep -q '^function$' 2>/dev/null; then + $setter + + # Copy file in right place + elif [[ "$source" == "settings" ]] ; then + ynh_app_setting_set $app $short_setting "$new[$short_setting]" + + # Get value from a kind of key/value file + elif [[ "$source" == *":"* ]] + then + local source_key="$(echo "$source" | cut -d: -f1)" + source_key=${source_key:-$short_setting} + local source_file="$(echo "$source" | cut -d: -f2)" + ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]" + + # Specific case for files (all content of the file is the source) + else + cp "$new[$short_setting]" "$source" + fi + done +} + +_ynh_panel_show() { + for short_setting in "${!old[@]}" + do + local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" + ynh_return "$key=${old[$short_setting]}" + done +} + +_ynh_panel_validate() { + # Change detection + local is_error=true + #for changed_status in "${!changed[@]}" + for short_setting in "${!dot_settings[@]}" + do + #TODO file hash + file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) + file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) + if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] + then + changed[$setting]=true + fi + if [[ "$new[$short_setting]" == "$old[$short_setting]" ]] + then + changed[$short_setting]=false + else + changed[$short_setting]=true + is_error=false + fi + done + + # Run validation if something is changed + if [[ "$is_error" == "false" ]] + then + + for short_setting in "${!dot_settings[@]}" + do + local result="$(validate__$short_setting)" + local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" + if [ -n "$result" ] + then + ynh_return "$key=$result" + is_error=true + fi + done + fi + + if [[ "$is_error" == "true" ]] + then + ynh_die + fi + +} + +ynh_panel_get() { + _ynh_panel_get +} + +ynh_panel_init() { + declare -A old=() + declare -A changed=() + declare -A file_hash=() + + ynh_panel_get +} + +ynh_panel_show() { + _ynh_panel_show +} + +ynh_panel_validate() { + _ynh_panel_validate +} + +ynh_panel_apply() { + _ynh_panel_apply +} + From 5fec35ccea72b9073ef75b18570c8df0e38aff7b Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 1 Jun 2021 01:29:26 +0200 Subject: [PATCH 006/119] [fix] No validate function in config panel --- data/helpers.d/configpanel | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index f648826e4..83130cfe6 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -123,7 +123,7 @@ EOL local _dot_setting=$(echo "$param_source" | cut -d= -f1) local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)" local short_setting=$(echo "$_dot_setting" | cut -d. -f3) - local _getter="get__${short_setting}" + local getter="get__${short_setting}" local source="$(echo $param_source | cut -d= -f2)" # Get value from getter if exists @@ -195,12 +195,12 @@ _ynh_panel_validate() { for short_setting in "${!dot_settings[@]}" do #TODO file hash - file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) - file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) - if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] - then - changed[$setting]=true - fi + file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) + file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) + if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] + then + changed[$setting]=true + fi if [[ "$new[$short_setting]" == "$old[$short_setting]" ]] then changed[$short_setting]=false @@ -216,10 +216,13 @@ _ynh_panel_validate() { for short_setting in "${!dot_settings[@]}" do - local result="$(validate__$short_setting)" - local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" + local result="" + if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then + result="$(validate__$short_setting)" + fi if [ -n "$result" ] then + local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" ynh_return "$key=$result" is_error=true fi @@ -237,14 +240,6 @@ ynh_panel_get() { _ynh_panel_get } -ynh_panel_init() { - declare -A old=() - declare -A changed=() - declare -A file_hash=() - - ynh_panel_get -} - ynh_panel_show() { _ynh_panel_show } @@ -257,3 +252,15 @@ ynh_panel_apply() { _ynh_panel_apply } +ynh_panel_run() { + declare -A old=() + declare -A changed=() + declare -A file_hash=() + + ynh_panel_get + case $1 in + show) ynh_panel_show;; + apply) ynh_panel_validate && ynh_panel_apply;; + esac +} + From 619b26f73c2356b5d256909c84142c64a8b06020 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 13 Aug 2021 13:38:06 +0200 Subject: [PATCH 007/119] [fix] tons of things --- data/helpers.d/configpanel | 110 ++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 45 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 83130cfe6..5ab199aea 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -99,13 +99,13 @@ ynh_value_set() { } _ynh_panel_get() { - + set +x # From settings local params_sources params_sources=`python3 << EOL import toml from collections import OrderedDict -with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f: +with open("../config_panel.toml", "r") as f: file_content = f.read() loaded_toml = toml.loads(file_content, _dict=OrderedDict) @@ -114,20 +114,23 @@ for panel_name,panel in loaded_toml.items(): for section_name, section in panel.items(): if isinstance(section, dict): for name, param in section.items(): - if isinstance(param, dict) and param.get('source', '') == 'settings': + if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']: print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings'))) EOL ` - for param_source in params_sources + for param_source in $params_sources do local _dot_setting=$(echo "$param_source" | cut -d= -f1) - local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)" + local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $_dot_setting)" local short_setting=$(echo "$_dot_setting" | cut -d. -f3) local getter="get__${short_setting}" local source="$(echo $param_source | cut -d= -f2)" + sources[${short_setting}]="$source" + file_hash[${short_setting}]="" + dot_settings[${short_setting}]="${_dot_setting}" # Get value from getter if exists - if type $getter | grep -q '^function$' 2>/dev/null; then + if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then old[$short_setting]="$($getter)" # Get value from app settings @@ -144,38 +147,43 @@ EOL # Specific case for files (all content of the file is the source) else old[$short_setting]="$source" + file_hash[$short_setting]="true" fi - + set +u + new[$short_setting]="${!_snake_setting}" + set -u done + set -x } _ynh_panel_apply() { - for short_setting in "${!dot_settings[@]}" + for short_setting in "${!old[@]}" do local setter="set__${short_setting}" - local source="$sources[$short_setting]" - - # Apply setter if exists - if type $setter | grep -q '^function$' 2>/dev/null; then - $setter + local source="${sources[$short_setting]}" + if [ "${changed[$short_setting]}" == "true" ] ; then + # Apply setter if exists + if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + $setter - # Copy file in right place - elif [[ "$source" == "settings" ]] ; then - ynh_app_setting_set $app $short_setting "$new[$short_setting]" - - # Get value from a kind of key/value file - elif [[ "$source" == *":"* ]] - then - local source_key="$(echo "$source" | cut -d: -f1)" - source_key=${source_key:-$short_setting} - local source_file="$(echo "$source" | cut -d: -f2)" - ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]" + # Copy file in right place + elif [[ "$source" == "settings" ]] ; then + ynh_app_setting_set $app $short_setting "${new[$short_setting]}" + + # Get value from a kind of key/value file + elif [[ "$source" == *":"* ]] + then + local source_key="$(echo "$source" | cut -d: -f1)" + source_key=${source_key:-$short_setting} + local source_file="$(echo "$source" | cut -d: -f2)" + ynh_value_set --file="${source_file}" --key="${source_key}" --value="${new[$short_setting]}" - # Specific case for files (all content of the file is the source) - else - cp "$new[$short_setting]" "$source" + # Specific case for files (all content of the file is the source) + else + cp "${new[$short_setting]}" "$source" + fi fi done } @@ -189,24 +197,32 @@ _ynh_panel_show() { } _ynh_panel_validate() { + set +x # Change detection local is_error=true #for changed_status in "${!changed[@]}" - for short_setting in "${!dot_settings[@]}" + for short_setting in "${!old[@]}" do - #TODO file hash - file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) - file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) - if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] - then - changed[$setting]=true - fi - if [[ "$new[$short_setting]" == "$old[$short_setting]" ]] - then - changed[$short_setting]=false + changed[$short_setting]=false + if [ ! -z "${file_hash[${short_setting}]}" ] ; then + file_hash[old__$short_setting]="" + file_hash[new__$short_setting]="" + if [ -f "${old[$short_setting]}" ] ; then + file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) + fi + if [ -f "${new[$short_setting]}" ] ; then + file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1) + if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] + then + changed[$short_setting]=true + fi + fi else - changed[$short_setting]=true - is_error=false + if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]] + then + changed[$short_setting]=true + is_error=false + fi fi done @@ -214,10 +230,10 @@ _ynh_panel_validate() { if [[ "$is_error" == "false" ]] then - for short_setting in "${!dot_settings[@]}" + for short_setting in "${!old[@]}" do local result="" - if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then + if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then result="$(validate__$short_setting)" fi if [ -n "$result" ] @@ -233,6 +249,7 @@ _ynh_panel_validate() { then ynh_die fi + set -x } @@ -253,9 +270,12 @@ ynh_panel_apply() { } ynh_panel_run() { - declare -A old=() - declare -A changed=() - declare -A file_hash=() + declare -Ag old=() + declare -Ag new=() + declare -Ag changed=() + declare -Ag file_hash=() + declare -Ag sources=() + declare -Ag dot_settings=() ynh_panel_get case $1 in From 596d05ae81d24712c87ec0de72f8deb8248bca9a Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 14 Aug 2021 14:38:45 +0200 Subject: [PATCH 008/119] [fix] Missing name or bad format management --- src/yunohost/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 49033d8b4..cf218823f 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2145,7 +2145,7 @@ def _get_app_config_panel(app_id): for key, value in panels: panel = { "id": key, - "name": value["name"], + "name": value.get("name", ""), "sections": [], } @@ -2158,7 +2158,7 @@ def _get_app_config_panel(app_id): for section_key, section_value in sections: section = { "id": section_key, - "name": section_value["name"], + "name": section_value.get("name", ""), "options": [], } From d8cdc20e0ec4f3af992d3e7f22ed1d9218bd38c2 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 11 May 2020 19:13:10 +0200 Subject: [PATCH 009/119] [wip] Allow file upload from config-panel --- src/yunohost/app.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a48400a8e..a8cd6aa40 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -33,6 +33,7 @@ import re import subprocess import glob import urllib.parse +import base64 import tempfile from collections import OrderedDict @@ -1874,6 +1875,7 @@ def app_config_apply(operation_logger, app, args): } args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} + upload_dir = None for tab in config_panel.get("panel", []): tab_id = tab["id"] # this makes things easier to debug on crash for section in tab.get("sections", []): @@ -1885,6 +1887,23 @@ def app_config_apply(operation_logger, app, args): ).upper() if generated_name in args: + # Upload files from API + # A file arg contains a string with "FILENAME:BASE64_CONTENT" + if option["type"] == "file" and msettings.get('interface') == 'api': + if upload_dir is None: + upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') + filename, args[generated_name] = args[generated_name].split(':') + logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) + file_path = os.join(upload_dir, filename) + try: + with open(file_path, 'wb') as f: + f.write(args[generated_name]) + except IOError as e: + raise YunohostError("cannot_write_file", file=file_path, error=str(e)) + except Exception as e: + raise YunohostError("error_writing_file", file=file_path, error=str(e)) + args[generated_name] = file_path + logger.debug( "include into env %s=%s", generated_name, args[generated_name] ) @@ -1906,6 +1925,11 @@ def app_config_apply(operation_logger, app, args): env=env, )[0] + # Delete files uploaded from API + if msettings.get('interface') == 'api': + if upload_dir is not None: + shutil.rmtree(upload_dir) + if return_code != 0: msg = ( "'script/config apply' return value code: %s (considered as an error)" From fb0d23533e2712f4c08e09b9febae1ced8877459 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 8 Jun 2020 19:18:22 +0200 Subject: [PATCH 010/119] [fix] Several files with same name --- src/yunohost/app.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a8cd6aa40..1a0ee9087 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1889,15 +1889,21 @@ def app_config_apply(operation_logger, app, args): if generated_name in args: # Upload files from API # A file arg contains a string with "FILENAME:BASE64_CONTENT" - if option["type"] == "file" and msettings.get('interface') == 'api': + if 'type' in option and option["type"] == "file" \ + and msettings.get('interface') == 'api': if upload_dir is None: upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') - filename, args[generated_name] = args[generated_name].split(':') + filename = args[generated_name + '[name]'] + content = args[generated_name] logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) - file_path = os.join(upload_dir, filename) + file_path = os.path.join(upload_dir, filename) + i = 2 + while os.path.exists(file_path): + file_path = os.path.join(upload_dir, filename + (".%d" % i)) + i += 1 try: with open(file_path, 'wb') as f: - f.write(args[generated_name]) + f.write(content.decode("base64")) except IOError as e: raise YunohostError("cannot_write_file", file=file_path, error=str(e)) except Exception as e: @@ -1914,7 +1920,7 @@ def app_config_apply(operation_logger, app, args): # for debug purpose for key in args: if key not in env: - logger.warning( + logger.debug( "Ignore key '%s' from arguments because it is not in the config", key ) From 975bf4edcbdeabd2f9487dca2f9e56926eb1f64b Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 7 Oct 2020 00:31:20 +0200 Subject: [PATCH 011/119] [enh] Replace os.path.join to improve security --- src/yunohost/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 1a0ee9087..324159859 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1896,10 +1896,14 @@ def app_config_apply(operation_logger, app, args): filename = args[generated_name + '[name]'] content = args[generated_name] logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) - file_path = os.path.join(upload_dir, filename) + + # Filename is given by user of the API. For security reason, we have replaced + # os.path.join to avoid the user to be able to rewrite a file in filesystem + # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" + file_path = os.path.normpath(upload_dir + "/" + filename) i = 2 while os.path.exists(file_path): - file_path = os.path.join(upload_dir, filename + (".%d" % i)) + file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) i += 1 try: with open(file_path, 'wb') as f: From 284554598521af8305f521b0eeabf2a42f95167e Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 31 May 2021 16:32:19 +0200 Subject: [PATCH 012/119] [fix] Base64 python3 change --- src/yunohost/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 324159859..4006c1ec4 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1852,7 +1852,7 @@ def app_config_apply(operation_logger, app, args): logger.warning(m18n.n("experimental_feature")) from yunohost.hook import hook_exec - + from base64 import b64decode installed = _is_installed(app) if not installed: raise YunohostValidationError( @@ -1896,8 +1896,8 @@ def app_config_apply(operation_logger, app, args): filename = args[generated_name + '[name]'] content = args[generated_name] logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) - - # Filename is given by user of the API. For security reason, we have replaced + + # Filename is given by user of the API. For security reason, we have replaced # os.path.join to avoid the user to be able to rewrite a file in filesystem # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" file_path = os.path.normpath(upload_dir + "/" + filename) @@ -1907,7 +1907,7 @@ def app_config_apply(operation_logger, app, args): i += 1 try: with open(file_path, 'wb') as f: - f.write(content.decode("base64")) + f.write(b64decode(content)) except IOError as e: raise YunohostError("cannot_write_file", file=file_path, error=str(e)) except Exception as e: From 1c636fe1ca8b4838ad99e8f69358e8b8e9e4e4c7 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 1 Jun 2021 00:41:37 +0200 Subject: [PATCH 013/119] [enh] Add configpanel helpers --- data/helpers.d/configpanel | 259 +++++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 data/helpers.d/configpanel diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel new file mode 100644 index 000000000..f648826e4 --- /dev/null +++ b/data/helpers.d/configpanel @@ -0,0 +1,259 @@ +#!/bin/bash + +ynh_lowerdot_to_uppersnake() { + local lowerdot + lowerdot=$(echo "$1" | cut -d= -f1 | sed "s/\./_/g") + echo "${lowerdot^^}" +} + +# Get a value from heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_value_get --file=PATH --key=KEY +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to get +# +# This helpers match several var affectation use case in several languages +# We don't use jq or equivalent to keep comments and blank space in files +# This helpers work line by line, it is not able to work correctly +# if you have several identical keys in your files +# +# Example of line this helpers can managed correctly +# .yml +# title: YunoHost documentation +# email: 'yunohost@yunohost.org' +# .json +# "theme": "colib'ris", +# "port": 8102 +# "some_boolean": false, +# "user": null +# .ini +# some_boolean = On +# action = "Clear" +# port = 20 +# .php +# $user= +# user => 20 +# .py +# USER = 8102 +# user = 'https://donate.local' +# CUSTOM['user'] = 'YunoHost' +# Requires YunoHost version 4.3 or higher. +ynh_value_get() { + # Declare an array to define the options of this helper. + local legacy_args=fk + local -A args_array=( [f]=file= [k]=key= ) + local file + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + + local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + + local first_char="${crazy_value:0:1}" + if [[ "$first_char" == '"' ]] ; then + echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + elif [[ "$first_char" == "'" ]] ; then + echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + else + echo "$crazy_value" + fi +} + +# Set a value into heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_value_set --file=PATH --key=KEY --value=VALUE +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to set +# | arg: -v, --value= - the value to set +# +# Requires YunoHost version 4.3 or higher. +ynh_value_set() { + # Declare an array to define the options of this helper. + local legacy_args=fkv + local -A args_array=( [f]=file= [k]=key= [v]=value=) + local file + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + + local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + local first_char="${crazy_value:0:1}" + if [[ "$first_char" == '"' ]] ; then + value="$(echo "$value" | sed 's/"/\\"/g')" + sed -ri "s%^(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file} + elif [[ "$first_char" == "'" ]] ; then + value="$(echo "$value" | sed "s/'/\\\\'/g")" + sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + else + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then + value="\"$(echo "$value" | sed 's/"/\\"/g')\"" + fi + sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + fi +} + +_ynh_panel_get() { + + # From settings + local params_sources + params_sources=`python3 << EOL +import toml +from collections import OrderedDict +with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f: + file_content = f.read() +loaded_toml = toml.loads(file_content, _dict=OrderedDict) + +for panel_name,panel in loaded_toml.items(): + if isinstance(panel, dict): + for section_name, section in panel.items(): + if isinstance(section, dict): + for name, param in section.items(): + if isinstance(param, dict) and param.get('source', '') == 'settings': + print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings'))) +EOL +` + for param_source in params_sources + do + local _dot_setting=$(echo "$param_source" | cut -d= -f1) + local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)" + local short_setting=$(echo "$_dot_setting" | cut -d. -f3) + local _getter="get__${short_setting}" + local source="$(echo $param_source | cut -d= -f2)" + + # Get value from getter if exists + if type $getter | grep -q '^function$' 2>/dev/null; then + old[$short_setting]="$($getter)" + + # Get value from app settings + elif [[ "$source" == "settings" ]] ; then + old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" + + # Get value from a kind of key/value file + elif [[ "$source" == *":"* ]] ; then + local source_key="$(echo "$source" | cut -d: -f1)" + source_key=${source_key:-$short_setting} + local source_file="$(echo "$source" | cut -d: -f2)" + old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")" + + # Specific case for files (all content of the file is the source) + else + old[$short_setting]="$source" + fi + + done + + +} + +_ynh_panel_apply() { + for short_setting in "${!dot_settings[@]}" + do + local setter="set__${short_setting}" + local source="$sources[$short_setting]" + + # Apply setter if exists + if type $setter | grep -q '^function$' 2>/dev/null; then + $setter + + # Copy file in right place + elif [[ "$source" == "settings" ]] ; then + ynh_app_setting_set $app $short_setting "$new[$short_setting]" + + # Get value from a kind of key/value file + elif [[ "$source" == *":"* ]] + then + local source_key="$(echo "$source" | cut -d: -f1)" + source_key=${source_key:-$short_setting} + local source_file="$(echo "$source" | cut -d: -f2)" + ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]" + + # Specific case for files (all content of the file is the source) + else + cp "$new[$short_setting]" "$source" + fi + done +} + +_ynh_panel_show() { + for short_setting in "${!old[@]}" + do + local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" + ynh_return "$key=${old[$short_setting]}" + done +} + +_ynh_panel_validate() { + # Change detection + local is_error=true + #for changed_status in "${!changed[@]}" + for short_setting in "${!dot_settings[@]}" + do + #TODO file hash + file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) + file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) + if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] + then + changed[$setting]=true + fi + if [[ "$new[$short_setting]" == "$old[$short_setting]" ]] + then + changed[$short_setting]=false + else + changed[$short_setting]=true + is_error=false + fi + done + + # Run validation if something is changed + if [[ "$is_error" == "false" ]] + then + + for short_setting in "${!dot_settings[@]}" + do + local result="$(validate__$short_setting)" + local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" + if [ -n "$result" ] + then + ynh_return "$key=$result" + is_error=true + fi + done + fi + + if [[ "$is_error" == "true" ]] + then + ynh_die + fi + +} + +ynh_panel_get() { + _ynh_panel_get +} + +ynh_panel_init() { + declare -A old=() + declare -A changed=() + declare -A file_hash=() + + ynh_panel_get +} + +ynh_panel_show() { + _ynh_panel_show +} + +ynh_panel_validate() { + _ynh_panel_validate +} + +ynh_panel_apply() { + _ynh_panel_apply +} + From caf2a9d6d13b9c5be0068585d0a3617c2fdc35a7 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 1 Jun 2021 01:29:26 +0200 Subject: [PATCH 014/119] [fix] No validate function in config panel --- data/helpers.d/configpanel | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index f648826e4..83130cfe6 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -123,7 +123,7 @@ EOL local _dot_setting=$(echo "$param_source" | cut -d= -f1) local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)" local short_setting=$(echo "$_dot_setting" | cut -d. -f3) - local _getter="get__${short_setting}" + local getter="get__${short_setting}" local source="$(echo $param_source | cut -d= -f2)" # Get value from getter if exists @@ -195,12 +195,12 @@ _ynh_panel_validate() { for short_setting in "${!dot_settings[@]}" do #TODO file hash - file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) - file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) - if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] - then - changed[$setting]=true - fi + file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) + file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) + if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] + then + changed[$setting]=true + fi if [[ "$new[$short_setting]" == "$old[$short_setting]" ]] then changed[$short_setting]=false @@ -216,10 +216,13 @@ _ynh_panel_validate() { for short_setting in "${!dot_settings[@]}" do - local result="$(validate__$short_setting)" - local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" + local result="" + if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then + result="$(validate__$short_setting)" + fi if [ -n "$result" ] then + local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" ynh_return "$key=$result" is_error=true fi @@ -237,14 +240,6 @@ ynh_panel_get() { _ynh_panel_get } -ynh_panel_init() { - declare -A old=() - declare -A changed=() - declare -A file_hash=() - - ynh_panel_get -} - ynh_panel_show() { _ynh_panel_show } @@ -257,3 +252,15 @@ ynh_panel_apply() { _ynh_panel_apply } +ynh_panel_run() { + declare -A old=() + declare -A changed=() + declare -A file_hash=() + + ynh_panel_get + case $1 in + show) ynh_panel_show;; + apply) ynh_panel_validate && ynh_panel_apply;; + esac +} + From ed0915cf81a42c1ae12b9897cd5b2827c5b96ff6 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 13 Aug 2021 13:38:06 +0200 Subject: [PATCH 015/119] [fix] tons of things --- data/helpers.d/configpanel | 110 ++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 45 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 83130cfe6..5ab199aea 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -99,13 +99,13 @@ ynh_value_set() { } _ynh_panel_get() { - + set +x # From settings local params_sources params_sources=`python3 << EOL import toml from collections import OrderedDict -with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f: +with open("../config_panel.toml", "r") as f: file_content = f.read() loaded_toml = toml.loads(file_content, _dict=OrderedDict) @@ -114,20 +114,23 @@ for panel_name,panel in loaded_toml.items(): for section_name, section in panel.items(): if isinstance(section, dict): for name, param in section.items(): - if isinstance(param, dict) and param.get('source', '') == 'settings': + if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']: print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings'))) EOL ` - for param_source in params_sources + for param_source in $params_sources do local _dot_setting=$(echo "$param_source" | cut -d= -f1) - local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)" + local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $_dot_setting)" local short_setting=$(echo "$_dot_setting" | cut -d. -f3) local getter="get__${short_setting}" local source="$(echo $param_source | cut -d= -f2)" + sources[${short_setting}]="$source" + file_hash[${short_setting}]="" + dot_settings[${short_setting}]="${_dot_setting}" # Get value from getter if exists - if type $getter | grep -q '^function$' 2>/dev/null; then + if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then old[$short_setting]="$($getter)" # Get value from app settings @@ -144,38 +147,43 @@ EOL # Specific case for files (all content of the file is the source) else old[$short_setting]="$source" + file_hash[$short_setting]="true" fi - + set +u + new[$short_setting]="${!_snake_setting}" + set -u done + set -x } _ynh_panel_apply() { - for short_setting in "${!dot_settings[@]}" + for short_setting in "${!old[@]}" do local setter="set__${short_setting}" - local source="$sources[$short_setting]" - - # Apply setter if exists - if type $setter | grep -q '^function$' 2>/dev/null; then - $setter + local source="${sources[$short_setting]}" + if [ "${changed[$short_setting]}" == "true" ] ; then + # Apply setter if exists + if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + $setter - # Copy file in right place - elif [[ "$source" == "settings" ]] ; then - ynh_app_setting_set $app $short_setting "$new[$short_setting]" - - # Get value from a kind of key/value file - elif [[ "$source" == *":"* ]] - then - local source_key="$(echo "$source" | cut -d: -f1)" - source_key=${source_key:-$short_setting} - local source_file="$(echo "$source" | cut -d: -f2)" - ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]" + # Copy file in right place + elif [[ "$source" == "settings" ]] ; then + ynh_app_setting_set $app $short_setting "${new[$short_setting]}" + + # Get value from a kind of key/value file + elif [[ "$source" == *":"* ]] + then + local source_key="$(echo "$source" | cut -d: -f1)" + source_key=${source_key:-$short_setting} + local source_file="$(echo "$source" | cut -d: -f2)" + ynh_value_set --file="${source_file}" --key="${source_key}" --value="${new[$short_setting]}" - # Specific case for files (all content of the file is the source) - else - cp "$new[$short_setting]" "$source" + # Specific case for files (all content of the file is the source) + else + cp "${new[$short_setting]}" "$source" + fi fi done } @@ -189,24 +197,32 @@ _ynh_panel_show() { } _ynh_panel_validate() { + set +x # Change detection local is_error=true #for changed_status in "${!changed[@]}" - for short_setting in "${!dot_settings[@]}" + for short_setting in "${!old[@]}" do - #TODO file hash - file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1) - file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1) - if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]] - then - changed[$setting]=true - fi - if [[ "$new[$short_setting]" == "$old[$short_setting]" ]] - then - changed[$short_setting]=false + changed[$short_setting]=false + if [ ! -z "${file_hash[${short_setting}]}" ] ; then + file_hash[old__$short_setting]="" + file_hash[new__$short_setting]="" + if [ -f "${old[$short_setting]}" ] ; then + file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) + fi + if [ -f "${new[$short_setting]}" ] ; then + file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1) + if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] + then + changed[$short_setting]=true + fi + fi else - changed[$short_setting]=true - is_error=false + if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]] + then + changed[$short_setting]=true + is_error=false + fi fi done @@ -214,10 +230,10 @@ _ynh_panel_validate() { if [[ "$is_error" == "false" ]] then - for short_setting in "${!dot_settings[@]}" + for short_setting in "${!old[@]}" do local result="" - if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then + if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then result="$(validate__$short_setting)" fi if [ -n "$result" ] @@ -233,6 +249,7 @@ _ynh_panel_validate() { then ynh_die fi + set -x } @@ -253,9 +270,12 @@ ynh_panel_apply() { } ynh_panel_run() { - declare -A old=() - declare -A changed=() - declare -A file_hash=() + declare -Ag old=() + declare -Ag new=() + declare -Ag changed=() + declare -Ag file_hash=() + declare -Ag sources=() + declare -Ag dot_settings=() ynh_panel_get case $1 in From 65fc06e3e743ed6bff30ace4ca0800c54e58b253 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 14 Aug 2021 14:38:45 +0200 Subject: [PATCH 016/119] [fix] Missing name or bad format management --- src/yunohost/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4006c1ec4..d8e4748f6 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2152,7 +2152,7 @@ def _get_app_config_panel(app_id): for key, value in panels: panel = { "id": key, - "name": value["name"], + "name": value.get("name", ""), "sections": [], } @@ -2165,7 +2165,7 @@ def _get_app_config_panel(app_id): for section_key, section_value in sections: section = { "id": section_key, - "name": section_value["name"], + "name": section_value.get("name", ""), "options": [], } From fddb79e841470a58f8bea31293417f80c3e1dd7b Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 17 Aug 2021 17:07:26 +0200 Subject: [PATCH 017/119] [wip] Reduce config panel var --- data/helpers.d/configpanel | 18 ++---- src/yunohost/app.py | 116 ++++++++++++++++++------------------- 2 files changed, 60 insertions(+), 74 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 5ab199aea..685f30a98 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -115,19 +115,16 @@ for panel_name,panel in loaded_toml.items(): if isinstance(section, dict): for name, param in section.items(): if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']: - print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings'))) + print("%s=%s" % (name, param.get('source', 'settings'))) EOL ` for param_source in $params_sources do - local _dot_setting=$(echo "$param_source" | cut -d= -f1) - local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $_dot_setting)" - local short_setting=$(echo "$_dot_setting" | cut -d. -f3) + local short_setting="$(echo $param_source | cut -d= -f1)" local getter="get__${short_setting}" local source="$(echo $param_source | cut -d= -f2)" sources[${short_setting}]="$source" file_hash[${short_setting}]="" - dot_settings[${short_setting}]="${_dot_setting}" # Get value from getter if exists if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then @@ -149,9 +146,6 @@ EOL old[$short_setting]="$source" file_hash[$short_setting]="true" fi - set +u - new[$short_setting]="${!_snake_setting}" - set -u done set -x @@ -191,8 +185,7 @@ _ynh_panel_apply() { _ynh_panel_show() { for short_setting in "${!old[@]}" do - local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" - ynh_return "$key=${old[$short_setting]}" + ynh_return "${short_setting}=${old[$short_setting]}" done } @@ -238,7 +231,7 @@ _ynh_panel_validate() { fi if [ -n "$result" ] then - local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" + local key="YNH_ERROR_${$short_setting}" ynh_return "$key=$result" is_error=true fi @@ -247,7 +240,7 @@ _ynh_panel_validate() { if [[ "$is_error" == "true" ]] then - ynh_die + ynh_die "" fi set -x @@ -275,7 +268,6 @@ ynh_panel_run() { declare -Ag changed=() declare -Ag file_hash=() declare -Ag sources=() - declare -Ag dot_settings=() ynh_panel_get case $1 in diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d8e4748f6..faa5098c9 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1782,59 +1782,49 @@ def app_config_show_panel(operation_logger, app): } env = { - "YNH_APP_ID": app_id, - "YNH_APP_INSTANCE_NAME": app, - "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), + "app_id": app_id, + "app": app, + "app_instance_nb": str(app_instance_nb), } - # FIXME: this should probably be ran in a tmp workdir... - return_code, parsed_values = hook_exec( - config_script, args=["show"], env=env, return_format="plain_dict" - ) - - if return_code != 0: - raise Exception( - "script/config show return value code: %s (considered as an error)", - return_code, + try: + ret, parsed_values = hook_exec( + config_script, args=["show"], env=env, return_format="plain_dict" ) + # Here again, calling hook_exec could fail miserably, or get + # manually interrupted (by mistake or because script was stuck) + except (KeyboardInterrupt, EOFError, Exception): + raise YunohostError("unexpected_error") logger.debug("Generating global variables:") for tab in config_panel.get("panel", []): - tab_id = tab["id"] # this makes things easier to debug on crash for section in tab.get("sections", []): - section_id = section["id"] for option in section.get("options", []): - option_name = option["name"] - generated_name = ( - "YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name) - ).upper() - option["name"] = generated_name logger.debug( - " * '%s'.'%s'.'%s' -> %s", + " * '%s'.'%s'.'%s'", tab.get("name"), section.get("name"), option.get("name"), - generated_name, ) - if generated_name in parsed_values: + if option['name'] in parsed_values: # code is not adapted for that so we have to mock expected format :/ if option.get("type") == "boolean": - if parsed_values[generated_name].lower() in ("true", "1", "y"): - option["default"] = parsed_values[generated_name] + if parsed_values[option['name']].lower() in ("true", "1", "y"): + option["default"] = parsed_values[option['name']] else: del option["default"] else: - option["default"] = parsed_values[generated_name] + option["default"] = parsed_values[option['name']] args_dict = _parse_args_in_yunohost_format( - {option["name"]: parsed_values[generated_name]}, [option] + parsed_values, [option] ) option["default"] = args_dict[option["name"]][0] else: logger.debug( "Variable '%s' is not declared by config script, using default", - generated_name, + option['name'], ) # do nothing, we'll use the default if present @@ -1869,32 +1859,26 @@ def app_config_apply(operation_logger, app, args): operation_logger.start() app_id, app_instance_nb = _parse_app_instance_name(app) env = { - "YNH_APP_ID": app_id, - "YNH_APP_INSTANCE_NAME": app, - "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), + "app_id": app_id, + "app": app, + "app_instance_nb": str(app_instance_nb), } args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} upload_dir = None for tab in config_panel.get("panel", []): - tab_id = tab["id"] # this makes things easier to debug on crash for section in tab.get("sections", []): - section_id = section["id"] for option in section.get("options", []): - option_name = option["name"] - generated_name = ( - "YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name) - ).upper() - if generated_name in args: + if option['name'] in args: # Upload files from API # A file arg contains a string with "FILENAME:BASE64_CONTENT" if 'type' in option and option["type"] == "file" \ and msettings.get('interface') == 'api': if upload_dir is None: upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') - filename = args[generated_name + '[name]'] - content = args[generated_name] + filename = args[option['name'] + '[name]'] + content = args[option['name']] logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) # Filename is given by user of the API. For security reason, we have replaced @@ -1912,14 +1896,14 @@ def app_config_apply(operation_logger, app, args): raise YunohostError("cannot_write_file", file=file_path, error=str(e)) except Exception as e: raise YunohostError("error_writing_file", file=file_path, error=str(e)) - args[generated_name] = file_path + args[option['name']] = file_path logger.debug( - "include into env %s=%s", generated_name, args[generated_name] + "include into env %s=%s", option['name'], args[option['name']] ) - env[generated_name] = args[generated_name] + env[option['name']] = args[option['name']] else: - logger.debug("no value for key id %s", generated_name) + logger.debug("no value for key id %s", option['name']) # for debug purpose for key in args: @@ -1928,25 +1912,21 @@ def app_config_apply(operation_logger, app, args): "Ignore key '%s' from arguments because it is not in the config", key ) - # FIXME: this should probably be ran in a tmp workdir... - return_code = hook_exec( - config_script, - args=["apply"], - env=env, - )[0] - - # Delete files uploaded from API - if msettings.get('interface') == 'api': - if upload_dir is not None: - shutil.rmtree(upload_dir) - - if return_code != 0: - msg = ( - "'script/config apply' return value code: %s (considered as an error)" - % return_code + try: + hook_exec( + config_script, + args=["apply"], + env=env ) - operation_logger.error(msg) - raise Exception(msg) + # Here again, calling hook_exec could fail miserably, or get + # manually interrupted (by mistake or because script was stuck) + except (KeyboardInterrupt, EOFError, Exception): + raise YunohostError("unexpected_error") + finally: + # Delete files uploaded from API + if msettings.get('interface') == 'api': + if upload_dir is not None: + shutil.rmtree(upload_dir) logger.success("Config updated as expected") return { @@ -2991,16 +2971,30 @@ class DisplayTextArgumentParser(YunoHostArgumentFormatParser): def parse(self, question, user_answers): print(question["ask"]) +class FileArgumentParser(YunoHostArgumentFormatParser): + argument_type = "file" + + ARGUMENTS_TYPE_PARSERS = { "string": StringArgumentParser, + "text": StringArgumentParser, + "select": StringArgumentParser, + "tags": StringArgumentParser, + "email": StringArgumentParser, + "url": StringArgumentParser, + "date": StringArgumentParser, + "time": StringArgumentParser, + "color": StringArgumentParser, "password": PasswordArgumentParser, "path": PathArgumentParser, "boolean": BooleanArgumentParser, "domain": DomainArgumentParser, "user": UserArgumentParser, "number": NumberArgumentParser, + "range": NumberArgumentParser, "display_text": DisplayTextArgumentParser, + "file": FileArgumentParser, } From a89dd4827c1f072b0e808b6b4b669909aad58179 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 20 Aug 2021 16:35:02 +0200 Subject: [PATCH 018/119] [enh] Use yaml for reading hook output --- src/yunohost/hook.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 33f5885e2..4d497de76 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -34,7 +34,7 @@ from importlib import import_module from moulinette import m18n, msettings from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils import log -from moulinette.utils.filesystem import read_json +from moulinette.utils.filesystem import read_yaml HOOK_FOLDER = "/usr/share/yunohost/hooks/" CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/" @@ -326,7 +326,7 @@ def hook_exec( chdir=None, env=None, user="root", - return_format="json", + return_format="yaml", ): """ Execute hook from a file with arguments @@ -447,10 +447,10 @@ def _hook_exec_bash(path, args, chdir, env, user, return_format, loggers): raw_content = f.read() returncontent = {} - if return_format == "json": + if return_format == "yaml": if raw_content != "": try: - returncontent = read_json(stdreturn) + returncontent = read_yaml(stdreturn) except Exception as e: raise YunohostError( "hook_json_return_error", From 98ca514f8fdff264eda071dbeb5244e8548e1657 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 20 Aug 2021 16:35:51 +0200 Subject: [PATCH 019/119] [enh] Rewrite config show, get, set actions --- data/actionsmap/yunohost.yml | 45 +++- data/helpers.d/configpanel | 29 +-- locales/en.json | 8 +- src/yunohost/app.py | 444 ++++++++++++++++++++++------------- 4 files changed, 327 insertions(+), 199 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 5df1c0877..d9e3a50d0 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -831,24 +831,47 @@ app: subcategory_help: Applications configuration panel actions: - ### app_config_show_panel() - show-panel: + ### app_config_show() + show: action_help: show config panel for the application api: GET /apps//config-panel arguments: - app: - help: App name + app: + help: App name + panel: + help: Select a specific panel + nargs: '?' + -f: + full: --full + help: Display all info known about the config-panel. + action: store_true - ### app_config_apply() - apply: + ### app_config_get() + get: + action_help: show config panel for the application + api: GET /apps//config-panel/ + arguments: + app: + help: App name + key: + help: The question identifier + + ### app_config_set() + set: action_help: apply the new configuration api: PUT /apps//config arguments: - app: - help: App name - -a: - full: --args - help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path") + app: + help: App name + key: + help: The question or panel key + nargs: '?' + -v: + full: --value + help: new value + -a: + full: --args + help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path") ############################# # Backup # diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 685f30a98..5b290629d 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -1,10 +1,5 @@ #!/bin/bash -ynh_lowerdot_to_uppersnake() { - local lowerdot - lowerdot=$(echo "$1" | cut -d= -f1 | sed "s/\./_/g") - echo "${lowerdot^^}" -} # Get a value from heterogeneous file (yaml, json, php, python...) # @@ -99,7 +94,6 @@ ynh_value_set() { } _ynh_panel_get() { - set +x # From settings local params_sources params_sources=`python3 << EOL @@ -114,7 +108,7 @@ for panel_name,panel in loaded_toml.items(): for section_name, section in panel.items(): if isinstance(section, dict): for name, param in section.items(): - if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']: + if isinstance(param, dict) and param.get('type', 'string') not in ['success', 'info', 'warning', 'danger', 'display_text', 'markdown']: print("%s=%s" % (name, param.get('source', 'settings'))) EOL ` @@ -147,7 +141,6 @@ EOL file_hash[$short_setting]="true" fi done - set -x } @@ -164,7 +157,7 @@ _ynh_panel_apply() { # Copy file in right place elif [[ "$source" == "settings" ]] ; then - ynh_app_setting_set $app $short_setting "${new[$short_setting]}" + ynh_app_setting_set $app $short_setting "${!short_setting}" # Get value from a kind of key/value file elif [[ "$source" == *":"* ]] @@ -172,11 +165,11 @@ _ynh_panel_apply() { local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2)" - ynh_value_set --file="${source_file}" --key="${source_key}" --value="${new[$short_setting]}" + ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}" # Specific case for files (all content of the file is the source) else - cp "${new[$short_setting]}" "$source" + cp "${!short_setting}" "$source" fi fi done @@ -185,25 +178,25 @@ _ynh_panel_apply() { _ynh_panel_show() { for short_setting in "${!old[@]}" do - ynh_return "${short_setting}=${old[$short_setting]}" + ynh_return "${short_setting}: \"${old[$short_setting]}\"" done } _ynh_panel_validate() { - set +x # Change detection local is_error=true #for changed_status in "${!changed[@]}" for short_setting in "${!old[@]}" do changed[$short_setting]=false + [ -z ${!short_setting+x} ] && continue if [ ! -z "${file_hash[${short_setting}]}" ] ; then file_hash[old__$short_setting]="" file_hash[new__$short_setting]="" if [ -f "${old[$short_setting]}" ] ; then file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) fi - if [ -f "${new[$short_setting]}" ] ; then + if [ -f "${!short_setting}" ] ; then file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1) if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] then @@ -211,7 +204,7 @@ _ynh_panel_validate() { fi fi else - if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]] + if [[ "${!short_setting}" != "${old[$short_setting]}" ]] then changed[$short_setting]=true is_error=false @@ -242,7 +235,6 @@ _ynh_panel_validate() { then ynh_die "" fi - set -x } @@ -264,15 +256,14 @@ ynh_panel_apply() { ynh_panel_run() { declare -Ag old=() - declare -Ag new=() declare -Ag changed=() declare -Ag file_hash=() declare -Ag sources=() ynh_panel_get case $1 in - show) ynh_panel_show;; - apply) ynh_panel_validate && ynh_panel_apply;; + show) ynh_panel_get && ynh_panel_show;; + apply) ynh_panel_get && ynh_panel_validate && ynh_panel_apply;; esac } diff --git a/locales/en.json b/locales/en.json index 693e9d24d..1f13b3e90 100644 --- a/locales/en.json +++ b/locales/en.json @@ -13,7 +13,7 @@ "app_already_installed": "{app} is already installed", "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.", "app_already_up_to_date": "{app} is already up-to-date", - "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}'", + "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'", "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason", "app_argument_required": "Argument '{name}' is required", @@ -143,6 +143,7 @@ "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system… If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system… If you are willing to take that risk anyway, type '{answers}'", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app}", + "danger": "Danger:", "diagnosis_basesystem_hardware": "Server hardware architecture is {virt} {arch}", "diagnosis_basesystem_hardware_model": "Server model is {model}", "diagnosis_basesystem_host": "Server is running Debian {debian_version}", @@ -382,8 +383,9 @@ "log_app_upgrade": "Upgrade the '{}' app", "log_app_makedefault": "Make '{}' the default app", "log_app_action_run": "Run action of the '{}' app", - "log_app_config_show_panel": "Show the config panel of the '{}' app", - "log_app_config_apply": "Apply config to the '{}' app", + "log_app_config_show": "Show the config panel of the '{}' app", + "log_app_config_get": "Get a specific setting from config panel of the '{}' app", + "log_app_config_set": "Apply config to the '{}' app", "log_available_on_yunopaste": "This log is now available via {url}", "log_backup_create": "Create a backup archive", "log_backup_restore_system": "Restore system from a backup archive", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index faa5098c9..c489cceaa 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -38,6 +38,7 @@ import tempfile from collections import OrderedDict from moulinette import msignals, m18n, msettings +from moulinette.interfaces.cli import colorize from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger from moulinette.utils.network import download_json @@ -190,10 +191,7 @@ def app_info(app, full=False): """ from yunohost.permission import user_permission_list - if not _is_installed(app): - raise YunohostValidationError( - "app_not_installed", app=app, all_apps=_get_all_installed_apps_id() - ) + _assert_is_installed(app) local_manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app)) permissions = user_permission_list(full=True, absolute_urls=True, apps=[app])[ @@ -534,10 +532,8 @@ def app_upgrade(app=[], url=None, file=None, force=False): apps = [app_ for i, app_ in enumerate(apps) if app_ not in apps[:i]] # Abort if any of those app is in fact not installed.. - for app in [app_ for app_ in apps if not _is_installed(app_)]: - raise YunohostValidationError( - "app_not_installed", app=app, all_apps=_get_all_installed_apps_id() - ) + for app_ in apps: + _assert_is_installed(app_) if len(apps) == 0: raise YunohostValidationError("apps_already_up_to_date") @@ -750,7 +746,6 @@ def app_upgrade(app=[], url=None, file=None, force=False): for file_to_copy in [ "actions.json", "actions.toml", - "config_panel.json", "config_panel.toml", "conf", ]: @@ -970,7 +965,6 @@ def app_install( for file_to_copy in [ "actions.json", "actions.toml", - "config_panel.json", "config_panel.toml", "conf", ]: @@ -1759,165 +1753,143 @@ def app_action_run(operation_logger, app, action, args=None): # * docstrings # * merge translations on the json once the workflow is in place @is_unit_operation() -def app_config_show_panel(operation_logger, app): - logger.warning(m18n.n("experimental_feature")) +def app_config_show(operation_logger, app, panel='', full=False): + # logger.warning(m18n.n("experimental_feature")) - from yunohost.hook import hook_exec - - # this will take care of checking if the app is installed - app_info_dict = app_info(app) + # Check app is installed + _assert_is_installed(app) + panel = panel if panel else '' operation_logger.start() - config_panel = _get_app_config_panel(app) - config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config") - app_id, app_instance_nb = _parse_app_instance_name(app) + # Read config panel toml + config_panel = _get_app_config_panel(app, filter_key=panel) - if not config_panel or not os.path.exists(config_script): - return { - "app_id": app_id, - "app": app, - "app_name": app_info_dict["name"], - "config_panel": [], + if not config_panel: + return None + + # Call config script to extract current values + parsed_values = _call_config_script(app, 'show') + + # # Check and transform values if needed + # options = [option for _, _, option in _get_options_iterator(config_panel)] + # args_dict = _parse_args_in_yunohost_format( + # parsed_values, options, False + # ) + + # Hydrate + logger.debug("Hydrating config with current value") + for _, _, option in _get_options_iterator(config_panel): + if option['name'] in parsed_values: + option["value"] = parsed_values[option['name']] #args_dict[option["name"]][0] + + # Format result in full or reduce mode + if full: + operation_logger.success() + return config_panel + + result = OrderedDict() + for panel, section, option in _get_options_iterator(config_panel): + if panel['id'] not in result: + r_panel = result[panel['id']] = OrderedDict() + if section['id'] not in r_panel: + r_section = r_panel[section['id']] = OrderedDict() + r_option = r_section[option['name']] = { + "ask": option['ask']['en'] } + if not option.get('optional', False): + r_option['ask'] += ' *' + if option.get('value', None) is not None: + r_option['value'] = option['value'] - env = { - "app_id": app_id, - "app": app, - "app_instance_nb": str(app_instance_nb), - } - - try: - ret, parsed_values = hook_exec( - config_script, args=["show"], env=env, return_format="plain_dict" - ) - # Here again, calling hook_exec could fail miserably, or get - # manually interrupted (by mistake or because script was stuck) - except (KeyboardInterrupt, EOFError, Exception): - raise YunohostError("unexpected_error") - - logger.debug("Generating global variables:") - for tab in config_panel.get("panel", []): - for section in tab.get("sections", []): - for option in section.get("options", []): - logger.debug( - " * '%s'.'%s'.'%s'", - tab.get("name"), - section.get("name"), - option.get("name"), - ) - - if option['name'] in parsed_values: - # code is not adapted for that so we have to mock expected format :/ - if option.get("type") == "boolean": - if parsed_values[option['name']].lower() in ("true", "1", "y"): - option["default"] = parsed_values[option['name']] - else: - del option["default"] - else: - option["default"] = parsed_values[option['name']] - - args_dict = _parse_args_in_yunohost_format( - parsed_values, [option] - ) - option["default"] = args_dict[option["name"]][0] - else: - logger.debug( - "Variable '%s' is not declared by config script, using default", - option['name'], - ) - # do nothing, we'll use the default if present - - return { - "app_id": app_id, - "app": app, - "app_name": app_info_dict["name"], - "config_panel": config_panel, - "logs": operation_logger.success(), - } + operation_logger.success() + return result @is_unit_operation() -def app_config_apply(operation_logger, app, args): - logger.warning(m18n.n("experimental_feature")) - - from yunohost.hook import hook_exec - from base64 import b64decode - installed = _is_installed(app) - if not installed: - raise YunohostValidationError( - "app_not_installed", app=app, all_apps=_get_all_installed_apps_id() - ) - - config_panel = _get_app_config_panel(app) - config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config") - - if not config_panel or not os.path.exists(config_script): - # XXX real exception - raise Exception("Not config-panel.json nor scripts/config") +def app_config_get(operation_logger, app, key): + # Check app is installed + _assert_is_installed(app) operation_logger.start() - app_id, app_instance_nb = _parse_app_instance_name(app) - env = { - "app_id": app_id, - "app": app, - "app_instance_nb": str(app_instance_nb), - } - args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} + + # Read config panel toml + config_panel = _get_app_config_panel(app, filter_key=key) + + if not config_panel: + raise YunohostError("app_config_no_panel") + + # Call config script to extract current values + parsed_values = _call_config_script(app, 'show') + + logger.debug("Searching value") + short_key = key.split('.')[-1] + if short_key not in parsed_values: + return None + + return parsed_values[short_key] + + # for panel, section, option in _get_options_iterator(config_panel): + # if option['name'] == short_key: + # # Check and transform values if needed + # args_dict = _parse_args_in_yunohost_format( + # parsed_values, [option], False + # ) + # operation_logger.success() + + # return args_dict[short_key][0] + + # return None + + +@is_unit_operation() +def app_config_set(operation_logger, app, key=None, value=None, args=None): + # Check app is installed + _assert_is_installed(app) + + filter_key = key if key else '' + + # Read config panel toml + config_panel = _get_app_config_panel(app, filter_key=filter_key) + + if not config_panel: + raise YunohostError("app_config_no_panel") + + if args is not None and value is not None: + raise YunohostError("app_config_args_value") + + operation_logger.start() + + # Prepare pre answered questions + args = {} + if args: + args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} + elif value is not None: + args = {key: value} upload_dir = None - for tab in config_panel.get("panel", []): - for section in tab.get("sections", []): - for option in section.get("options", []): - if option['name'] in args: - # Upload files from API - # A file arg contains a string with "FILENAME:BASE64_CONTENT" - if 'type' in option and option["type"] == "file" \ - and msettings.get('interface') == 'api': - if upload_dir is None: - upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') - filename = args[option['name'] + '[name]'] - content = args[option['name']] - logger.debug("Save uploaded file %s from API into %s", filename, upload_dir) + for panel in config_panel.get("panel", []): - # Filename is given by user of the API. For security reason, we have replaced - # os.path.join to avoid the user to be able to rewrite a file in filesystem - # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" - file_path = os.path.normpath(upload_dir + "/" + filename) - i = 2 - while os.path.exists(file_path): - file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) - i += 1 - try: - with open(file_path, 'wb') as f: - f.write(b64decode(content)) - except IOError as e: - raise YunohostError("cannot_write_file", file=file_path, error=str(e)) - except Exception as e: - raise YunohostError("error_writing_file", file=file_path, error=str(e)) - args[option['name']] = file_path + if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: + msignals.display(colorize("\n" + "=" * 40, 'purple')) + msignals.display(colorize(f">>>> {panel['name']}", 'purple')) + msignals.display(colorize("=" * 40, 'purple')) + for section in panel.get("sections", []): + if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: + msignals.display(colorize(f"\n# {section['name']}", 'purple')) - logger.debug( - "include into env %s=%s", option['name'], args[option['name']] - ) - env[option['name']] = args[option['name']] - else: - logger.debug("no value for key id %s", option['name']) - - # for debug purpose - for key in args: - if key not in env: - logger.debug( - "Ignore key '%s' from arguments because it is not in the config", key + # Check and ask unanswered questions + args_dict = _parse_args_in_yunohost_format( + args, section['options'] ) + # Call config script to extract current values + logger.info("Running config script...") + env = {key: value[0] for key, value in args_dict.items()} + try: - hook_exec( - config_script, - args=["apply"], - env=env - ) + errors = _call_config_script(app, 'apply', env=env) # Here again, calling hook_exec could fail miserably, or get # manually interrupted (by mistake or because script was stuck) except (KeyboardInterrupt, EOFError, Exception): @@ -1931,10 +1903,51 @@ def app_config_apply(operation_logger, app, args): logger.success("Config updated as expected") return { "app": app, + "errors": errors, "logs": operation_logger.success(), } +def _get_options_iterator(config_panel): + for panel in config_panel.get("panel", []): + for section in panel.get("sections", []): + for option in section.get("options", []): + yield (panel, section, option) + + +def _call_config_script(app, action, env={}): + from yunohost.hook import hook_exec + + # Add default config script if needed + config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config") + if not os.path.exists(config_script): + logger.debug("Adding a default config script") + default_script = """#!/bin/bash +source /usr/share/yunohost/helpers +ynh_abort_if_errors +final_path=$(ynh_app_setting_get $app final_path) +ynh_panel_run $1 +""" + write_to_file(config_script, default_script) + + # Call config script to extract current values + logger.debug("Calling 'show' action from config script") + app_id, app_instance_nb = _parse_app_instance_name(app) + env.update({ + "app_id": app_id, + "app": app, + "app_instance_nb": str(app_instance_nb), + }) + + try: + _, parsed_values = hook_exec( + config_script, args=[action], env=env + ) + except (KeyboardInterrupt, EOFError, Exception): + logger.error('Unable to extract some values') + parsed_values = {} + return parsed_values + def _get_all_installed_apps_id(): """ Return something like: @@ -2036,14 +2049,11 @@ def _get_app_actions(app_id): return None -def _get_app_config_panel(app_id): +def _get_app_config_panel(app_id, filter_key=''): "Get app config panel stored in json or in toml" config_panel_toml_path = os.path.join( APPS_SETTING_PATH, app_id, "config_panel.toml" ) - config_panel_json_path = os.path.join( - APPS_SETTING_PATH, app_id, "config_panel.json" - ) # sample data to get an idea of what is going on # this toml extract: @@ -2121,6 +2131,10 @@ def _get_app_config_panel(app_id): "version": toml_config_panel["version"], "panel": [], } + filter_key = filter_key.split('.') + filter_panel = filter_key.pop(0) + filter_section = filter_key.pop(0) if len(filter_key) > 0 else False + filter_option = filter_key.pop(0) if len(filter_key) > 0 else False panels = [ key_value @@ -2130,6 +2144,9 @@ def _get_app_config_panel(app_id): ] for key, value in panels: + if filter_panel and key != filter_panel: + continue + panel = { "id": key, "name": value.get("name", ""), @@ -2143,9 +2160,14 @@ def _get_app_config_panel(app_id): ] for section_key, section_value in sections: + + if filter_section and section_key != filter_section: + continue + section = { "id": section_key, "name": section_value.get("name", ""), + "optional": section_value.get("optional", True), "options": [], } @@ -2156,7 +2178,11 @@ def _get_app_config_panel(app_id): ] for option_key, option_value in options: + if filter_option and option_key != filter_option: + continue + option = dict(option_value) + option["optional"] = option_value.get("optional", section['optional']) option["name"] = option_key option["ask"] = {"en": option["ask"]} if "help" in option: @@ -2169,9 +2195,6 @@ def _get_app_config_panel(app_id): return config_panel - elif os.path.exists(config_panel_json_path): - return json.load(open(config_panel_json_path)) - return None @@ -2615,6 +2638,13 @@ def _is_installed(app): return os.path.isdir(APPS_SETTING_PATH + app) +def _assert_is_installed(app): + if not _is_installed(app): + raise YunohostValidationError( + "app_not_installed", app=app, all_apps=_get_all_installed_apps_id() + ) + + def _installed_apps(): return os.listdir(APPS_SETTING_PATH) @@ -2727,10 +2757,13 @@ class YunoHostArgumentFormatParser(object): parsed_question = Question() parsed_question.name = question["name"] + parsed_question.type = question.get("type", 'string') parsed_question.default = question.get("default", None) parsed_question.choices = question.get("choices", []) parsed_question.optional = question.get("optional", False) parsed_question.ask = question.get("ask") + parsed_question.help = question.get("help") + parsed_question.helpLink = question.get("helpLink") parsed_question.value = user_answers.get(parsed_question.name) if parsed_question.ask is None: @@ -2742,24 +2775,28 @@ class YunoHostArgumentFormatParser(object): return parsed_question - def parse(self, question, user_answers): + def parse(self, question, user_answers, check_required=True): question = self.parse_question(question, user_answers) - if question.value is None: + if question.value is None and not getattr(self, "readonly", False): text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( question ) - try: question.value = msignals.prompt( - text_for_user_input_in_cli, self.hide_user_input_in_prompt + message=text_for_user_input_in_cli, + is_password=self.hide_user_input_in_prompt, + confirm=self.hide_user_input_in_prompt ) except NotImplementedError: question.value = None + if getattr(self, "readonly", False): + msignals.display(self._format_text_for_user_input_in_cli(question)) + # we don't have an answer, check optional and default_value if question.value is None or question.value == "": - if not question.optional and question.default is None: + if not question.optional and question.default is None and check_required: raise YunohostValidationError( "app_argument_required", name=question.name ) @@ -2785,6 +2822,7 @@ class YunoHostArgumentFormatParser(object): raise YunohostValidationError( "app_argument_choice_invalid", name=question.name, + value=question.value, choices=", ".join(question.choices), ) @@ -2796,7 +2834,15 @@ class YunoHostArgumentFormatParser(object): if question.default is not None: text_for_user_input_in_cli += " (default: {0})".format(question.default) - + if question.help or question.helpLink: + text_for_user_input_in_cli += ":\033[m" + if question.help: + text_for_user_input_in_cli += "\n - " + text_for_user_input_in_cli += question.help['en'] + if question.helpLink: + if not isinstance(question.helpLink, dict): + question.helpLink = {'href': question.helpLink} + text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}" return text_for_user_input_in_cli def _post_parse_value(self, question): @@ -2884,6 +2930,7 @@ class BooleanArgumentParser(YunoHostArgumentFormatParser): raise YunohostValidationError( "app_argument_choice_invalid", name=question.name, + value=question.value, choices="yes, no, y, n, 1, 0", ) @@ -2967,13 +3014,73 @@ class NumberArgumentParser(YunoHostArgumentFormatParser): class DisplayTextArgumentParser(YunoHostArgumentFormatParser): argument_type = "display_text" + readonly = True - def parse(self, question, user_answers): - print(question["ask"]) + def parse_question(self, question, user_answers): + question = super(DisplayTextArgumentParser, self).parse_question( + question, user_answers + ) + + question.optional = True + + return question + + def _format_text_for_user_input_in_cli(self, question): + text = question.ask['en'] + if question.type in ['info', 'warning', 'danger']: + color = { + 'info': 'cyan', + 'warning': 'yellow', + 'danger': 'red' + } + return colorize(m18n.g(question.type), color[question.type]) + f" {text}" + else: + return text class FileArgumentParser(YunoHostArgumentFormatParser): argument_type = "file" + def parse_question(self, question, user_answers): + question = super(FileArgumentParser, self).parse_question( + question, user_answers + ) + if msettings.get('interface') == 'api': + question.value = { + 'content': user_answers[question.name], + 'filename': user_answers.get(f"{question.name}[name]", question.name), + } if user_answers[question.name] else None + return question + + def _post_parse_value(self, question): + from base64 import b64decode + # Upload files from API + # A file arg contains a string with "FILENAME:BASE64_CONTENT" + if not question.value: + return question.value + + if msettings.get('interface') == 'api': + upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') + filename = question.value['filename'] + logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}") + + # Filename is given by user of the API. For security reason, we have replaced + # os.path.join to avoid the user to be able to rewrite a file in filesystem + # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" + file_path = os.path.normpath(upload_dir + "/" + filename) + i = 2 + while os.path.exists(file_path): + file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) + i += 1 + content = question.value['content'] + try: + with open(file_path, 'wb') as f: + f.write(b64decode(content)) + except IOError as e: + raise YunohostError("cannot_write_file", file=file_path, error=str(e)) + except Exception as e: + raise YunohostError("error_writing_file", file=file_path, error=str(e)) + question.value = file_path + return question.value ARGUMENTS_TYPE_PARSERS = { @@ -2994,11 +3101,16 @@ ARGUMENTS_TYPE_PARSERS = { "number": NumberArgumentParser, "range": NumberArgumentParser, "display_text": DisplayTextArgumentParser, + "success": DisplayTextArgumentParser, + "danger": DisplayTextArgumentParser, + "warning": DisplayTextArgumentParser, + "info": DisplayTextArgumentParser, + "markdown": DisplayTextArgumentParser, "file": FileArgumentParser, } -def _parse_args_in_yunohost_format(user_answers, argument_questions): +def _parse_args_in_yunohost_format(user_answers, argument_questions, check_required=True): """Parse arguments store in either manifest.json or actions.json or from a config panel against the user answers when they are present. @@ -3014,7 +3126,7 @@ def _parse_args_in_yunohost_format(user_answers, argument_questions): for question in argument_questions: parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]() - answer = parser.parse(question=question, user_answers=user_answers) + answer = parser.parse(question=question, user_answers=user_answers, check_required=check_required) if answer is not None: parsed_answers_dict[question["name"]] = answer From 2ac4e1c5bf37c0a33704da92c305a51fa5b94750 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 20 Aug 2021 17:26:26 +0200 Subject: [PATCH 020/119] [fix] I like regexp --- data/helpers.d/configpanel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 5b290629d..efbd5248f 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -42,9 +42,9 @@ ynh_value_get() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" local first_char="${crazy_value:0:1}" if [[ "$first_char" == '"' ]] ; then From b79d5ae416ffaaf9fc8ddebf1db70dce0d462b21 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 20 Aug 2021 18:02:54 +0200 Subject: [PATCH 021/119] [enh] Support __FINALPATH__ in file source --- data/helpers.d/configpanel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index efbd5248f..fcb1bd0d1 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -132,7 +132,7 @@ EOL elif [[ "$source" == *":"* ]] ; then local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} - local source_file="$(echo "$source" | cut -d: -f2)" + local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")" # Specific case for files (all content of the file is the source) From 10c8babf8c0ea2bfaab4e7aeb913a4750ed53c34 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 20 Aug 2021 18:03:32 +0200 Subject: [PATCH 022/119] [enh] Fail if script fail --- src/yunohost/app.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index c489cceaa..676e07f5a 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1939,15 +1939,12 @@ ynh_panel_run $1 "app_instance_nb": str(app_instance_nb), }) - try: - _, parsed_values = hook_exec( - config_script, args=[action], env=env - ) - except (KeyboardInterrupt, EOFError, Exception): - logger.error('Unable to extract some values') - parsed_values = {} + _, parsed_values = hook_exec( + config_script, args=[action], env=env + ) return parsed_values + def _get_all_installed_apps_id(): """ Return something like: From ef058c07f7fd011ff605b13780e933dc3772a8bc Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 20 Aug 2021 20:12:15 +0200 Subject: [PATCH 023/119] [fix] File question in config panel with cli --- data/helpers.d/configpanel | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index fcb1bd0d1..67c7a92f7 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -44,7 +44,8 @@ ynh_value_get() { local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" + local crazy_value="$((grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL) | head -n1)" + #" local first_char="${crazy_value:0:1}" if [[ "$first_char" == '"' ]] ; then @@ -74,22 +75,22 @@ ynh_value_set() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + local crazy_value="$(grep -i -o -P "${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" local first_char="${crazy_value:0:1}" if [[ "$first_char" == '"' ]] ; then value="$(echo "$value" | sed 's/"/\\"/g')" - sed -ri "s%^(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file} + sed -ri "s%(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file} elif [[ "$first_char" == "'" ]] ; then value="$(echo "$value" | sed "s/'/\\\\'/g")" - sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} else if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then value="\"$(echo "$value" | sed 's/"/\\"/g')\"" fi - sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} fi } @@ -124,12 +125,12 @@ EOL if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then old[$short_setting]="$($getter)" - # Get value from app settings - elif [[ "$source" == "settings" ]] ; then - old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" + # Get value from app settings or from another file + elif [[ "$source" == "settings" ]] || [[ "$source" == *":"* ]] ; then + if [[ "$source" == "settings" ]] ; then + source=":/etc/yunohost/apps/$app/settings.yml" + fi - # Get value from a kind of key/value file - elif [[ "$source" == *":"* ]] ; then local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" @@ -137,7 +138,8 @@ EOL # Specific case for files (all content of the file is the source) else - old[$short_setting]="$source" + + old[$short_setting]="$(ls $source 2> /dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" fi done @@ -164,12 +166,13 @@ _ynh_panel_apply() { then local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} - local source_file="$(echo "$source" | cut -d: -f2)" + local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}" # Specific case for files (all content of the file is the source) else - cp "${!short_setting}" "$source" + local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + cp "${!short_setting}" "$source_file" fi fi done @@ -178,7 +181,9 @@ _ynh_panel_apply() { _ynh_panel_show() { for short_setting in "${!old[@]}" do - ynh_return "${short_setting}: \"${old[$short_setting]}\"" + if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then + ynh_return "${short_setting}: \"${old[$short_setting]}\"" + fi done } @@ -197,10 +202,11 @@ _ynh_panel_validate() { file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) fi if [ -f "${!short_setting}" ] ; then - file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1) + file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1) if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] then changed[$short_setting]=true + is_error=false fi fi else @@ -218,6 +224,7 @@ _ynh_panel_validate() { for short_setting in "${!old[@]}" do + [[ "${changed[$short_setting]}" == "false" ]] && continue local result="" if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then result="$(validate__$short_setting)" @@ -225,7 +232,7 @@ _ynh_panel_validate() { if [ -n "$result" ] then local key="YNH_ERROR_${$short_setting}" - ynh_return "$key=$result" + ynh_return "$key: $result" is_error=true fi done From 0de69104b153bf24b18f533c89f1f7b007734a18 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 24 Aug 2021 16:06:46 +0200 Subject: [PATCH 024/119] [fix] Bad merge --- data/helpers.d/configpanel | 51 -------------------------------------- 1 file changed, 51 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index e8dbaafe9..67c7a92f7 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -181,39 +181,26 @@ _ynh_panel_apply() { _ynh_panel_show() { for short_setting in "${!old[@]}" do -<<<<<<< HEAD if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then ynh_return "${short_setting}: \"${old[$short_setting]}\"" fi -======= - local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" - ynh_return "$key=${old[$short_setting]}" ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a done } _ynh_panel_validate() { -<<<<<<< HEAD -======= - set +x ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a # Change detection local is_error=true #for changed_status in "${!changed[@]}" for short_setting in "${!old[@]}" do changed[$short_setting]=false -<<<<<<< HEAD [ -z ${!short_setting+x} ] && continue -======= ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a if [ ! -z "${file_hash[${short_setting}]}" ] ; then file_hash[old__$short_setting]="" file_hash[new__$short_setting]="" if [ -f "${old[$short_setting]}" ] ; then file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) fi -<<<<<<< HEAD if [ -f "${!short_setting}" ] ; then file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1) if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] @@ -224,17 +211,6 @@ _ynh_panel_validate() { fi else if [[ "${!short_setting}" != "${old[$short_setting]}" ]] -======= - if [ -f "${new[$short_setting]}" ] ; then - file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1) - if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] - then - changed[$short_setting]=true - fi - fi - else - if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]] ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a then changed[$short_setting]=true is_error=false @@ -248,23 +224,15 @@ _ynh_panel_validate() { for short_setting in "${!old[@]}" do -<<<<<<< HEAD [[ "${changed[$short_setting]}" == "false" ]] && continue -======= ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a local result="" if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then result="$(validate__$short_setting)" fi if [ -n "$result" ] then -<<<<<<< HEAD local key="YNH_ERROR_${$short_setting}" ynh_return "$key: $result" -======= - local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])" - ynh_return "$key=$result" ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a is_error=true fi done @@ -272,14 +240,8 @@ _ynh_panel_validate() { if [[ "$is_error" == "true" ]] then -<<<<<<< HEAD ynh_die "" fi -======= - ynh_die - fi - set -x ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a } @@ -301,7 +263,6 @@ ynh_panel_apply() { ynh_panel_run() { declare -Ag old=() -<<<<<<< HEAD declare -Ag changed=() declare -Ag file_hash=() declare -Ag sources=() @@ -310,18 +271,6 @@ ynh_panel_run() { case $1 in show) ynh_panel_get && ynh_panel_show;; apply) ynh_panel_get && ynh_panel_validate && ynh_panel_apply;; -======= - declare -Ag new=() - declare -Ag changed=() - declare -Ag file_hash=() - declare -Ag sources=() - declare -Ag dot_settings=() - - ynh_panel_get - case $1 in - show) ynh_panel_show;; - apply) ynh_panel_validate && ynh_panel_apply;; ->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a esac } From 4c46f41036b505d51faeab7f1545e7f3db421e1e Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 24 Aug 2021 17:36:54 +0200 Subject: [PATCH 025/119] [fix] Args always empty in config panel --- src/yunohost/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 676e07f5a..770706190 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1861,11 +1861,13 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): operation_logger.start() # Prepare pre answered questions - args = {} if args: args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} elif value is not None: args = {key: value} + else: + args = {} + upload_dir = None From 4547a6ec077f7d09a7334a08b92ccc7da4ba2827 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 25 Aug 2021 18:13:17 +0200 Subject: [PATCH 026/119] [fix] Multiple fixes in config panel --- data/helpers.d/configpanel | 73 ++++++++++++++++++++++++-------------- src/yunohost/app.py | 16 +++++---- 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 67c7a92f7..7e26811f5 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -189,12 +189,18 @@ _ynh_panel_show() { _ynh_panel_validate() { # Change detection + ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1 local is_error=true #for changed_status in "${!changed[@]}" for short_setting in "${!old[@]}" do changed[$short_setting]=false - [ -z ${!short_setting+x} ] && continue + if [ -z ${!short_setting+x} ]; then + # Assign the var with the old value in order to allows multiple + # args validation + declare "$short_setting"="${old[$short_setting]}" + continue + fi if [ ! -z "${file_hash[${short_setting}]}" ] ; then file_hash[old__$short_setting]="" file_hash[new__$short_setting]="" @@ -217,30 +223,32 @@ _ynh_panel_validate() { fi fi done - - # Run validation if something is changed - if [[ "$is_error" == "false" ]] - then - - for short_setting in "${!old[@]}" - do - [[ "${changed[$short_setting]}" == "false" ]] && continue - local result="" - if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then - result="$(validate__$short_setting)" - fi - if [ -n "$result" ] - then - local key="YNH_ERROR_${$short_setting}" - ynh_return "$key: $result" - is_error=true - fi - done - fi - if [[ "$is_error" == "true" ]] then - ynh_die "" + ynh_die "Nothing has changed" + fi + + # Run validation if something is changed + ynh_script_progression --message="Validating the new configuration..." --weight=1 + + for short_setting in "${!old[@]}" + do + [[ "${changed[$short_setting]}" == "false" ]] && continue + local result="" + if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then + result="$(validate__$short_setting)" + fi + if [ -n "$result" ] + then + local key="YNH_ERROR_${short_setting}" + ynh_return "$key: $result" + is_error=true + fi + done + + if [[ "$is_error" == "true" ]] + then + exit 0 fi } @@ -266,11 +274,22 @@ ynh_panel_run() { declare -Ag changed=() declare -Ag file_hash=() declare -Ag sources=() - - ynh_panel_get case $1 in - show) ynh_panel_get && ynh_panel_show;; - apply) ynh_panel_get && ynh_panel_validate && ynh_panel_apply;; + show) + ynh_panel_get + ynh_panel_show + ;; + apply) + max_progression=4 + ynh_script_progression --message="Reading config panel description and current configuration..." --weight=1 + ynh_panel_get + + ynh_panel_validate + + ynh_script_progression --message="Applying the new configuration..." --weight=1 + ynh_panel_apply + ynh_script_progression --message="Configuration of $app completed" --last + ;; esac } diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 770706190..92254d1d0 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1769,7 +1769,7 @@ def app_config_show(operation_logger, app, panel='', full=False): return None # Call config script to extract current values - parsed_values = _call_config_script(app, 'show') + parsed_values = _call_config_script(operation_logger, app, 'show') # # Check and transform values if needed # options = [option for _, _, option in _get_options_iterator(config_panel)] @@ -1820,7 +1820,7 @@ def app_config_get(operation_logger, app, key): raise YunohostError("app_config_no_panel") # Call config script to extract current values - parsed_values = _call_config_script(app, 'show') + parsed_values = _call_config_script(operation_logger, app, 'show') logger.debug("Searching value") short_key = key.split('.')[-1] @@ -1891,7 +1891,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): env = {key: value[0] for key, value in args_dict.items()} try: - errors = _call_config_script(app, 'apply', env=env) + errors = _call_config_script(operation_logger, app, 'apply', env=env) # Here again, calling hook_exec could fail miserably, or get # manually interrupted (by mistake or because script was stuck) except (KeyboardInterrupt, EOFError, Exception): @@ -1917,7 +1917,7 @@ def _get_options_iterator(config_panel): yield (panel, section, option) -def _call_config_script(app, action, env={}): +def _call_config_script(operation_logger, app, action, env={}): from yunohost.hook import hook_exec # Add default config script if needed @@ -1933,7 +1933,7 @@ ynh_panel_run $1 write_to_file(config_script, default_script) # Call config script to extract current values - logger.debug("Calling 'show' action from config script") + logger.debug(f"Calling '{action}' action from config script") app_id, app_instance_nb = _parse_app_instance_name(app) env.update({ "app_id": app_id, @@ -1941,9 +1941,11 @@ ynh_panel_run $1 "app_instance_nb": str(app_instance_nb), }) - _, parsed_values = hook_exec( + ret, parsed_values = hook_exec( config_script, args=[action], env=env ) + if ret != 0: + operation_logger.error(parsed_values) return parsed_values @@ -3047,7 +3049,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): question.value = { 'content': user_answers[question.name], 'filename': user_answers.get(f"{question.name}[name]", question.name), - } if user_answers[question.name] else None + } if user_answers.get(question.name) else None return question def _post_parse_value(self, question): From 86c099812363d96142d6fdb9a53b2168365a247a Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 25 Aug 2021 19:18:15 +0200 Subject: [PATCH 027/119] [enh] Add a pattern validation for config panel --- src/yunohost/app.py | 74 ++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 92254d1d0..1853a63f4 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2764,6 +2764,7 @@ class YunoHostArgumentFormatParser(object): parsed_question.optional = question.get("optional", False) parsed_question.ask = question.get("ask") parsed_question.help = question.get("help") + parsed_question.pattern = question.get("pattern") parsed_question.helpLink = question.get("helpLink") parsed_question.value = user_answers.get(parsed_question.name) @@ -2776,49 +2777,66 @@ class YunoHostArgumentFormatParser(object): return parsed_question - def parse(self, question, user_answers, check_required=True): + def parse(self, question, user_answers): question = self.parse_question(question, user_answers) - if question.value is None and not getattr(self, "readonly", False): - text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( - question - ) - try: - question.value = msignals.prompt( - message=text_for_user_input_in_cli, - is_password=self.hide_user_input_in_prompt, - confirm=self.hide_user_input_in_prompt + while True: + # Display question if no value filled or if it's a readonly message + if msettings.get('interface') == 'cli': + text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( + question ) - except NotImplementedError: - question.value = None + if getattr(self, "readonly", False): + msignals.display(text_for_user_input_in_cli) - if getattr(self, "readonly", False): - msignals.display(self._format_text_for_user_input_in_cli(question)) + elif question.value is None: + question.value = msignals.prompt( + message=text_for_user_input_in_cli, + is_password=self.hide_user_input_in_prompt, + confirm=self.hide_user_input_in_prompt + ) - # we don't have an answer, check optional and default_value - if question.value is None or question.value == "": - if not question.optional and question.default is None and check_required: - raise YunohostValidationError( - "app_argument_required", name=question.name - ) - else: + + # Apply default value + if question.value in [None, ""] and question.default is not None: question.value = ( getattr(self, "default_value", None) if question.default is None else question.default ) - # we have an answer, do some post checks - if question.value is not None: - if question.choices and question.value not in question.choices: - self._raise_invalid_answer(question) - + # Prevalidation + try: + self._prevalidate(question) + except YunoHostValidationError: + if msettings.get('interface') == 'api': + raise + question.value = None + continue + break # this is done to enforce a certain formating like for boolean # by default it doesn't do anything question.value = self._post_parse_value(question) return (question.value, self.argument_type) + def _prevalidate(self, question): + if question.value in [None, ""] and not question.optional: + raise YunohostValidationError( + "app_argument_required", name=question.name + ) + + # we have an answer, do some post checks + if question.value is not None: + if question.choices and question.value not in question.choices: + self._raise_invalid_answer(question) + if question.pattern and re.match(question.pattern['regexp'], str(question.value)): + raise YunohostValidationError( + question.pattern['error'], + name=question.name, + value=question.value, + ) + def _raise_invalid_answer(self, question): raise YunohostValidationError( "app_argument_choice_invalid", @@ -3111,7 +3129,7 @@ ARGUMENTS_TYPE_PARSERS = { } -def _parse_args_in_yunohost_format(user_answers, argument_questions, check_required=True): +def _parse_args_in_yunohost_format(user_answers, argument_questions): """Parse arguments store in either manifest.json or actions.json or from a config panel against the user answers when they are present. @@ -3127,7 +3145,7 @@ def _parse_args_in_yunohost_format(user_answers, argument_questions, check_requi for question in argument_questions: parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]() - answer = parser.parse(question=question, user_answers=user_answers, check_required=check_required) + answer = parser.parse(question=question, user_answers=user_answers) if answer is not None: parsed_answers_dict[question["name"]] = answer From 5dc1ee62660c2b06be840ddd20cf6b051ace86f0 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 25 Aug 2021 20:10:11 +0200 Subject: [PATCH 028/119] [enh] Services key in config panel --- src/yunohost/app.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 1853a63f4..3d775ea5b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -54,7 +54,7 @@ from moulinette.utils.filesystem import ( mkdir, ) -from yunohost.service import service_status, _run_service_command +from yunohost.service import service_status, _run_service_command, _get_services from yunohost.utils import packages from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory @@ -1870,9 +1870,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): upload_dir = None - for panel in config_panel.get("panel", []): - if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: msignals.display(colorize("\n" + "=" * 40, 'purple')) msignals.display(colorize(f">>>> {panel['name']}", 'purple')) @@ -1902,7 +1900,29 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): if upload_dir is not None: shutil.rmtree(upload_dir) - logger.success("Config updated as expected") + # Reload services + services_to_reload = set([]) + for panel in config_panel.get("panel", []): + services_to_reload |= set(panel.get('services', [])) + for section in panel.get("sections", []): + services_to_reload |= set(section.get('services', [])) + for option in section.get("options", []): + services_to_reload |= set(section.get('options', [])) + + services_to_reload = list(services_to_reload) + services_to_reload.sort(key = 'nginx'.__eq__) + for service in services_to_reload: + if not _run_service_command('reload_or_restart', service): + services = _get_services() + test_conf = services[service].get('test_conf') + errors = check_output(f"{test_conf}; exit 0") if test_conf else '' + raise YunohostError( + "app_config_failed_service_reload", + service=service, errors=errors + ) + + if not errors: + logger.success("Config updated as expected") return { "app": app, "errors": errors, @@ -2760,17 +2780,14 @@ class YunoHostArgumentFormatParser(object): parsed_question.name = question["name"] parsed_question.type = question.get("type", 'string') parsed_question.default = question.get("default", None) - parsed_question.choices = question.get("choices", []) parsed_question.optional = question.get("optional", False) - parsed_question.ask = question.get("ask") - parsed_question.help = question.get("help") + parsed_question.choices = question.get("choices", []) parsed_question.pattern = question.get("pattern") + parsed_question.ask = question.get("ask", {'en': f"Enter value for '{parsed_question.name}':"}) + parsed_question.help = question.get("help") parsed_question.helpLink = question.get("helpLink") parsed_question.value = user_answers.get(parsed_question.name) - if parsed_question.ask is None: - parsed_question.ask = "Enter value for '%s':" % parsed_question.name - # Empty value is parsed as empty string if parsed_question.default == "": parsed_question.default = None @@ -2857,7 +2874,7 @@ class YunoHostArgumentFormatParser(object): text_for_user_input_in_cli += ":\033[m" if question.help: text_for_user_input_in_cli += "\n - " - text_for_user_input_in_cli += question.help['en'] + text_for_user_input_in_cli += _value_for_locale(question.help) if question.helpLink: if not isinstance(question.helpLink, dict): question.helpLink = {'href': question.helpLink} From 97128d7ddbaaf4806c6bfe1d5e9847c78aa9c0c0 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 25 Aug 2021 20:16:31 +0200 Subject: [PATCH 029/119] [enh] Support __APP__ in services config panel key --- src/yunohost/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 3d775ea5b..fba8499c0 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1912,6 +1912,8 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): services_to_reload = list(services_to_reload) services_to_reload.sort(key = 'nginx'.__eq__) for service in services_to_reload: + if service == "__APP__": + service = app if not _run_service_command('reload_or_restart', service): services = _get_services() test_conf = services[service].get('test_conf') From 5a64a063b285d99e84bd22097cbae8f63a74121a Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 25 Aug 2021 20:51:50 +0200 Subject: [PATCH 030/119] [fix] Clean properly config panel upload dir --- src/yunohost/app.py | 71 +++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index fba8499c0..aa8f0d864 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1861,34 +1861,29 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): operation_logger.start() # Prepare pre answered questions - if args: - args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} - elif value is not None: + args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} + if value is not None: args = {key: value} - else: - args = {} - - - upload_dir = None - for panel in config_panel.get("panel", []): - if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: - msignals.display(colorize("\n" + "=" * 40, 'purple')) - msignals.display(colorize(f">>>> {panel['name']}", 'purple')) - msignals.display(colorize("=" * 40, 'purple')) - for section in panel.get("sections", []): - if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: - msignals.display(colorize(f"\n# {section['name']}", 'purple')) - - # Check and ask unanswered questions - args_dict = _parse_args_in_yunohost_format( - args, section['options'] - ) - - # Call config script to extract current values - logger.info("Running config script...") - env = {key: value[0] for key, value in args_dict.items()} try: + for panel in config_panel.get("panel", []): + if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: + msignals.display(colorize("\n" + "=" * 40, 'purple')) + msignals.display(colorize(f">>>> {panel['name']}", 'purple')) + msignals.display(colorize("=" * 40, 'purple')) + for section in panel.get("sections", []): + if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: + msignals.display(colorize(f"\n# {section['name']}", 'purple')) + + # Check and ask unanswered questions + args_dict = _parse_args_in_yunohost_format( + args, section['options'] + ) + + # Call config script to extract current values + logger.info("Running config script...") + env = {key: value[0] for key, value in args_dict.items()} + errors = _call_config_script(operation_logger, app, 'apply', env=env) # Here again, calling hook_exec could fail miserably, or get # manually interrupted (by mistake or because script was stuck) @@ -1896,11 +1891,16 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): raise YunohostError("unexpected_error") finally: # Delete files uploaded from API - if msettings.get('interface') == 'api': - if upload_dir is not None: - shutil.rmtree(upload_dir) + FileArgumentParser.clean_upload_dirs() + + if errors: + return { + "app": app, + "errors": errors, + } # Reload services + logger.info("Reloading services...") services_to_reload = set([]) for panel in config_panel.get("panel", []): services_to_reload |= set(panel.get('services', [])) @@ -1923,11 +1923,10 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): service=service, errors=errors ) - if not errors: - logger.success("Config updated as expected") + logger.success("Config updated as expected") return { "app": app, - "errors": errors, + "errors": [], "logs": operation_logger.success(), } @@ -3077,6 +3076,14 @@ class DisplayTextArgumentParser(YunoHostArgumentFormatParser): class FileArgumentParser(YunoHostArgumentFormatParser): argument_type = "file" + upload_dirs = [] + + @classmethod + def clean_upload_dirs(cls): + # Delete files uploaded from API + if msettings.get('interface') == 'api': + for upload_dir in cls.upload_dirs: + shutil.rmtree(upload_dir) def parse_question(self, question, user_answers): question = super(FileArgumentParser, self).parse_question( @@ -3097,7 +3104,9 @@ class FileArgumentParser(YunoHostArgumentFormatParser): return question.value if msettings.get('interface') == 'api': + upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') + FileArgumentParser.upload_dirs += [upload_dir] filename = question.value['filename'] logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}") From 8d364029a03bbaf87f9cc491e3e7d5975a1f49bc Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 25 Aug 2021 20:58:15 +0200 Subject: [PATCH 031/119] [fix] Services key not properly converted --- src/yunohost/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index aa8f0d864..f94e22462 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1866,6 +1866,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): args = {key: value} try: + logger.debug("Asking unanswered question and prevalidating...") for panel in config_panel.get("panel", []): if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: msignals.display(colorize("\n" + "=" * 40, 'purple')) @@ -2172,6 +2173,7 @@ def _get_app_config_panel(app_id, filter_key=''): panel = { "id": key, "name": value.get("name", ""), + "services": value.get("services", []), "sections": [], } @@ -2190,6 +2192,7 @@ def _get_app_config_panel(app_id, filter_key=''): "id": section_key, "name": section_value.get("name", ""), "optional": section_value.get("optional", True), + "services": value.get("services", []), "options": [], } From f5529c584d8fd4d91a9177f2fa9f946790011b0c Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 25 Aug 2021 21:16:31 +0200 Subject: [PATCH 032/119] [wip] Min max in number questions --- src/yunohost/app.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index f94e22462..44c93e6d9 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1908,14 +1908,15 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): for section in panel.get("sections", []): services_to_reload |= set(section.get('services', [])) for option in section.get("options", []): - services_to_reload |= set(section.get('options', [])) + services_to_reload |= set(option.get('services', [])) services_to_reload = list(services_to_reload) services_to_reload.sort(key = 'nginx'.__eq__) for service in services_to_reload: if service == "__APP__": service = app - if not _run_service_command('reload_or_restart', service): + logger.debug(f"Reloading {service}") + if not _run_service_command('reload-or-restart', service): services = _get_services() test_conf = services[service].get('test_conf') errors = check_output(f"{test_conf}; exit 0") if test_conf else '' @@ -2937,7 +2938,7 @@ class BooleanArgumentParser(YunoHostArgumentFormatParser): default_value = False def parse_question(self, question, user_answers): - question = super(BooleanArgumentParser, self).parse_question( + question = super().parse_question( question, user_answers ) @@ -3031,18 +3032,33 @@ class NumberArgumentParser(YunoHostArgumentFormatParser): default_value = "" def parse_question(self, question, user_answers): - question = super(NumberArgumentParser, self).parse_question( + question_parsed = super().parse_question( question, user_answers ) - + question_parsed.min = question.get('min', None) + question_parsed.max = question.get('max', None) if question.default is None: - question.default = 0 + question_parsed.default = 0 - return question + return question_parsed + def _prevalidate(self, question): + super()._prevalidate(question) + if question.min is not None and question.value < question.min: + raise YunohostValidationError( + "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + ) + if question.max is not None and question.value > question.max: + raise YunohostValidationError( + "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + ) + if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()): + raise YunohostValidationError( + "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + ) def _post_parse_value(self, question): if isinstance(question.value, int): - return super(NumberArgumentParser, self)._post_parse_value(question) + return super()._post_parse_value(question) if isinstance(question.value, str) and question.value.isdigit(): return int(question.value) From 9eb9ec1804e45856b07962e16393f0100bf6d113 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 26 Aug 2021 17:39:33 +0200 Subject: [PATCH 033/119] [fix] Several fixes in config panel --- src/yunohost/app.py | 59 +++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 44c93e6d9..0e945c64c 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -70,6 +70,7 @@ APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" APPS_CATALOG_API_VERSION = 2 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" +APPS_CONFIG_PANEL_VERSION_SUPPORTED = 1.0 re_app_instance_name = re.compile( r"^(?P[\w-]+?)(__(?P[1-9][0-9]*))?$" ) @@ -1863,7 +1864,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): # Prepare pre answered questions args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} if value is not None: - args = {key: value} + args = {filter_key.split('.')[-1]: value} try: logger.debug("Asking unanswered question and prevalidating...") @@ -1883,13 +1884,23 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): # Call config script to extract current values logger.info("Running config script...") - env = {key: value[0] for key, value in args_dict.items()} + env = {key: str(value[0]) for key, value in args_dict.items()} errors = _call_config_script(operation_logger, app, 'apply', env=env) - # Here again, calling hook_exec could fail miserably, or get - # manually interrupted (by mistake or because script was stuck) - except (KeyboardInterrupt, EOFError, Exception): - raise YunohostError("unexpected_error") + # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception + except (KeyboardInterrupt, EOFError): + error = m18n.n("operation_interrupted") + logger.error(m18n.n("app_config_failed", app=app, error=error)) + failure_message_with_debug_instructions = operation_logger.error(error) + raise + # Something wrong happened in Yunohost's code (most probably hook_exec) + except Exception: + import traceback + + error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) + logger.error(m18n.n("app_config_failed", app=app, error=error)) + failure_message_with_debug_instructions = operation_logger.error(error) + raise finally: # Delete files uploaded from API FileArgumentParser.clean_upload_dirs() @@ -2148,6 +2159,11 @@ def _get_app_config_panel(app_id, filter_key=''): toml_config_panel = toml.load( open(config_panel_toml_path, "r"), _dict=OrderedDict ) + if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED: + raise YunohostError( + "app_config_too_old_version", app=app_id, + version=toml_config_panel["version"] + ) # transform toml format into json format config_panel = { @@ -2219,6 +2235,13 @@ def _get_app_config_panel(app_id, filter_key=''): config_panel["panel"].append(panel) + if (filter_panel and len(config_panel['panel']) == 0) or \ + (filter_section and len(config_panel['panel'][0]['sections']) == 0) or \ + (filter_option and len(config_panel['panel'][0]['sections'][0]['options']) == 0): + raise YunohostError( + "app_config_bad_filter_key", app=app_id, filter_key=filter_key + ) + return config_panel return None @@ -2830,9 +2853,10 @@ class YunoHostArgumentFormatParser(object): # Prevalidation try: self._prevalidate(question) - except YunoHostValidationError: + except YunohostValidationError as e: if msettings.get('interface') == 'api': raise + msignals.display(str(e), 'error') question.value = None continue break @@ -3037,25 +3061,28 @@ class NumberArgumentParser(YunoHostArgumentFormatParser): ) question_parsed.min = question.get('min', None) question_parsed.max = question.get('max', None) - if question.default is None: + if question_parsed.default is None: question_parsed.default = 0 return question_parsed def _prevalidate(self, question): super()._prevalidate(question) - if question.min is not None and question.value < question.min: - raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") - ) - if question.max is not None and question.value > question.max: - raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") - ) if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()): raise YunohostValidationError( "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") ) + + if question.min is not None and int(question.value) < question.min: + raise YunohostValidationError( + "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + ) + + if question.max is not None and int(question.value) > question.max: + raise YunohostValidationError( + "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + ) + def _post_parse_value(self, question): if isinstance(question.value, int): return super()._post_parse_value(question) From 574b01bcf4ae164518333220d4fd9fdc8964aa24 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 26 Aug 2021 17:48:08 +0200 Subject: [PATCH 034/119] [fix] Pattern key not working --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0e945c64c..b5b4128dc 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2876,7 +2876,7 @@ class YunoHostArgumentFormatParser(object): if question.value is not None: if question.choices and question.value not in question.choices: self._raise_invalid_answer(question) - if question.pattern and re.match(question.pattern['regexp'], str(question.value)): + if question.pattern and not re.match(question.pattern['regexp'], str(question.value)): raise YunohostValidationError( question.pattern['error'], name=question.name, From b0f6a07372f4e5d5ebcaa37c4c9a5dad6c674713 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 27 Aug 2021 01:05:20 +0200 Subject: [PATCH 035/119] [fix] Source getter for config panel --- data/helpers.d/configpanel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 7e26811f5..e1a96b866 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -139,7 +139,7 @@ EOL # Specific case for files (all content of the file is the source) else - old[$short_setting]="$(ls $source 2> /dev/null || echo YNH_NULL)" + old[$short_setting]="$(ls $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" fi done From bc725e9768639bcf4330627156af9b84c750f1f8 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 27 Aug 2021 01:05:59 +0200 Subject: [PATCH 036/119] [fix] File through config panel API --- src/yunohost/app.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index b5b4128dc..4e20c6950 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2937,7 +2937,7 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser): return question - def _post_parse_value(self, question): + def _prevalidate(self, question): if any(char in question.value for char in self.forbidden_chars): raise YunohostValidationError( "pattern_password_app", forbidden_chars=self.forbidden_chars @@ -2949,8 +2949,6 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser): assert_password_is_strong_enough("user", question.value) - return super(PasswordArgumentParser, self)._post_parse_value(question) - class PathArgumentParser(YunoHostArgumentFormatParser): argument_type = "path" @@ -3129,18 +3127,40 @@ class FileArgumentParser(YunoHostArgumentFormatParser): # Delete files uploaded from API if msettings.get('interface') == 'api': for upload_dir in cls.upload_dirs: - shutil.rmtree(upload_dir) + if os.path.exists(upload_dir): + shutil.rmtree(upload_dir) def parse_question(self, question, user_answers): - question = super(FileArgumentParser, self).parse_question( + question_parsed = super().parse_question( question, user_answers ) + if question.get('accept'): + question_parsed.accept = question.get('accept').replace(' ', '').split(',') + else: + question.accept = [] if msettings.get('interface') == 'api': - question.value = { - 'content': user_answers[question.name], - 'filename': user_answers.get(f"{question.name}[name]", question.name), - } if user_answers.get(question.name) else None - return question + if user_answers.get(question_parsed.name): + question_parsed.value = { + 'content': question_parsed.value, + 'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name), + } + return question_parsed + + def _prevalidate(self, question): + super()._prevalidate(question) + if isinstance(question.value, str) and not os.path.exists(question.value): + raise YunohostValidationError( + "app_argument_invalid", name=question.name, error=m18n.n("invalid_number1") + ) + if question.value is None or not question.accept: + return + + filename = question.value if isinstance(question.value, str) else question.value['filename'] + if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept: + raise YunohostValidationError( + "app_argument_invalid", name=question.name, error=m18n.n("invalid_number2") + ) + def _post_parse_value(self, question): from base64 import b64decode From f1e5309d40a429513354e2c79857ee7d6623a8df Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 30 Aug 2021 13:14:17 +0200 Subject: [PATCH 037/119] Multiline, file, tags management + prefilled cli --- data/actionsmap/yunohost.yml | 4 +- data/helpers.d/configpanel | 112 ++++++++++++++++++-------- src/yunohost/app.py | 147 +++++++++++++++++++++++------------ 3 files changed, 177 insertions(+), 86 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index d9e3a50d0..28b713b03 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -838,8 +838,8 @@ app: arguments: app: help: App name - panel: - help: Select a specific panel + key: + help: Select a specific panel, section or a question nargs: '?' -f: full: --full diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index e1a96b866..0c3469c14 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -96,51 +96,72 @@ ynh_value_set() { _ynh_panel_get() { # From settings - local params_sources - params_sources=`python3 << EOL + local lines + lines=`python3 << EOL import toml from collections import OrderedDict with open("../config_panel.toml", "r") as f: file_content = f.read() loaded_toml = toml.loads(file_content, _dict=OrderedDict) -for panel_name,panel in loaded_toml.items(): - if isinstance(panel, dict): - for section_name, section in panel.items(): - if isinstance(section, dict): - for name, param in section.items(): - if isinstance(param, dict) and param.get('type', 'string') not in ['success', 'info', 'warning', 'danger', 'display_text', 'markdown']: - print("%s=%s" % (name, param.get('source', 'settings'))) +for panel_name, panel in loaded_toml.items(): + if not isinstance(panel, dict): continue + for section_name, section in panel.items(): + if not isinstance(section, dict): continue + for name, param in section.items(): + if not isinstance(param, dict): + continue + print(';'.join([ + name, + param.get('type', 'string'), + param.get('source', 'settings' if param.get('type', 'string') != 'file' else '') + ])) EOL ` - for param_source in $params_sources + for line in $lines do - local short_setting="$(echo $param_source | cut -d= -f1)" + IFS=';' read short_setting type source <<< "$line" local getter="get__${short_setting}" - local source="$(echo $param_source | cut -d= -f2)" sources[${short_setting}]="$source" + types[${short_setting}]="$type" file_hash[${short_setting}]="" - + formats[${short_setting}]="" # Get value from getter if exists if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then old[$short_setting]="$($getter)" + formats[${short_setting}]="yaml" + elif [[ "$source" == "" ]] ; then + old[$short_setting]="YNH_NULL" + # Get value from app settings or from another file - elif [[ "$source" == "settings" ]] || [[ "$source" == *":"* ]] ; then + elif [[ "$type" == "file" ]] ; then + if [[ "$source" == "settings" ]] ; then + ynh_die "File '${short_setting}' can't be stored in settings" + fi + old[$short_setting]="$(ls $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" + file_hash[$short_setting]="true" + + # Get multiline text from settings or from a full file + elif [[ "$type" == "text" ]] ; then + if [[ "$source" == "settings" ]] ; then + old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" + elif [[ "$source" == *":"* ]] ; then + ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + else + old[$short_setting]="$(cat $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" + fi + + # Get value from a kind of key/value file + else if [[ "$source" == "settings" ]] ; then source=":/etc/yunohost/apps/$app/settings.yml" fi - local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")" - # Specific case for files (all content of the file is the source) - else - - old[$short_setting]="$(ls $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" - file_hash[$short_setting]="true" fi done @@ -152,27 +173,42 @@ _ynh_panel_apply() { do local setter="set__${short_setting}" local source="${sources[$short_setting]}" + local type="${types[$short_setting]}" if [ "${changed[$short_setting]}" == "true" ] ; then # Apply setter if exists if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then $setter - # Copy file in right place + elif [[ "$source" == "" ]] ; then + continue + + # Save in a file + elif [[ "$type" == "file" ]] ; then + if [[ "$source" == "settings" ]] ; then + ynh_die "File '${short_setting}' can't be stored in settings" + fi + local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + cp "${!short_setting}" "$source_file" + + # Save value in app settings elif [[ "$source" == "settings" ]] ; then ynh_app_setting_set $app $short_setting "${!short_setting}" - # Get value from a kind of key/value file - elif [[ "$source" == *":"* ]] - then + # Save multiline text in a file + elif [[ "$type" == "text" ]] ; then + if [[ "$source" == *":"* ]] ; then + ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + fi + local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + echo "${!short_setting}" > "$source_file" + + # Set value into a kind of key/value file + else local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}" - # Specific case for files (all content of the file is the source) - else - local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - cp "${!short_setting}" "$source_file" fi fi done @@ -182,7 +218,13 @@ _ynh_panel_show() { for short_setting in "${!old[@]}" do if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then - ynh_return "${short_setting}: \"${old[$short_setting]}\"" + if [[ "${formats[$short_setting]}" == "yaml" ]] ; then + ynh_return "${short_setting}:" + ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/ /g')" + else + ynh_return "${short_setting}: \"$(echo "${old[$short_setting]}" | sed ':a;N;$!ba;s/\n/\n\n/g')\"" + + fi fi done } @@ -225,7 +267,8 @@ _ynh_panel_validate() { done if [[ "$is_error" == "true" ]] then - ynh_die "Nothing has changed" + ynh_print_info "Nothing has changed" + exit 0 fi # Run validation if something is changed @@ -241,7 +284,7 @@ _ynh_panel_validate() { if [ -n "$result" ] then local key="YNH_ERROR_${short_setting}" - ynh_return "$key: $result" + ynh_return "$key: \"$result\"" is_error=true fi done @@ -274,6 +317,9 @@ ynh_panel_run() { declare -Ag changed=() declare -Ag file_hash=() declare -Ag sources=() + declare -Ag types=() + declare -Ag formats=() + case $1 in show) ynh_panel_get @@ -281,12 +327,12 @@ ynh_panel_run() { ;; apply) max_progression=4 - ynh_script_progression --message="Reading config panel description and current configuration..." --weight=1 + ynh_script_progression --message="Reading config panel description and current configuration..." ynh_panel_get ynh_panel_validate - ynh_script_progression --message="Applying the new configuration..." --weight=1 + ynh_script_progression --message="Applying the new configuration..." ynh_panel_apply ynh_script_progression --message="Configuration of $app completed" --last ;; diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4e20c6950..2acdcb679 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -35,6 +35,7 @@ import glob import urllib.parse import base64 import tempfile +import readline from collections import OrderedDict from moulinette import msignals, m18n, msettings @@ -1754,36 +1755,21 @@ def app_action_run(operation_logger, app, action, args=None): # * docstrings # * merge translations on the json once the workflow is in place @is_unit_operation() -def app_config_show(operation_logger, app, panel='', full=False): +def app_config_show(operation_logger, app, key='', full=False): # logger.warning(m18n.n("experimental_feature")) # Check app is installed _assert_is_installed(app) - panel = panel if panel else '' - operation_logger.start() + key = key if key else '' # Read config panel toml - config_panel = _get_app_config_panel(app, filter_key=panel) + config_panel = _get_app_hydrated_config_panel(operation_logger, + app, filter_key=key) if not config_panel: return None - # Call config script to extract current values - parsed_values = _call_config_script(operation_logger, app, 'show') - - # # Check and transform values if needed - # options = [option for _, _, option in _get_options_iterator(config_panel)] - # args_dict = _parse_args_in_yunohost_format( - # parsed_values, options, False - # ) - - # Hydrate - logger.debug("Hydrating config with current value") - for _, _, option in _get_options_iterator(config_panel): - if option['name'] in parsed_values: - option["value"] = parsed_values[option['name']] #args_dict[option["name"]][0] - # Format result in full or reduce mode if full: operation_logger.success() @@ -1800,8 +1786,8 @@ def app_config_show(operation_logger, app, panel='', full=False): } if not option.get('optional', False): r_option['ask'] += ' *' - if option.get('value', None) is not None: - r_option['value'] = option['value'] + if option.get('current_value', None) is not None: + r_option['value'] = option['current_value'] operation_logger.success() return result @@ -1812,7 +1798,6 @@ def app_config_get(operation_logger, app, key): # Check app is installed _assert_is_installed(app) - operation_logger.start() # Read config panel toml config_panel = _get_app_config_panel(app, filter_key=key) @@ -1820,6 +1805,8 @@ def app_config_get(operation_logger, app, key): if not config_panel: raise YunohostError("app_config_no_panel") + operation_logger.start() + # Call config script to extract current values parsed_values = _call_config_script(operation_logger, app, 'show') @@ -1851,7 +1838,8 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): filter_key = key if key else '' # Read config panel toml - config_panel = _get_app_config_panel(app, filter_key=filter_key) + config_panel = _get_app_hydrated_config_panel(operation_logger, + app, filter_key=filter_key) if not config_panel: raise YunohostError("app_config_no_panel") @@ -1862,12 +1850,16 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): operation_logger.start() # Prepare pre answered questions - args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} + if args: + args = { key: ','.join(value) for key, value in urllib.parse.parse_qs(args, keep_blank_values=True).items() } + else: + args = {} if value is not None: args = {filter_key.split('.')[-1]: value} try: logger.debug("Asking unanswered question and prevalidating...") + args_dict = {} for panel in config_panel.get("panel", []): if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3: msignals.display(colorize("\n" + "=" * 40, 'purple')) @@ -1878,13 +1870,13 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): msignals.display(colorize(f"\n# {section['name']}", 'purple')) # Check and ask unanswered questions - args_dict = _parse_args_in_yunohost_format( + args_dict.update(_parse_args_in_yunohost_format( args, section['options'] - ) + )) # Call config script to extract current values logger.info("Running config script...") - env = {key: str(value[0]) for key, value in args_dict.items()} + env = {key: str(value[0]) for key, value in args_dict.items() if not value[0] is None} errors = _call_config_script(operation_logger, app, 'apply', env=env) # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception @@ -2246,6 +2238,37 @@ def _get_app_config_panel(app_id, filter_key=''): return None +def _get_app_hydrated_config_panel(operation_logger, app, filter_key=''): + + # Read config panel toml + config_panel = _get_app_config_panel(app, filter_key=filter_key) + + if not config_panel: + return None + + operation_logger.start() + + # Call config script to extract current values + parsed_values = _call_config_script(operation_logger, app, 'show') + + # # Check and transform values if needed + # options = [option for _, _, option in _get_options_iterator(config_panel)] + # args_dict = _parse_args_in_yunohost_format( + # parsed_values, options, False + # ) + + # Hydrate + logger.debug("Hydrating config with current value") + for _, _, option in _get_options_iterator(config_panel): + if option['name'] in parsed_values: + value = parsed_values[option['name']] + if isinstance(value, dict): + option.update(value) + else: + option["current_value"] = value #args_dict[option["name"]][0] + + return config_panel + def _get_app_settings(app_id): """ @@ -2808,6 +2831,7 @@ class YunoHostArgumentFormatParser(object): parsed_question.name = question["name"] parsed_question.type = question.get("type", 'string') parsed_question.default = question.get("default", None) + parsed_question.current_value = question.get("current_value") parsed_question.optional = question.get("optional", False) parsed_question.choices = question.get("choices", []) parsed_question.pattern = question.get("pattern") @@ -2835,11 +2859,20 @@ class YunoHostArgumentFormatParser(object): msignals.display(text_for_user_input_in_cli) elif question.value is None: - question.value = msignals.prompt( - message=text_for_user_input_in_cli, - is_password=self.hide_user_input_in_prompt, - confirm=self.hide_user_input_in_prompt - ) + prefill = None + if question.current_value is not None: + prefill = question.current_value + elif question.default is not None: + prefill = question.default + readline.set_startup_hook(lambda: readline.insert_text(prefill)) + try: + question.value = msignals.prompt( + message=text_for_user_input_in_cli, + is_password=self.hide_user_input_in_prompt, + confirm=self.hide_user_input_in_prompt + ) + finally: + readline.set_startup_hook() # Apply default value @@ -2897,8 +2930,6 @@ class YunoHostArgumentFormatParser(object): if question.choices: text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices)) - if question.default is not None: - text_for_user_input_in_cli += " (default: {0})".format(question.default) if question.help or question.helpLink: text_for_user_input_in_cli += ":\033[m" if question.help: @@ -2919,6 +2950,18 @@ class StringArgumentParser(YunoHostArgumentFormatParser): default_value = "" +class TagsArgumentParser(YunoHostArgumentFormatParser): + argument_type = "tags" + + def _prevalidate(self, question): + values = question.value + for value in values.split(','): + question.value = value + super()._prevalidate(question) + question.value = values + + + class PasswordArgumentParser(YunoHostArgumentFormatParser): hide_user_input_in_prompt = True argument_type = "password" @@ -2938,13 +2981,15 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser): return question def _prevalidate(self, question): - if any(char in question.value for char in self.forbidden_chars): - raise YunohostValidationError( - "pattern_password_app", forbidden_chars=self.forbidden_chars - ) + super()._prevalidate(question) - # If it's an optional argument the value should be empty or strong enough - if not question.optional or question.value: + if question.value is not None: + if any(char in question.value for char in self.forbidden_chars): + raise YunohostValidationError( + "pattern_password_app", forbidden_chars=self.forbidden_chars + ) + + # If it's an optional argument the value should be empty or strong enough from yunohost.utils.password import assert_password_is_strong_enough assert_password_is_strong_enough("user", question.value) @@ -3098,23 +3143,26 @@ class DisplayTextArgumentParser(YunoHostArgumentFormatParser): readonly = True def parse_question(self, question, user_answers): - question = super(DisplayTextArgumentParser, self).parse_question( + question_parsed = super().parse_question( question, user_answers ) - question.optional = True + question_parsed.optional = True + question_parsed.style = question.get('style', 'info') - return question + return question_parsed def _format_text_for_user_input_in_cli(self, question): text = question.ask['en'] - if question.type in ['info', 'warning', 'danger']: + + if question.style in ['success', 'info', 'warning', 'danger']: color = { + 'success': 'green', 'info': 'cyan', 'warning': 'yellow', 'danger': 'red' } - return colorize(m18n.g(question.type), color[question.type]) + f" {text}" + return colorize(m18n.g(question.style), color[question.style]) + f" {text}" else: return text @@ -3137,7 +3185,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): if question.get('accept'): question_parsed.accept = question.get('accept').replace(' ', '').split(',') else: - question.accept = [] + question_parsed.accept = [] if msettings.get('interface') == 'api': if user_answers.get(question_parsed.name): question_parsed.value = { @@ -3200,7 +3248,7 @@ ARGUMENTS_TYPE_PARSERS = { "string": StringArgumentParser, "text": StringArgumentParser, "select": StringArgumentParser, - "tags": StringArgumentParser, + "tags": TagsArgumentParser, "email": StringArgumentParser, "url": StringArgumentParser, "date": StringArgumentParser, @@ -3214,10 +3262,7 @@ ARGUMENTS_TYPE_PARSERS = { "number": NumberArgumentParser, "range": NumberArgumentParser, "display_text": DisplayTextArgumentParser, - "success": DisplayTextArgumentParser, - "danger": DisplayTextArgumentParser, - "warning": DisplayTextArgumentParser, - "info": DisplayTextArgumentParser, + "alert": DisplayTextArgumentParser, "markdown": DisplayTextArgumentParser, "file": FileArgumentParser, } From 766711069f674f7d48bc9d0ce406aae757cb8f7c Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 30 Aug 2021 13:25:32 +0200 Subject: [PATCH 038/119] [fix] Bad call to Moulinette.get --- src/yunohost/app.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 79944d340..f5e5dfd48 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1867,12 +1867,12 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): logger.debug("Asking unanswered question and prevalidating...") args_dict = {} for panel in config_panel.get("panel", []): - if Moulinette.get('interface') == 'cli' and len(filter_key.split('.')) < 3: + if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3: Moulinette.display(colorize("\n" + "=" * 40, 'purple')) Moulinette.display(colorize(f">>>> {panel['name']}", 'purple')) Moulinette.display(colorize("=" * 40, 'purple')) for section in panel.get("sections", []): - if Moulinette.get('interface') == 'cli' and len(filter_key.split('.')) < 3: + if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3: Moulinette.display(colorize(f"\n# {section['name']}", 'purple')) # Check and ask unanswered questions @@ -2855,9 +2855,10 @@ class YunoHostArgumentFormatParser(object): def parse(self, question, user_answers): question = self.parse_question(question, user_answers) +<<<<<<< HEAD while True: # Display question if no value filled or if it's a readonly message - if Moulinette.get('interface') == 'cli': + if Moulinette.interface == 'cli': text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( question ) @@ -2893,7 +2894,7 @@ class YunoHostArgumentFormatParser(object): try: self._prevalidate(question) except YunohostValidationError as e: - if Moulinette.get('interface') == 'api': + if Moulinette.interface == 'api': raise Moulinette.display(str(e), 'error') question.value = None @@ -3179,7 +3180,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): @classmethod def clean_upload_dirs(cls): # Delete files uploaded from API - if Moulinette.get('interface') == 'api': + if Moulinette.interface == 'api': for upload_dir in cls.upload_dirs: if os.path.exists(upload_dir): shutil.rmtree(upload_dir) @@ -3192,7 +3193,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): question_parsed.accept = question.get('accept').replace(' ', '').split(',') else: question_parsed.accept = [] - if Moulinette.get('interface') == 'api': + if Moulinette.interface == 'api': if user_answers.get(question_parsed.name): question_parsed.value = { 'content': question_parsed.value, @@ -3223,7 +3224,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): if not question.value: return question.value - if Moulinette.get('interface') == 'api': + if Moulinette.interface == 'api': upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') FileArgumentParser.upload_dirs += [upload_dir] From 8d9f8c7123dbc8286025ae2b8acbafa8c03c746d Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 30 Aug 2021 14:04:12 +0200 Subject: [PATCH 039/119] [fix] Bad call to Moulinette.interface --- src/yunohost/app.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index f5e5dfd48..7ddbdc7f3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1867,12 +1867,12 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): logger.debug("Asking unanswered question and prevalidating...") args_dict = {} for panel in config_panel.get("panel", []): - if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3: + if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3: Moulinette.display(colorize("\n" + "=" * 40, 'purple')) Moulinette.display(colorize(f">>>> {panel['name']}", 'purple')) Moulinette.display(colorize("=" * 40, 'purple')) for section in panel.get("sections", []): - if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3: + if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3: Moulinette.display(colorize(f"\n# {section['name']}", 'purple')) # Check and ask unanswered questions @@ -2855,10 +2855,9 @@ class YunoHostArgumentFormatParser(object): def parse(self, question, user_answers): question = self.parse_question(question, user_answers) -<<<<<<< HEAD while True: # Display question if no value filled or if it's a readonly message - if Moulinette.interface == 'cli': + if Moulinette.interface.type== 'cli': text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( question ) @@ -2894,7 +2893,7 @@ class YunoHostArgumentFormatParser(object): try: self._prevalidate(question) except YunohostValidationError as e: - if Moulinette.interface == 'api': + if Moulinette.interface.type== 'api': raise Moulinette.display(str(e), 'error') question.value = None @@ -3180,7 +3179,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): @classmethod def clean_upload_dirs(cls): # Delete files uploaded from API - if Moulinette.interface == 'api': + if Moulinette.interface.type== 'api': for upload_dir in cls.upload_dirs: if os.path.exists(upload_dir): shutil.rmtree(upload_dir) @@ -3193,7 +3192,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): question_parsed.accept = question.get('accept').replace(' ', '').split(',') else: question_parsed.accept = [] - if Moulinette.interface == 'api': + if Moulinette.interface.type== 'api': if user_answers.get(question_parsed.name): question_parsed.value = { 'content': question_parsed.value, @@ -3224,7 +3223,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): if not question.value: return question.value - if Moulinette.interface == 'api': + if Moulinette.interface.type== 'api': upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') FileArgumentParser.upload_dirs += [upload_dir] From 969564eec659d51cade553d4bfe092b2c4ba888c Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 30 Aug 2021 19:41:07 +0200 Subject: [PATCH 040/119] [fix] simple/double quotes into source --- data/helpers.d/configpanel | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index 0c3469c14..cfdbfc331 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -75,22 +75,21 @@ ynh_value_set() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' + local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - local crazy_value="$(grep -i -o -P "${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" - local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*" + local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" local first_char="${crazy_value:0:1}" if [[ "$first_char" == '"' ]] ; then - value="$(echo "$value" | sed 's/"/\\"/g')" - sed -ri "s%(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file} + value="$(echo "$value" | sed 's/"/\"/g')" + sed -ri 's%^('"${var_part}"'")[^"]*("[ \t;,]*)$%\1'"${value}"'\3%i' ${file} elif [[ "$first_char" == "'" ]] ; then - value="$(echo "$value" | sed "s/'/\\\\'/g")" - sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + value="$(echo "$value" | sed "s/'/"'\'"'/g")" + sed -ri "s%^(${var_part}')[^']*('"'[ \t,;]*)$%\1'"${value}"'\3%i' ${file} else if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then - value="\"$(echo "$value" | sed 's/"/\\"/g')\"" + value='\"'"$(echo "$value" | sed 's/"/\"/g')"'\"' fi - sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file} + sed -ri "s%^(${var_part}).*"'$%\1'"${value}"'%i' ${file} fi } @@ -222,7 +221,7 @@ _ynh_panel_show() { ynh_return "${short_setting}:" ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/ /g')" else - ynh_return "${short_setting}: \"$(echo "${old[$short_setting]}" | sed ':a;N;$!ba;s/\n/\n\n/g')\"" + ynh_return "${short_setting}: "'"'"$(echo "${old[$short_setting]}" | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\n\n/g')"'"' fi fi From c20226fc54f3e1772e7231907796e076074b7198 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 30 Aug 2021 19:41:30 +0200 Subject: [PATCH 041/119] [enh] Move prefill feature into moulinette --- src/yunohost/app.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7ddbdc7f3..2ea1a3b70 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -35,7 +35,6 @@ import glob import urllib.parse import base64 import tempfile -import readline from collections import OrderedDict from moulinette.interfaces.cli import colorize @@ -2865,20 +2864,18 @@ class YunoHostArgumentFormatParser(object): Moulinette.display(text_for_user_input_in_cli) elif question.value is None: - prefill = None + prefill = "" if question.current_value is not None: prefill = question.current_value elif question.default is not None: prefill = question.default - readline.set_startup_hook(lambda: readline.insert_text(prefill)) - try: - question.value = Moulinette.prompt( - message=text_for_user_input_in_cli, - is_password=self.hide_user_input_in_prompt, - confirm=self.hide_user_input_in_prompt - ) - finally: - readline.set_startup_hook() + question.value = Moulinette.prompt( + message=text_for_user_input_in_cli, + is_password=self.hide_user_input_in_prompt, + confirm=self.hide_user_input_in_prompt, + prefill=prefill, + is_multiline=(question.type == "text") + ) # Apply default value From bb11b5dcacc6ad8f6dfeb9ccb9e98a4bca7a39bd Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 31 Aug 2021 04:20:21 +0200 Subject: [PATCH 042/119] [enh] Be able to delete source file --- data/helpers.d/configpanel | 10 +++++++++- src/yunohost/app.py | 10 +++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index cfdbfc331..e99a66d4c 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -187,7 +187,11 @@ _ynh_panel_apply() { ynh_die "File '${short_setting}' can't be stored in settings" fi local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - cp "${!short_setting}" "$source_file" + if [[ "${!short_setting}" == "" ]] ; then + rm -f "$source_file" + else + cp "${!short_setting}" "$source_file" + fi # Save value in app settings elif [[ "$source" == "settings" ]] ; then @@ -247,6 +251,10 @@ _ynh_panel_validate() { file_hash[new__$short_setting]="" if [ -f "${old[$short_setting]}" ] ; then file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) + if [ -z "${!short_setting}" ] ; then + changed[$short_setting]=true + is_error=false + fi fi if [ -f "${!short_setting}" ] ; then file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 2ea1a3b70..425f04023 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -3190,20 +3190,24 @@ class FileArgumentParser(YunoHostArgumentFormatParser): else: question_parsed.accept = [] if Moulinette.interface.type== 'api': - if user_answers.get(question_parsed.name): + if user_answers.get(f"{question_parsed.name}[name]"): question_parsed.value = { 'content': question_parsed.value, 'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name), } + # If path file are the same + if question_parsed.value and str(question_parsed.value) == question_parsed.current_value: + question_parsed.value = None + return question_parsed def _prevalidate(self, question): super()._prevalidate(question) - if isinstance(question.value, str) and not os.path.exists(question.value): + if isinstance(question.value, str) and question.value and not os.path.exists(question.value): raise YunohostValidationError( "app_argument_invalid", name=question.name, error=m18n.n("invalid_number1") ) - if question.value is None or not question.accept: + if question.value in [None, ''] or not question.accept: return filename = question.value if isinstance(question.value, str) else question.value['filename'] From 08e7fcc48ef94aa48bc328878f06bb10d81af82d Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 31 Aug 2021 17:44:42 +0200 Subject: [PATCH 043/119] [enh] Be able to correctly display the error --- src/yunohost/utils/error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/utils/error.py b/src/yunohost/utils/error.py index f9b4ac61a..8405830e7 100644 --- a/src/yunohost/utils/error.py +++ b/src/yunohost/utils/error.py @@ -59,4 +59,4 @@ class YunohostValidationError(YunohostError): def content(self): - return {"error": self.strerror, "error_key": self.key} + return {"error": self.strerror, "error_key": self.key, **self.kwargs} From 6d16e22f8779c2d7741c6e6b20c5ef301ac11d57 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 31 Aug 2021 17:45:13 +0200 Subject: [PATCH 044/119] [enh] VisibleIf on config panel section --- src/yunohost/app.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 425f04023..69c65046a 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2206,9 +2206,11 @@ def _get_app_config_panel(app_id, filter_key=''): "id": section_key, "name": section_value.get("name", ""), "optional": section_value.get("optional", True), - "services": value.get("services", []), + "services": section_value.get("services", []), "options": [], } + if section_value.get('visibleIf'): + section['visibleIf'] = section_value.get('visibleIf') options = [ k_v @@ -2952,6 +2954,11 @@ class StringArgumentParser(YunoHostArgumentFormatParser): argument_type = "string" default_value = "" + def _prevalidate(self, question): + super()._prevalidate(question) + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2") + ) class TagsArgumentParser(YunoHostArgumentFormatParser): argument_type = "tags" @@ -3065,7 +3072,7 @@ class DomainArgumentParser(YunoHostArgumentFormatParser): def _raise_invalid_answer(self, question): raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("domain_unknown") + "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown") ) @@ -3092,7 +3099,7 @@ class UserArgumentParser(YunoHostArgumentFormatParser): def _raise_invalid_answer(self, question): raise YunohostValidationError( "app_argument_invalid", - name=question.name, + field=question.name, error=m18n.n("user_unknown", user=question.value), ) @@ -3116,17 +3123,17 @@ class NumberArgumentParser(YunoHostArgumentFormatParser): super()._prevalidate(question) if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()): raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") ) if question.min is not None and int(question.value) < question.min: raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") ) if question.max is not None and int(question.value) > question.max: raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") ) def _post_parse_value(self, question): @@ -3137,7 +3144,7 @@ class NumberArgumentParser(YunoHostArgumentFormatParser): return int(question.value) raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") ) @@ -3205,7 +3212,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): super()._prevalidate(question) if isinstance(question.value, str) and question.value and not os.path.exists(question.value): raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number1") + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1") ) if question.value in [None, ''] or not question.accept: return @@ -3213,7 +3220,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): filename = question.value if isinstance(question.value, str) else question.value['filename'] if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept: raise YunohostValidationError( - "app_argument_invalid", name=question.name, error=m18n.n("invalid_number2") + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2") ) From 7d26b1477fe12756b798defcec0d3126e3a2368f Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 2 Sep 2021 02:27:22 +0200 Subject: [PATCH 045/119] [enh] Some refactoring for config panel --- data/actionsmap/yunohost.yml | 36 ++- data/helpers.d/configpanel | 32 ++- locales/en.json | 2 +- src/yunohost/app.py | 456 +++++++++++++++++------------------ 4 files changed, 263 insertions(+), 263 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 2c91651bd..adc6aa93a 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -831,34 +831,28 @@ app: subcategory_help: Applications configuration panel actions: - ### app_config_show() - show: - action_help: show config panel for the application + ### app_config_get() + get: + action_help: Display an app configuration api: GET /apps//config-panel arguments: app: help: App name key: - help: Select a specific panel, section or a question + help: A specific panel, section or a question identifier nargs: '?' - -f: - full: --full - help: Display all info known about the config-panel. - action: store_true - - ### app_config_get() - get: - action_help: show config panel for the application - api: GET /apps//config-panel/ - arguments: - app: - help: App name - key: - help: The question identifier + -m: + full: --mode + help: Display mode to use + choices: + - classic + - full + - export + default: classic ### app_config_set() set: - action_help: apply the new configuration + action_help: Apply a new configuration api: PUT /apps//config arguments: app: @@ -872,6 +866,10 @@ app: -a: full: --args help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path") + -f: + full: --args-file + help: YAML or JSON file with key/value couples + type: open ############################# # Backup # diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index e99a66d4c..b55b0b0fd 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -74,23 +74,29 @@ ynh_value_set() { local value # Manage arguments with getopts ynh_handle_getopts_args "$@" - local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" + # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" local first_char="${crazy_value:0:1}" if [[ "$first_char" == '"' ]] ; then - value="$(echo "$value" | sed 's/"/\"/g')" - sed -ri 's%^('"${var_part}"'")[^"]*("[ \t;,]*)$%\1'"${value}"'\3%i' ${file} + # \ and sed is quite complex you need 2 \\ to get one in a sed + # So we need \\\\ to go through 2 sed + value="$(echo "$value" | sed 's/"/\\\\"/g')" + sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file} elif [[ "$first_char" == "'" ]] ; then - value="$(echo "$value" | sed "s/'/"'\'"'/g")" - sed -ri "s%^(${var_part}')[^']*('"'[ \t,;]*)$%\1'"${value}"'\3%i' ${file} + # \ and sed is quite complex you need 2 \\ to get one in a sed + # However double quotes implies to double \\ to + # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str + value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" + sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file} else if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then - value='\"'"$(echo "$value" | sed 's/"/\"/g')"'\"' + value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' fi - sed -ri "s%^(${var_part}).*"'$%\1'"${value}"'%i' ${file} + sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file} fi + ynh_print_info "Configuration key '$key' edited into $file" } _ynh_panel_get() { @@ -189,13 +195,16 @@ _ynh_panel_apply() { local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]] ; then rm -f "$source_file" + ynh_print_info "File '$source_file' removed" else cp "${!short_setting}" "$source_file" + ynh_print_info "File '$source_file' overwrited with ${!short_setting}" fi # Save value in app settings elif [[ "$source" == "settings" ]] ; then ynh_app_setting_set $app $short_setting "${!short_setting}" + ynh_print_info "Configuration key '$short_setting' edited in app settings" # Save multiline text in a file elif [[ "$type" == "text" ]] ; then @@ -204,13 +213,20 @@ _ynh_panel_apply() { fi local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" echo "${!short_setting}" > "$source_file" + ynh_print_info "File '$source_file' overwrited with the content you provieded in '${short_setting}' question" # Set value into a kind of key/value file else local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + + ynh_backup_if_checksum_is_different --file="$source_file" ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}" + ynh_store_file_checksum --file="$source_file" + + # We stored the info in settings in order to be able to upgrade the app + ynh_app_setting_set $app $short_setting "${!short_setting}" fi fi diff --git a/locales/en.json b/locales/en.json index 3498c00e7..dcbf0f866 100644 --- a/locales/en.json +++ b/locales/en.json @@ -14,7 +14,7 @@ "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.", "app_already_up_to_date": "{app} is already up-to-date", "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'", - "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", + "app_argument_invalid": "Pick a valid value for the argument '{field}': {error}", "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason", "app_argument_required": "Argument '{name}' is required", "app_change_url_failed_nginx_reload": "Could not reload NGINX. Here is the output of 'nginx -t':\n{nginx_errors}", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 69c65046a..a4c215328 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1756,135 +1756,122 @@ def app_action_run(operation_logger, app, action, args=None): return logger.success("Action successed!") -# Config panel todo list: -# * docstrings -# * merge translations on the json once the workflow is in place @is_unit_operation() -def app_config_show(operation_logger, app, key='', full=False): - # logger.warning(m18n.n("experimental_feature")) +def app_config_get(operation_logger, app, key='', mode='classic'): + """ + Display an app configuration in classic, full or export mode + """ # Check app is installed _assert_is_installed(app) - key = key if key else '' + filter_key = key or '' # Read config panel toml - config_panel = _get_app_hydrated_config_panel(operation_logger, - app, filter_key=key) + config_panel = _get_app_config_panel(app, filter_key=filter_key) if not config_panel: - return None + raise YunohostError("app_config_no_panel") - # Format result in full or reduce mode - if full: + # Call config script in order to hydrate config panel with current values + values = _call_config_script(operation_logger, app, 'show', config_panel=config_panel) + + # Format result in full mode + if mode == 'full': operation_logger.success() return config_panel - result = OrderedDict() - for panel, section, option in _get_options_iterator(config_panel): - if panel['id'] not in result: - r_panel = result[panel['id']] = OrderedDict() - if section['id'] not in r_panel: - r_section = r_panel[section['id']] = OrderedDict() - r_option = r_section[option['name']] = { - "ask": option['ask']['en'] - } - if not option.get('optional', False): - r_option['ask'] += ' *' - if option.get('current_value', None) is not None: - r_option['value'] = option['current_value'] + # In 'classic' mode, we display the current value if key refer to an option + if filter_key.count('.') == 2 and mode == 'classic': + option = filter_key.split('.')[-1] + operation_logger.success() + return values.get(option, None) + + # Format result in 'classic' or 'export' mode + logger.debug(f"Formating result in '{mode}' mode") + result = {} + for panel, section, option in _get_config_iterator(config_panel): + key = f"{panel['id']}.{section['id']}.{option['id']}" + if mode == 'export': + result[option['id']] = option.get('current_value') + else: + result[key] = { 'ask': _value_for_locale(option['ask']) } + if 'current_value' in option: + result[key]['value'] = option['current_value'] operation_logger.success() return result @is_unit_operation() -def app_config_get(operation_logger, app, key): +def app_config_set(operation_logger, app, key=None, value=None, args=None, args_file=None): + """ + Apply a new app configuration + """ + # Check app is installed _assert_is_installed(app) + filter_key = key or '' # Read config panel toml - config_panel = _get_app_config_panel(app, filter_key=key) + config_panel = _get_app_config_panel(app, filter_key=filter_key) if not config_panel: raise YunohostError("app_config_no_panel") - operation_logger.start() - - # Call config script to extract current values - parsed_values = _call_config_script(operation_logger, app, 'show') - - logger.debug("Searching value") - short_key = key.split('.')[-1] - if short_key not in parsed_values: - return None - - return parsed_values[short_key] - - # for panel, section, option in _get_options_iterator(config_panel): - # if option['name'] == short_key: - # # Check and transform values if needed - # args_dict = _parse_args_in_yunohost_format( - # parsed_values, [option], False - # ) - # operation_logger.success() - - # return args_dict[short_key][0] - - # return None - - -@is_unit_operation() -def app_config_set(operation_logger, app, key=None, value=None, args=None): - # Check app is installed - _assert_is_installed(app) - - filter_key = key if key else '' - - # Read config panel toml - config_panel = _get_app_hydrated_config_panel(operation_logger, - app, filter_key=filter_key) - - if not config_panel: - raise YunohostError("app_config_no_panel") - - if args is not None and value is not None: + if (args is not None or args_file is not None) and value is not None: raise YunohostError("app_config_args_value") - operation_logger.start() + if filter_key.count('.') != 2 and not value is None: + raise YunohostError("app_config_set_value_on_section") + + # Import and parse pre-answered options + logger.debug("Import and parse pre-answered options") + args = urllib.parse.parse_qs(args or '', keep_blank_values=True) + args = { key: ','.join(value_) for key, value_ in args.items() } + + if args_file: + # Import YAML / JSON file but keep --args values + args = { **read_yaml(args_file), **args } - # Prepare pre answered questions - if args: - args = { key: ','.join(value) for key, value in urllib.parse.parse_qs(args, keep_blank_values=True).items() } - else: - args = {} if value is not None: args = {filter_key.split('.')[-1]: value} + # Call config script in order to hydrate config panel with current values + _call_config_script(operation_logger, app, 'show', config_panel=config_panel) + + # Ask unanswered question and prevalidate + logger.debug("Ask unanswered question and prevalidate data") + def display_header(message): + """ CLI panel/section header display + """ + if Moulinette.interface.type == 'cli' and filter_key.count('.') < 2: + Moulinette.display(colorize(message, 'purple')) + try: - logger.debug("Asking unanswered question and prevalidating...") - args_dict = {} - for panel in config_panel.get("panel", []): - if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3: - Moulinette.display(colorize("\n" + "=" * 40, 'purple')) - Moulinette.display(colorize(f">>>> {panel['name']}", 'purple')) - Moulinette.display(colorize("=" * 40, 'purple')) - for section in panel.get("sections", []): - if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3: - Moulinette.display(colorize(f"\n# {section['name']}", 'purple')) + env = {} + for panel, section, obj in _get_config_iterator(config_panel, + ['panel', 'section']): + if panel == obj: + name = _value_for_locale(panel['name']) + display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}") + continue + name = _value_for_locale(section['name']) + display_header(f"\n# {name}") - # Check and ask unanswered questions - args_dict.update(_parse_args_in_yunohost_format( - args, section['options'] - )) + # Check and ask unanswered questions + env.update(_parse_args_in_yunohost_format( + args, section['options'] + )) - # Call config script to extract current values + # Call config script in 'apply' mode logger.info("Running config script...") - env = {key: str(value[0]) for key, value in args_dict.items() if not value[0] is None} + env = {key: str(value[0]) for key, value in env.items() if not value[0] is None} errors = _call_config_script(operation_logger, app, 'apply', env=env) - # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception + # Script got manually interrupted ... + # N.B. : KeyboardInterrupt does not inherit from Exception except (KeyboardInterrupt, EOFError): error = m18n.n("operation_interrupted") logger.error(m18n.n("app_config_failed", app=app, error=error)) @@ -1904,25 +1891,20 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): if errors: return { - "app": app, "errors": errors, } # Reload services logger.info("Reloading services...") - services_to_reload = set([]) - for panel in config_panel.get("panel", []): - services_to_reload |= set(panel.get('services', [])) - for section in panel.get("sections", []): - services_to_reload |= set(section.get('services', [])) - for option in section.get("options", []): - services_to_reload |= set(option.get('services', [])) + services_to_reload = set() + for panel, section, obj in _get_config_iterator(config_panel, + ['panel', 'section', 'option']): + services_to_reload |= set(obj.get('services', [])) services_to_reload = list(services_to_reload) services_to_reload.sort(key = 'nginx'.__eq__) for service in services_to_reload: - if service == "__APP__": - service = app + service = service.replace('__APP__', app) logger.debug(f"Reloading {service}") if not _run_service_command('reload-or-restart', service): services = _get_services() @@ -1934,23 +1916,27 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None): ) logger.success("Config updated as expected") - return { - "app": app, - "errors": [], - "logs": operation_logger.success(), - } + return {} -def _get_options_iterator(config_panel): - for panel in config_panel.get("panel", []): +def _get_config_iterator(config_panel, trigger=['option']): + for panel in config_panel.get("panels", []): + if 'panel' in trigger: + yield (panel, None, panel) for section in panel.get("sections", []): - for option in section.get("options", []): - yield (panel, section, option) + if 'section' in trigger: + yield (panel, section, section) + if 'option' in trigger: + for option in section.get("options", []): + yield (panel, section, option) -def _call_config_script(operation_logger, app, action, env={}): +def _call_config_script(operation_logger, app, action, env={}, config_panel=None): from yunohost.hook import hook_exec + YunoHostArgumentFormatParser.operation_logger = operation_logger + operation_logger.start() + # Add default config script if needed config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config") if not os.path.exists(config_script): @@ -1976,7 +1962,27 @@ ynh_panel_run $1 config_script, args=[action], env=env ) if ret != 0: - operation_logger.error(parsed_values) + if action == 'show': + raise YunohostError("app_config_unable_to_read_values") + else: + raise YunohostError("app_config_unable_to_apply_values_correctly") + + return parsed_values + + if not config_panel: + return parsed_values + + # Hydrating config panel with current value + logger.debug("Hydrating config with current values") + for _, _, option in _get_config_iterator(config_panel): + if option['name'] not in parsed_values: + continue + value = parsed_values[option['name']] + # In general, the value is just a simple value. + # Sometimes it could be a dict used to overwrite the option itself + value = value if isinstance(value, dict) else {'current_value': value } + option.update(value) + return parsed_values @@ -2083,6 +2089,13 @@ def _get_app_actions(app_id): def _get_app_config_panel(app_id, filter_key=''): "Get app config panel stored in json or in toml" + + # Split filter_key + filter_key = dict(enumerate(filter_key.split('.'))) + if len(filter_key) > 3: + raise YunohostError("app_config_too_much_sub_keys") + + # Open TOML config_panel_toml_path = os.path.join( APPS_SETTING_PATH, app_id, "config_panel.toml" ) @@ -2103,7 +2116,7 @@ def _get_app_config_panel(app_id, filter_key=''): # name = "Choose the sources of packages to automatically upgrade." # default = "Security only" # type = "text" - # help = "We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates." + # help = "We can't use a choices field for now. In the meantime[...]" # # choices = ["Security only", "Security and updates"] # [main.unattended_configuration.ynh_update] @@ -2143,7 +2156,7 @@ def _get_app_config_panel(app_id, filter_key=''): # u'name': u'50unattended-upgrades configuration file', # u'options': [{u'//': u'"choices" : ["Security only", "Security and updates"]', # u'default': u'Security only', - # u'help': u"We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates.", + # u'help': u"We can't use a choices field for now. In the meantime[...]", # u'id': u'upgrade_level', # u'name': u'Choose the sources of packages to automatically upgrade.', # u'type': u'text'}, @@ -2152,127 +2165,81 @@ def _get_app_config_panel(app_id, filter_key=''): # u'name': u'Would you like to update YunoHost packages automatically ?', # u'type': u'bool'}, - if os.path.exists(config_panel_toml_path): - toml_config_panel = toml.load( - open(config_panel_toml_path, "r"), _dict=OrderedDict - ) - if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED: - raise YunohostError( - "app_config_too_old_version", app=app_id, - version=toml_config_panel["version"] - ) - - # transform toml format into json format - config_panel = { - "name": toml_config_panel["name"], - "version": toml_config_panel["version"], - "panel": [], - } - filter_key = filter_key.split('.') - filter_panel = filter_key.pop(0) - filter_section = filter_key.pop(0) if len(filter_key) > 0 else False - filter_option = filter_key.pop(0) if len(filter_key) > 0 else False - - panels = [ - key_value - for key_value in toml_config_panel.items() - if key_value[0] not in ("name", "version") - and isinstance(key_value[1], OrderedDict) - ] - - for key, value in panels: - if filter_panel and key != filter_panel: - continue - - panel = { - "id": key, - "name": value.get("name", ""), - "services": value.get("services", []), - "sections": [], - } - - sections = [ - k_v1 - for k_v1 in value.items() - if k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict) - ] - - for section_key, section_value in sections: - - if filter_section and section_key != filter_section: - continue - - section = { - "id": section_key, - "name": section_value.get("name", ""), - "optional": section_value.get("optional", True), - "services": section_value.get("services", []), - "options": [], - } - if section_value.get('visibleIf'): - section['visibleIf'] = section_value.get('visibleIf') - - options = [ - k_v - for k_v in section_value.items() - if k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict) - ] - - for option_key, option_value in options: - if filter_option and option_key != filter_option: - continue - - option = dict(option_value) - option["optional"] = option_value.get("optional", section['optional']) - option["name"] = option_key - option["ask"] = {"en": option["ask"]} - if "help" in option: - option["help"] = {"en": option["help"]} - section["options"].append(option) - - panel["sections"].append(section) - - config_panel["panel"].append(panel) - - if (filter_panel and len(config_panel['panel']) == 0) or \ - (filter_section and len(config_panel['panel'][0]['sections']) == 0) or \ - (filter_option and len(config_panel['panel'][0]['sections'][0]['options']) == 0): - raise YunohostError( - "app_config_bad_filter_key", app=app_id, filter_key=filter_key - ) - - return config_panel - - return None - -def _get_app_hydrated_config_panel(operation_logger, app, filter_key=''): - - # Read config panel toml - config_panel = _get_app_config_panel(app, filter_key=filter_key) - - if not config_panel: + if not os.path.exists(config_panel_toml_path): return None + toml_config_panel = read_toml(config_panel_toml_path) - operation_logger.start() + # Check TOML config panel is in a supported version + if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED: + raise YunohostError( + "app_config_too_old_version", app=app_id, + version=toml_config_panel["version"] + ) - # Call config script to extract current values - parsed_values = _call_config_script(operation_logger, app, 'show') + # Transform toml format into internal format + defaults = { + 'toml': { + 'version': 1.0 + }, + 'panels': { + 'name': '', + 'services': [], + 'actions': {'apply': {'en': 'Apply'}} + }, # help + 'sections': { + 'name': '', + 'services': [], + 'optional': True + }, # visibleIf help + 'options': {} + # ask type source help helpLink example style icon placeholder visibleIf + # optional choices pattern limit min max step accept redact + } - # # Check and transform values if needed - # options = [option for _, _, option in _get_options_iterator(config_panel)] - # args_dict = _parse_args_in_yunohost_format( - # parsed_values, options, False - # ) + def convert(toml_node, node_type): + """Convert TOML in internal format ('full' mode used by webadmin) - # Hydrate - logger.debug("Hydrating config with current value") - for _, _, option in _get_options_iterator(config_panel): - if option['name'] in parsed_values: - value = parsed_values[option['name']] - if isinstance(value, dict): - option.update(value) + Here are some properties of 1.0 config panel in toml: + - node properties and node children are mixed, + - text are in english only + - some properties have default values + This function detects all children nodes and put them in a list + """ + # Prefill the node default keys if needed + default = defaults[node_type] + node = {key: toml_node.get(key, value) for key, value in default.items()} + + # Define the filter_key part to use and the children type + i = list(defaults).index(node_type) + search_key = filter_key.get(i) + subnode_type = list(defaults)[i+1] if node_type != 'options' else None + + for key, value in toml_node.items(): + # Key/value are a child node + if isinstance(value, OrderedDict) and key not in default and subnode_type: + # We exclude all nodes not referenced by the filter_key + if search_key and key != search_key: + continue + subnode = convert(value, subnode_type) + subnode['id'] = key + if node_type == 'sections': + subnode['name'] = key # legacy + subnode.setdefault('optional', toml_node.get('optional', True)) + node.setdefault(subnode_type, []).append(subnode) + # Key/value are a property else: - option["current_value"] = value #args_dict[option["name"]][0] + # Todo search all i18n keys + node[key] = value if key not in ['ask', 'help', 'name'] else { 'en': value } + return node + + config_panel = convert(toml_config_panel, 'toml') + + try: + config_panel['panels'][0]['sections'][0]['options'][0] + except (KeyError, IndexError): + raise YunohostError( + "app_config_empty_or_bad_filter_key", app=app_id, filter_key=filter_key + ) return config_panel @@ -2831,6 +2798,7 @@ class Question: class YunoHostArgumentFormatParser(object): hide_user_input_in_prompt = False + operation_logger = None def parse_question(self, question, user_answers): parsed_question = Question() @@ -2842,10 +2810,11 @@ class YunoHostArgumentFormatParser(object): parsed_question.optional = question.get("optional", False) parsed_question.choices = question.get("choices", []) parsed_question.pattern = question.get("pattern") - parsed_question.ask = question.get("ask", {'en': f"Enter value for '{parsed_question.name}':"}) + parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"}) parsed_question.help = question.get("help") parsed_question.helpLink = question.get("helpLink") parsed_question.value = user_answers.get(parsed_question.name) + parsed_question.redact = question.get('redact', False) # Empty value is parsed as empty string if parsed_question.default == "": @@ -2947,6 +2916,27 @@ class YunoHostArgumentFormatParser(object): return text_for_user_input_in_cli def _post_parse_value(self, question): + if not question.redact: + return question.value + + # Tell the operation_logger to redact all password-type / secret args + # Also redact the % escaped version of the password that might appear in + # the 'args' section of metadata (relevant for password with non-alphanumeric char) + data_to_redact = [] + if question.value and isinstance(question.value, str): + data_to_redact.append(question.value) + if question.current_value and isinstance(question.current_value, str): + data_to_redact.append(question.current_value) + data_to_redact += [ + urllib.parse.quote(data) + for data in data_to_redact + if urllib.parse.quote(data) != data + ] + if self.operation_logger: + self.operation_logger.data_to_redact.extend(data_to_redact) + elif data_to_redact: + raise YunohostError("app_argument_cant_redact", arg=question.name) + return question.value @@ -2954,12 +2944,6 @@ class StringArgumentParser(YunoHostArgumentFormatParser): argument_type = "string" default_value = "" - def _prevalidate(self, question): - super()._prevalidate(question) - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2") - ) - class TagsArgumentParser(YunoHostArgumentFormatParser): argument_type = "tags" @@ -2982,7 +2966,7 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser): question = super(PasswordArgumentParser, self).parse_question( question, user_answers ) - + question.redact = True if question.default is not None: raise YunohostValidationError( "app_argument_password_no_default", name=question.name @@ -3242,6 +3226,8 @@ class FileArgumentParser(YunoHostArgumentFormatParser): # os.path.join to avoid the user to be able to rewrite a file in filesystem # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" file_path = os.path.normpath(upload_dir + "/" + filename) + if not file_path.startswith(upload_dir + "/"): + raise YunohostError("relative_parent_path_in_filename_forbidden") i = 2 while os.path.exists(file_path): file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) From 60564d55c8899d51dbff7cec32e5b7107c045959 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 2 Sep 2021 17:14:27 +0200 Subject: [PATCH 046/119] (enh] Config panel helpers wording --- data/helpers.d/configpanel | 139 ++++++------------------------------- data/helpers.d/utils | 98 ++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 118 deletions(-) diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel index b55b0b0fd..7727c8d55 100644 --- a/data/helpers.d/configpanel +++ b/data/helpers.d/configpanel @@ -1,105 +1,7 @@ #!/bin/bash -# Get a value from heterogeneous file (yaml, json, php, python...) -# -# usage: ynh_value_get --file=PATH --key=KEY -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to get -# -# This helpers match several var affectation use case in several languages -# We don't use jq or equivalent to keep comments and blank space in files -# This helpers work line by line, it is not able to work correctly -# if you have several identical keys in your files -# -# Example of line this helpers can managed correctly -# .yml -# title: YunoHost documentation -# email: 'yunohost@yunohost.org' -# .json -# "theme": "colib'ris", -# "port": 8102 -# "some_boolean": false, -# "user": null -# .ini -# some_boolean = On -# action = "Clear" -# port = 20 -# .php -# $user= -# user => 20 -# .py -# USER = 8102 -# user = 'https://donate.local' -# CUSTOM['user'] = 'YunoHost' -# Requires YunoHost version 4.3 or higher. -ynh_value_get() { - # Declare an array to define the options of this helper. - local legacy_args=fk - local -A args_array=( [f]=file= [k]=key= ) - local file - local key - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - - local crazy_value="$((grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL) | head -n1)" - #" - - local first_char="${crazy_value:0:1}" - if [[ "$first_char" == '"' ]] ; then - echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' - elif [[ "$first_char" == "'" ]] ; then - echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" - else - echo "$crazy_value" - fi -} - -# Set a value into heterogeneous file (yaml, json, php, python...) -# -# usage: ynh_value_set --file=PATH --key=KEY --value=VALUE -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to set -# | arg: -v, --value= - the value to set -# -# Requires YunoHost version 4.3 or higher. -ynh_value_set() { - # Declare an array to define the options of this helper. - local legacy_args=fkv - local -A args_array=( [f]=file= [k]=key= [v]=value=) - local file - local key - local value - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - - local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" - # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" - local first_char="${crazy_value:0:1}" - if [[ "$first_char" == '"' ]] ; then - # \ and sed is quite complex you need 2 \\ to get one in a sed - # So we need \\\\ to go through 2 sed - value="$(echo "$value" | sed 's/"/\\\\"/g')" - sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file} - elif [[ "$first_char" == "'" ]] ; then - # \ and sed is quite complex you need 2 \\ to get one in a sed - # However double quotes implies to double \\ to - # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str - value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" - sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file} - else - if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then - value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' - fi - sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file} - fi - ynh_print_info "Configuration key '$key' edited into $file" -} - -_ynh_panel_get() { +_ynh_app_config_get() { # From settings local lines lines=`python3 << EOL @@ -165,7 +67,7 @@ EOL local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")" + old[$short_setting]="$(ynh_get_var --file="${source_file}" --key="${source_key}")" fi done @@ -173,7 +75,7 @@ EOL } -_ynh_panel_apply() { +_ynh_app_config_apply() { for short_setting in "${!old[@]}" do local setter="set__${short_setting}" @@ -222,18 +124,19 @@ _ynh_panel_apply() { local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$source_file" - ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}" + ynh_set_var --file="${source_file}" --key="${source_key}" --value="${!short_setting}" ynh_store_file_checksum --file="$source_file" # We stored the info in settings in order to be able to upgrade the app ynh_app_setting_set $app $short_setting "${!short_setting}" + ynh_print_info "Configuration key '$source_key' edited into $source_file" fi fi done } -_ynh_panel_show() { +_ynh_app_config_show() { for short_setting in "${!old[@]}" do if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then @@ -248,7 +151,7 @@ _ynh_panel_show() { done } -_ynh_panel_validate() { +_ynh_app_config_validate() { # Change detection ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1 local is_error=true @@ -319,23 +222,23 @@ _ynh_panel_validate() { } -ynh_panel_get() { - _ynh_panel_get +ynh_app_config_get() { + _ynh_app_config_get } -ynh_panel_show() { - _ynh_panel_show +ynh_app_config_show() { + _ynh_app_config_show } -ynh_panel_validate() { - _ynh_panel_validate +ynh_app_config_validate() { + _ynh_app_config_validate } -ynh_panel_apply() { - _ynh_panel_apply +ynh_app_config_apply() { + _ynh_app_config_apply } -ynh_panel_run() { +ynh_app_config_run() { declare -Ag old=() declare -Ag changed=() declare -Ag file_hash=() @@ -345,18 +248,18 @@ ynh_panel_run() { case $1 in show) - ynh_panel_get - ynh_panel_show + ynh_app_config_get + ynh_app_config_show ;; apply) max_progression=4 ynh_script_progression --message="Reading config panel description and current configuration..." - ynh_panel_get + ynh_app_config_get - ynh_panel_validate + ynh_app_config_validate ynh_script_progression --message="Applying the new configuration..." - ynh_panel_apply + ynh_app_config_apply ynh_script_progression --message="Configuration of $app completed" --last ;; esac diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 00bec89ac..1c4f73ddf 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -473,6 +473,104 @@ ynh_replace_vars () { done } +# Get a value from heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_get_var --file=PATH --key=KEY +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to get +# +# This helpers match several var affectation use case in several languages +# We don't use jq or equivalent to keep comments and blank space in files +# This helpers work line by line, it is not able to work correctly +# if you have several identical keys in your files +# +# Example of line this helpers can managed correctly +# .yml +# title: YunoHost documentation +# email: 'yunohost@yunohost.org' +# .json +# "theme": "colib'ris", +# "port": 8102 +# "some_boolean": false, +# "user": null +# .ini +# some_boolean = On +# action = "Clear" +# port = 20 +# .php +# $user= +# user => 20 +# .py +# USER = 8102 +# user = 'https://donate.local' +# CUSTOM['user'] = 'YunoHost' +# Requires YunoHost version 4.3 or higher. +ynh_get_var() { + # Declare an array to define the options of this helper. + local legacy_args=fk + local -A args_array=( [f]=file= [k]=key= ) + local file + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' + + local crazy_value="$((grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL) | head -n1)" + #" + + local first_char="${crazy_value:0:1}" + if [[ "$first_char" == '"' ]] ; then + echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + elif [[ "$first_char" == "'" ]] ; then + echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + else + echo "$crazy_value" + fi +} + +# Set a value into heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_set_var --file=PATH --key=KEY --value=VALUE +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to set +# | arg: -v, --value= - the value to set +# +# Requires YunoHost version 4.3 or higher. +ynh_set_var() { + # Declare an array to define the options of this helper. + local legacy_args=fkv + local -A args_array=( [f]=file= [k]=key= [v]=value=) + local file + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' + + local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" + # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" + local first_char="${crazy_value:0:1}" + if [[ "$first_char" == '"' ]] ; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # So we need \\\\ to go through 2 sed + value="$(echo "$value" | sed 's/"/\\\\"/g')" + sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file} + elif [[ "$first_char" == "'" ]] ; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # However double quotes implies to double \\ to + # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str + value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" + sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file} + else + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then + value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' + fi + sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file} + fi +} + + # Render templates with Jinja2 # # [internal] From ea2026a0297c75f8616373b5120cc7b28a5379f2 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 2 Sep 2021 17:15:38 +0200 Subject: [PATCH 047/119] (enh] Config panel helpers wording --- data/helpers.d/{configpanel => config} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/helpers.d/{configpanel => config} (100%) diff --git a/data/helpers.d/configpanel b/data/helpers.d/config similarity index 100% rename from data/helpers.d/configpanel rename to data/helpers.d/config From edef077c7419782b90436098413a682bc65febb6 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 2 Sep 2021 17:19:30 +0200 Subject: [PATCH 048/119] (enh] Config panel helpers wording --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a4c215328..828175393 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1945,7 +1945,7 @@ def _call_config_script(operation_logger, app, action, env={}, config_panel=None source /usr/share/yunohost/helpers ynh_abort_if_errors final_path=$(ynh_app_setting_get $app final_path) -ynh_panel_run $1 +ynh_app_config_run $1 """ write_to_file(config_script, default_script) From c5885000ecac21d6b3620def3f784c0617b9f0d1 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 2 Sep 2021 19:46:33 +0200 Subject: [PATCH 049/119] [enh] Better upgrade management --- data/helpers.d/backup | 15 ++++++++++++++- data/helpers.d/config | 10 ++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/backup b/data/helpers.d/backup index ae746a37b..21ca2d7f0 100644 --- a/data/helpers.d/backup +++ b/data/helpers.d/backup @@ -326,12 +326,25 @@ ynh_bind_or_cp() { ynh_store_file_checksum () { # Declare an array to define the options of this helper. local legacy_args=f - local -A args_array=( [f]=file= ) + local -A args_array=( [f]=file= [u]=update_only ) local file + local update_only + update_only="${update_only:-0}" + # Manage arguments with getopts ynh_handle_getopts_args "$@" local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + + # If update only, we don't save the new checksum if no old checksum exist + if [ $update_only -eq 1 ] ; then + local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + if [ -z "${checksum_value}" ] ; then + unset backup_file_checksum + return 0 + fi + fi + ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup diff --git a/data/helpers.d/config b/data/helpers.d/config index 7727c8d55..52454ff91 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -96,10 +96,14 @@ _ynh_app_config_apply() { fi local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]] ; then + ynh_backup_if_checksum_is_different --file="$source_file" rm -f "$source_file" + ynh_delete_file_checksum --file="$source_file" --update_only ynh_print_info "File '$source_file' removed" else + ynh_backup_if_checksum_is_different --file="$source_file" cp "${!short_setting}" "$source_file" + ynh_store_file_checksum --file="$source_file" --update_only ynh_print_info "File '$source_file' overwrited with ${!short_setting}" fi @@ -114,7 +118,9 @@ _ynh_app_config_apply() { ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + ynh_backup_if_checksum_is_different --file="$source_file" echo "${!short_setting}" > "$source_file" + ynh_store_file_checksum --file="$source_file" --update_only ynh_print_info "File '$source_file' overwrited with the content you provieded in '${short_setting}' question" # Set value into a kind of key/value file @@ -122,10 +128,10 @@ _ynh_app_config_apply() { local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - + ynh_backup_if_checksum_is_different --file="$source_file" ynh_set_var --file="${source_file}" --key="${source_key}" --value="${!short_setting}" - ynh_store_file_checksum --file="$source_file" + ynh_store_file_checksum --file="$source_file" --update_only # We stored the info in settings in order to be able to upgrade the app ynh_app_setting_set $app $short_setting "${!short_setting}" From e0fe82f566e15fc50376869e22a593aa91446fc4 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Thu, 2 Sep 2021 20:09:41 +0200 Subject: [PATCH 050/119] [fix] Some service has no test_conf Co-authored-by: Alexandre Aubin --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 828175393..de6df6579 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1908,7 +1908,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None, args_ logger.debug(f"Reloading {service}") if not _run_service_command('reload-or-restart', service): services = _get_services() - test_conf = services[service].get('test_conf') + test_conf = services[service].get('test_conf', 'true') errors = check_output(f"{test_conf}; exit 0") if test_conf else '' raise YunohostError( "app_config_failed_service_reload", From b28cf8cbce8a9126a5a97af4f1fd58d21e73e8df Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 3 Sep 2021 14:26:34 +0200 Subject: [PATCH 051/119] [enh] Prepare config panel for domain --- data/helpers.d/utils | 7 +- src/yunohost/app.py | 947 +++-------------------------------- src/yunohost/utils/config.py | 814 ++++++++++++++++++++++++++++++ src/yunohost/utils/i18n.py | 46 ++ 4 files changed, 921 insertions(+), 893 deletions(-) create mode 100644 src/yunohost/utils/config.py create mode 100644 src/yunohost/utils/i18n.py diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 1c4f73ddf..14e7ebe4a 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -551,22 +551,23 @@ ynh_set_var() { local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" local first_char="${crazy_value:0:1}" + delimiter=$'\001' if [[ "$first_char" == '"' ]] ; then # \ and sed is quite complex you need 2 \\ to get one in a sed # So we need \\\\ to go through 2 sed value="$(echo "$value" | sed 's/"/\\\\"/g')" - sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file} + sed -ri s$delimiter'^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} elif [[ "$first_char" == "'" ]] ; then # \ and sed is quite complex you need 2 \\ to get one in a sed # However double quotes implies to double \\ to # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" - sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file} + sed -ri "s$delimiter^(${var_part}')([^']|\\')*('"'[ \t,;]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} else if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' fi - sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file} + sed -ri "s$delimiter^(${var_part}).*"'$'$delimiter'\1'"${value}"$delimiter'i' ${file} fi } diff --git a/src/yunohost/app.py b/src/yunohost/app.py index de6df6579..522f695e2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -33,7 +33,6 @@ import re import subprocess import glob import urllib.parse -import base64 import tempfile from collections import OrderedDict @@ -54,8 +53,10 @@ from moulinette.utils.filesystem import ( mkdir, ) -from yunohost.service import service_status, _run_service_command, _get_services -from yunohost.utils import packages +from yunohost.service import service_status, _run_service_command +from yunohost.utils import packages, config +from yunohost.utils.config import ConfigPanel, parse_args_in_yunohost_format, YunoHostArgumentFormatParser +from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory from yunohost.log import is_unit_operation, OperationLogger @@ -70,7 +71,6 @@ APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" APPS_CATALOG_API_VERSION = 2 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" -APPS_CONFIG_PANEL_VERSION_SUPPORTED = 1.0 re_app_instance_name = re.compile( r"^(?P[\w-]+?)(__(?P[1-9][0-9]*))?$" ) @@ -1756,51 +1756,12 @@ def app_action_run(operation_logger, app, action, args=None): return logger.success("Action successed!") -@is_unit_operation() -def app_config_get(operation_logger, app, key='', mode='classic'): +def app_config_get(app, key='', mode='classic'): """ Display an app configuration in classic, full or export mode """ - - # Check app is installed - _assert_is_installed(app) - - filter_key = key or '' - - # Read config panel toml - config_panel = _get_app_config_panel(app, filter_key=filter_key) - - if not config_panel: - raise YunohostError("app_config_no_panel") - - # Call config script in order to hydrate config panel with current values - values = _call_config_script(operation_logger, app, 'show', config_panel=config_panel) - - # Format result in full mode - if mode == 'full': - operation_logger.success() - return config_panel - - # In 'classic' mode, we display the current value if key refer to an option - if filter_key.count('.') == 2 and mode == 'classic': - option = filter_key.split('.')[-1] - operation_logger.success() - return values.get(option, None) - - # Format result in 'classic' or 'export' mode - logger.debug(f"Formating result in '{mode}' mode") - result = {} - for panel, section, option in _get_config_iterator(config_panel): - key = f"{panel['id']}.{section['id']}.{option['id']}" - if mode == 'export': - result[option['id']] = option.get('current_value') - else: - result[key] = { 'ask': _value_for_locale(option['ask']) } - if 'current_value' in option: - result[key]['value'] = option['current_value'] - - operation_logger.success() - return result + config = AppConfigPanel(app) + return config.get(key, mode) @is_unit_operation() @@ -1809,182 +1770,65 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None, args_ Apply a new app configuration """ - # Check app is installed - _assert_is_installed(app) - - filter_key = key or '' - - # Read config panel toml - config_panel = _get_app_config_panel(app, filter_key=filter_key) - - if not config_panel: - raise YunohostError("app_config_no_panel") - - if (args is not None or args_file is not None) and value is not None: - raise YunohostError("app_config_args_value") - - if filter_key.count('.') != 2 and not value is None: - raise YunohostError("app_config_set_value_on_section") - - # Import and parse pre-answered options - logger.debug("Import and parse pre-answered options") - args = urllib.parse.parse_qs(args or '', keep_blank_values=True) - args = { key: ','.join(value_) for key, value_ in args.items() } - - if args_file: - # Import YAML / JSON file but keep --args values - args = { **read_yaml(args_file), **args } - - if value is not None: - args = {filter_key.split('.')[-1]: value} - - # Call config script in order to hydrate config panel with current values - _call_config_script(operation_logger, app, 'show', config_panel=config_panel) - - # Ask unanswered question and prevalidate - logger.debug("Ask unanswered question and prevalidate data") - def display_header(message): - """ CLI panel/section header display - """ - if Moulinette.interface.type == 'cli' and filter_key.count('.') < 2: - Moulinette.display(colorize(message, 'purple')) - - try: - env = {} - for panel, section, obj in _get_config_iterator(config_panel, - ['panel', 'section']): - if panel == obj: - name = _value_for_locale(panel['name']) - display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}") - continue - name = _value_for_locale(section['name']) - display_header(f"\n# {name}") - - # Check and ask unanswered questions - env.update(_parse_args_in_yunohost_format( - args, section['options'] - )) - - # Call config script in 'apply' mode - logger.info("Running config script...") - env = {key: str(value[0]) for key, value in env.items() if not value[0] is None} - - errors = _call_config_script(operation_logger, app, 'apply', env=env) - # Script got manually interrupted ... - # N.B. : KeyboardInterrupt does not inherit from Exception - except (KeyboardInterrupt, EOFError): - error = m18n.n("operation_interrupted") - logger.error(m18n.n("app_config_failed", app=app, error=error)) - failure_message_with_debug_instructions = operation_logger.error(error) - raise - # Something wrong happened in Yunohost's code (most probably hook_exec) - except Exception: - import traceback - - error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("app_config_failed", app=app, error=error)) - failure_message_with_debug_instructions = operation_logger.error(error) - raise - finally: - # Delete files uploaded from API - FileArgumentParser.clean_upload_dirs() - - if errors: - return { - "errors": errors, - } - - # Reload services - logger.info("Reloading services...") - services_to_reload = set() - for panel, section, obj in _get_config_iterator(config_panel, - ['panel', 'section', 'option']): - services_to_reload |= set(obj.get('services', [])) - - services_to_reload = list(services_to_reload) - services_to_reload.sort(key = 'nginx'.__eq__) - for service in services_to_reload: - service = service.replace('__APP__', app) - logger.debug(f"Reloading {service}") - if not _run_service_command('reload-or-restart', service): - services = _get_services() - test_conf = services[service].get('test_conf', 'true') - errors = check_output(f"{test_conf}; exit 0") if test_conf else '' - raise YunohostError( - "app_config_failed_service_reload", - service=service, errors=errors - ) - - logger.success("Config updated as expected") - return {} - - -def _get_config_iterator(config_panel, trigger=['option']): - for panel in config_panel.get("panels", []): - if 'panel' in trigger: - yield (panel, None, panel) - for section in panel.get("sections", []): - if 'section' in trigger: - yield (panel, section, section) - if 'option' in trigger: - for option in section.get("options", []): - yield (panel, section, option) - - -def _call_config_script(operation_logger, app, action, env={}, config_panel=None): - from yunohost.hook import hook_exec + config = AppConfigPanel(app) YunoHostArgumentFormatParser.operation_logger = operation_logger operation_logger.start() - # Add default config script if needed - config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config") - if not os.path.exists(config_script): - logger.debug("Adding a default config script") - default_script = """#!/bin/bash + result = config.set(key, value, args, args_file) + if "errors" not in result: + operation_logger.success() + return result + +class AppConfigPanel(ConfigPanel): + def __init__(self, app): + + # Check app is installed + _assert_is_installed(app) + + self.app = app + config_path = os.path.join(APPS_SETTING_PATH, app, "config_panel.toml") + super().__init__(config_path=config_path) + + def _load_current_values(self): + self.values = self._call_config_script('show') + + def _apply(self): + self.errors = self._call_config_script('apply', self.new_values) + + def _call_config_script(self, action, env={}): + from yunohost.hook import hook_exec + + # Add default config script if needed + config_script = os.path.join(APPS_SETTING_PATH, self.app, "scripts", "config") + if not os.path.exists(config_script): + logger.debug("Adding a default config script") + default_script = """#!/bin/bash source /usr/share/yunohost/helpers ynh_abort_if_errors final_path=$(ynh_app_setting_get $app final_path) ynh_app_config_run $1 """ - write_to_file(config_script, default_script) + write_to_file(config_script, default_script) - # Call config script to extract current values - logger.debug(f"Calling '{action}' action from config script") - app_id, app_instance_nb = _parse_app_instance_name(app) - env.update({ - "app_id": app_id, - "app": app, - "app_instance_nb": str(app_instance_nb), - }) - - ret, parsed_values = hook_exec( - config_script, args=[action], env=env - ) - if ret != 0: - if action == 'show': - raise YunohostError("app_config_unable_to_read_values") - else: - raise YunohostError("app_config_unable_to_apply_values_correctly") - - return parsed_values - - if not config_panel: - return parsed_values - - # Hydrating config panel with current value - logger.debug("Hydrating config with current values") - for _, _, option in _get_config_iterator(config_panel): - if option['name'] not in parsed_values: - continue - value = parsed_values[option['name']] - # In general, the value is just a simple value. - # Sometimes it could be a dict used to overwrite the option itself - value = value if isinstance(value, dict) else {'current_value': value } - option.update(value) - - return parsed_values + # Call config script to extract current values + logger.debug(f"Calling '{action}' action from config script") + app_id, app_instance_nb = _parse_app_instance_name(self.app) + env.update({ + "app_id": app_id, + "app": self.app, + "app_instance_nb": str(app_instance_nb), + }) + ret, values = hook_exec( + config_script, args=[action], env=env + ) + if ret != 0: + if action == 'show': + raise YunohostError("app_config_unable_to_read_values") + else: + raise YunohostError("app_config_unable_to_apply_values_correctly") + return values def _get_all_installed_apps_id(): """ @@ -2087,163 +1931,6 @@ def _get_app_actions(app_id): return None -def _get_app_config_panel(app_id, filter_key=''): - "Get app config panel stored in json or in toml" - - # Split filter_key - filter_key = dict(enumerate(filter_key.split('.'))) - if len(filter_key) > 3: - raise YunohostError("app_config_too_much_sub_keys") - - # Open TOML - config_panel_toml_path = os.path.join( - APPS_SETTING_PATH, app_id, "config_panel.toml" - ) - - # sample data to get an idea of what is going on - # this toml extract: - # - # version = "0.1" - # name = "Unattended-upgrades configuration panel" - # - # [main] - # name = "Unattended-upgrades configuration" - # - # [main.unattended_configuration] - # name = "50unattended-upgrades configuration file" - # - # [main.unattended_configuration.upgrade_level] - # name = "Choose the sources of packages to automatically upgrade." - # default = "Security only" - # type = "text" - # help = "We can't use a choices field for now. In the meantime[...]" - # # choices = ["Security only", "Security and updates"] - - # [main.unattended_configuration.ynh_update] - # name = "Would you like to update YunoHost packages automatically ?" - # type = "bool" - # default = true - # - # will be parsed into this: - # - # OrderedDict([(u'version', u'0.1'), - # (u'name', u'Unattended-upgrades configuration panel'), - # (u'main', - # OrderedDict([(u'name', u'Unattended-upgrades configuration'), - # (u'unattended_configuration', - # OrderedDict([(u'name', - # u'50unattended-upgrades configuration file'), - # (u'upgrade_level', - # OrderedDict([(u'name', - # u'Choose the sources of packages to automatically upgrade.'), - # (u'default', - # u'Security only'), - # (u'type', u'text'), - # (u'help', - # u"We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates.")])), - # (u'ynh_update', - # OrderedDict([(u'name', - # u'Would you like to update YunoHost packages automatically ?'), - # (u'type', u'bool'), - # (u'default', True)])), - # - # and needs to be converted into this: - # - # {u'name': u'Unattended-upgrades configuration panel', - # u'panel': [{u'id': u'main', - # u'name': u'Unattended-upgrades configuration', - # u'sections': [{u'id': u'unattended_configuration', - # u'name': u'50unattended-upgrades configuration file', - # u'options': [{u'//': u'"choices" : ["Security only", "Security and updates"]', - # u'default': u'Security only', - # u'help': u"We can't use a choices field for now. In the meantime[...]", - # u'id': u'upgrade_level', - # u'name': u'Choose the sources of packages to automatically upgrade.', - # u'type': u'text'}, - # {u'default': True, - # u'id': u'ynh_update', - # u'name': u'Would you like to update YunoHost packages automatically ?', - # u'type': u'bool'}, - - if not os.path.exists(config_panel_toml_path): - return None - toml_config_panel = read_toml(config_panel_toml_path) - - # Check TOML config panel is in a supported version - if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED: - raise YunohostError( - "app_config_too_old_version", app=app_id, - version=toml_config_panel["version"] - ) - - # Transform toml format into internal format - defaults = { - 'toml': { - 'version': 1.0 - }, - 'panels': { - 'name': '', - 'services': [], - 'actions': {'apply': {'en': 'Apply'}} - }, # help - 'sections': { - 'name': '', - 'services': [], - 'optional': True - }, # visibleIf help - 'options': {} - # ask type source help helpLink example style icon placeholder visibleIf - # optional choices pattern limit min max step accept redact - } - - def convert(toml_node, node_type): - """Convert TOML in internal format ('full' mode used by webadmin) - - Here are some properties of 1.0 config panel in toml: - - node properties and node children are mixed, - - text are in english only - - some properties have default values - This function detects all children nodes and put them in a list - """ - # Prefill the node default keys if needed - default = defaults[node_type] - node = {key: toml_node.get(key, value) for key, value in default.items()} - - # Define the filter_key part to use and the children type - i = list(defaults).index(node_type) - search_key = filter_key.get(i) - subnode_type = list(defaults)[i+1] if node_type != 'options' else None - - for key, value in toml_node.items(): - # Key/value are a child node - if isinstance(value, OrderedDict) and key not in default and subnode_type: - # We exclude all nodes not referenced by the filter_key - if search_key and key != search_key: - continue - subnode = convert(value, subnode_type) - subnode['id'] = key - if node_type == 'sections': - subnode['name'] = key # legacy - subnode.setdefault('optional', toml_node.get('optional', True)) - node.setdefault(subnode_type, []).append(subnode) - # Key/value are a property - else: - # Todo search all i18n keys - node[key] = value if key not in ['ask', 'help', 'name'] else { 'en': value } - return node - - config_panel = convert(toml_config_panel, 'toml') - - try: - config_panel['panels'][0]['sections'][0]['options'][0] - except (KeyError, IndexError): - raise YunohostError( - "app_config_empty_or_bad_filter_key", app=app_id, filter_key=filter_key - ) - - return config_panel - - def _get_app_settings(app_id): """ Get settings of an installed app @@ -2695,30 +2382,6 @@ def _installed_apps(): return os.listdir(APPS_SETTING_PATH) -def _value_for_locale(values): - """ - Return proper value for current locale - - Keyword arguments: - values -- A dict of values associated to their locale - - Returns: - An utf-8 encoded string - - """ - if not isinstance(values, dict): - return values - - for lang in [m18n.locale, m18n.default_locale]: - try: - return values[lang] - except KeyError: - continue - - # Fallback to first value - return list(values.values())[0] - - def _check_manifest_requirements(manifest, app_instance_name): """Check if required packages are met from the manifest""" @@ -2765,7 +2428,7 @@ def _parse_args_from_manifest(manifest, action, args={}): return OrderedDict() action_args = manifest["arguments"][action] - return _parse_args_in_yunohost_format(args, action_args) + return parse_args_in_yunohost_format(args, action_args) def _parse_args_for_action(action, args={}): @@ -2789,507 +2452,11 @@ def _parse_args_for_action(action, args={}): action_args = action["arguments"] - return _parse_args_in_yunohost_format(args, action_args) + return parse_args_in_yunohost_format(args, action_args) -class Question: - "empty class to store questions information" -class YunoHostArgumentFormatParser(object): - hide_user_input_in_prompt = False - operation_logger = None - - def parse_question(self, question, user_answers): - parsed_question = Question() - - parsed_question.name = question["name"] - parsed_question.type = question.get("type", 'string') - parsed_question.default = question.get("default", None) - parsed_question.current_value = question.get("current_value") - parsed_question.optional = question.get("optional", False) - parsed_question.choices = question.get("choices", []) - parsed_question.pattern = question.get("pattern") - parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"}) - parsed_question.help = question.get("help") - parsed_question.helpLink = question.get("helpLink") - parsed_question.value = user_answers.get(parsed_question.name) - parsed_question.redact = question.get('redact', False) - - # Empty value is parsed as empty string - if parsed_question.default == "": - parsed_question.default = None - - return parsed_question - - def parse(self, question, user_answers): - question = self.parse_question(question, user_answers) - - while True: - # Display question if no value filled or if it's a readonly message - if Moulinette.interface.type== 'cli': - text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( - question - ) - if getattr(self, "readonly", False): - Moulinette.display(text_for_user_input_in_cli) - - elif question.value is None: - prefill = "" - if question.current_value is not None: - prefill = question.current_value - elif question.default is not None: - prefill = question.default - question.value = Moulinette.prompt( - message=text_for_user_input_in_cli, - is_password=self.hide_user_input_in_prompt, - confirm=self.hide_user_input_in_prompt, - prefill=prefill, - is_multiline=(question.type == "text") - ) - - - # Apply default value - if question.value in [None, ""] and question.default is not None: - question.value = ( - getattr(self, "default_value", None) - if question.default is None - else question.default - ) - - # Prevalidation - try: - self._prevalidate(question) - except YunohostValidationError as e: - if Moulinette.interface.type== 'api': - raise - Moulinette.display(str(e), 'error') - question.value = None - continue - break - # this is done to enforce a certain formating like for boolean - # by default it doesn't do anything - question.value = self._post_parse_value(question) - - return (question.value, self.argument_type) - - def _prevalidate(self, question): - if question.value in [None, ""] and not question.optional: - raise YunohostValidationError( - "app_argument_required", name=question.name - ) - - # we have an answer, do some post checks - if question.value is not None: - if question.choices and question.value not in question.choices: - self._raise_invalid_answer(question) - if question.pattern and not re.match(question.pattern['regexp'], str(question.value)): - raise YunohostValidationError( - question.pattern['error'], - name=question.name, - value=question.value, - ) - - def _raise_invalid_answer(self, question): - raise YunohostValidationError( - "app_argument_choice_invalid", - name=question.name, - value=question.value, - choices=", ".join(question.choices), - ) - - def _format_text_for_user_input_in_cli(self, question): - text_for_user_input_in_cli = _value_for_locale(question.ask) - - if question.choices: - text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices)) - - if question.help or question.helpLink: - text_for_user_input_in_cli += ":\033[m" - if question.help: - text_for_user_input_in_cli += "\n - " - text_for_user_input_in_cli += _value_for_locale(question.help) - if question.helpLink: - if not isinstance(question.helpLink, dict): - question.helpLink = {'href': question.helpLink} - text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}" - return text_for_user_input_in_cli - - def _post_parse_value(self, question): - if not question.redact: - return question.value - - # Tell the operation_logger to redact all password-type / secret args - # Also redact the % escaped version of the password that might appear in - # the 'args' section of metadata (relevant for password with non-alphanumeric char) - data_to_redact = [] - if question.value and isinstance(question.value, str): - data_to_redact.append(question.value) - if question.current_value and isinstance(question.current_value, str): - data_to_redact.append(question.current_value) - data_to_redact += [ - urllib.parse.quote(data) - for data in data_to_redact - if urllib.parse.quote(data) != data - ] - if self.operation_logger: - self.operation_logger.data_to_redact.extend(data_to_redact) - elif data_to_redact: - raise YunohostError("app_argument_cant_redact", arg=question.name) - - return question.value - - -class StringArgumentParser(YunoHostArgumentFormatParser): - argument_type = "string" - default_value = "" - -class TagsArgumentParser(YunoHostArgumentFormatParser): - argument_type = "tags" - - def _prevalidate(self, question): - values = question.value - for value in values.split(','): - question.value = value - super()._prevalidate(question) - question.value = values - - - -class PasswordArgumentParser(YunoHostArgumentFormatParser): - hide_user_input_in_prompt = True - argument_type = "password" - default_value = "" - forbidden_chars = "{}" - - def parse_question(self, question, user_answers): - question = super(PasswordArgumentParser, self).parse_question( - question, user_answers - ) - question.redact = True - if question.default is not None: - raise YunohostValidationError( - "app_argument_password_no_default", name=question.name - ) - - return question - - def _prevalidate(self, question): - super()._prevalidate(question) - - if question.value is not None: - if any(char in question.value for char in self.forbidden_chars): - raise YunohostValidationError( - "pattern_password_app", forbidden_chars=self.forbidden_chars - ) - - # If it's an optional argument the value should be empty or strong enough - from yunohost.utils.password import assert_password_is_strong_enough - - assert_password_is_strong_enough("user", question.value) - - -class PathArgumentParser(YunoHostArgumentFormatParser): - argument_type = "path" - default_value = "" - - -class BooleanArgumentParser(YunoHostArgumentFormatParser): - argument_type = "boolean" - default_value = False - - def parse_question(self, question, user_answers): - question = super().parse_question( - question, user_answers - ) - - if question.default is None: - question.default = False - - return question - - def _format_text_for_user_input_in_cli(self, question): - text_for_user_input_in_cli = _value_for_locale(question.ask) - - text_for_user_input_in_cli += " [yes | no]" - - if question.default is not None: - formatted_default = "yes" if question.default else "no" - text_for_user_input_in_cli += " (default: {0})".format(formatted_default) - - return text_for_user_input_in_cli - - def _post_parse_value(self, question): - if isinstance(question.value, bool): - return 1 if question.value else 0 - - if str(question.value).lower() in ["1", "yes", "y", "true"]: - return 1 - - if str(question.value).lower() in ["0", "no", "n", "false"]: - return 0 - - raise YunohostValidationError( - "app_argument_choice_invalid", - name=question.name, - value=question.value, - choices="yes, no, y, n, 1, 0", - ) - - -class DomainArgumentParser(YunoHostArgumentFormatParser): - argument_type = "domain" - - def parse_question(self, question, user_answers): - from yunohost.domain import domain_list, _get_maindomain - - question = super(DomainArgumentParser, self).parse_question( - question, user_answers - ) - - if question.default is None: - question.default = _get_maindomain() - - question.choices = domain_list()["domains"] - - return question - - def _raise_invalid_answer(self, question): - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown") - ) - - -class UserArgumentParser(YunoHostArgumentFormatParser): - argument_type = "user" - - def parse_question(self, question, user_answers): - from yunohost.user import user_list, user_info - from yunohost.domain import _get_maindomain - - question = super(UserArgumentParser, self).parse_question( - question, user_answers - ) - question.choices = user_list()["users"] - if question.default is None: - root_mail = "root@%s" % _get_maindomain() - for user in question.choices.keys(): - if root_mail in user_info(user).get("mail-aliases", []): - question.default = user - break - - return question - - def _raise_invalid_answer(self, question): - raise YunohostValidationError( - "app_argument_invalid", - field=question.name, - error=m18n.n("user_unknown", user=question.value), - ) - - -class NumberArgumentParser(YunoHostArgumentFormatParser): - argument_type = "number" - default_value = "" - - def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) - question_parsed.min = question.get('min', None) - question_parsed.max = question.get('max', None) - if question_parsed.default is None: - question_parsed.default = 0 - - return question_parsed - - def _prevalidate(self, question): - super()._prevalidate(question) - if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()): - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") - ) - - if question.min is not None and int(question.value) < question.min: - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") - ) - - if question.max is not None and int(question.value) > question.max: - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") - ) - - def _post_parse_value(self, question): - if isinstance(question.value, int): - return super()._post_parse_value(question) - - if isinstance(question.value, str) and question.value.isdigit(): - return int(question.value) - - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") - ) - - -class DisplayTextArgumentParser(YunoHostArgumentFormatParser): - argument_type = "display_text" - readonly = True - - def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) - - question_parsed.optional = True - question_parsed.style = question.get('style', 'info') - - return question_parsed - - def _format_text_for_user_input_in_cli(self, question): - text = question.ask['en'] - - if question.style in ['success', 'info', 'warning', 'danger']: - color = { - 'success': 'green', - 'info': 'cyan', - 'warning': 'yellow', - 'danger': 'red' - } - return colorize(m18n.g(question.style), color[question.style]) + f" {text}" - else: - return text - -class FileArgumentParser(YunoHostArgumentFormatParser): - argument_type = "file" - upload_dirs = [] - - @classmethod - def clean_upload_dirs(cls): - # Delete files uploaded from API - if Moulinette.interface.type== 'api': - for upload_dir in cls.upload_dirs: - if os.path.exists(upload_dir): - shutil.rmtree(upload_dir) - - def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) - if question.get('accept'): - question_parsed.accept = question.get('accept').replace(' ', '').split(',') - else: - question_parsed.accept = [] - if Moulinette.interface.type== 'api': - if user_answers.get(f"{question_parsed.name}[name]"): - question_parsed.value = { - 'content': question_parsed.value, - 'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name), - } - # If path file are the same - if question_parsed.value and str(question_parsed.value) == question_parsed.current_value: - question_parsed.value = None - - return question_parsed - - def _prevalidate(self, question): - super()._prevalidate(question) - if isinstance(question.value, str) and question.value and not os.path.exists(question.value): - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1") - ) - if question.value in [None, ''] or not question.accept: - return - - filename = question.value if isinstance(question.value, str) else question.value['filename'] - if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept: - raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2") - ) - - - def _post_parse_value(self, question): - from base64 import b64decode - # Upload files from API - # A file arg contains a string with "FILENAME:BASE64_CONTENT" - if not question.value: - return question.value - - if Moulinette.interface.type== 'api': - - upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') - FileArgumentParser.upload_dirs += [upload_dir] - filename = question.value['filename'] - logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}") - - # Filename is given by user of the API. For security reason, we have replaced - # os.path.join to avoid the user to be able to rewrite a file in filesystem - # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" - file_path = os.path.normpath(upload_dir + "/" + filename) - if not file_path.startswith(upload_dir + "/"): - raise YunohostError("relative_parent_path_in_filename_forbidden") - i = 2 - while os.path.exists(file_path): - file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) - i += 1 - content = question.value['content'] - try: - with open(file_path, 'wb') as f: - f.write(b64decode(content)) - except IOError as e: - raise YunohostError("cannot_write_file", file=file_path, error=str(e)) - except Exception as e: - raise YunohostError("error_writing_file", file=file_path, error=str(e)) - question.value = file_path - return question.value - - -ARGUMENTS_TYPE_PARSERS = { - "string": StringArgumentParser, - "text": StringArgumentParser, - "select": StringArgumentParser, - "tags": TagsArgumentParser, - "email": StringArgumentParser, - "url": StringArgumentParser, - "date": StringArgumentParser, - "time": StringArgumentParser, - "color": StringArgumentParser, - "password": PasswordArgumentParser, - "path": PathArgumentParser, - "boolean": BooleanArgumentParser, - "domain": DomainArgumentParser, - "user": UserArgumentParser, - "number": NumberArgumentParser, - "range": NumberArgumentParser, - "display_text": DisplayTextArgumentParser, - "alert": DisplayTextArgumentParser, - "markdown": DisplayTextArgumentParser, - "file": FileArgumentParser, -} - - -def _parse_args_in_yunohost_format(user_answers, argument_questions): - """Parse arguments store in either manifest.json or actions.json or from a - config panel against the user answers when they are present. - - Keyword arguments: - user_answers -- a dictionnary of arguments from the user (generally - empty in CLI, filed from the admin interface) - argument_questions -- the arguments description store in yunohost - format from actions.json/toml, manifest.json/toml - or config_panel.json/toml - """ - parsed_answers_dict = OrderedDict() - - for question in argument_questions: - parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]() - - answer = parser.parse(question=question, user_answers=user_answers) - if answer is not None: - parsed_answers_dict[question["name"]] = answer - - return parsed_answers_dict - def _validate_and_normalize_webpath(args_dict, app_folder): diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py new file mode 100644 index 000000000..34883dcf7 --- /dev/null +++ b/src/yunohost/utils/config.py @@ -0,0 +1,814 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2018 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" + +import os +import re +import toml +import urllib.parse +import tempfile +from collections import OrderedDict + +from moulinette.interfaces.cli import colorize +from moulinette import Moulinette, m18n +from moulinette.utils.log import getActionLogger +from moulinette.utils.process import check_output +from moulinette.utils.filesystem import ( + read_toml, + read_yaml, + write_to_yaml, + mkdir, +) + +from yunohost.service import _get_services +from yunohost.service import _run_service_command, _get_services +from yunohost.utils.i18n import _value_for_locale +from yunohost.utils.error import YunohostError, YunohostValidationError + +logger = getActionLogger("yunohost.config") +CONFIG_PANEL_VERSION_SUPPORTED = 1.0 + +class ConfigPanel: + + def __init__(self, config_path, save_path=None): + self.config_path = config_path + self.save_path = save_path + self.config = {} + self.values = {} + self.new_values = {} + + def get(self, key='', mode='classic'): + self.filter_key = key or '' + + # Read config panel toml + self._get_config_panel() + + if not self.config: + raise YunohostError("config_no_panel") + + # Read or get values and hydrate the config + self._load_current_values() + self._hydrate() + + # Format result in full mode + if mode == 'full': + return self.config + + # In 'classic' mode, we display the current value if key refer to an option + if self.filter_key.count('.') == 2 and mode == 'classic': + option = self.filter_key.split('.')[-1] + return self.values.get(option, None) + + # Format result in 'classic' or 'export' mode + logger.debug(f"Formating result in '{mode}' mode") + result = {} + for panel, section, option in self._iterate(): + key = f"{panel['id']}.{section['id']}.{option['id']}" + if mode == 'export': + result[option['id']] = option.get('current_value') + else: + result[key] = { 'ask': _value_for_locale(option['ask']) } + if 'current_value' in option: + result[key]['value'] = option['current_value'] + + return result + + def set(self, key=None, value=None, args=None, args_file=None): + self.filter_key = key or '' + + # Read config panel toml + self._get_config_panel() + + if not self.config: + raise YunohostError("config_no_panel") + + if (args is not None or args_file is not None) and value is not None: + raise YunohostError("config_args_value") + + if self.filter_key.count('.') != 2 and not value is None: + raise YunohostError("config_set_value_on_section") + + # Import and parse pre-answered options + logger.debug("Import and parse pre-answered options") + args = urllib.parse.parse_qs(args or '', keep_blank_values=True) + self.args = { key: ','.join(value_) for key, value_ in args.items() } + + if args_file: + # Import YAML / JSON file but keep --args values + self.args = { **read_yaml(args_file), **self.args } + + if value is not None: + self.args = {self.filter_key.split('.')[-1]: value} + + # Read or get values and hydrate the config + self._load_current_values() + self._hydrate() + + try: + self._ask() + self._apply() + + # Script got manually interrupted ... + # N.B. : KeyboardInterrupt does not inherit from Exception + except (KeyboardInterrupt, EOFError): + error = m18n.n("operation_interrupted") + logger.error(m18n.n("config_failed", error=error)) + raise + # Something wrong happened in Yunohost's code (most probably hook_exec) + except Exception: + import traceback + + error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) + logger.error(m18n.n("config_failed", error=error)) + raise + finally: + # Delete files uploaded from API + FileArgumentParser.clean_upload_dirs() + + if self.errors: + return { + "errors": errors, + } + + self._reload_services() + + logger.success("Config updated as expected") + return {} + + def _get_toml(self): + return read_toml(self.config_path) + + + def _get_config_panel(self): + # Split filter_key + filter_key = dict(enumerate(self.filter_key.split('.'))) + if len(filter_key) > 3: + raise YunohostError("config_too_much_sub_keys") + + if not os.path.exists(self.config_path): + return None + toml_config_panel = self._get_toml() + + # Check TOML config panel is in a supported version + if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED: + raise YunohostError( + "config_too_old_version", version=toml_config_panel["version"] + ) + + # Transform toml format into internal format + defaults = { + 'toml': { + 'version': 1.0 + }, + 'panels': { + 'name': '', + 'services': [], + 'actions': {'apply': {'en': 'Apply'}} + }, # help + 'sections': { + 'name': '', + 'services': [], + 'optional': True + }, # visibleIf help + 'options': {} + # ask type source help helpLink example style icon placeholder visibleIf + # optional choices pattern limit min max step accept redact + } + + def convert(toml_node, node_type): + """Convert TOML in internal format ('full' mode used by webadmin) + Here are some properties of 1.0 config panel in toml: + - node properties and node children are mixed, + - text are in english only + - some properties have default values + This function detects all children nodes and put them in a list + """ + # Prefill the node default keys if needed + default = defaults[node_type] + node = {key: toml_node.get(key, value) for key, value in default.items()} + + # Define the filter_key part to use and the children type + i = list(defaults).index(node_type) + search_key = filter_key.get(i) + subnode_type = list(defaults)[i+1] if node_type != 'options' else None + + for key, value in toml_node.items(): + # Key/value are a child node + if isinstance(value, OrderedDict) and key not in default and subnode_type: + # We exclude all nodes not referenced by the filter_key + if search_key and key != search_key: + continue + subnode = convert(value, subnode_type) + subnode['id'] = key + if node_type == 'sections': + subnode['name'] = key # legacy + subnode.setdefault('optional', toml_node.get('optional', True)) + node.setdefault(subnode_type, []).append(subnode) + # Key/value are a property + else: + # Todo search all i18n keys + node[key] = value if key not in ['ask', 'help', 'name'] else { 'en': value } + return node + + self.config = convert(toml_config_panel, 'toml') + + try: + self.config['panels'][0]['sections'][0]['options'][0] + except (KeyError, IndexError): + raise YunohostError( + "config_empty_or_bad_filter_key", filter_key=self.filter_key + ) + + return self.config + + def _hydrate(self): + # Hydrating config panel with current value + logger.debug("Hydrating config with current values") + for _, _, option in self._iterate(): + if option['name'] not in self.values: + continue + value = self.values[option['name']] + # In general, the value is just a simple value. + # Sometimes it could be a dict used to overwrite the option itself + value = value if isinstance(value, dict) else {'current_value': value } + option.update(value) + + return self.values + + def _ask(self): + logger.debug("Ask unanswered question and prevalidate data") + def display_header(message): + """ CLI panel/section header display + """ + if Moulinette.interface.type == 'cli' and self.filter_key.count('.') < 2: + Moulinette.display(colorize(message, 'purple')) + for panel, section, obj in self._iterate(['panel', 'section']): + if panel == obj: + name = _value_for_locale(panel['name']) + display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}") + continue + name = _value_for_locale(section['name']) + display_header(f"\n# {name}") + + # Check and ask unanswered questions + self.new_values.update(parse_args_in_yunohost_format( + self.args, section['options'] + )) + self.new_values = {key: str(value[0]) for key, value in self.new_values.items() if not value[0] is None} + + def _apply(self): + logger.info("Running config script...") + dir_path = os.path.dirname(os.path.realpath(self.save_path)) + if not os.path.exists(dir_path): + mkdir(dir_path, mode=0o700) + # Save the settings to the .yaml file + write_to_yaml(self.save_path, self.new_values) + + + def _reload_services(self): + logger.info("Reloading services...") + services_to_reload = set() + for panel, section, obj in self._iterate(['panel', 'section', 'option']): + services_to_reload |= set(obj.get('services', [])) + + services_to_reload = list(services_to_reload) + services_to_reload.sort(key = 'nginx'.__eq__) + for service in services_to_reload: + if '__APP__': + service = service.replace('__APP__', self.app) + logger.debug(f"Reloading {service}") + if not _run_service_command('reload-or-restart', service): + services = _get_services() + test_conf = services[service].get('test_conf', 'true') + errors = check_output(f"{test_conf}; exit 0") if test_conf else '' + raise YunohostError( + "config_failed_service_reload", + service=service, errors=errors + ) + + def _iterate(self, trigger=['option']): + for panel in self.config.get("panels", []): + if 'panel' in trigger: + yield (panel, None, panel) + for section in panel.get("sections", []): + if 'section' in trigger: + yield (panel, section, section) + if 'option' in trigger: + for option in section.get("options", []): + yield (panel, section, option) + + +class Question: + "empty class to store questions information" + + +class YunoHostArgumentFormatParser(object): + hide_user_input_in_prompt = False + operation_logger = None + + def parse_question(self, question, user_answers): + parsed_question = Question() + + parsed_question.name = question["name"] + parsed_question.type = question.get("type", 'string') + parsed_question.default = question.get("default", None) + parsed_question.current_value = question.get("current_value") + parsed_question.optional = question.get("optional", False) + parsed_question.choices = question.get("choices", []) + parsed_question.pattern = question.get("pattern") + parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"}) + parsed_question.help = question.get("help") + parsed_question.helpLink = question.get("helpLink") + parsed_question.value = user_answers.get(parsed_question.name) + parsed_question.redact = question.get('redact', False) + + # Empty value is parsed as empty string + if parsed_question.default == "": + parsed_question.default = None + + return parsed_question + + def parse(self, question, user_answers): + question = self.parse_question(question, user_answers) + + while True: + # Display question if no value filled or if it's a readonly message + if Moulinette.interface.type== 'cli': + text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( + question + ) + if getattr(self, "readonly", False): + Moulinette.display(text_for_user_input_in_cli) + + elif question.value is None: + prefill = "" + if question.current_value is not None: + prefill = question.current_value + elif question.default is not None: + prefill = question.default + question.value = Moulinette.prompt( + message=text_for_user_input_in_cli, + is_password=self.hide_user_input_in_prompt, + confirm=self.hide_user_input_in_prompt, + prefill=prefill, + is_multiline=(question.type == "text") + ) + + + # Apply default value + if question.value in [None, ""] and question.default is not None: + question.value = ( + getattr(self, "default_value", None) + if question.default is None + else question.default + ) + + # Prevalidation + try: + self._prevalidate(question) + except YunohostValidationError as e: + if Moulinette.interface.type== 'api': + raise + Moulinette.display(str(e), 'error') + question.value = None + continue + break + # this is done to enforce a certain formating like for boolean + # by default it doesn't do anything + question.value = self._post_parse_value(question) + + return (question.value, self.argument_type) + + def _prevalidate(self, question): + if question.value in [None, ""] and not question.optional: + raise YunohostValidationError( + "app_argument_required", name=question.name + ) + + # we have an answer, do some post checks + if question.value is not None: + if question.choices and question.value not in question.choices: + self._raise_invalid_answer(question) + if question.pattern and not re.match(question.pattern['regexp'], str(question.value)): + raise YunohostValidationError( + question.pattern['error'], + name=question.name, + value=question.value, + ) + + def _raise_invalid_answer(self, question): + raise YunohostValidationError( + "app_argument_choice_invalid", + name=question.name, + value=question.value, + choices=", ".join(question.choices), + ) + + def _format_text_for_user_input_in_cli(self, question): + text_for_user_input_in_cli = _value_for_locale(question.ask) + + if question.choices: + text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices)) + + if question.help or question.helpLink: + text_for_user_input_in_cli += ":\033[m" + if question.help: + text_for_user_input_in_cli += "\n - " + text_for_user_input_in_cli += _value_for_locale(question.help) + if question.helpLink: + if not isinstance(question.helpLink, dict): + question.helpLink = {'href': question.helpLink} + text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}" + return text_for_user_input_in_cli + + def _post_parse_value(self, question): + if not question.redact: + return question.value + + # Tell the operation_logger to redact all password-type / secret args + # Also redact the % escaped version of the password that might appear in + # the 'args' section of metadata (relevant for password with non-alphanumeric char) + data_to_redact = [] + if question.value and isinstance(question.value, str): + data_to_redact.append(question.value) + if question.current_value and isinstance(question.current_value, str): + data_to_redact.append(question.current_value) + data_to_redact += [ + urllib.parse.quote(data) + for data in data_to_redact + if urllib.parse.quote(data) != data + ] + if self.operation_logger: + self.operation_logger.data_to_redact.extend(data_to_redact) + elif data_to_redact: + raise YunohostError("app_argument_cant_redact", arg=question.name) + + return question.value + + +class StringArgumentParser(YunoHostArgumentFormatParser): + argument_type = "string" + default_value = "" + +class TagsArgumentParser(YunoHostArgumentFormatParser): + argument_type = "tags" + + def _prevalidate(self, question): + values = question.value + for value in values.split(','): + question.value = value + super()._prevalidate(question) + question.value = values + + + +class PasswordArgumentParser(YunoHostArgumentFormatParser): + hide_user_input_in_prompt = True + argument_type = "password" + default_value = "" + forbidden_chars = "{}" + + def parse_question(self, question, user_answers): + question = super(PasswordArgumentParser, self).parse_question( + question, user_answers + ) + question.redact = True + if question.default is not None: + raise YunohostValidationError( + "app_argument_password_no_default", name=question.name + ) + + return question + + def _prevalidate(self, question): + super()._prevalidate(question) + + if question.value is not None: + if any(char in question.value for char in self.forbidden_chars): + raise YunohostValidationError( + "pattern_password_app", forbidden_chars=self.forbidden_chars + ) + + # If it's an optional argument the value should be empty or strong enough + from yunohost.utils.password import assert_password_is_strong_enough + + assert_password_is_strong_enough("user", question.value) + + +class PathArgumentParser(YunoHostArgumentFormatParser): + argument_type = "path" + default_value = "" + + +class BooleanArgumentParser(YunoHostArgumentFormatParser): + argument_type = "boolean" + default_value = False + + def parse_question(self, question, user_answers): + question = super().parse_question( + question, user_answers + ) + + if question.default is None: + question.default = False + + return question + + def _format_text_for_user_input_in_cli(self, question): + text_for_user_input_in_cli = _value_for_locale(question.ask) + + text_for_user_input_in_cli += " [yes | no]" + + if question.default is not None: + formatted_default = "yes" if question.default else "no" + text_for_user_input_in_cli += " (default: {0})".format(formatted_default) + + return text_for_user_input_in_cli + + def _post_parse_value(self, question): + if isinstance(question.value, bool): + return 1 if question.value else 0 + + if str(question.value).lower() in ["1", "yes", "y", "true"]: + return 1 + + if str(question.value).lower() in ["0", "no", "n", "false"]: + return 0 + + raise YunohostValidationError( + "app_argument_choice_invalid", + name=question.name, + value=question.value, + choices="yes, no, y, n, 1, 0", + ) + + +class DomainArgumentParser(YunoHostArgumentFormatParser): + argument_type = "domain" + + def parse_question(self, question, user_answers): + from yunohost.domain import domain_list, _get_maindomain + + question = super(DomainArgumentParser, self).parse_question( + question, user_answers + ) + + if question.default is None: + question.default = _get_maindomain() + + question.choices = domain_list()["domains"] + + return question + + def _raise_invalid_answer(self, question): + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown") + ) + + +class UserArgumentParser(YunoHostArgumentFormatParser): + argument_type = "user" + + def parse_question(self, question, user_answers): + from yunohost.user import user_list, user_info + from yunohost.domain import _get_maindomain + + question = super(UserArgumentParser, self).parse_question( + question, user_answers + ) + question.choices = user_list()["users"] + if question.default is None: + root_mail = "root@%s" % _get_maindomain() + for user in question.choices.keys(): + if root_mail in user_info(user).get("mail-aliases", []): + question.default = user + break + + return question + + def _raise_invalid_answer(self, question): + raise YunohostValidationError( + "app_argument_invalid", + field=question.name, + error=m18n.n("user_unknown", user=question.value), + ) + + +class NumberArgumentParser(YunoHostArgumentFormatParser): + argument_type = "number" + default_value = "" + + def parse_question(self, question, user_answers): + question_parsed = super().parse_question( + question, user_answers + ) + question_parsed.min = question.get('min', None) + question_parsed.max = question.get('max', None) + if question_parsed.default is None: + question_parsed.default = 0 + + return question_parsed + + def _prevalidate(self, question): + super()._prevalidate(question) + if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()): + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + ) + + if question.min is not None and int(question.value) < question.min: + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + ) + + if question.max is not None and int(question.value) > question.max: + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + ) + + def _post_parse_value(self, question): + if isinstance(question.value, int): + return super()._post_parse_value(question) + + if isinstance(question.value, str) and question.value.isdigit(): + return int(question.value) + + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + ) + + +class DisplayTextArgumentParser(YunoHostArgumentFormatParser): + argument_type = "display_text" + readonly = True + + def parse_question(self, question, user_answers): + question_parsed = super().parse_question( + question, user_answers + ) + + question_parsed.optional = True + question_parsed.style = question.get('style', 'info') + + return question_parsed + + def _format_text_for_user_input_in_cli(self, question): + text = question.ask['en'] + + if question.style in ['success', 'info', 'warning', 'danger']: + color = { + 'success': 'green', + 'info': 'cyan', + 'warning': 'yellow', + 'danger': 'red' + } + return colorize(m18n.g(question.style), color[question.style]) + f" {text}" + else: + return text + +class FileArgumentParser(YunoHostArgumentFormatParser): + argument_type = "file" + upload_dirs = [] + + @classmethod + def clean_upload_dirs(cls): + # Delete files uploaded from API + if Moulinette.interface.type== 'api': + for upload_dir in cls.upload_dirs: + if os.path.exists(upload_dir): + shutil.rmtree(upload_dir) + + def parse_question(self, question, user_answers): + question_parsed = super().parse_question( + question, user_answers + ) + if question.get('accept'): + question_parsed.accept = question.get('accept').replace(' ', '').split(',') + else: + question_parsed.accept = [] + if Moulinette.interface.type== 'api': + if user_answers.get(f"{question_parsed.name}[name]"): + question_parsed.value = { + 'content': question_parsed.value, + 'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name), + } + # If path file are the same + if question_parsed.value and str(question_parsed.value) == question_parsed.current_value: + question_parsed.value = None + + return question_parsed + + def _prevalidate(self, question): + super()._prevalidate(question) + if isinstance(question.value, str) and question.value and not os.path.exists(question.value): + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1") + ) + if question.value in [None, ''] or not question.accept: + return + + filename = question.value if isinstance(question.value, str) else question.value['filename'] + if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept: + raise YunohostValidationError( + "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2") + ) + + + def _post_parse_value(self, question): + from base64 import b64decode + # Upload files from API + # A file arg contains a string with "FILENAME:BASE64_CONTENT" + if not question.value: + return question.value + + if Moulinette.interface.type== 'api': + + upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') + FileArgumentParser.upload_dirs += [upload_dir] + filename = question.value['filename'] + logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}") + + # Filename is given by user of the API. For security reason, we have replaced + # os.path.join to avoid the user to be able to rewrite a file in filesystem + # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" + file_path = os.path.normpath(upload_dir + "/" + filename) + if not file_path.startswith(upload_dir + "/"): + raise YunohostError("relative_parent_path_in_filename_forbidden") + i = 2 + while os.path.exists(file_path): + file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) + i += 1 + content = question.value['content'] + try: + with open(file_path, 'wb') as f: + f.write(b64decode(content)) + except IOError as e: + raise YunohostError("cannot_write_file", file=file_path, error=str(e)) + except Exception as e: + raise YunohostError("error_writing_file", file=file_path, error=str(e)) + question.value = file_path + return question.value + + +ARGUMENTS_TYPE_PARSERS = { + "string": StringArgumentParser, + "text": StringArgumentParser, + "select": StringArgumentParser, + "tags": TagsArgumentParser, + "email": StringArgumentParser, + "url": StringArgumentParser, + "date": StringArgumentParser, + "time": StringArgumentParser, + "color": StringArgumentParser, + "password": PasswordArgumentParser, + "path": PathArgumentParser, + "boolean": BooleanArgumentParser, + "domain": DomainArgumentParser, + "user": UserArgumentParser, + "number": NumberArgumentParser, + "range": NumberArgumentParser, + "display_text": DisplayTextArgumentParser, + "alert": DisplayTextArgumentParser, + "markdown": DisplayTextArgumentParser, + "file": FileArgumentParser, +} + +def parse_args_in_yunohost_format(user_answers, argument_questions): + """Parse arguments store in either manifest.json or actions.json or from a + config panel against the user answers when they are present. + + Keyword arguments: + user_answers -- a dictionnary of arguments from the user (generally + empty in CLI, filed from the admin interface) + argument_questions -- the arguments description store in yunohost + format from actions.json/toml, manifest.json/toml + or config_panel.json/toml + """ + parsed_answers_dict = OrderedDict() + + for question in argument_questions: + parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]() + + answer = parser.parse(question=question, user_answers=user_answers) + if answer is not None: + parsed_answers_dict[question["name"]] = answer + + return parsed_answers_dict + diff --git a/src/yunohost/utils/i18n.py b/src/yunohost/utils/i18n.py new file mode 100644 index 000000000..89d1d0b34 --- /dev/null +++ b/src/yunohost/utils/i18n.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2018 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" +from moulinette import Moulinette, m18n + +def _value_for_locale(values): + """ + Return proper value for current locale + + Keyword arguments: + values -- A dict of values associated to their locale + + Returns: + An utf-8 encoded string + + """ + if not isinstance(values, dict): + return values + + for lang in [m18n.locale, m18n.default_locale]: + try: + return values[lang] + except KeyError: + continue + + # Fallback to first value + return list(values.values())[0] + + From 31631794641253a3439d0f35ee4c06ed92f2258c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 3 Sep 2021 17:13:31 +0200 Subject: [PATCH 052/119] config helpers: get_var / set_var -> read/write_var_in_file --- data/helpers.d/config | 4 ++-- data/helpers.d/utils | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 52454ff91..6223a17b2 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -67,7 +67,7 @@ EOL local source_key="$(echo "$source" | cut -d: -f1)" source_key=${source_key:-$short_setting} local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - old[$short_setting]="$(ynh_get_var --file="${source_file}" --key="${source_key}")" + old[$short_setting]="$(ynh_read_var_in_file --file="${source_file}" --key="${source_key}")" fi done @@ -130,7 +130,7 @@ _ynh_app_config_apply() { local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$source_file" - ynh_set_var --file="${source_file}" --key="${source_key}" --value="${!short_setting}" + ynh_write_var_in_file --file="${source_file}" --key="${source_key}" --value="${!short_setting}" ynh_store_file_checksum --file="$source_file" --update_only # We stored the info in settings in order to be able to upgrade the app diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 14e7ebe4a..3389101a6 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -475,7 +475,7 @@ ynh_replace_vars () { # Get a value from heterogeneous file (yaml, json, php, python...) # -# usage: ynh_get_var --file=PATH --key=KEY +# usage: ynh_read_var_in_file --file=PATH --key=KEY # | arg: -f, --file= - the path to the file # | arg: -k, --key= - the key to get # @@ -504,8 +504,9 @@ ynh_replace_vars () { # USER = 8102 # user = 'https://donate.local' # CUSTOM['user'] = 'YunoHost' +# # Requires YunoHost version 4.3 or higher. -ynh_get_var() { +ynh_read_var_in_file() { # Declare an array to define the options of this helper. local legacy_args=fk local -A args_array=( [f]=file= [k]=key= ) @@ -515,10 +516,9 @@ ynh_get_var() { ynh_handle_getopts_args "$@" local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - - local crazy_value="$((grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL) | head -n1)" - #" - + + local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL | head -n1)" + local first_char="${crazy_value:0:1}" if [[ "$first_char" == '"' ]] ; then echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' @@ -531,13 +531,13 @@ ynh_get_var() { # Set a value into heterogeneous file (yaml, json, php, python...) # -# usage: ynh_set_var --file=PATH --key=KEY --value=VALUE +# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE # | arg: -f, --file= - the path to the file # | arg: -k, --key= - the key to set # | arg: -v, --value= - the value to set # # Requires YunoHost version 4.3 or higher. -ynh_set_var() { +ynh_write_var_in_file() { # Declare an array to define the options of this helper. local legacy_args=fkv local -A args_array=( [f]=file= [k]=key= [v]=value=) @@ -547,7 +547,7 @@ ynh_set_var() { # Manage arguments with getopts ynh_handle_getopts_args "$@" local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' - + local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" local first_char="${crazy_value:0:1}" From d74bc485ddaf06e2ca7e1834a16f3a2027e15948 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 3 Sep 2021 20:23:12 +0200 Subject: [PATCH 053/119] helpers/config: Add unit tests for read/write var from json/php/yaml --- tests/test_helpers.d/ynhtest_config.sh | 394 +++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 tests/test_helpers.d/ynhtest_config.sh diff --git a/tests/test_helpers.d/ynhtest_config.sh b/tests/test_helpers.d/ynhtest_config.sh new file mode 100644 index 000000000..69e715229 --- /dev/null +++ b/tests/test_helpers.d/ynhtest_config.sh @@ -0,0 +1,394 @@ +_make_dummy_files() { + + local_dummy_dir="$1" + + cat << EOF > $dummy_dir/dummy.ini +# Some comment +foo = +enabled = False +# title = Old title +title = Lorem Ipsum +email = root@example.com +theme = colib'ris + port = 1234 +url = https://yunohost.org +[dict] + ldap_base = ou=users,dc=yunohost,dc=org +EOF + + cat << EOF > $dummy_dir/dummy.py +# Some comment +FOO = None +ENABLED = False +# TITLE = "Old title" +TITLE = "Lorem Ipsum" +THEME = "colib'ris" +EMAIL = "root@example.com" +PORT = 1234 +URL = 'https://yunohost.org' +DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" +EOF + +} + +_ynh_read_yaml_with_python() { + local file="$1" + local key="$2" + python3 -c "import yaml; print(yaml.safe_load(open('$file'))['$key'])" +} + +_ynh_read_json_with_python() { + local file="$1" + local key="$2" + python3 -c "import json; print(json.load(open('$file'))['$key'])" +} + +_ynh_read_php_with_php() { + local file="$1" + local key="$2" + php -r "include '$file'; echo var_export(\$$key);" | sed "s/^'//g" | sed "s/'$//g" +} + + +ynhtest_config_read_yaml() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.yml" + + cat << EOF > $file +# Some comment +foo: +enabled: false +# title: old title +title: Lorem Ipsum +theme: colib'ris +email: root@example.com +port: 1234 +url: https://yunohost.org +dict: + ldap_base: ou=users,dc=yunohost,dc=org +EOF + + test "$(_ynh_read_yaml_with_python "$file" "foo")" == "None" + test "$(ynh_read_var_in_file "$file" "foo")" == "" + + test "$(_ynh_read_yaml_with_python "$file" "enabled")" == "False" + test "$(ynh_read_var_in_file "$file" "enabled")" == "false" + + test "$(_ynh_read_yaml_with_python "$file" "title")" == "Lorem Ipsum" + test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" + + test "$(_ynh_read_yaml_with_python "$file" "theme")" == "colib'ris" + test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" + + test "$(_ynh_read_yaml_with_python "$file" "email")" == "root@example.com" + test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" + + test "$(_ynh_read_yaml_with_python "$file" "port")" == "1234" + test "$(ynh_read_var_in_file "$file" "port")" == "1234" + + test "$(_ynh_read_yaml_with_python "$file" "url")" == "https://yunohost.org" + test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" + + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ! _ynh_read_yaml_with_python "$file" "nonexistent" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ! _ynh_read_yaml_with_python "$file" "enable" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" +} + + +ynhtest_config_write_yaml() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.yml" + + cat << EOF > $file +# Some comment +foo: +enabled: false +# title: old title +title: Lorem Ipsum +theme: colib'ris +email: root@example.com +port: 1234 +url: https://yunohost.org +dict: + ldap_base: ou=users,dc=yunohost,dc=org +EOF + + + + #ynh_write_var_in_file "$file" "foo" "bar" + # cat $dummy_dir/dummy.yml # to debug + #! test "$(_ynh_read_yaml_with_python "$file" "foo")" == "bar" # FIXME FIXME FIXME : writing broke the yaml syntax... "foo:bar" (no space aftr :) + #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" + + ynh_write_var_in_file "$file" "enabled" "true" + test "$(_ynh_read_yaml_with_python "$file" "enabled")" == "True" + test "$(ynh_read_var_in_file "$file" "enabled")" == "true" + + ynh_write_var_in_file "$file" "title" "Foo Bar" + test "$(_ynh_read_yaml_with_python "$file" "title")" == "Foo Bar" + test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" + + ynh_write_var_in_file "$file" "theme" "super-awesome-theme" + test "$(_ynh_read_yaml_with_python "$file" "theme")" == "super-awesome-theme" + test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" + + ynh_write_var_in_file "$file" "email" "sam@domain.tld" + test "$(_ynh_read_yaml_with_python "$file" "email")" == "sam@domain.tld" + test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" + + ynh_write_var_in_file "$file" "port" "5678" + test "$(_ynh_read_yaml_with_python "$file" "port")" == "5678" + test "$(ynh_read_var_in_file "$file" "port")" == "5678" + + ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" + test "$(_ynh_read_yaml_with_python "$file" "url")" == "https://domain.tld/foobar" + test "$(ynh_read_var_in_file "$file" "url")" == "https://domain.tld/foobar" + + ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" + + ynh_write_var_in_file "$file" "nonexistent" "foobar" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ynh_write_var_in_file "$file" "enable" "foobar" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" + test "$(ynh_read_var_in_file "$file" "enabled")" == "true" +} + +ynhtest_config_read_json() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.json" + + cat << EOF > $file +{ + "foo": null, + "enabled": false, + "title": "Lorem Ipsum", + "theme": "colib'ris", + "email": "root@example.com", + "port": 1234, + "url": "https://yunohost.org", + "dict": { + "ldap_base": "ou=users,dc=yunohost,dc=org" + } +} +EOF + + + test "$(_ynh_read_json_with_python "$file" "foo")" == "None" + test "$(ynh_read_var_in_file "$file" "foo")" == "null," # FIXME FIXME FIXME trailing , + + test "$(_ynh_read_json_with_python "$file" "enabled")" == "False" + test "$(ynh_read_var_in_file "$file" "enabled")" == "false," # FIXME FIXME FIXME trailing , + + test "$(_ynh_read_json_with_python "$file" "title")" == "Lorem Ipsum" + test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" + + test "$(_ynh_read_json_with_python "$file" "theme")" == "colib'ris" + test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" + + test "$(_ynh_read_json_with_python "$file" "email")" == "root@example.com" + test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" + + test "$(_ynh_read_json_with_python "$file" "port")" == "1234" + test "$(ynh_read_var_in_file "$file" "port")" == "1234," # FIXME FIXME FIXME trailing , + + test "$(_ynh_read_json_with_python "$file" "url")" == "https://yunohost.org" + test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" + + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ! _ynh_read_json_with_python "$file" "nonexistent" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ! _ynh_read_json_with_python "$file" "enable" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" +} + + +ynhtest_config_write_json() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.json" + + cat << EOF > $file +{ + "foo": null, + "enabled": false, + "title": "Lorem Ipsum", + "theme": "colib'ris", + "email": "root@example.com", + "port": 1234, + "url": "https://yunohost.org", + "dict": { + "ldap_base": "ou=users,dc=yunohost,dc=org" + } +} +EOF + + #ynh_write_var_in_file "$file" "foo" "bar" + #cat $file + #test "$(_ynh_read_json_with_python "$file" "foo")" == "bar" # FIXME FIXME FIXME + #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" + + #ynh_write_var_in_file "$file" "enabled" "true" + #test "$(_ynh_read_json_with_python "$file" "enabled")" == "True" # FIXME FIXME FIXME + #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" + + ynh_write_var_in_file "$file" "title" "Foo Bar" + cat $file + test "$(_ynh_read_json_with_python "$file" "title")" == "Foo Bar" + test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" + + ynh_write_var_in_file "$file" "theme" "super-awesome-theme" + cat $file + test "$(_ynh_read_json_with_python "$file" "theme")" == "super-awesome-theme" + test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" + + ynh_write_var_in_file "$file" "email" "sam@domain.tld" + cat $file + test "$(_ynh_read_json_with_python "$file" "email")" == "sam@domain.tld" + test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" + + #ynh_write_var_in_file "$file" "port" "5678" + #cat $file + #test "$(_ynh_read_json_with_python "$file" "port")" == "5678" # FIXME FIXME FIXME + #test "$(ynh_read_var_in_file "$file" "port")" == "5678" + + ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" + test "$(_ynh_read_json_with_python "$file" "url")" == "https://domain.tld/foobar" + test "$(ynh_read_var_in_file "$file" "url")" == "https://domain.tld/foobar" + + ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" + + ynh_write_var_in_file "$file" "nonexistent" "foobar" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ynh_write_var_in_file "$file" "enable" "foobar" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" + #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME +} + + + +ynhtest_config_read_php() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.php" + + cat << EOF > $file + "ou=users,dc=yunohost,dc=org", + ]; +?> +EOF + + test "$(_ynh_read_php_with_php "$file" "foo")" == "NULL" + test "$(ynh_read_var_in_file "$file" "foo")" == "NULL;" # FIXME FIXME FIXME trailing ; + + test "$(_ynh_read_php_with_php "$file" "enabled")" == "false" + test "$(ynh_read_var_in_file "$file" "enabled")" == "false;" # FIXME FIXME FIXME trailing ; + + test "$(_ynh_read_php_with_php "$file" "title")" == "Lorem Ipsum" + test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" + + test "$(_ynh_read_php_with_php "$file" "theme")" == "colib\\'ris" + test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" + + test "$(_ynh_read_php_with_php "$file" "email")" == "root@example.com" + test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" + + test "$(_ynh_read_php_with_php "$file" "port")" == "1234" + test "$(ynh_read_var_in_file "$file" "port")" == "1234;" # FIXME FIXME FIXME trailing ; + + test "$(_ynh_read_php_with_php "$file" "url")" == "https://yunohost.org" + test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" + + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ! _ynh_read_php_with_php "$file" "nonexistent" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ! _ynh_read_php_with_php "$file" "enable" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" +} + + +ynhtest_config_write_php() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.php" + + cat << EOF > $file + "ou=users,dc=yunohost,dc=org", + ]; +?> +EOF + + #ynh_write_var_in_file "$file" "foo" "bar" + #cat $file + #test "$(_ynh_read_php_with_php "$file" "foo")" == "bar" + #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" # FIXME FIXME FIXME + + #ynh_write_var_in_file "$file" "enabled" "true" + #cat $file + #test "$(_ynh_read_php_with_php "$file" "enabled")" == "true" + #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME FIXME FIXME + + ynh_write_var_in_file "$file" "title" "Foo Bar" + cat $file + test "$(_ynh_read_php_with_php "$file" "title")" == "Foo Bar" + test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" + + ynh_write_var_in_file "$file" "theme" "super-awesome-theme" + cat $file + test "$(_ynh_read_php_with_php "$file" "theme")" == "super-awesome-theme" + test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" + + ynh_write_var_in_file "$file" "email" "sam@domain.tld" + cat $file + test "$(_ynh_read_php_with_php "$file" "email")" == "sam@domain.tld" + test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" + + #ynh_write_var_in_file "$file" "port" "5678" + #cat $file + #test "$(_ynh_read_php_with_php "$file" "port")" == "5678" # FIXME FIXME FIXME + #test "$(ynh_read_var_in_file "$file" "port")" == "5678" + + ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" + test "$(_ynh_read_php_with_php "$file" "url")" == "https://domain.tld/foobar" + test "$(ynh_read_var_in_file "$file" "url")" == "https://domain.tld/foobar" + + ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" + + ynh_write_var_in_file "$file" "nonexistent" "foobar" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ynh_write_var_in_file "$file" "enable" "foobar" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" + #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME +} From cc8247acfdc24786c1547246db0072536fc6eba8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 3 Sep 2021 23:36:54 +0200 Subject: [PATCH 054/119] helpers/config: Add unit tests for read/write var from py/ini --- tests/test_helpers.d/ynhtest_config.sh | 509 ++++++++++++++++++------- 1 file changed, 380 insertions(+), 129 deletions(-) diff --git a/tests/test_helpers.d/ynhtest_config.sh b/tests/test_helpers.d/ynhtest_config.sh index 69e715229..7b749adf5 100644 --- a/tests/test_helpers.d/ynhtest_config.sh +++ b/tests/test_helpers.d/ynhtest_config.sh @@ -1,20 +1,24 @@ -_make_dummy_files() { - local_dummy_dir="$1" +################# +# _ __ _ _ # +# | '_ \| | | | # +# | |_) | |_| | # +# | .__/ \__, | # +# | | __/ | # +# |_| |___/ # +# # +################# - cat << EOF > $dummy_dir/dummy.ini -# Some comment -foo = -enabled = False -# title = Old title -title = Lorem Ipsum -email = root@example.com -theme = colib'ris - port = 1234 -url = https://yunohost.org -[dict] - ldap_base = ou=users,dc=yunohost,dc=org -EOF +_read_py() { + local file="$1" + local key="$2" + python3 -c "exec(open('$file').read()); print($key)" +} + +ynhtest_config_read_py() { + + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.py" cat << EOF > $dummy_dir/dummy.py # Some comment @@ -26,30 +30,245 @@ THEME = "colib'ris" EMAIL = "root@example.com" PORT = 1234 URL = 'https://yunohost.org' +DICT = {} DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" EOF + test "$(_read_py "$file" "FOO")" == "None" + test "$(ynh_read_var_in_file "$file" "FOO")" == "None" + + test "$(_read_py "$file" "ENABLED")" == "False" + test "$(ynh_read_var_in_file "$file" "ENABLED")" == "False" + + test "$(_read_py "$file" "TITLE")" == "Lorem Ipsum" + test "$(ynh_read_var_in_file "$file" "TITLE")" == "Lorem Ipsum" + + test "$(_read_py "$file" "THEME")" == "colib'ris" + test "$(ynh_read_var_in_file "$file" "THEME")" == "colib'ris" + + test "$(_read_py "$file" "EMAIL")" == "root@example.com" + test "$(ynh_read_var_in_file "$file" "EMAIL")" == "root@example.com" + + test "$(_read_py "$file" "PORT")" == "1234" + test "$(ynh_read_var_in_file "$file" "PORT")" == "1234" + + test "$(_read_py "$file" "URL")" == "https://yunohost.org" + test "$(ynh_read_var_in_file "$file" "URL")" == "https://yunohost.org" + + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ! _read_py "$file" "NONEXISTENT" + test "$(ynh_read_var_in_file "$file" "NONEXISTENT")" == "YNH_NULL" + + ! _read_py "$file" "ENABLE" + test "$(ynh_read_var_in_file "$file" "ENABLE")" == "YNH_NULL" } -_ynh_read_yaml_with_python() { +ynhtest_config_write_py() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.py" + + cat << EOF > $dummy_dir/dummy.py +# Some comment +FOO = None +ENABLED = False +# TITLE = "Old title" +TITLE = "Lorem Ipsum" +THEME = "colib'ris" +EMAIL = "root@example.com" +PORT = 1234 +URL = 'https://yunohost.org' +DICT = {} +DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" +EOF + + #ynh_write_var_in_file "$file" "FOO" "bar" + #test "$(_read_py "$file" "FOO")" == "bar" # FIXME FIXME FIXME + #test "$(ynh_read_var_in_file "$file" "FOO")" == "bar" + + ynh_write_var_in_file "$file" "ENABLED" "True" + test "$(_read_py "$file" "ENABLED")" == "True" + test "$(ynh_read_var_in_file "$file" "ENABLED")" == "True" + + ynh_write_var_in_file "$file" "TITLE" "Foo Bar" + test "$(_read_py "$file" "TITLE")" == "Foo Bar" + test "$(ynh_read_var_in_file "$file" "TITLE")" == "Foo Bar" + + ynh_write_var_in_file "$file" "THEME" "super-awesome-theme" + test "$(_read_py "$file" "THEME")" == "super-awesome-theme" + test "$(ynh_read_var_in_file "$file" "THEME")" == "super-awesome-theme" + + ynh_write_var_in_file "$file" "EMAIL" "sam@domain.tld" + test "$(_read_py "$file" "EMAIL")" == "sam@domain.tld" + test "$(ynh_read_var_in_file "$file" "EMAIL")" == "sam@domain.tld" + + ynh_write_var_in_file "$file" "PORT" "5678" + test "$(_read_py "$file" "PORT")" == "5678" + test "$(ynh_read_var_in_file "$file" "PORT")" == "5678" + + ynh_write_var_in_file "$file" "URL" "https://domain.tld/foobar" + test "$(_read_py "$file" "URL")" == "https://domain.tld/foobar" + test "$(ynh_read_var_in_file "$file" "URL")" == "https://domain.tld/foobar" + + ynh_write_var_in_file "$file" "ldap_base" "ou=users,dc=yunohost,dc=org" + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ynh_write_var_in_file "$file" "NONEXISTENT" "foobar" + ! _read_py "$file" "NONEXISTENT" + test "$(ynh_read_var_in_file "$file" "NONEXISTENT")" == "YNH_NULL" + + ynh_write_var_in_file "$file" "ENABLE" "foobar" + ! _read_py "$file" "ENABLE" + test "$(ynh_read_var_in_file "$file" "ENABLE")" == "YNH_NULL" + +} + +############### +# _ _ # +# (_) (_) # +# _ _ __ _ # +# | | '_ \| | # +# | | | | | | # +# |_|_| |_|_| # +# # +############### + +_read_ini() { + local file="$1" + local key="$2" + python3 -c "import configparser; c = configparser.ConfigParser(); c.read('$file'); print(c['main']['$key'])" +} + +ynhtest_config_read_ini() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.yml" + + cat << EOF > $file +# Some comment +; Another comment +[main] +foo = null +enabled = False +# title = Old title +title = Lorem Ipsum +theme = colib'ris +email = root@example.com +port = 1234 +url = https://yunohost.org +[dict] + ldap_base = ou=users,dc=yunohost,dc=org +EOF + + test "$(_read_ini "$file" "foo")" == "null" + test "$(ynh_read_var_in_file "$file" "foo")" == "null" + + test "$(_read_ini "$file" "enabled")" == "False" + test "$(ynh_read_var_in_file "$file" "enabled")" == "False" + + test "$(_read_ini "$file" "title")" == "Lorem Ipsum" + test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" + + test "$(_read_ini "$file" "theme")" == "colib'ris" + test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" + + test "$(_read_ini "$file" "email")" == "root@example.com" + test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" + + test "$(_read_ini "$file" "port")" == "1234" + test "$(ynh_read_var_in_file "$file" "port")" == "1234" + + test "$(_read_ini "$file" "url")" == "https://yunohost.org" + test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" + + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ! _read_ini "$file" "nonexistent" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ! _read_ini "$file" "enable" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" + +} + +ynhtest_config_write_ini() { + local dummy_dir="$(mktemp -d -p $VAR_WWW)" + file="$dummy_dir/dummy.ini" + + cat << EOF > $file +# Some comment +; Another comment +[main] +foo = null +enabled = False +# title = Old title +title = Lorem Ipsum +theme = colib'ris +email = root@example.com +port = 1234 +url = https://yunohost.org +[dict] + ldap_base = ou=users,dc=yunohost,dc=org +EOF + + ynh_write_var_in_file "$file" "foo" "bar" + test "$(_read_ini "$file" "foo")" == "bar" + test "$(ynh_read_var_in_file "$file" "foo")" == "bar" + + ynh_write_var_in_file "$file" "enabled" "True" + test "$(_read_ini "$file" "enabled")" == "True" + test "$(ynh_read_var_in_file "$file" "enabled")" == "True" + + ynh_write_var_in_file "$file" "title" "Foo Bar" + test "$(_read_ini "$file" "title")" == "Foo Bar" + test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" + + ynh_write_var_in_file "$file" "theme" "super-awesome-theme" + test "$(_read_ini "$file" "theme")" == "super-awesome-theme" + test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" + + ynh_write_var_in_file "$file" "email" "sam@domain.tld" + test "$(_read_ini "$file" "email")" == "sam@domain.tld" + test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" + + ynh_write_var_in_file "$file" "port" "5678" + test "$(_read_ini "$file" "port")" == "5678" + test "$(ynh_read_var_in_file "$file" "port")" == "5678" + + ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" + test "$(_read_ini "$file" "url")" == "https://domain.tld/foobar" + test "$(ynh_read_var_in_file "$file" "url")" == "https://domain.tld/foobar" + + ynh_write_var_in_file "$file" "ldap_base" "ou=users,dc=yunohost,dc=org" + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ynh_write_var_in_file "$file" "nonexistent" "foobar" + ! _read_ini "$file" "nonexistent" + test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" + + ynh_write_var_in_file "$file" "enable" "foobar" + ! _read_ini "$file" "enable" + test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" + +} + +############################# +# _ # +# | | # +# _ _ __ _ _ __ ___ | | # +# | | | |/ _` | '_ ` _ \| | # +# | |_| | (_| | | | | | | | # +# \__, |\__,_|_| |_| |_|_| # +# __/ | # +# |___/ # +# # +############################# + +_read_yaml() { local file="$1" local key="$2" python3 -c "import yaml; print(yaml.safe_load(open('$file'))['$key'])" } -_ynh_read_json_with_python() { - local file="$1" - local key="$2" - python3 -c "import json; print(json.load(open('$file'))['$key'])" -} - -_ynh_read_php_with_php() { - local file="$1" - local key="$2" - php -r "include '$file'; echo var_export(\$$key);" | sed "s/^'//g" | sed "s/'$//g" -} - - ynhtest_config_read_yaml() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.yml" @@ -68,33 +287,33 @@ dict: ldap_base: ou=users,dc=yunohost,dc=org EOF - test "$(_ynh_read_yaml_with_python "$file" "foo")" == "None" + test "$(_read_yaml "$file" "foo")" == "None" test "$(ynh_read_var_in_file "$file" "foo")" == "" - - test "$(_ynh_read_yaml_with_python "$file" "enabled")" == "False" + + test "$(_read_yaml "$file" "enabled")" == "False" test "$(ynh_read_var_in_file "$file" "enabled")" == "false" - - test "$(_ynh_read_yaml_with_python "$file" "title")" == "Lorem Ipsum" + + test "$(_read_yaml "$file" "title")" == "Lorem Ipsum" test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" - - test "$(_ynh_read_yaml_with_python "$file" "theme")" == "colib'ris" + + test "$(_read_yaml "$file" "theme")" == "colib'ris" test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" - - test "$(_ynh_read_yaml_with_python "$file" "email")" == "root@example.com" + + test "$(_read_yaml "$file" "email")" == "root@example.com" test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" - - test "$(_ynh_read_yaml_with_python "$file" "port")" == "1234" + + test "$(_read_yaml "$file" "port")" == "1234" test "$(ynh_read_var_in_file "$file" "port")" == "1234" - - test "$(_ynh_read_yaml_with_python "$file" "url")" == "https://yunohost.org" + + test "$(_read_yaml "$file" "url")" == "https://yunohost.org" test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" - + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" - - ! _ynh_read_yaml_with_python "$file" "nonexistent" + + ! _read_yaml "$file" "nonexistent" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - - ! _ynh_read_yaml_with_python "$file" "enable" + + ! _read_yaml "$file" "enable" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" } @@ -117,48 +336,64 @@ dict: ldap_base: ou=users,dc=yunohost,dc=org EOF - - #ynh_write_var_in_file "$file" "foo" "bar" # cat $dummy_dir/dummy.yml # to debug - #! test "$(_ynh_read_yaml_with_python "$file" "foo")" == "bar" # FIXME FIXME FIXME : writing broke the yaml syntax... "foo:bar" (no space aftr :) + #! test "$(_read_yaml "$file" "foo")" == "bar" # FIXME FIXME FIXME : writing broke the yaml syntax... "foo:bar" (no space aftr :) #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" ynh_write_var_in_file "$file" "enabled" "true" - test "$(_ynh_read_yaml_with_python "$file" "enabled")" == "True" + test "$(_read_yaml "$file" "enabled")" == "True" test "$(ynh_read_var_in_file "$file" "enabled")" == "true" ynh_write_var_in_file "$file" "title" "Foo Bar" - test "$(_ynh_read_yaml_with_python "$file" "title")" == "Foo Bar" + test "$(_read_yaml "$file" "title")" == "Foo Bar" test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" - + ynh_write_var_in_file "$file" "theme" "super-awesome-theme" - test "$(_ynh_read_yaml_with_python "$file" "theme")" == "super-awesome-theme" + test "$(_read_yaml "$file" "theme")" == "super-awesome-theme" test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" - + ynh_write_var_in_file "$file" "email" "sam@domain.tld" - test "$(_ynh_read_yaml_with_python "$file" "email")" == "sam@domain.tld" + test "$(_read_yaml "$file" "email")" == "sam@domain.tld" test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" - + ynh_write_var_in_file "$file" "port" "5678" - test "$(_ynh_read_yaml_with_python "$file" "port")" == "5678" + test "$(_read_yaml "$file" "port")" == "5678" test "$(ynh_read_var_in_file "$file" "port")" == "5678" - + ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" - test "$(_ynh_read_yaml_with_python "$file" "url")" == "https://domain.tld/foobar" + test "$(_read_yaml "$file" "url")" == "https://domain.tld/foobar" test "$(ynh_read_var_in_file "$file" "url")" == "https://domain.tld/foobar" - + ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" - + ynh_write_var_in_file "$file" "nonexistent" "foobar" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - + ynh_write_var_in_file "$file" "enable" "foobar" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" test "$(ynh_read_var_in_file "$file" "enabled")" == "true" } +######################### +# _ # +# (_) # +# _ ___ ___ _ __ # +# | / __|/ _ \| '_ \ # +# | \__ \ (_) | | | | # +# | |___/\___/|_| |_| # +# _/ | # +# |__/ # +# # +######################### + +_read_json() { + local file="$1" + local key="$2" + python3 -c "import json; print(json.load(open('$file'))['$key'])" +} + ynhtest_config_read_json() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.json" @@ -179,33 +414,33 @@ ynhtest_config_read_json() { EOF - test "$(_ynh_read_json_with_python "$file" "foo")" == "None" + test "$(_read_json "$file" "foo")" == "None" test "$(ynh_read_var_in_file "$file" "foo")" == "null," # FIXME FIXME FIXME trailing , - - test "$(_ynh_read_json_with_python "$file" "enabled")" == "False" + + test "$(_read_json "$file" "enabled")" == "False" test "$(ynh_read_var_in_file "$file" "enabled")" == "false," # FIXME FIXME FIXME trailing , - - test "$(_ynh_read_json_with_python "$file" "title")" == "Lorem Ipsum" + + test "$(_read_json "$file" "title")" == "Lorem Ipsum" test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" - - test "$(_ynh_read_json_with_python "$file" "theme")" == "colib'ris" + + test "$(_read_json "$file" "theme")" == "colib'ris" test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" - - test "$(_ynh_read_json_with_python "$file" "email")" == "root@example.com" + + test "$(_read_json "$file" "email")" == "root@example.com" test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" - - test "$(_ynh_read_json_with_python "$file" "port")" == "1234" + + test "$(_read_json "$file" "port")" == "1234" test "$(ynh_read_var_in_file "$file" "port")" == "1234," # FIXME FIXME FIXME trailing , - - test "$(_ynh_read_json_with_python "$file" "url")" == "https://yunohost.org" + + test "$(_read_json "$file" "url")" == "https://yunohost.org" test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" - + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" - - ! _ynh_read_json_with_python "$file" "nonexistent" + + ! _read_json "$file" "nonexistent" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - - ! _ynh_read_json_with_python "$file" "enable" + + ! _read_json "$file" "enable" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" } @@ -231,49 +466,65 @@ EOF #ynh_write_var_in_file "$file" "foo" "bar" #cat $file - #test "$(_ynh_read_json_with_python "$file" "foo")" == "bar" # FIXME FIXME FIXME + #test "$(_read_json "$file" "foo")" == "bar" # FIXME FIXME FIXME #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" #ynh_write_var_in_file "$file" "enabled" "true" - #test "$(_ynh_read_json_with_python "$file" "enabled")" == "True" # FIXME FIXME FIXME + #test "$(_read_json "$file" "enabled")" == "True" # FIXME FIXME FIXME #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" ynh_write_var_in_file "$file" "title" "Foo Bar" cat $file - test "$(_ynh_read_json_with_python "$file" "title")" == "Foo Bar" + test "$(_read_json "$file" "title")" == "Foo Bar" test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" - + ynh_write_var_in_file "$file" "theme" "super-awesome-theme" cat $file - test "$(_ynh_read_json_with_python "$file" "theme")" == "super-awesome-theme" + test "$(_read_json "$file" "theme")" == "super-awesome-theme" test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" - + ynh_write_var_in_file "$file" "email" "sam@domain.tld" cat $file - test "$(_ynh_read_json_with_python "$file" "email")" == "sam@domain.tld" + test "$(_read_json "$file" "email")" == "sam@domain.tld" test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" - + #ynh_write_var_in_file "$file" "port" "5678" #cat $file - #test "$(_ynh_read_json_with_python "$file" "port")" == "5678" # FIXME FIXME FIXME + #test "$(_read_json "$file" "port")" == "5678" # FIXME FIXME FIXME #test "$(ynh_read_var_in_file "$file" "port")" == "5678" - + ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" - test "$(_ynh_read_json_with_python "$file" "url")" == "https://domain.tld/foobar" + test "$(_read_json "$file" "url")" == "https://domain.tld/foobar" test "$(ynh_read_var_in_file "$file" "url")" == "https://domain.tld/foobar" - + ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" - + ynh_write_var_in_file "$file" "nonexistent" "foobar" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - + ynh_write_var_in_file "$file" "enable" "foobar" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME } +####################### +# _ # +# | | # +# _ __ | |__ _ __ # +# | '_ \| '_ \| '_ \ # +# | |_) | | | | |_) | # +# | .__/|_| |_| .__/ # +# | | | | # +# |_| |_| # +# # +####################### +_read_php() { + local file="$1" + local key="$2" + php -r "include '$file'; echo var_export(\$$key);" | sed "s/^'//g" | sed "s/'$//g" +} ynhtest_config_read_php() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" @@ -296,33 +547,33 @@ ynhtest_config_read_php() { ?> EOF - test "$(_ynh_read_php_with_php "$file" "foo")" == "NULL" + test "$(_read_php "$file" "foo")" == "NULL" test "$(ynh_read_var_in_file "$file" "foo")" == "NULL;" # FIXME FIXME FIXME trailing ; - - test "$(_ynh_read_php_with_php "$file" "enabled")" == "false" + + test "$(_read_php "$file" "enabled")" == "false" test "$(ynh_read_var_in_file "$file" "enabled")" == "false;" # FIXME FIXME FIXME trailing ; - - test "$(_ynh_read_php_with_php "$file" "title")" == "Lorem Ipsum" + + test "$(_read_php "$file" "title")" == "Lorem Ipsum" test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" - - test "$(_ynh_read_php_with_php "$file" "theme")" == "colib\\'ris" + + test "$(_read_php "$file" "theme")" == "colib\\'ris" test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" - - test "$(_ynh_read_php_with_php "$file" "email")" == "root@example.com" + + test "$(_read_php "$file" "email")" == "root@example.com" test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" - - test "$(_ynh_read_php_with_php "$file" "port")" == "1234" + + test "$(_read_php "$file" "port")" == "1234" test "$(ynh_read_var_in_file "$file" "port")" == "1234;" # FIXME FIXME FIXME trailing ; - - test "$(_ynh_read_php_with_php "$file" "url")" == "https://yunohost.org" + + test "$(_read_php "$file" "url")" == "https://yunohost.org" test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" - + test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" - - ! _ynh_read_php_with_php "$file" "nonexistent" + + ! _read_php "$file" "nonexistent" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - - ! _ynh_read_php_with_php "$file" "enable" + + ! _read_php "$file" "enable" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" } @@ -350,44 +601,44 @@ EOF #ynh_write_var_in_file "$file" "foo" "bar" #cat $file - #test "$(_ynh_read_php_with_php "$file" "foo")" == "bar" + #test "$(_read_php "$file" "foo")" == "bar" #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" # FIXME FIXME FIXME - + #ynh_write_var_in_file "$file" "enabled" "true" #cat $file - #test "$(_ynh_read_php_with_php "$file" "enabled")" == "true" + #test "$(_read_php "$file" "enabled")" == "true" #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME FIXME FIXME - + ynh_write_var_in_file "$file" "title" "Foo Bar" cat $file - test "$(_ynh_read_php_with_php "$file" "title")" == "Foo Bar" - test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" - + test "$(_read_php "$file" "title")" == "Foo Bar" + test "$(ynh_read_var_in_file "$file" "title")" == "Foo Bar" + ynh_write_var_in_file "$file" "theme" "super-awesome-theme" cat $file - test "$(_ynh_read_php_with_php "$file" "theme")" == "super-awesome-theme" + test "$(_read_php "$file" "theme")" == "super-awesome-theme" test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" - + ynh_write_var_in_file "$file" "email" "sam@domain.tld" cat $file - test "$(_ynh_read_php_with_php "$file" "email")" == "sam@domain.tld" + test "$(_read_php "$file" "email")" == "sam@domain.tld" test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" #ynh_write_var_in_file "$file" "port" "5678" #cat $file - #test "$(_ynh_read_php_with_php "$file" "port")" == "5678" # FIXME FIXME FIXME + #test "$(_read_php "$file" "port")" == "5678" # FIXME FIXME FIXME #test "$(ynh_read_var_in_file "$file" "port")" == "5678" - + ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" - test "$(_ynh_read_php_with_php "$file" "url")" == "https://domain.tld/foobar" + test "$(_read_php "$file" "url")" == "https://domain.tld/foobar" test "$(ynh_read_var_in_file "$file" "url")" == "https://domain.tld/foobar" - + ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" - + ynh_write_var_in_file "$file" "nonexistent" "foobar" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - + ynh_write_var_in_file "$file" "enable" "foobar" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME From 0a52430186a1743416ad928f4fb22f16fef46407 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 4 Sep 2021 18:39:39 +0200 Subject: [PATCH 055/119] Black --- data/hooks/diagnosis/80-apps.py | 34 +++- src/yunohost/app.py | 39 +++-- src/yunohost/utils/config.py | 291 +++++++++++++++++--------------- src/yunohost/utils/i18n.py | 3 +- 4 files changed, 208 insertions(+), 159 deletions(-) diff --git a/data/hooks/diagnosis/80-apps.py b/data/hooks/diagnosis/80-apps.py index 4ab5a6c0d..177ec590f 100644 --- a/data/hooks/diagnosis/80-apps.py +++ b/data/hooks/diagnosis/80-apps.py @@ -6,6 +6,7 @@ from yunohost.app import app_list from yunohost.diagnosis import Diagnoser + class AppDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] @@ -30,13 +31,17 @@ class AppDiagnoser(Diagnoser): if not app["issues"]: continue - level = "ERROR" if any(issue[0] == "error" for issue in app["issues"]) else "WARNING" + level = ( + "ERROR" + if any(issue[0] == "error" for issue in app["issues"]) + else "WARNING" + ) yield dict( meta={"test": "apps", "app": app["name"]}, status=level, summary="diagnosis_apps_issue", - details=[issue[1] for issue in app["issues"]] + details=[issue[1] for issue in app["issues"]], ) def issues(self, app): @@ -45,14 +50,19 @@ class AppDiagnoser(Diagnoser): if not app.get("from_catalog") or app["from_catalog"].get("state") != "working": yield ("error", "diagnosis_apps_not_in_app_catalog") - elif not isinstance(app["from_catalog"].get("level"), int) or app["from_catalog"]["level"] == 0: + elif ( + not isinstance(app["from_catalog"].get("level"), int) + or app["from_catalog"]["level"] == 0 + ): yield ("error", "diagnosis_apps_broken") elif app["from_catalog"]["level"] <= 4: yield ("warning", "diagnosis_apps_bad_quality") # Check for super old, deprecated practices - yunohost_version_req = app["manifest"].get("requirements", {}).get("yunohost", "").strip(">= ") + yunohost_version_req = ( + app["manifest"].get("requirements", {}).get("yunohost", "").strip(">= ") + ) if yunohost_version_req.startswith("2."): yield ("error", "diagnosis_apps_outdated_ynh_requirement") @@ -64,11 +74,21 @@ class AppDiagnoser(Diagnoser): "yunohost tools port-available", ] for deprecated_helper in deprecated_helpers: - if os.system(f"grep -nr -q '{deprecated_helper}' {app['setting_path']}/scripts/") == 0: + if ( + os.system( + f"grep -nr -q '{deprecated_helper}' {app['setting_path']}/scripts/" + ) + == 0 + ): yield ("error", "diagnosis_apps_deprecated_practices") - old_arg_regex = r'^domain=\${?[0-9]' - if os.system(f"grep -q '{old_arg_regex}' {app['setting_path']}/scripts/install") == 0: + old_arg_regex = r"^domain=\${?[0-9]" + if ( + os.system( + f"grep -q '{old_arg_regex}' {app['setting_path']}/scripts/install" + ) + == 0 + ): yield ("error", "diagnosis_apps_deprecated_practices") diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 522f695e2..0b8e2565e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -55,7 +55,11 @@ from moulinette.utils.filesystem import ( from yunohost.service import service_status, _run_service_command from yunohost.utils import packages, config -from yunohost.utils.config import ConfigPanel, parse_args_in_yunohost_format, YunoHostArgumentFormatParser +from yunohost.utils.config import ( + ConfigPanel, + parse_args_in_yunohost_format, + YunoHostArgumentFormatParser, +) from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory @@ -1756,7 +1760,7 @@ def app_action_run(operation_logger, app, action, args=None): return logger.success("Action successed!") -def app_config_get(app, key='', mode='classic'): +def app_config_get(app, key="", mode="classic"): """ Display an app configuration in classic, full or export mode """ @@ -1765,7 +1769,9 @@ def app_config_get(app, key='', mode='classic'): @is_unit_operation() -def app_config_set(operation_logger, app, key=None, value=None, args=None, args_file=None): +def app_config_set( + operation_logger, app, key=None, value=None, args=None, args_file=None +): """ Apply a new app configuration """ @@ -1780,6 +1786,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None, args_ operation_logger.success() return result + class AppConfigPanel(ConfigPanel): def __init__(self, app): @@ -1791,10 +1798,10 @@ class AppConfigPanel(ConfigPanel): super().__init__(config_path=config_path) def _load_current_values(self): - self.values = self._call_config_script('show') + self.values = self._call_config_script("show") def _apply(self): - self.errors = self._call_config_script('apply', self.new_values) + self.errors = self._call_config_script("apply", self.new_values) def _call_config_script(self, action, env={}): from yunohost.hook import hook_exec @@ -1814,22 +1821,23 @@ ynh_app_config_run $1 # Call config script to extract current values logger.debug(f"Calling '{action}' action from config script") app_id, app_instance_nb = _parse_app_instance_name(self.app) - env.update({ - "app_id": app_id, - "app": self.app, - "app_instance_nb": str(app_instance_nb), - }) - - ret, values = hook_exec( - config_script, args=[action], env=env + env.update( + { + "app_id": app_id, + "app": self.app, + "app_instance_nb": str(app_instance_nb), + } ) + + ret, values = hook_exec(config_script, args=[action], env=env) if ret != 0: - if action == 'show': + if action == "show": raise YunohostError("app_config_unable_to_read_values") else: raise YunohostError("app_config_unable_to_apply_values_correctly") return values + def _get_all_installed_apps_id(): """ Return something like: @@ -2455,9 +2463,6 @@ def _parse_args_for_action(action, args={}): return parse_args_in_yunohost_format(args, action_args) - - - def _validate_and_normalize_webpath(args_dict, app_folder): # If there's only one "domain" and "path", validate that domain/path diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 34883dcf7..4528fb708 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -45,8 +45,8 @@ from yunohost.utils.error import YunohostError, YunohostValidationError logger = getActionLogger("yunohost.config") CONFIG_PANEL_VERSION_SUPPORTED = 1.0 -class ConfigPanel: +class ConfigPanel: def __init__(self, config_path, save_path=None): self.config_path = config_path self.save_path = save_path @@ -54,8 +54,8 @@ class ConfigPanel: self.values = {} self.new_values = {} - def get(self, key='', mode='classic'): - self.filter_key = key or '' + def get(self, key="", mode="classic"): + self.filter_key = key or "" # Read config panel toml self._get_config_panel() @@ -68,12 +68,12 @@ class ConfigPanel: self._hydrate() # Format result in full mode - if mode == 'full': + if mode == "full": return self.config # In 'classic' mode, we display the current value if key refer to an option - if self.filter_key.count('.') == 2 and mode == 'classic': - option = self.filter_key.split('.')[-1] + if self.filter_key.count(".") == 2 and mode == "classic": + option = self.filter_key.split(".")[-1] return self.values.get(option, None) # Format result in 'classic' or 'export' mode @@ -81,17 +81,17 @@ class ConfigPanel: result = {} for panel, section, option in self._iterate(): key = f"{panel['id']}.{section['id']}.{option['id']}" - if mode == 'export': - result[option['id']] = option.get('current_value') + if mode == "export": + result[option["id"]] = option.get("current_value") else: - result[key] = { 'ask': _value_for_locale(option['ask']) } - if 'current_value' in option: - result[key]['value'] = option['current_value'] + result[key] = {"ask": _value_for_locale(option["ask"])} + if "current_value" in option: + result[key]["value"] = option["current_value"] return result def set(self, key=None, value=None, args=None, args_file=None): - self.filter_key = key or '' + self.filter_key = key or "" # Read config panel toml self._get_config_panel() @@ -102,20 +102,20 @@ class ConfigPanel: if (args is not None or args_file is not None) and value is not None: raise YunohostError("config_args_value") - if self.filter_key.count('.') != 2 and not value is None: + if self.filter_key.count(".") != 2 and not value is None: raise YunohostError("config_set_value_on_section") # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") - args = urllib.parse.parse_qs(args or '', keep_blank_values=True) - self.args = { key: ','.join(value_) for key, value_ in args.items() } + args = urllib.parse.parse_qs(args or "", keep_blank_values=True) + self.args = {key: ",".join(value_) for key, value_ in args.items()} if args_file: # Import YAML / JSON file but keep --args values - self.args = { **read_yaml(args_file), **self.args } + self.args = {**read_yaml(args_file), **self.args} if value is not None: - self.args = {self.filter_key.split('.')[-1]: value} + self.args = {self.filter_key.split(".")[-1]: value} # Read or get values and hydrate the config self._load_current_values() @@ -155,10 +155,9 @@ class ConfigPanel: def _get_toml(self): return read_toml(self.config_path) - def _get_config_panel(self): # Split filter_key - filter_key = dict(enumerate(self.filter_key.split('.'))) + filter_key = dict(enumerate(self.filter_key.split("."))) if len(filter_key) > 3: raise YunohostError("config_too_much_sub_keys") @@ -174,20 +173,18 @@ class ConfigPanel: # Transform toml format into internal format defaults = { - 'toml': { - 'version': 1.0 - }, - 'panels': { - 'name': '', - 'services': [], - 'actions': {'apply': {'en': 'Apply'}} - }, # help - 'sections': { - 'name': '', - 'services': [], - 'optional': True - }, # visibleIf help - 'options': {} + "toml": {"version": 1.0}, + "panels": { + "name": "", + "services": [], + "actions": {"apply": {"en": "Apply"}}, + }, # help + "sections": { + "name": "", + "services": [], + "optional": True, + }, # visibleIf help + "options": {} # ask type source help helpLink example style icon placeholder visibleIf # optional choices pattern limit min max step accept redact } @@ -207,30 +204,36 @@ class ConfigPanel: # Define the filter_key part to use and the children type i = list(defaults).index(node_type) search_key = filter_key.get(i) - subnode_type = list(defaults)[i+1] if node_type != 'options' else None + subnode_type = list(defaults)[i + 1] if node_type != "options" else None for key, value in toml_node.items(): # Key/value are a child node - if isinstance(value, OrderedDict) and key not in default and subnode_type: + if ( + isinstance(value, OrderedDict) + and key not in default + and subnode_type + ): # We exclude all nodes not referenced by the filter_key if search_key and key != search_key: continue subnode = convert(value, subnode_type) - subnode['id'] = key - if node_type == 'sections': - subnode['name'] = key # legacy - subnode.setdefault('optional', toml_node.get('optional', True)) + subnode["id"] = key + if node_type == "sections": + subnode["name"] = key # legacy + subnode.setdefault("optional", toml_node.get("optional", True)) node.setdefault(subnode_type, []).append(subnode) # Key/value are a property else: # Todo search all i18n keys - node[key] = value if key not in ['ask', 'help', 'name'] else { 'en': value } + node[key] = ( + value if key not in ["ask", "help", "name"] else {"en": value} + ) return node - self.config = convert(toml_config_panel, 'toml') + self.config = convert(toml_config_panel, "toml") try: - self.config['panels'][0]['sections'][0]['options'][0] + self.config["panels"][0]["sections"][0]["options"][0] except (KeyError, IndexError): raise YunohostError( "config_empty_or_bad_filter_key", filter_key=self.filter_key @@ -242,36 +245,41 @@ class ConfigPanel: # Hydrating config panel with current value logger.debug("Hydrating config with current values") for _, _, option in self._iterate(): - if option['name'] not in self.values: + if option["name"] not in self.values: continue - value = self.values[option['name']] + value = self.values[option["name"]] # In general, the value is just a simple value. # Sometimes it could be a dict used to overwrite the option itself - value = value if isinstance(value, dict) else {'current_value': value } + value = value if isinstance(value, dict) else {"current_value": value} option.update(value) return self.values def _ask(self): logger.debug("Ask unanswered question and prevalidate data") + def display_header(message): - """ CLI panel/section header display - """ - if Moulinette.interface.type == 'cli' and self.filter_key.count('.') < 2: - Moulinette.display(colorize(message, 'purple')) - for panel, section, obj in self._iterate(['panel', 'section']): + """CLI panel/section header display""" + if Moulinette.interface.type == "cli" and self.filter_key.count(".") < 2: + Moulinette.display(colorize(message, "purple")) + + for panel, section, obj in self._iterate(["panel", "section"]): if panel == obj: - name = _value_for_locale(panel['name']) + name = _value_for_locale(panel["name"]) display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}") continue - name = _value_for_locale(section['name']) + name = _value_for_locale(section["name"]) display_header(f"\n# {name}") # Check and ask unanswered questions - self.new_values.update(parse_args_in_yunohost_format( - self.args, section['options'] - )) - self.new_values = {key: str(value[0]) for key, value in self.new_values.items() if not value[0] is None} + self.new_values.update( + parse_args_in_yunohost_format(self.args, section["options"]) + ) + self.new_values = { + key: str(value[0]) + for key, value in self.new_values.items() + if not value[0] is None + } def _apply(self): logger.info("Running config script...") @@ -281,36 +289,34 @@ class ConfigPanel: # Save the settings to the .yaml file write_to_yaml(self.save_path, self.new_values) - def _reload_services(self): logger.info("Reloading services...") services_to_reload = set() - for panel, section, obj in self._iterate(['panel', 'section', 'option']): - services_to_reload |= set(obj.get('services', [])) + for panel, section, obj in self._iterate(["panel", "section", "option"]): + services_to_reload |= set(obj.get("services", [])) services_to_reload = list(services_to_reload) - services_to_reload.sort(key = 'nginx'.__eq__) + services_to_reload.sort(key="nginx".__eq__) for service in services_to_reload: - if '__APP__': - service = service.replace('__APP__', self.app) + if "__APP__": + service = service.replace("__APP__", self.app) logger.debug(f"Reloading {service}") - if not _run_service_command('reload-or-restart', service): + if not _run_service_command("reload-or-restart", service): services = _get_services() - test_conf = services[service].get('test_conf', 'true') - errors = check_output(f"{test_conf}; exit 0") if test_conf else '' + test_conf = services[service].get("test_conf", "true") + errors = check_output(f"{test_conf}; exit 0") if test_conf else "" raise YunohostError( - "config_failed_service_reload", - service=service, errors=errors + "config_failed_service_reload", service=service, errors=errors ) - def _iterate(self, trigger=['option']): + def _iterate(self, trigger=["option"]): for panel in self.config.get("panels", []): - if 'panel' in trigger: + if "panel" in trigger: yield (panel, None, panel) for section in panel.get("sections", []): - if 'section' in trigger: + if "section" in trigger: yield (panel, section, section) - if 'option' in trigger: + if "option" in trigger: for option in section.get("options", []): yield (panel, section, option) @@ -327,17 +333,17 @@ class YunoHostArgumentFormatParser(object): parsed_question = Question() parsed_question.name = question["name"] - parsed_question.type = question.get("type", 'string') + parsed_question.type = question.get("type", "string") parsed_question.default = question.get("default", None) parsed_question.current_value = question.get("current_value") parsed_question.optional = question.get("optional", False) parsed_question.choices = question.get("choices", []) parsed_question.pattern = question.get("pattern") - parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"}) + parsed_question.ask = question.get("ask", {"en": f"{parsed_question.name}"}) parsed_question.help = question.get("help") parsed_question.helpLink = question.get("helpLink") parsed_question.value = user_answers.get(parsed_question.name) - parsed_question.redact = question.get('redact', False) + parsed_question.redact = question.get("redact", False) # Empty value is parsed as empty string if parsed_question.default == "": @@ -350,7 +356,7 @@ class YunoHostArgumentFormatParser(object): while True: # Display question if no value filled or if it's a readonly message - if Moulinette.interface.type== 'cli': + if Moulinette.interface.type == "cli": text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( question ) @@ -368,10 +374,9 @@ class YunoHostArgumentFormatParser(object): is_password=self.hide_user_input_in_prompt, confirm=self.hide_user_input_in_prompt, prefill=prefill, - is_multiline=(question.type == "text") + is_multiline=(question.type == "text"), ) - # Apply default value if question.value in [None, ""] and question.default is not None: question.value = ( @@ -384,9 +389,9 @@ class YunoHostArgumentFormatParser(object): try: self._prevalidate(question) except YunohostValidationError as e: - if Moulinette.interface.type== 'api': + if Moulinette.interface.type == "api": raise - Moulinette.display(str(e), 'error') + Moulinette.display(str(e), "error") question.value = None continue break @@ -398,17 +403,17 @@ class YunoHostArgumentFormatParser(object): def _prevalidate(self, question): if question.value in [None, ""] and not question.optional: - raise YunohostValidationError( - "app_argument_required", name=question.name - ) + raise YunohostValidationError("app_argument_required", name=question.name) # we have an answer, do some post checks if question.value is not None: if question.choices and question.value not in question.choices: self._raise_invalid_answer(question) - if question.pattern and not re.match(question.pattern['regexp'], str(question.value)): + if question.pattern and not re.match( + question.pattern["regexp"], str(question.value) + ): raise YunohostValidationError( - question.pattern['error'], + question.pattern["error"], name=question.name, value=question.value, ) @@ -434,7 +439,7 @@ class YunoHostArgumentFormatParser(object): text_for_user_input_in_cli += _value_for_locale(question.help) if question.helpLink: if not isinstance(question.helpLink, dict): - question.helpLink = {'href': question.helpLink} + question.helpLink = {"href": question.helpLink} text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}" return text_for_user_input_in_cli @@ -467,18 +472,18 @@ class StringArgumentParser(YunoHostArgumentFormatParser): argument_type = "string" default_value = "" + class TagsArgumentParser(YunoHostArgumentFormatParser): argument_type = "tags" def _prevalidate(self, question): values = question.value - for value in values.split(','): + for value in values.split(","): question.value = value super()._prevalidate(question) question.value = values - class PasswordArgumentParser(YunoHostArgumentFormatParser): hide_user_input_in_prompt = True argument_type = "password" @@ -522,9 +527,7 @@ class BooleanArgumentParser(YunoHostArgumentFormatParser): default_value = False def parse_question(self, question, user_answers): - question = super().parse_question( - question, user_answers - ) + question = super().parse_question(question, user_answers) if question.default is None: question.default = False @@ -616,11 +619,9 @@ class NumberArgumentParser(YunoHostArgumentFormatParser): default_value = "" def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) - question_parsed.min = question.get('min', None) - question_parsed.max = question.get('max', None) + question_parsed = super().parse_question(question, user_answers) + question_parsed.min = question.get("min", None) + question_parsed.max = question.get("max", None) if question_parsed.default is None: question_parsed.default = 0 @@ -628,19 +629,27 @@ class NumberArgumentParser(YunoHostArgumentFormatParser): def _prevalidate(self, question): super()._prevalidate(question) - if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()): + if not isinstance(question.value, int) and not ( + isinstance(question.value, str) and question.value.isdigit() + ): raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", + field=question.name, + error=m18n.n("invalid_number"), ) if question.min is not None and int(question.value) < question.min: raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", + field=question.name, + error=m18n.n("invalid_number"), ) if question.max is not None and int(question.value) > question.max: raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", + field=question.name, + error=m18n.n("invalid_number"), ) def _post_parse_value(self, question): @@ -660,29 +669,28 @@ class DisplayTextArgumentParser(YunoHostArgumentFormatParser): readonly = True def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) + question_parsed = super().parse_question(question, user_answers) question_parsed.optional = True - question_parsed.style = question.get('style', 'info') + question_parsed.style = question.get("style", "info") return question_parsed def _format_text_for_user_input_in_cli(self, question): - text = question.ask['en'] + text = question.ask["en"] - if question.style in ['success', 'info', 'warning', 'danger']: + if question.style in ["success", "info", "warning", "danger"]: color = { - 'success': 'green', - 'info': 'cyan', - 'warning': 'yellow', - 'danger': 'red' + "success": "green", + "info": "cyan", + "warning": "yellow", + "danger": "red", } return colorize(m18n.g(question.style), color[question.style]) + f" {text}" else: return text + class FileArgumentParser(YunoHostArgumentFormatParser): argument_type = "file" upload_dirs = [] @@ -690,60 +698,77 @@ class FileArgumentParser(YunoHostArgumentFormatParser): @classmethod def clean_upload_dirs(cls): # Delete files uploaded from API - if Moulinette.interface.type== 'api': + if Moulinette.interface.type == "api": for upload_dir in cls.upload_dirs: if os.path.exists(upload_dir): shutil.rmtree(upload_dir) def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) - if question.get('accept'): - question_parsed.accept = question.get('accept').replace(' ', '').split(',') + question_parsed = super().parse_question(question, user_answers) + if question.get("accept"): + question_parsed.accept = question.get("accept").replace(" ", "").split(",") else: question_parsed.accept = [] - if Moulinette.interface.type== 'api': + if Moulinette.interface.type == "api": if user_answers.get(f"{question_parsed.name}[name]"): question_parsed.value = { - 'content': question_parsed.value, - 'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name), + "content": question_parsed.value, + "filename": user_answers.get( + f"{question_parsed.name}[name]", question_parsed.name + ), } # If path file are the same - if question_parsed.value and str(question_parsed.value) == question_parsed.current_value: + if ( + question_parsed.value + and str(question_parsed.value) == question_parsed.current_value + ): question_parsed.value = None return question_parsed def _prevalidate(self, question): super()._prevalidate(question) - if isinstance(question.value, str) and question.value and not os.path.exists(question.value): + if ( + isinstance(question.value, str) + and question.value + and not os.path.exists(question.value) + ): raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1") + "app_argument_invalid", + field=question.name, + error=m18n.n("invalid_number1"), ) - if question.value in [None, ''] or not question.accept: + if question.value in [None, ""] or not question.accept: return - filename = question.value if isinstance(question.value, str) else question.value['filename'] - if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept: + filename = ( + question.value + if isinstance(question.value, str) + else question.value["filename"] + ) + if "." not in filename or "." + filename.split(".")[-1] not in question.accept: raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2") + "app_argument_invalid", + field=question.name, + error=m18n.n("invalid_number2"), ) - def _post_parse_value(self, question): from base64 import b64decode + # Upload files from API # A file arg contains a string with "FILENAME:BASE64_CONTENT" if not question.value: return question.value - if Moulinette.interface.type== 'api': + if Moulinette.interface.type == "api": - upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') + upload_dir = tempfile.mkdtemp(prefix="tmp_configpanel_") FileArgumentParser.upload_dirs += [upload_dir] - filename = question.value['filename'] - logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}") + filename = question.value["filename"] + logger.debug( + f"Save uploaded file {question.value['filename']} from API into {upload_dir}" + ) # Filename is given by user of the API. For security reason, we have replaced # os.path.join to avoid the user to be able to rewrite a file in filesystem @@ -755,9 +780,9 @@ class FileArgumentParser(YunoHostArgumentFormatParser): while os.path.exists(file_path): file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) i += 1 - content = question.value['content'] + content = question.value["content"] try: - with open(file_path, 'wb') as f: + with open(file_path, "wb") as f: f.write(b64decode(content)) except IOError as e: raise YunohostError("cannot_write_file", file=file_path, error=str(e)) @@ -790,6 +815,7 @@ ARGUMENTS_TYPE_PARSERS = { "file": FileArgumentParser, } + def parse_args_in_yunohost_format(user_answers, argument_questions): """Parse arguments store in either manifest.json or actions.json or from a config panel against the user answers when they are present. @@ -811,4 +837,3 @@ def parse_args_in_yunohost_format(user_answers, argument_questions): parsed_answers_dict[question["name"]] = answer return parsed_answers_dict - diff --git a/src/yunohost/utils/i18n.py b/src/yunohost/utils/i18n.py index 89d1d0b34..894841439 100644 --- a/src/yunohost/utils/i18n.py +++ b/src/yunohost/utils/i18n.py @@ -20,6 +20,7 @@ """ from moulinette import Moulinette, m18n + def _value_for_locale(values): """ Return proper value for current locale @@ -42,5 +43,3 @@ def _value_for_locale(values): # Fallback to first value return list(values.values())[0] - - From aeeac12309139175d5b3d37effd2153a4c30bbdb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 4 Sep 2021 18:45:45 +0200 Subject: [PATCH 056/119] Flake8 --- src/yunohost/app.py | 11 +++++------ src/yunohost/utils/config.py | 9 +++++---- src/yunohost/utils/i18n.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0b8e2565e..83ff27cdf 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -36,7 +36,6 @@ import urllib.parse import tempfile from collections import OrderedDict -from moulinette.interfaces.cli import colorize from moulinette import Moulinette, m18n from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger @@ -54,7 +53,7 @@ from moulinette.utils.filesystem import ( ) from yunohost.service import service_status, _run_service_command -from yunohost.utils import packages, config +from yunohost.utils import packages from yunohost.utils.config import ( ConfigPanel, parse_args_in_yunohost_format, @@ -1764,8 +1763,8 @@ def app_config_get(app, key="", mode="classic"): """ Display an app configuration in classic, full or export mode """ - config = AppConfigPanel(app) - return config.get(key, mode) + config_ = AppConfigPanel(app) + return config_.get(key, mode) @is_unit_operation() @@ -1776,12 +1775,12 @@ def app_config_set( Apply a new app configuration """ - config = AppConfigPanel(app) + config_ = AppConfigPanel(app) YunoHostArgumentFormatParser.operation_logger = operation_logger operation_logger.start() - result = config.set(key, value, args, args_file) + result = config_.set(key, value, args, args_file) if "errors" not in result: operation_logger.success() return result diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 4528fb708..8fcf493ed 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -21,9 +21,9 @@ import os import re -import toml import urllib.parse import tempfile +import shutil from collections import OrderedDict from moulinette.interfaces.cli import colorize @@ -37,8 +37,6 @@ from moulinette.utils.filesystem import ( mkdir, ) -from yunohost.service import _get_services -from yunohost.service import _run_service_command, _get_services from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError @@ -144,7 +142,7 @@ class ConfigPanel: if self.errors: return { - "errors": errors, + "errors": self.errors, } self._reload_services() @@ -290,6 +288,9 @@ class ConfigPanel: write_to_yaml(self.save_path, self.new_values) def _reload_services(self): + + from yunohost.service import _run_service_command, _get_services + logger.info("Reloading services...") services_to_reload = set() for panel, section, obj in self._iterate(["panel", "section", "option"]): diff --git a/src/yunohost/utils/i18n.py b/src/yunohost/utils/i18n.py index 894841439..a0daf8181 100644 --- a/src/yunohost/utils/i18n.py +++ b/src/yunohost/utils/i18n.py @@ -18,7 +18,7 @@ along with this program; if not, see http://www.gnu.org/licenses """ -from moulinette import Moulinette, m18n +from moulinette import m18n def _value_for_locale(values): From b5aca9895d64544805f9159ba6be39b2f82b3b8d Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 4 Sep 2021 19:03:11 +0200 Subject: [PATCH 057/119] [enh] yes/no args on boolean aquestion + semantic --- src/yunohost/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 522f695e2..e9df193c7 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -55,7 +55,7 @@ from moulinette.utils.filesystem import ( from yunohost.service import service_status, _run_service_command from yunohost.utils import packages, config -from yunohost.utils.config import ConfigPanel, parse_args_in_yunohost_format, YunoHostArgumentFormatParser +from yunohost.utils.config import ConfigPanel, parse_args_in_yunohost_format, Question from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory @@ -1772,7 +1772,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None, args_ config = AppConfigPanel(app) - YunoHostArgumentFormatParser.operation_logger = operation_logger + Question.operation_logger = operation_logger operation_logger.start() result = config.set(key, value, args, args_file) From 8eaaf0975afd8d3bfb15b8a65d8d3a851774c8c8 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 4 Sep 2021 19:03:07 +0200 Subject: [PATCH 058/119] [enh] yes/no args on boolean aquestion + semantic --- src/yunohost/utils/config.py | 592 +++++++++++++++++++---------------- 1 file changed, 322 insertions(+), 270 deletions(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 8fcf493ed..def083cdc 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -79,12 +79,15 @@ class ConfigPanel: result = {} for panel, section, option in self._iterate(): key = f"{panel['id']}.{section['id']}.{option['id']}" - if mode == "export": - result[option["id"]] = option.get("current_value") - else: - result[key] = {"ask": _value_for_locale(option["ask"])} - if "current_value" in option: - result[key]["value"] = option["current_value"] + if mode == 'export': + result[option['id']] = option.get('current_value') + elif 'ask' in option: + result[key] = {'ask': _value_for_locale(option['ask'])} + elif 'i18n' in self.config: + result[key] = {'ask': m18n.n(self.config['i18n'] + '_' + option['id'])} + if 'current_value' in option: + question_class = ARGUMENTS_TYPE_PARSERS[option.get("type", "string")] + result[key]['value'] = question_class.humanize(option['current_value'], option) return result @@ -138,7 +141,7 @@ class ConfigPanel: raise finally: # Delete files uploaded from API - FileArgumentParser.clean_upload_dirs() + FileQuestion.clean_upload_dirs() if self.errors: return { @@ -274,16 +277,42 @@ class ConfigPanel: parse_args_in_yunohost_format(self.args, section["options"]) ) self.new_values = { - key: str(value[0]) + key: value[0] for key, value in self.new_values.items() if not value[0] is None } + self.errors = None + + def _get_default_values(self): + return { option['id']: option['default'] + for _, _, option in self._iterate() if 'default' in option } + + def _load_current_values(self): + """ + Retrieve entries in YAML file + And set default values if needed + """ + + # Retrieve entries in the YAML + on_disk_settings = {} + if os.path.exists(self.save_path) and os.path.isfile(self.save_path): + on_disk_settings = read_yaml(self.save_path) or {} + + # Inject defaults if needed (using the magic .update() ;)) + self.values = self._get_default_values() + self.values.update(on_disk_settings) def _apply(self): - logger.info("Running config script...") + logger.info("Saving the new configuration...") dir_path = os.path.dirname(os.path.realpath(self.save_path)) if not os.path.exists(dir_path): mkdir(dir_path, mode=0o700) + + values_to_save = {**self.values, **self.new_values} + if self.save_mode == 'diff': + defaults = self._get_default_values() + values_to_save = {k: v for k, v in values_to_save.items() if defaults.get(k) != v} + # Save the settings to the .yaml file write_to_yaml(self.save_path, self.new_values) @@ -291,15 +320,16 @@ class ConfigPanel: from yunohost.service import _run_service_command, _get_services - logger.info("Reloading services...") services_to_reload = set() for panel, section, obj in self._iterate(["panel", "section", "option"]): services_to_reload |= set(obj.get("services", [])) services_to_reload = list(services_to_reload) services_to_reload.sort(key="nginx".__eq__) + if services_to_reload: + logger.info("Reloading services...") for service in services_to_reload: - if "__APP__": + if "__APP__" in service: service = service.replace("__APP__", self.app) logger.debug(f"Reloading {service}") if not _run_service_command("reload-or-restart", service): @@ -322,140 +352,138 @@ class ConfigPanel: yield (panel, section, option) -class Question: - "empty class to store questions information" - - -class YunoHostArgumentFormatParser(object): +class Question(object): hide_user_input_in_prompt = False operation_logger = None - def parse_question(self, question, user_answers): - parsed_question = Question() - - parsed_question.name = question["name"] - parsed_question.type = question.get("type", "string") - parsed_question.default = question.get("default", None) - parsed_question.current_value = question.get("current_value") - parsed_question.optional = question.get("optional", False) - parsed_question.choices = question.get("choices", []) - parsed_question.pattern = question.get("pattern") - parsed_question.ask = question.get("ask", {"en": f"{parsed_question.name}"}) - parsed_question.help = question.get("help") - parsed_question.helpLink = question.get("helpLink") - parsed_question.value = user_answers.get(parsed_question.name) - parsed_question.redact = question.get("redact", False) + def __init__(self, question, user_answers): + self.name = question["name"] + self.type = question.get("type", 'string') + self.default = question.get("default", None) + self.current_value = question.get("current_value") + self.optional = question.get("optional", False) + self.choices = question.get("choices", []) + self.pattern = question.get("pattern") + self.ask = question.get("ask", {'en': self.name}) + self.help = question.get("help") + self.helpLink = question.get("helpLink") + self.value = user_answers.get(self.name) + self.redact = question.get('redact', False) # Empty value is parsed as empty string - if parsed_question.default == "": - parsed_question.default = None + if self.default == "": + self.default = None - return parsed_question + @staticmethod + def humanize(value, option={}): + return str(value) - def parse(self, question, user_answers): - question = self.parse_question(question, user_answers) + @staticmethod + def normalize(value, option={}): + return value + + def ask_if_needed(self): while True: # Display question if no value filled or if it's a readonly message - if Moulinette.interface.type == "cli": - text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( - question - ) + if Moulinette.interface.type== 'cli': + text_for_user_input_in_cli = self._format_text_for_user_input_in_cli() if getattr(self, "readonly", False): Moulinette.display(text_for_user_input_in_cli) - elif question.value is None: + elif self.value is None: prefill = "" - if question.current_value is not None: - prefill = question.current_value - elif question.default is not None: - prefill = question.default - question.value = Moulinette.prompt( + if self.current_value is not None: + prefill = self.humanize(self.current_value, self) + elif self.default is not None: + prefill = self.default + self.value = Moulinette.prompt( message=text_for_user_input_in_cli, is_password=self.hide_user_input_in_prompt, confirm=self.hide_user_input_in_prompt, prefill=prefill, - is_multiline=(question.type == "text"), + is_multiline=(self.type == "text"), ) + # Normalization + # This is done to enforce a certain formating like for boolean + self.value = self.normalize(self.value, self) + # Apply default value - if question.value in [None, ""] and question.default is not None: - question.value = ( + if self.value in [None, ""] and self.default is not None: + self.value = ( getattr(self, "default_value", None) - if question.default is None - else question.default + if self.default is None + else self.default ) # Prevalidation try: - self._prevalidate(question) + self._prevalidate() except YunohostValidationError as e: if Moulinette.interface.type == "api": raise Moulinette.display(str(e), "error") - question.value = None + self.value = None continue break - # this is done to enforce a certain formating like for boolean - # by default it doesn't do anything - question.value = self._post_parse_value(question) + self.value = self._post_parse_value() - return (question.value, self.argument_type) + return (self.value, self.argument_type) - def _prevalidate(self, question): - if question.value in [None, ""] and not question.optional: - raise YunohostValidationError("app_argument_required", name=question.name) + + def _prevalidate(self): + if self.value in [None, ""] and not self.optional: + raise YunohostValidationError("app_argument_required", name=self.name) # we have an answer, do some post checks - if question.value is not None: - if question.choices and question.value not in question.choices: - self._raise_invalid_answer(question) - if question.pattern and not re.match( - question.pattern["regexp"], str(question.value) - ): + if self.value is not None: + if self.choices and self.value not in self.choices: + self._raise_invalid_answer() + if self.pattern and not re.match(self.pattern['regexp'], str(self.value)): raise YunohostValidationError( - question.pattern["error"], - name=question.name, - value=question.value, + self.pattern['error'], + name=self.name, + value=self.value, ) - def _raise_invalid_answer(self, question): + def _raise_invalid_answer(self): raise YunohostValidationError( "app_argument_choice_invalid", - name=question.name, - value=question.value, - choices=", ".join(question.choices), + name=self.name, + value=self.value, + choices=", ".join(self.choices), ) - def _format_text_for_user_input_in_cli(self, question): - text_for_user_input_in_cli = _value_for_locale(question.ask) + def _format_text_for_user_input_in_cli(self): + text_for_user_input_in_cli = _value_for_locale(self.ask) - if question.choices: - text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices)) + if self.choices: + text_for_user_input_in_cli += " [{0}]".format(" | ".join(self.choices)) - if question.help or question.helpLink: + if self.help or self.helpLink: text_for_user_input_in_cli += ":\033[m" - if question.help: + if self.help: text_for_user_input_in_cli += "\n - " - text_for_user_input_in_cli += _value_for_locale(question.help) - if question.helpLink: - if not isinstance(question.helpLink, dict): - question.helpLink = {"href": question.helpLink} - text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}" + text_for_user_input_in_cli += _value_for_locale(self.help) + if self.helpLink: + if not isinstance(self.helpLink, dict): + self.helpLink = {"href": self.helpLink} + text_for_user_input_in_cli += f"\n - See {self.helpLink['href']}" return text_for_user_input_in_cli - def _post_parse_value(self, question): - if not question.redact: - return question.value + def _post_parse_value(self): + if not self.redact: + return self.value # Tell the operation_logger to redact all password-type / secret args # Also redact the % escaped version of the password that might appear in # the 'args' section of metadata (relevant for password with non-alphanumeric char) data_to_redact = [] - if question.value and isinstance(question.value, str): - data_to_redact.append(question.value) - if question.current_value and isinstance(question.current_value, str): - data_to_redact.append(question.current_value) + if self.value and isinstance(self.value, str): + data_to_redact.append(self.value) + if self.current_value and isinstance(self.current_value, str): + data_to_redact.append(self.current_value) data_to_redact += [ urllib.parse.quote(data) for data in data_to_redact @@ -464,50 +492,60 @@ class YunoHostArgumentFormatParser(object): if self.operation_logger: self.operation_logger.data_to_redact.extend(data_to_redact) elif data_to_redact: - raise YunohostError("app_argument_cant_redact", arg=question.name) + raise YunohostError("app_argument_cant_redact", arg=self.name) - return question.value + return self.value -class StringArgumentParser(YunoHostArgumentFormatParser): +class StringQuestion(Question): argument_type = "string" default_value = "" -class TagsArgumentParser(YunoHostArgumentFormatParser): +class TagsQuestion(Question): argument_type = "tags" - def _prevalidate(self, question): - values = question.value - for value in values.split(","): - question.value = value - super()._prevalidate(question) - question.value = values + @staticmethod + def humanize(value, option={}): + if isinstance(value, list): + return ','.join(value) + return value + + def _prevalidate(self): + values = self.value + if isinstance(values, str): + values = values.split(",") + for value in values: + self.value = value + super()._prevalidate() + self.value = values -class PasswordArgumentParser(YunoHostArgumentFormatParser): +class PasswordQuestion(Question): hide_user_input_in_prompt = True argument_type = "password" default_value = "" forbidden_chars = "{}" - def parse_question(self, question, user_answers): - question = super(PasswordArgumentParser, self).parse_question( - question, user_answers - ) - question.redact = True - if question.default is not None: + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + self.redact = True + if self.default is not None: raise YunohostValidationError( - "app_argument_password_no_default", name=question.name + "app_argument_password_no_default", name=self.name ) - return question + @staticmethod + def humanize(value, option={}): + if value: + return '***' # Avoid to display the password on screen + return "" - def _prevalidate(self, question): - super()._prevalidate(question) + def _prevalidate(self): + super()._prevalidate() - if question.value is not None: - if any(char in question.value for char in self.forbidden_chars): + if self.value is not None: + if any(char in self.value for char in self.forbidden_chars): raise YunohostValidationError( "pattern_password_app", forbidden_chars=self.forbidden_chars ) @@ -515,184 +553,214 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser): # If it's an optional argument the value should be empty or strong enough from yunohost.utils.password import assert_password_is_strong_enough - assert_password_is_strong_enough("user", question.value) + assert_password_is_strong_enough("user", self.value) -class PathArgumentParser(YunoHostArgumentFormatParser): +class PathQuestion(Question): argument_type = "path" default_value = "" -class BooleanArgumentParser(YunoHostArgumentFormatParser): +class BooleanQuestion(Question): argument_type = "boolean" default_value = False + yes_answers = ["1", "yes", "y", "true", "t", "on"] + no_answers = ["0", "no", "n", "false", "f", "off"] - def parse_question(self, question, user_answers): - question = super().parse_question(question, user_answers) + @staticmethod + def humanize(value, option={}): + yes = option.get('yes', 1) + no = option.get('no', 0) + value = str(value).lower() + if value == str(yes).lower(): + return 'yes' + if value == str(no).lower(): + return 'no' + if value in BooleanQuestion.yes_answers: + return 'yes' + if value in BooleanQuestion.no_answers: + return 'no' - if question.default is None: - question.default = False + if value in ['none', ""]: + return '' - return question + raise YunohostValidationError( + "app_argument_choice_invalid", + name=self.name, + value=self.value, + choices="yes, no, y, n, 1, 0", + ) - def _format_text_for_user_input_in_cli(self, question): - text_for_user_input_in_cli = _value_for_locale(question.ask) + @staticmethod + def normalize(value, option={}): + yes = option.get('yes', 1) + no = option.get('no', 0) + + if str(value).lower() in BooleanQuestion.yes_answers: + return yes + + if str(value).lower() in BooleanQuestion.no_answers: + return no + + if value in [None, ""]: + return None + raise YunohostValidationError( + "app_argument_choice_invalid", + name=self.name, + value=self.value, + choices="yes, no, y, n, 1, 0", + ) + + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + self.yes = question.get('yes', 1) + self.no = question.get('no', 0) + if self.default is None: + self.default = False + + + def _format_text_for_user_input_in_cli(self): + text_for_user_input_in_cli = _value_for_locale(self.ask) text_for_user_input_in_cli += " [yes | no]" - if question.default is not None: - formatted_default = "yes" if question.default else "no" + if self.default is not None: + formatted_default = self.humanize(self.default) text_for_user_input_in_cli += " (default: {0})".format(formatted_default) return text_for_user_input_in_cli - def _post_parse_value(self, question): - if isinstance(question.value, bool): - return 1 if question.value else 0 - - if str(question.value).lower() in ["1", "yes", "y", "true"]: - return 1 - - if str(question.value).lower() in ["0", "no", "n", "false"]: - return 0 - - raise YunohostValidationError( - "app_argument_choice_invalid", - name=question.name, - value=question.value, - choices="yes, no, y, n, 1, 0", - ) + def get(self, key, default=None): + try: + return getattr(self, key) + except AttributeError: + return default -class DomainArgumentParser(YunoHostArgumentFormatParser): +class DomainQuestion(Question): argument_type = "domain" - def parse_question(self, question, user_answers): + def __init__(self, question, user_answers): from yunohost.domain import domain_list, _get_maindomain - question = super(DomainArgumentParser, self).parse_question( - question, user_answers - ) + super().__init__(question, user_answers) - if question.default is None: - question.default = _get_maindomain() + if self.default is None: + self.default = _get_maindomain() - question.choices = domain_list()["domains"] + self.choices = domain_list()["domains"] - return question - def _raise_invalid_answer(self, question): + def _raise_invalid_answer(self): raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown") + "app_argument_invalid", field=self.name, error=m18n.n("domain_unknown") ) -class UserArgumentParser(YunoHostArgumentFormatParser): +class UserQuestion(Question): argument_type = "user" - def parse_question(self, question, user_answers): + def __init__(self, question, user_answers): from yunohost.user import user_list, user_info from yunohost.domain import _get_maindomain - question = super(UserArgumentParser, self).parse_question( - question, user_answers - ) - question.choices = user_list()["users"] - if question.default is None: + super().__init__(question, user_answers) + self.choices = user_list()["users"] + if self.default is None: root_mail = "root@%s" % _get_maindomain() - for user in question.choices.keys(): + for user in self.choices.keys(): if root_mail in user_info(user).get("mail-aliases", []): - question.default = user + self.default = user break - return question - def _raise_invalid_answer(self, question): + def _raise_invalid_answer(self): raise YunohostValidationError( "app_argument_invalid", - field=question.name, - error=m18n.n("user_unknown", user=question.value), + field=self.name, + error=m18n.n("user_unknown", user=self.value), ) -class NumberArgumentParser(YunoHostArgumentFormatParser): +class NumberQuestion(Question): argument_type = "number" default_value = "" - def parse_question(self, question, user_answers): - question_parsed = super().parse_question(question, user_answers) - question_parsed.min = question.get("min", None) - question_parsed.max = question.get("max", None) - if question_parsed.default is None: - question_parsed.default = 0 + @staticmethod + def humanize(value, option={}): + return str(value) - return question_parsed + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + self.min = question.get("min", None) + self.max = question.get("max", None) + self.step = question.get("step", None) - def _prevalidate(self, question): - super()._prevalidate(question) - if not isinstance(question.value, int) and not ( - isinstance(question.value, str) and question.value.isdigit() + + def _prevalidate(self): + super()._prevalidate() + if not isinstance(self.value, int) and not ( + isinstance(self.value, str) and self.value.isdigit() ): raise YunohostValidationError( "app_argument_invalid", - field=question.name, + field=self.name, error=m18n.n("invalid_number"), ) - if question.min is not None and int(question.value) < question.min: + if self.min is not None and int(self.value) < self.min: raise YunohostValidationError( "app_argument_invalid", - field=question.name, + field=self.name, error=m18n.n("invalid_number"), ) - if question.max is not None and int(question.value) > question.max: + if self.max is not None and int(self.value) > self.max: raise YunohostValidationError( "app_argument_invalid", - field=question.name, + field=self.name, error=m18n.n("invalid_number"), ) - def _post_parse_value(self, question): - if isinstance(question.value, int): - return super()._post_parse_value(question) + def _post_parse_value(self): + if isinstance(self.value, int): + return super()._post_parse_value() - if isinstance(question.value, str) and question.value.isdigit(): - return int(question.value) + if isinstance(self.value, str) and self.value.isdigit(): + return int(self.value) raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number") ) -class DisplayTextArgumentParser(YunoHostArgumentFormatParser): +class DisplayTextQuestion(Question): argument_type = "display_text" readonly = True - def parse_question(self, question, user_answers): - question_parsed = super().parse_question(question, user_answers) + def __init__(self, question, user_answers): + super().__init__(question, user_answers) - question_parsed.optional = True - question_parsed.style = question.get("style", "info") + self.optional = True + self.style = question.get("style", "info") - return question_parsed - def _format_text_for_user_input_in_cli(self, question): - text = question.ask["en"] + def _format_text_for_user_input_in_cli(self): + text = self.ask["en"] - if question.style in ["success", "info", "warning", "danger"]: + if self.style in ["success", "info", "warning", "danger"]: color = { "success": "green", "info": "cyan", "warning": "yellow", "danger": "red", } - return colorize(m18n.g(question.style), color[question.style]) + f" {text}" + return colorize(m18n.g(self.style), color[self.style]) + f" {text}" else: return text -class FileArgumentParser(YunoHostArgumentFormatParser): +class FileQuestion(Question): argument_type = "file" upload_dirs = [] @@ -704,71 +772,54 @@ class FileArgumentParser(YunoHostArgumentFormatParser): if os.path.exists(upload_dir): shutil.rmtree(upload_dir) - def parse_question(self, question, user_answers): - question_parsed = super().parse_question(question, user_answers) - if question.get("accept"): - question_parsed.accept = question.get("accept").replace(" ", "").split(",") + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + if self.get("accept"): + self.accept = question.get("accept").replace(" ", "").split(",") else: - question_parsed.accept = [] - if Moulinette.interface.type == "api": - if user_answers.get(f"{question_parsed.name}[name]"): - question_parsed.value = { - "content": question_parsed.value, - "filename": user_answers.get( - f"{question_parsed.name}[name]", question_parsed.name - ), + self.accept = [] + if Moulinette.interface.type== "api": + if user_answers.get(f"{self.name}[name]"): + self.value = { + "content": self.value, + "filename": user_answers.get(f"{self.name}[name]", self.name), } # If path file are the same - if ( - question_parsed.value - and str(question_parsed.value) == question_parsed.current_value - ): - question_parsed.value = None + if self.value and str(self.value) == self.current_value: + self.value = None - return question_parsed - def _prevalidate(self, question): - super()._prevalidate(question) - if ( - isinstance(question.value, str) - and question.value - and not os.path.exists(question.value) - ): + def _prevalidate(self): + super()._prevalidate() + if isinstance(self.value, str) and self.value and not os.path.exists(self.value): raise YunohostValidationError( - "app_argument_invalid", - field=question.name, - error=m18n.n("invalid_number1"), + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number1") ) - if question.value in [None, ""] or not question.accept: + if self.value in [None, ""] or not self.accept: return - filename = ( - question.value - if isinstance(question.value, str) - else question.value["filename"] - ) - if "." not in filename or "." + filename.split(".")[-1] not in question.accept: + filename = self.value if isinstance(self.value, str) else self.value["filename"] + if "." not in filename or "." + filename.split(".")[-1] not in self.accept: raise YunohostValidationError( - "app_argument_invalid", - field=question.name, - error=m18n.n("invalid_number2"), + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number2") ) - def _post_parse_value(self, question): + + def _post_parse_value(self): from base64 import b64decode # Upload files from API # A file arg contains a string with "FILENAME:BASE64_CONTENT" - if not question.value: - return question.value + if not self.value: + return self.value if Moulinette.interface.type == "api": upload_dir = tempfile.mkdtemp(prefix="tmp_configpanel_") - FileArgumentParser.upload_dirs += [upload_dir] - filename = question.value["filename"] + FileQuestion.upload_dirs += [upload_dir] + filename = self.value["filename"] logger.debug( - f"Save uploaded file {question.value['filename']} from API into {upload_dir}" + f"Save uploaded file {self.value['filename']} from API into {upload_dir}" ) # Filename is given by user of the API. For security reason, we have replaced @@ -781,7 +832,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): while os.path.exists(file_path): file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) i += 1 - content = question.value["content"] + content = self.value["content"] try: with open(file_path, "wb") as f: f.write(b64decode(content)) @@ -789,31 +840,31 @@ class FileArgumentParser(YunoHostArgumentFormatParser): raise YunohostError("cannot_write_file", file=file_path, error=str(e)) except Exception as e: raise YunohostError("error_writing_file", file=file_path, error=str(e)) - question.value = file_path - return question.value + self.value = file_path + return self.value ARGUMENTS_TYPE_PARSERS = { - "string": StringArgumentParser, - "text": StringArgumentParser, - "select": StringArgumentParser, - "tags": TagsArgumentParser, - "email": StringArgumentParser, - "url": StringArgumentParser, - "date": StringArgumentParser, - "time": StringArgumentParser, - "color": StringArgumentParser, - "password": PasswordArgumentParser, - "path": PathArgumentParser, - "boolean": BooleanArgumentParser, - "domain": DomainArgumentParser, - "user": UserArgumentParser, - "number": NumberArgumentParser, - "range": NumberArgumentParser, - "display_text": DisplayTextArgumentParser, - "alert": DisplayTextArgumentParser, - "markdown": DisplayTextArgumentParser, - "file": FileArgumentParser, + "string": StringQuestion, + "text": StringQuestion, + "select": StringQuestion, + "tags": TagsQuestion, + "email": StringQuestion, + "url": StringQuestion, + "date": StringQuestion, + "time": StringQuestion, + "color": StringQuestion, + "password": PasswordQuestion, + "path": PathQuestion, + "boolean": BooleanQuestion, + "domain": DomainQuestion, + "user": UserQuestion, + "number": NumberQuestion, + "range": NumberQuestion, + "display_text": DisplayTextQuestion, + "alert": DisplayTextQuestion, + "markdown": DisplayTextQuestion, + "file": FileQuestion, } @@ -831,10 +882,11 @@ def parse_args_in_yunohost_format(user_answers, argument_questions): parsed_answers_dict = OrderedDict() for question in argument_questions: - parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]() + question_class = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")] + question = question_class(question, user_answers) - answer = parser.parse(question=question, user_answers=user_answers) + answer = question.ask_if_needed() if answer is not None: - parsed_answers_dict[question["name"]] = answer + parsed_answers_dict[question.name] = answer return parsed_answers_dict From 4c9fcdc9e447f9fffbcd1e32127c849df304f912 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 4 Sep 2021 20:12:56 +0200 Subject: [PATCH 059/119] [fix] Get / set app config panel --- src/yunohost/app.py | 3 ++- src/yunohost/utils/config.py | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index bc84d0da0..5a8c6cd56 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1800,7 +1800,8 @@ class AppConfigPanel(ConfigPanel): self.values = self._call_config_script("show") def _apply(self): - self.errors = self._call_config_script("apply", self.new_values) + env = {key: str(value) for key, value in self.new_values.items()} + self.errors = self._call_config_script("apply", env=env) def _call_config_script(self, action, env={}): from yunohost.hook import hook_exec diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index def083cdc..2b7282259 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -81,10 +81,11 @@ class ConfigPanel: key = f"{panel['id']}.{section['id']}.{option['id']}" if mode == 'export': result[option['id']] = option.get('current_value') - elif 'ask' in option: - result[key] = {'ask': _value_for_locale(option['ask'])} - elif 'i18n' in self.config: - result[key] = {'ask': m18n.n(self.config['i18n'] + '_' + option['id'])} + else: + if 'ask' in option: + result[key] = {'ask': _value_for_locale(option['ask'])} + elif 'i18n' in self.config: + result[key] = {'ask': m18n.n(self.config['i18n'] + '_' + option['id'])} if 'current_value' in option: question_class = ARGUMENTS_TYPE_PARSERS[option.get("type", "string")] result[key]['value'] = question_class.humanize(option['current_value'], option) @@ -774,7 +775,7 @@ class FileQuestion(Question): def __init__(self, question, user_answers): super().__init__(question, user_answers) - if self.get("accept"): + if question.get("accept"): self.accept = question.get("accept").replace(" ", "").split(",") else: self.accept = [] From c8791a98340acd28566a818244bba4c886fdd68e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 4 Sep 2021 20:21:30 +0200 Subject: [PATCH 060/119] Add config check during service_reload_or_restart --- src/yunohost/app.py | 14 ++++---------- src/yunohost/service.py | 25 ++++++++++++++++++++++++- src/yunohost/tests/test_service.py | 22 ++++++++++++++++++++++ src/yunohost/utils/config.py | 14 +++----------- 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 83ff27cdf..23f3b011b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -52,7 +52,6 @@ from moulinette.utils.filesystem import ( mkdir, ) -from yunohost.service import service_status, _run_service_command from yunohost.utils import packages from yunohost.utils.config import ( ConfigPanel, @@ -424,6 +423,7 @@ def app_change_url(operation_logger, app, domain, path): """ from yunohost.hook import hook_exec, hook_callback + from yunohost.service import service_reload_or_restart installed = _is_installed(app) if not installed: @@ -492,15 +492,7 @@ def app_change_url(operation_logger, app, domain, path): app_ssowatconf() - # avoid common mistakes - if _run_service_command("reload", "nginx") is False: - # grab nginx errors - # the "exit 0" is here to avoid check_output to fail because 'nginx -t' - # will return != 0 since we are in a failed state - nginx_errors = check_output("nginx -t; exit 0") - raise YunohostError( - "app_change_url_failed_nginx_reload", nginx_errors=nginx_errors - ) + service_reload_or_restart("nginx") logger.success(m18n.n("app_change_url_success", app=app, domain=domain, path=path)) @@ -2883,6 +2875,8 @@ def unstable_apps(): def _assert_system_is_sane_for_app(manifest, when): + from yunohost.service import service_status + logger.debug("Checking that required services are up and running...") services = manifest.get("services", []) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index fb12e9053..da3bded3c 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -256,7 +256,7 @@ def service_restart(names): ) -def service_reload_or_restart(names): +def service_reload_or_restart(names, test_conf=True): """ Reload one or more services if they support it. If not, restart them instead. If the services are not running yet, they will be started. @@ -266,7 +266,30 @@ def service_reload_or_restart(names): """ if isinstance(names, str): names = [names] + + services = _get_services() + for name in names: + + logger.debug(f"Reloading service {name}") + + test_conf_cmd = services.get(service, {}).get("test_conf") + if test_conf and test_conf_cmd: + + p = subprocess.Popen( + test_conf_cmd, + shell=True, + executable="/bin/bash", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + out, _ = p.communicate() + if p.returncode != 0: + errors = out.strip().split("\n") + logger.error(m18n.("service_not_reloading_because_conf_broken", errors=errors)) + continue + if _run_service_command("reload-or-restart", name): logger.success(m18n.n("service_reloaded_or_restarted", service=name)) else: diff --git a/src/yunohost/tests/test_service.py b/src/yunohost/tests/test_service.py index 1f82dc8fd..1007419a1 100644 --- a/src/yunohost/tests/test_service.py +++ b/src/yunohost/tests/test_service.py @@ -9,6 +9,7 @@ from yunohost.service import ( service_add, service_remove, service_log, + service_reload_or_restart, ) @@ -38,6 +39,10 @@ def clean(): _save_services(services) + if os.path.exists("/etc/nginx/conf.d/broken.conf"): + os.remove("/etc/nginx/conf.d/broken.conf") + os.system("systemctl reload-or-restart nginx") + def test_service_status_all(): @@ -118,3 +123,20 @@ def test_service_update_to_remove_properties(): assert _get_services()["dummyservice"].get("test_status") == "false" service_add("dummyservice", description="dummy", test_status="") assert not _get_services()["dummyservice"].get("test_status") + + +def test_service_conf_broken(): + + os.system("echo pwet > /etc/nginx/conf.d/broken.conf") + + status = service_status("nginx") + assert status["status"] == "running" + assert status["configuration"] == "broken" + assert "broken.conf" in status["configuration-details"] + + # Service reload-or-restart should check that the conf ain't valid + # before reload-or-restart, hence the service should still be running + service_reload_or_restart("nginx") + assert status["status"] == "running" + + os.remove("/etc/nginx/conf.d/broken.conf") diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 8fcf493ed..9cdb30119 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -289,7 +289,7 @@ class ConfigPanel: def _reload_services(self): - from yunohost.service import _run_service_command, _get_services + from yunohost.service import service_reload_or_restart logger.info("Reloading services...") services_to_reload = set() @@ -299,16 +299,8 @@ class ConfigPanel: services_to_reload = list(services_to_reload) services_to_reload.sort(key="nginx".__eq__) for service in services_to_reload: - if "__APP__": - service = service.replace("__APP__", self.app) - logger.debug(f"Reloading {service}") - if not _run_service_command("reload-or-restart", service): - services = _get_services() - test_conf = services[service].get("test_conf", "true") - errors = check_output(f"{test_conf}; exit 0") if test_conf else "" - raise YunohostError( - "config_failed_service_reload", service=service, errors=errors - ) + service = service.replace("__APP__", self.app) + service_reload_or_restart(service) def _iterate(self, trigger=["option"]): for panel in self.config.get("panels", []): From 778c67540cbd6abe25c2735ef3a80494aa80e10e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 4 Sep 2021 20:22:28 +0200 Subject: [PATCH 061/119] Misc fixes, try to implement locale strings --- locales/en.json | 6 +++++- src/yunohost/app.py | 4 ++-- src/yunohost/utils/config.py | 37 ++++++++++++++++++++---------------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/locales/en.json b/locales/en.json index a5652f378..667a8d4b3 100644 --- a/locales/en.json +++ b/locales/en.json @@ -21,6 +21,8 @@ "app_change_url_identical_domains": "The old and new domain/url_path are identical ('{domain}{path}'), nothing to do.", "app_change_url_no_script": "The app '{app_name}' doesn't support URL modification yet. Maybe you should upgrade it.", "app_change_url_success": "{app} URL is now {domain}{path}", + "app_config_unable_to_apply": "Failed to apply config panel values.", + "app_config_unable_to_read": "Failed to read config panel values.", "app_extraction_failed": "Could not extract the installation files", "app_full_domain_unavailable": "Sorry, this app must be installed on a domain of its own, but other apps are already installed on the domain '{domain}'. You could use a subdomain dedicated to this app instead.", "app_id_invalid": "Invalid app ID", @@ -139,6 +141,7 @@ "certmanager_self_ca_conf_file_not_found": "Could not find configuration file for self-signing authority (file: {file})", "certmanager_unable_to_parse_self_CA_name": "Could not parse name of self-signing authority (file: {file})", "certmanager_warning_subdomain_dns_record": "Subdomain '{subdomain}' does not resolve to the same IP address as '{domain}'. Some features will not be available until you fix this and regenerate the certificate.", + "config_apply_failed": "Applying the new configuration failed: {error}", "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_warning": "Warning: This app may work, but is not well-integrated in YunoHost. Some features such as single sign-on and backup/restore might not be available. Install anyway? [{answers}] ", @@ -389,8 +392,8 @@ "log_app_change_url": "Change the URL of the '{}' app", "log_app_config_apply": "Apply config to the '{}' app", "log_app_config_get": "Get a specific setting from config panel of the '{}' app", - "log_app_config_show": "Show the config panel of the '{}' app", "log_app_config_set": "Apply config to the '{}' app", + "log_app_config_show": "Show the config panel of the '{}' app", "log_app_install": "Install the '{}' app", "log_app_makedefault": "Make '{}' the default app", "log_app_remove": "Remove the '{}' app", @@ -596,6 +599,7 @@ "service_disabled": "The service '{service}' will not be started anymore when system boots.", "service_enable_failed": "Could not make the service '{service}' automatically start at boot.\n\nRecent service logs:{logs}", "service_enabled": "The service '{service}' will now be automatically started during system boots.", + "service_not_reloading_because_conf_broken": "Not reloading/restarting service '{name}' because it configuration is broken: {errors}", "service_regen_conf_is_deprecated": "'yunohost service regen-conf' is deprecated! Please use 'yunohost tools regen-conf' instead.", "service_reload_failed": "Could not reload the service '{service}'\n\nRecent service logs:{logs}", "service_reload_or_restart_failed": "Could not reload or restart the service '{service}'\n\nRecent service logs:{logs}", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 23f3b011b..e24af1654 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1823,9 +1823,9 @@ ynh_app_config_run $1 ret, values = hook_exec(config_script, args=[action], env=env) if ret != 0: if action == "show": - raise YunohostError("app_config_unable_to_read_values") + raise YunohostError("app_config_unable_to_read") else: - raise YunohostError("app_config_unable_to_apply_values_correctly") + raise YunohostError("app_config_unable_to_apply") return values diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 9cdb30119..8c1b2948a 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -31,6 +31,7 @@ from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output from moulinette.utils.filesystem import ( + write_to_file, read_toml, read_yaml, write_to_yaml, @@ -127,14 +128,14 @@ class ConfigPanel: # N.B. : KeyboardInterrupt does not inherit from Exception except (KeyboardInterrupt, EOFError): error = m18n.n("operation_interrupted") - logger.error(m18n.n("config_failed", error=error)) + logger.error(m18n.n("config_apply_failed", error=error)) raise # Something wrong happened in Yunohost's code (most probably hook_exec) except Exception: import traceback error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("config_failed", error=error)) + logger.error(m18n.n("config_apply_failed", error=error)) raise finally: # Delete files uploaded from API @@ -154,10 +155,11 @@ class ConfigPanel: return read_toml(self.config_path) def _get_config_panel(self): + # Split filter_key - filter_key = dict(enumerate(self.filter_key.split("."))) + filter_key = self.filter_key.split(".") if len(filter_key) > 3: - raise YunohostError("config_too_much_sub_keys") + raise YunohostError("config_too_many_sub_keys", key=self.filter_key) if not os.path.exists(self.config_path): return None @@ -166,7 +168,7 @@ class ConfigPanel: # Check TOML config panel is in a supported version if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED: raise YunohostError( - "config_too_old_version", version=toml_config_panel["version"] + "config_version_not_supported", version=toml_config_panel["version"] ) # Transform toml format into internal format @@ -187,6 +189,13 @@ class ConfigPanel: # optional choices pattern limit min max step accept redact } + # + # FIXME : this is hella confusing ... + # from what I understand, the purpose is to have some sort of "deep_update" + # to apply the defaults onto the loaded toml ... + # in that case we probably want to get inspiration from + # https://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth + # def convert(toml_node, node_type): """Convert TOML in internal format ('full' mode used by webadmin) Here are some properties of 1.0 config panel in toml: @@ -456,7 +465,7 @@ class YunoHostArgumentFormatParser(object): if self.operation_logger: self.operation_logger.data_to_redact.extend(data_to_redact) elif data_to_redact: - raise YunohostError("app_argument_cant_redact", arg=question.name) + raise YunohostError(f"Can't redact {question.name} because no operation logger available in the context", raw_msg=True) return question.value @@ -729,7 +738,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): raise YunohostValidationError( "app_argument_invalid", field=question.name, - error=m18n.n("invalid_number1"), + error=m18n.n("file_does_not_exists"), ) if question.value in [None, ""] or not question.accept: return @@ -743,7 +752,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): raise YunohostValidationError( "app_argument_invalid", field=question.name, - error=m18n.n("invalid_number2"), + error=m18n.n("file_extension_not_accepted"), ) def _post_parse_value(self, question): @@ -768,19 +777,15 @@ class FileArgumentParser(YunoHostArgumentFormatParser): # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" file_path = os.path.normpath(upload_dir + "/" + filename) if not file_path.startswith(upload_dir + "/"): - raise YunohostError("relative_parent_path_in_filename_forbidden") + raise YunohostError("file_relative_parent_path_in_filename_forbidden") i = 2 while os.path.exists(file_path): file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) i += 1 content = question.value["content"] - try: - with open(file_path, "wb") as f: - f.write(b64decode(content)) - except IOError as e: - raise YunohostError("cannot_write_file", file=file_path, error=str(e)) - except Exception as e: - raise YunohostError("error_writing_file", file=file_path, error=str(e)) + + write_to_file(file_path, b64decode(content), file_mode="wb") + question.value = file_path return question.value From a062254402ba5463c5c12a4885fd45f91052d457 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 4 Sep 2021 20:34:56 +0200 Subject: [PATCH 062/119] Moar localization --- locales/en.json | 1 + src/yunohost/utils/config.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/locales/en.json b/locales/en.json index 667a8d4b3..faef3efc5 100644 --- a/locales/en.json +++ b/locales/en.json @@ -327,6 +327,7 @@ "extracting": "Extracting...", "field_invalid": "Invalid field '{}'", "file_does_not_exist": "The file {path} does not exist.", + "file_extension_not_accepted": "Refusing file '{path}' because its extension is not among the accepted extensions: {accept}", "firewall_reload_failed": "Could not reload the firewall", "firewall_reloaded": "Firewall reloaded", "firewall_rules_cmd_failed": "Some firewall rule commands have failed. More info in log.", diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index cb493be03..23a95b565 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -797,7 +797,7 @@ class FileQuestion(Question): raise YunohostValidationError( "app_argument_invalid", field=self.name, - error=m18n.n("file_does_not_exists"), + error=m18n.n("file_does_not_exist", path=self.value), ) if self.value in [None, ""] or not self.accept: return @@ -807,7 +807,7 @@ class FileQuestion(Question): raise YunohostValidationError( "app_argument_invalid", field=self.name, - error=m18n.n("file_extension_not_accepted"), + error=m18n.n("file_extension_not_accepted", file=filename, accept=self.accept), ) @@ -833,7 +833,7 @@ class FileQuestion(Question): # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" file_path = os.path.normpath(upload_dir + "/" + filename) if not file_path.startswith(upload_dir + "/"): - raise YunohostError("file_relative_parent_path_in_filename_forbidden") + raise YunohostError(f"Filename '{filename}' received from the API got a relative parent path, which is forbidden", raw_msg=True) i = 2 while os.path.exists(file_path): file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) From 1bf0196ff8fa33712894bd8e4dde7ca8df62f4c2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 00:20:12 +0200 Subject: [PATCH 063/119] Black + fix some invalid code issues --- src/yunohost/service.py | 4 +- src/yunohost/utils/config.py | 119 ++++++++++++++++++--------------- tests/autofix_locale_format.py | 12 +++- tests/reformat_locales.py | 54 ++++++++++----- 4 files changed, 116 insertions(+), 73 deletions(-) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index da3bded3c..2ef94878d 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -287,7 +287,9 @@ def service_reload_or_restart(names, test_conf=True): out, _ = p.communicate() if p.returncode != 0: errors = out.strip().split("\n") - logger.error(m18n.("service_not_reloading_because_conf_broken", errors=errors)) + logger.error( + m18n.n("service_not_reloading_because_conf_broken", errors=errors) + ) continue if _run_service_command("reload-or-restart", name): diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 23a95b565..5c81f5007 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -29,7 +29,6 @@ from collections import OrderedDict from moulinette.interfaces.cli import colorize from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output from moulinette.utils.filesystem import ( write_to_file, read_toml, @@ -80,16 +79,22 @@ class ConfigPanel: result = {} for panel, section, option in self._iterate(): key = f"{panel['id']}.{section['id']}.{option['id']}" - if mode == 'export': - result[option['id']] = option.get('current_value') + if mode == "export": + result[option["id"]] = option.get("current_value") else: - if 'ask' in option: - result[key] = {'ask': _value_for_locale(option['ask'])} - elif 'i18n' in self.config: - result[key] = {'ask': m18n.n(self.config['i18n'] + '_' + option['id'])} - if 'current_value' in option: - question_class = ARGUMENTS_TYPE_PARSERS[option.get("type", "string")] - result[key]['value'] = question_class.humanize(option['current_value'], option) + if "ask" in option: + result[key] = {"ask": _value_for_locale(option["ask"])} + elif "i18n" in self.config: + result[key] = { + "ask": m18n.n(self.config["i18n"] + "_" + option["id"]) + } + if "current_value" in option: + question_class = ARGUMENTS_TYPE_PARSERS[ + option.get("type", "string") + ] + result[key]["value"] = question_class.humanize( + option["current_value"], option + ) return result @@ -294,8 +299,11 @@ class ConfigPanel: self.errors = None def _get_default_values(self): - return { option['id']: option['default'] - for _, _, option in self._iterate() if 'default' in option } + return { + option["id"]: option["default"] + for _, _, option in self._iterate() + if "default" in option + } def _load_current_values(self): """ @@ -319,9 +327,11 @@ class ConfigPanel: mkdir(dir_path, mode=0o700) values_to_save = {**self.values, **self.new_values} - if self.save_mode == 'diff': + if self.save_mode == "diff": defaults = self._get_default_values() - values_to_save = {k: v for k, v in values_to_save.items() if defaults.get(k) != v} + values_to_save = { + k: v for k, v in values_to_save.items() if defaults.get(k) != v + } # Save the settings to the .yaml file write_to_yaml(self.save_path, self.new_values) @@ -360,17 +370,17 @@ class Question(object): def __init__(self, question, user_answers): self.name = question["name"] - self.type = question.get("type", 'string') + self.type = question.get("type", "string") self.default = question.get("default", None) self.current_value = question.get("current_value") self.optional = question.get("optional", False) self.choices = question.get("choices", []) self.pattern = question.get("pattern") - self.ask = question.get("ask", {'en': self.name}) + self.ask = question.get("ask", {"en": self.name}) self.help = question.get("help") self.helpLink = question.get("helpLink") self.value = user_answers.get(self.name) - self.redact = question.get('redact', False) + self.redact = question.get("redact", False) # Empty value is parsed as empty string if self.default == "": @@ -384,11 +394,10 @@ class Question(object): def normalize(value, option={}): return value - def ask_if_needed(self): while True: # Display question if no value filled or if it's a readonly message - if Moulinette.interface.type== 'cli': + if Moulinette.interface.type == "cli": text_for_user_input_in_cli = self._format_text_for_user_input_in_cli() if getattr(self, "readonly", False): Moulinette.display(text_for_user_input_in_cli) @@ -433,7 +442,6 @@ class Question(object): return (self.value, self.argument_type) - def _prevalidate(self): if self.value in [None, ""] and not self.optional: raise YunohostValidationError("app_argument_required", name=self.name) @@ -442,9 +450,9 @@ class Question(object): if self.value is not None: if self.choices and self.value not in self.choices: self._raise_invalid_answer() - if self.pattern and not re.match(self.pattern['regexp'], str(self.value)): + if self.pattern and not re.match(self.pattern["regexp"], str(self.value)): raise YunohostValidationError( - self.pattern['error'], + self.pattern["error"], name=self.name, value=self.value, ) @@ -494,7 +502,10 @@ class Question(object): if self.operation_logger: self.operation_logger.data_to_redact.extend(data_to_redact) elif data_to_redact: - raise YunohostError(f"Can't redact {question.name} because no operation logger available in the context", raw_msg=True) + raise YunohostError( + f"Can't redact {self.name} because no operation logger available in the context", + raw_msg=True, + ) return self.value @@ -510,7 +521,7 @@ class TagsQuestion(Question): @staticmethod def humanize(value, option={}): if isinstance(value, list): - return ','.join(value) + return ",".join(value) return value def _prevalidate(self): @@ -540,7 +551,7 @@ class PasswordQuestion(Question): @staticmethod def humanize(value, option={}): if value: - return '***' # Avoid to display the password on screen + return "********" # Avoid to display the password on screen return "" def _prevalidate(self): @@ -571,32 +582,32 @@ class BooleanQuestion(Question): @staticmethod def humanize(value, option={}): - yes = option.get('yes', 1) - no = option.get('no', 0) + yes = option.get("yes", 1) + no = option.get("no", 0) value = str(value).lower() if value == str(yes).lower(): - return 'yes' + return "yes" if value == str(no).lower(): - return 'no' + return "no" if value in BooleanQuestion.yes_answers: - return 'yes' + return "yes" if value in BooleanQuestion.no_answers: - return 'no' + return "no" - if value in ['none', ""]: - return '' + if value in ["none", ""]: + return "" raise YunohostValidationError( "app_argument_choice_invalid", - name=self.name, - value=self.value, + name=self.name, # FIXME ... + value=value, choices="yes, no, y, n, 1, 0", ) @staticmethod def normalize(value, option={}): - yes = option.get('yes', 1) - no = option.get('no', 0) + yes = option.get("yes", 1) + no = option.get("no", 0) if str(value).lower() in BooleanQuestion.yes_answers: return yes @@ -608,19 +619,18 @@ class BooleanQuestion(Question): return None raise YunohostValidationError( "app_argument_choice_invalid", - name=self.name, - value=self.value, + name=self.name, # FIXME.... + value=value, choices="yes, no, y, n, 1, 0", ) def __init__(self, question, user_answers): super().__init__(question, user_answers) - self.yes = question.get('yes', 1) - self.no = question.get('no', 0) + self.yes = question.get("yes", 1) + self.no = question.get("no", 0) if self.default is None: self.default = False - def _format_text_for_user_input_in_cli(self): text_for_user_input_in_cli = _value_for_locale(self.ask) @@ -652,7 +662,6 @@ class DomainQuestion(Question): self.choices = domain_list()["domains"] - def _raise_invalid_answer(self): raise YunohostValidationError( "app_argument_invalid", field=self.name, error=m18n.n("domain_unknown") @@ -675,7 +684,6 @@ class UserQuestion(Question): self.default = user break - def _raise_invalid_answer(self): raise YunohostValidationError( "app_argument_invalid", @@ -698,7 +706,6 @@ class NumberQuestion(Question): self.max = question.get("max", None) self.step = question.get("step", None) - def _prevalidate(self): super()._prevalidate() if not isinstance(self.value, int) and not ( @@ -744,8 +751,7 @@ class DisplayTextQuestion(Question): super().__init__(question, user_answers) self.optional = True - self.style = question.get("style", "info") - + self.style = question.get("style", "info") def _format_text_for_user_input_in_cli(self): text = self.ask["en"] @@ -780,7 +786,7 @@ class FileQuestion(Question): self.accept = question.get("accept").replace(" ", "").split(",") else: self.accept = [] - if Moulinette.interface.type== "api": + if Moulinette.interface.type == "api": if user_answers.get(f"{self.name}[name]"): self.value = { "content": self.value, @@ -790,10 +796,13 @@ class FileQuestion(Question): if self.value and str(self.value) == self.current_value: self.value = None - def _prevalidate(self): super()._prevalidate() - if isinstance(self.value, str) and self.value and not os.path.exists(self.value): + if ( + isinstance(self.value, str) + and self.value + and not os.path.exists(self.value) + ): raise YunohostValidationError( "app_argument_invalid", field=self.name, @@ -807,10 +816,11 @@ class FileQuestion(Question): raise YunohostValidationError( "app_argument_invalid", field=self.name, - error=m18n.n("file_extension_not_accepted", file=filename, accept=self.accept), + error=m18n.n( + "file_extension_not_accepted", file=filename, accept=self.accept + ), ) - def _post_parse_value(self): from base64 import b64decode @@ -833,7 +843,10 @@ class FileQuestion(Question): # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd" file_path = os.path.normpath(upload_dir + "/" + filename) if not file_path.startswith(upload_dir + "/"): - raise YunohostError(f"Filename '{filename}' received from the API got a relative parent path, which is forbidden", raw_msg=True) + raise YunohostError( + f"Filename '{filename}' received from the API got a relative parent path, which is forbidden", + raw_msg=True, + ) i = 2 while os.path.exists(file_path): file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) diff --git a/tests/autofix_locale_format.py b/tests/autofix_locale_format.py index f777f06f1..dd7812635 100644 --- a/tests/autofix_locale_format.py +++ b/tests/autofix_locale_format.py @@ -27,11 +27,17 @@ def fix_locale(locale_file): # should also be in the translated string, otherwise the .format # will trigger an exception! subkeys_in_ref = [k[0] for k in re.findall(r"{(\w+)(:\w)?}", string)] - subkeys_in_this_locale = [k[0] for k in re.findall(r"{(\w+)(:\w)?}", this_locale[key])] + subkeys_in_this_locale = [ + k[0] for k in re.findall(r"{(\w+)(:\w)?}", this_locale[key]) + ] - if set(subkeys_in_ref) != set(subkeys_in_this_locale) and (len(subkeys_in_ref) == len(subkeys_in_this_locale)): + if set(subkeys_in_ref) != set(subkeys_in_this_locale) and ( + len(subkeys_in_ref) == len(subkeys_in_this_locale) + ): for i, subkey in enumerate(subkeys_in_ref): - this_locale[key] = this_locale[key].replace('{%s}' % subkeys_in_this_locale[i], '{%s}' % subkey) + this_locale[key] = this_locale[key].replace( + "{%s}" % subkeys_in_this_locale[i], "{%s}" % subkey + ) fixed_stuff = True if fixed_stuff: diff --git a/tests/reformat_locales.py b/tests/reformat_locales.py index 90251d040..9119c7288 100644 --- a/tests/reformat_locales.py +++ b/tests/reformat_locales.py @@ -1,5 +1,6 @@ import re + def reformat(lang, transformations): locale = open(f"locales/{lang}.json").read() @@ -8,31 +9,52 @@ def reformat(lang, transformations): open(f"locales/{lang}.json", "w").write(locale) + ###################################################### -godamn_spaces_of_hell = ["\u00a0", "\u2000", "\u2001", "\u2002", "\u2003", "\u2004", "\u2005", "\u2006", "\u2007", "\u2008", "\u2009", "\u200A", "\u202f", "\u202F", "\u3000"] +godamn_spaces_of_hell = [ + "\u00a0", + "\u2000", + "\u2001", + "\u2002", + "\u2003", + "\u2004", + "\u2005", + "\u2006", + "\u2007", + "\u2008", + "\u2009", + "\u200A", + "\u202f", + "\u202F", + "\u3000", +] transformations = {s: " " for s in godamn_spaces_of_hell} -transformations.update({ - "…": "...", -}) +transformations.update( + { + "…": "...", + } +) reformat("en", transformations) ###################################################### -transformations.update({ - "courriel": "email", - "e-mail": "email", - "Courriel": "Email", - "E-mail": "Email", - "« ": "'", - "«": "'", - " »": "'", - "»": "'", - "’": "'", - #r"$(\w{1,2})'|( \w{1,2})'": r"\1\2’", -}) +transformations.update( + { + "courriel": "email", + "e-mail": "email", + "Courriel": "Email", + "E-mail": "Email", + "« ": "'", + "«": "'", + " »": "'", + "»": "'", + "’": "'", + # r"$(\w{1,2})'|( \w{1,2})'": r"\1\2’", + } +) reformat("fr", transformations) From 0122b7a1262cc63f250074d3a2a4930683e9fc94 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 15:37:46 +0200 Subject: [PATCH 064/119] Misc fixes --- .gitlab/ci/test.gitlab-ci.yml | 6 +- src/yunohost/service.py | 2 +- ...ps_arguments_parsing.py => test_config.py} | 250 +++++++++--------- src/yunohost/tests/test_service.py | 2 +- 4 files changed, 130 insertions(+), 130 deletions(-) rename src/yunohost/tests/{test_apps_arguments_parsing.py => test_config.py} (81%) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index e0e0e001a..78394f253 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -112,14 +112,14 @@ test-appurl: changes: - src/yunohost/app.py -test-apps-arguments-parsing: +test-config: extends: .test-stage script: - cd src/yunohost - - python3 -m pytest tests/test_apps_arguments_parsing.py + - python3 -m pytest tests/test_config.py only: changes: - - src/yunohost/app.py + - src/yunohost/utils/config.py test-changeurl: extends: .test-stage diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 2ef94878d..5f9f3e60a 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -273,7 +273,7 @@ def service_reload_or_restart(names, test_conf=True): logger.debug(f"Reloading service {name}") - test_conf_cmd = services.get(service, {}).get("test_conf") + test_conf_cmd = services.get(name, {}).get("test_conf") if test_conf and test_conf_cmd: p = subprocess.Popen( diff --git a/src/yunohost/tests/test_apps_arguments_parsing.py b/src/yunohost/tests/test_config.py similarity index 81% rename from src/yunohost/tests/test_apps_arguments_parsing.py rename to src/yunohost/tests/test_config.py index fe5c5f8cd..2e52c4086 100644 --- a/src/yunohost/tests/test_apps_arguments_parsing.py +++ b/src/yunohost/tests/test_config.py @@ -8,7 +8,7 @@ from collections import OrderedDict from moulinette import Moulinette from yunohost import domain, user -from yunohost.app import _parse_args_in_yunohost_format, PasswordArgumentParser +from yunohost.utils.config import parse_args_in_yunohost_format, PasswordQuestion from yunohost.utils.error import YunohostError @@ -36,7 +36,7 @@ User answers: def test_parse_args_in_yunohost_format_empty(): - assert _parse_args_in_yunohost_format({}, []) == {} + assert parse_args_in_yunohost_format({}, []) == {} def test_parse_args_in_yunohost_format_string(): @@ -48,7 +48,7 @@ def test_parse_args_in_yunohost_format_string(): ] answers = {"some_string": "some_value"} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_default_type(): @@ -59,7 +59,7 @@ def test_parse_args_in_yunohost_format_string_default_type(): ] answers = {"some_string": "some_value"} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_no_input(): @@ -71,7 +71,7 @@ def test_parse_args_in_yunohost_format_string_no_input(): answers = {} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_string_input(): @@ -85,7 +85,7 @@ def test_parse_args_in_yunohost_format_string_input(): expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_input_no_ask(): @@ -98,7 +98,7 @@ def test_parse_args_in_yunohost_format_string_input_no_ask(): expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_no_input_optional(): @@ -110,7 +110,7 @@ def test_parse_args_in_yunohost_format_string_no_input_optional(): ] answers = {} expected_result = OrderedDict({"some_string": ("", "string")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_optional_with_input(): @@ -125,7 +125,7 @@ def test_parse_args_in_yunohost_format_string_optional_with_input(): expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_optional_with_empty_input(): @@ -140,7 +140,7 @@ def test_parse_args_in_yunohost_format_string_optional_with_empty_input(): expected_result = OrderedDict({"some_string": ("", "string")}) with patch.object(Moulinette.interface, "prompt", return_value=""): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_optional_with_input_without_ask(): @@ -154,7 +154,7 @@ def test_parse_args_in_yunohost_format_string_optional_with_input_without_ask(): expected_result = OrderedDict({"some_string": ("some_value", "string")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_no_input_default(): @@ -167,7 +167,7 @@ def test_parse_args_in_yunohost_format_string_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_input_test_ask(): @@ -183,7 +183,7 @@ def test_parse_args_in_yunohost_format_string_input_test_ask(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with(ask_text, False) @@ -202,7 +202,7 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_default(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with("%s (default: %s)" % (ask_text, default_text), False) @@ -222,7 +222,7 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_example(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert example_text in prompt.call_args[0][0] @@ -243,7 +243,7 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_help(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert help_text in prompt.call_args[0][0] @@ -252,7 +252,7 @@ def test_parse_args_in_yunohost_format_string_with_choice(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "fr"} expected_result = OrderedDict({"some_string": ("fr", "string")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_with_choice_prompt(): @@ -260,7 +260,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_prompt(): answers = {"some_string": "fr"} expected_result = OrderedDict({"some_string": ("fr", "string")}) with patch.object(Moulinette.interface, "prompt", return_value="fr"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_string_with_choice_bad(): @@ -268,7 +268,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_bad(): answers = {"some_string": "bad"} with pytest.raises(YunohostError): - assert _parse_args_in_yunohost_format(answers, questions) + assert parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_string_with_choice_ask(): @@ -284,7 +284,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_ask(): answers = {} with patch.object(Moulinette.interface, "prompt", return_value="ru") as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] for choice in choices: @@ -302,7 +302,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_default(): ] answers = {} expected_result = OrderedDict({"some_string": ("en", "string")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password(): @@ -314,7 +314,7 @@ def test_parse_args_in_yunohost_format_password(): ] answers = {"some_password": "some_value"} expected_result = OrderedDict({"some_password": ("some_value", "password")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password_no_input(): @@ -327,7 +327,7 @@ def test_parse_args_in_yunohost_format_password_no_input(): answers = {} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_password_input(): @@ -342,7 +342,7 @@ def test_parse_args_in_yunohost_format_password_input(): expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password_input_no_ask(): @@ -356,7 +356,7 @@ def test_parse_args_in_yunohost_format_password_input_no_ask(): expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password_no_input_optional(): @@ -370,13 +370,13 @@ def test_parse_args_in_yunohost_format_password_no_input_optional(): answers = {} expected_result = OrderedDict({"some_password": ("", "password")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result questions = [ {"name": "some_password", "type": "password", "optional": True, "default": ""} ] - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password_optional_with_input(): @@ -392,7 +392,7 @@ def test_parse_args_in_yunohost_format_password_optional_with_input(): expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password_optional_with_empty_input(): @@ -408,7 +408,7 @@ def test_parse_args_in_yunohost_format_password_optional_with_empty_input(): expected_result = OrderedDict({"some_password": ("", "password")}) with patch.object(Moulinette.interface, "prompt", return_value=""): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password_optional_with_input_without_ask(): @@ -423,7 +423,7 @@ def test_parse_args_in_yunohost_format_password_optional_with_input_without_ask( expected_result = OrderedDict({"some_password": ("some_value", "password")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_password_no_input_default(): @@ -439,7 +439,7 @@ def test_parse_args_in_yunohost_format_password_no_input_default(): # no default for password! with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) @pytest.mark.skip # this should raises @@ -456,7 +456,7 @@ def test_parse_args_in_yunohost_format_password_no_input_example(): # no example for password! with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_password_input_test_ask(): @@ -473,7 +473,7 @@ def test_parse_args_in_yunohost_format_password_input_test_ask(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with(ask_text, True) @@ -494,7 +494,7 @@ def test_parse_args_in_yunohost_format_password_input_test_ask_with_example(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert example_text in prompt.call_args[0][0] @@ -516,7 +516,7 @@ def test_parse_args_in_yunohost_format_password_input_test_ask_with_help(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert help_text in prompt.call_args[0][0] @@ -531,9 +531,9 @@ def test_parse_args_in_yunohost_format_password_bad_chars(): } ] - for i in PasswordArgumentParser.forbidden_chars: + for i in PasswordQuestion.forbidden_chars: with pytest.raises(YunohostError): - _parse_args_in_yunohost_format({"some_password": i * 8}, questions) + parse_args_in_yunohost_format({"some_password": i * 8}, questions) def test_parse_args_in_yunohost_format_password_strong_enough(): @@ -548,10 +548,10 @@ def test_parse_args_in_yunohost_format_password_strong_enough(): with pytest.raises(YunohostError): # too short - _parse_args_in_yunohost_format({"some_password": "a"}, questions) + parse_args_in_yunohost_format({"some_password": "a"}, questions) with pytest.raises(YunohostError): - _parse_args_in_yunohost_format({"some_password": "password"}, questions) + parse_args_in_yunohost_format({"some_password": "password"}, questions) def test_parse_args_in_yunohost_format_password_optional_strong_enough(): @@ -566,10 +566,10 @@ def test_parse_args_in_yunohost_format_password_optional_strong_enough(): with pytest.raises(YunohostError): # too short - _parse_args_in_yunohost_format({"some_password": "a"}, questions) + parse_args_in_yunohost_format({"some_password": "a"}, questions) with pytest.raises(YunohostError): - _parse_args_in_yunohost_format({"some_password": "password"}, questions) + parse_args_in_yunohost_format({"some_password": "password"}, questions) def test_parse_args_in_yunohost_format_path(): @@ -581,7 +581,7 @@ def test_parse_args_in_yunohost_format_path(): ] answers = {"some_path": "some_value"} expected_result = OrderedDict({"some_path": ("some_value", "path")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_no_input(): @@ -594,7 +594,7 @@ def test_parse_args_in_yunohost_format_path_no_input(): answers = {} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_path_input(): @@ -609,7 +609,7 @@ def test_parse_args_in_yunohost_format_path_input(): expected_result = OrderedDict({"some_path": ("some_value", "path")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_input_no_ask(): @@ -623,7 +623,7 @@ def test_parse_args_in_yunohost_format_path_input_no_ask(): expected_result = OrderedDict({"some_path": ("some_value", "path")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_no_input_optional(): @@ -636,7 +636,7 @@ def test_parse_args_in_yunohost_format_path_no_input_optional(): ] answers = {} expected_result = OrderedDict({"some_path": ("", "path")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_optional_with_input(): @@ -652,7 +652,7 @@ def test_parse_args_in_yunohost_format_path_optional_with_input(): expected_result = OrderedDict({"some_path": ("some_value", "path")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_optional_with_empty_input(): @@ -668,7 +668,7 @@ def test_parse_args_in_yunohost_format_path_optional_with_empty_input(): expected_result = OrderedDict({"some_path": ("", "path")}) with patch.object(Moulinette.interface, "prompt", return_value=""): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_optional_with_input_without_ask(): @@ -683,7 +683,7 @@ def test_parse_args_in_yunohost_format_path_optional_with_input_without_ask(): expected_result = OrderedDict({"some_path": ("some_value", "path")}) with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_no_input_default(): @@ -697,7 +697,7 @@ def test_parse_args_in_yunohost_format_path_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_path": ("some_value", "path")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_path_input_test_ask(): @@ -714,7 +714,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with(ask_text, False) @@ -734,7 +734,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_default(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with("%s (default: %s)" % (ask_text, default_text), False) @@ -755,7 +755,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_example(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert example_text in prompt.call_args[0][0] @@ -777,7 +777,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_help(): with patch.object( Moulinette.interface, "prompt", return_value="some_value" ) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert help_text in prompt.call_args[0][0] @@ -791,7 +791,7 @@ def test_parse_args_in_yunohost_format_boolean(): ] answers = {"some_boolean": "y"} expected_result = OrderedDict({"some_boolean": (1, "boolean")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_all_yes(): @@ -803,47 +803,47 @@ def test_parse_args_in_yunohost_format_boolean_all_yes(): ] expected_result = OrderedDict({"some_boolean": (1, "boolean")}) assert ( - _parse_args_in_yunohost_format({"some_boolean": "y"}, questions) + parse_args_in_yunohost_format({"some_boolean": "y"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "Y"}, questions) + parse_args_in_yunohost_format({"some_boolean": "Y"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "yes"}, questions) + parse_args_in_yunohost_format({"some_boolean": "yes"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "Yes"}, questions) + parse_args_in_yunohost_format({"some_boolean": "Yes"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "YES"}, questions) + parse_args_in_yunohost_format({"some_boolean": "YES"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "1"}, questions) + parse_args_in_yunohost_format({"some_boolean": "1"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": 1}, questions) + parse_args_in_yunohost_format({"some_boolean": 1}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": True}, questions) + parse_args_in_yunohost_format({"some_boolean": True}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "True"}, questions) + parse_args_in_yunohost_format({"some_boolean": "True"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "TRUE"}, questions) + parse_args_in_yunohost_format({"some_boolean": "TRUE"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "true"}, questions) + parse_args_in_yunohost_format({"some_boolean": "true"}, questions) == expected_result ) @@ -857,47 +857,47 @@ def test_parse_args_in_yunohost_format_boolean_all_no(): ] expected_result = OrderedDict({"some_boolean": (0, "boolean")}) assert ( - _parse_args_in_yunohost_format({"some_boolean": "n"}, questions) + parse_args_in_yunohost_format({"some_boolean": "n"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "N"}, questions) + parse_args_in_yunohost_format({"some_boolean": "N"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "no"}, questions) + parse_args_in_yunohost_format({"some_boolean": "no"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "No"}, questions) + parse_args_in_yunohost_format({"some_boolean": "No"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "No"}, questions) + parse_args_in_yunohost_format({"some_boolean": "No"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "0"}, questions) + parse_args_in_yunohost_format({"some_boolean": "0"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": 0}, questions) + parse_args_in_yunohost_format({"some_boolean": 0}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": False}, questions) + parse_args_in_yunohost_format({"some_boolean": False}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "False"}, questions) + parse_args_in_yunohost_format({"some_boolean": "False"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "FALSE"}, questions) + parse_args_in_yunohost_format({"some_boolean": "FALSE"}, questions) == expected_result ) assert ( - _parse_args_in_yunohost_format({"some_boolean": "false"}, questions) + parse_args_in_yunohost_format({"some_boolean": "false"}, questions) == expected_result ) @@ -913,7 +913,7 @@ def test_parse_args_in_yunohost_format_boolean_no_input(): answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_bad_input(): @@ -926,7 +926,7 @@ def test_parse_args_in_yunohost_format_boolean_bad_input(): answers = {"some_boolean": "stuff"} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_boolean_input(): @@ -941,11 +941,11 @@ def test_parse_args_in_yunohost_format_boolean_input(): expected_result = OrderedDict({"some_boolean": (1, "boolean")}) with patch.object(Moulinette.interface, "prompt", return_value="y"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result expected_result = OrderedDict({"some_boolean": (0, "boolean")}) with patch.object(Moulinette.interface, "prompt", return_value="n"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_input_no_ask(): @@ -959,7 +959,7 @@ def test_parse_args_in_yunohost_format_boolean_input_no_ask(): expected_result = OrderedDict({"some_boolean": (1, "boolean")}) with patch.object(Moulinette.interface, "prompt", return_value="y"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_no_input_optional(): @@ -972,7 +972,7 @@ def test_parse_args_in_yunohost_format_boolean_no_input_optional(): ] answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) # default to false - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_optional_with_input(): @@ -988,7 +988,7 @@ def test_parse_args_in_yunohost_format_boolean_optional_with_input(): expected_result = OrderedDict({"some_boolean": (1, "boolean")}) with patch.object(Moulinette.interface, "prompt", return_value="y"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_optional_with_empty_input(): @@ -1004,7 +1004,7 @@ def test_parse_args_in_yunohost_format_boolean_optional_with_empty_input(): expected_result = OrderedDict({"some_boolean": (0, "boolean")}) # default to false with patch.object(Moulinette.interface, "prompt", return_value=""): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_optional_with_input_without_ask(): @@ -1019,7 +1019,7 @@ def test_parse_args_in_yunohost_format_boolean_optional_with_input_without_ask() expected_result = OrderedDict({"some_boolean": (0, "boolean")}) with patch.object(Moulinette.interface, "prompt", return_value="n"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_no_input_default(): @@ -1033,7 +1033,7 @@ def test_parse_args_in_yunohost_format_boolean_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_boolean_bad_default(): @@ -1047,7 +1047,7 @@ def test_parse_args_in_yunohost_format_boolean_bad_default(): ] answers = {} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_boolean_input_test_ask(): @@ -1062,7 +1062,7 @@ def test_parse_args_in_yunohost_format_boolean_input_test_ask(): answers = {} with patch.object(Moulinette.interface, "prompt", return_value=0) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with(ask_text + " [yes | no] (default: no)", False) @@ -1080,7 +1080,7 @@ def test_parse_args_in_yunohost_format_boolean_input_test_ask_with_default(): answers = {} with patch.object(Moulinette.interface, "prompt", return_value=1) as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with("%s [yes | no] (default: yes)" % ask_text, False) @@ -1098,7 +1098,7 @@ def test_parse_args_in_yunohost_format_domain_empty(): with patch.object( domain, "_get_maindomain", return_value="my_main_domain.com" ), patch.object(domain, "domain_list", return_value={"domains": [main_domain]}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_domain(): @@ -1117,7 +1117,7 @@ def test_parse_args_in_yunohost_format_domain(): with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_domain_two_domains(): @@ -1137,7 +1137,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains(): with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result answers = {"some_domain": main_domain} expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) @@ -1145,7 +1145,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains(): with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_domain_two_domains_wrong_answer(): @@ -1165,7 +1165,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains_wrong_answer(): domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_domain_two_domains_default_no_ask(): @@ -1185,7 +1185,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains_default_no_ask(): with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_domain_two_domains_default(): @@ -1200,7 +1200,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains_default(): with patch.object( domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_domain_two_domains_default_input(): @@ -1216,11 +1216,11 @@ def test_parse_args_in_yunohost_format_domain_two_domains_default_input(): ), patch.object(domain, "domain_list", return_value={"domains": domains}): expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object(Moulinette.interface, "prompt", return_value=main_domain): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result expected_result = OrderedDict({"some_domain": (other_domain, "domain")}) with patch.object(Moulinette.interface, "prompt", return_value=other_domain): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_user_empty(): @@ -1244,7 +1244,7 @@ def test_parse_args_in_yunohost_format_user_empty(): with patch.object(user, "user_list", return_value={"users": users}): with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_user(): @@ -1271,7 +1271,7 @@ def test_parse_args_in_yunohost_format_user(): with patch.object(user, "user_list", return_value={"users": users}): with patch.object(user, "user_info", return_value={}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_user_two_users(): @@ -1305,14 +1305,14 @@ def test_parse_args_in_yunohost_format_user_two_users(): with patch.object(user, "user_list", return_value={"users": users}): with patch.object(user, "user_info", return_value={}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result answers = {"some_user": username} expected_result = OrderedDict({"some_user": (username, "user")}) with patch.object(user, "user_list", return_value={"users": users}): with patch.object(user, "user_info", return_value={}): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_user_two_users_wrong_answer(): @@ -1345,7 +1345,7 @@ def test_parse_args_in_yunohost_format_user_two_users_wrong_answer(): with patch.object(user, "user_list", return_value={"users": users}): with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_user_two_users_no_default(): @@ -1373,7 +1373,7 @@ def test_parse_args_in_yunohost_format_user_two_users_no_default(): with patch.object(user, "user_list", return_value={"users": users}): with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_user_two_users_default_input(): @@ -1404,14 +1404,14 @@ def test_parse_args_in_yunohost_format_user_two_users_default_input(): expected_result = OrderedDict({"some_user": (username, "user")}) with patch.object(Moulinette.interface, "prompt", return_value=username): assert ( - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) == expected_result ) expected_result = OrderedDict({"some_user": (other_user, "user")}) with patch.object(Moulinette.interface, "prompt", return_value=other_user): assert ( - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) == expected_result ) @@ -1425,7 +1425,7 @@ def test_parse_args_in_yunohost_format_number(): ] answers = {"some_number": 1337} expected_result = OrderedDict({"some_number": (1337, "number")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_no_input(): @@ -1438,7 +1438,7 @@ def test_parse_args_in_yunohost_format_number_no_input(): answers = {} expected_result = OrderedDict({"some_number": (0, "number")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_bad_input(): @@ -1451,11 +1451,11 @@ def test_parse_args_in_yunohost_format_number_bad_input(): answers = {"some_number": "stuff"} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) answers = {"some_number": 1.5} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_number_input(): @@ -1470,14 +1470,14 @@ def test_parse_args_in_yunohost_format_number_input(): expected_result = OrderedDict({"some_number": (1337, "number")}) with patch.object(Moulinette.interface, "prompt", return_value="1337"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result with patch.object(Moulinette.interface, "prompt", return_value=1337): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result expected_result = OrderedDict({"some_number": (0, "number")}) with patch.object(Moulinette.interface, "prompt", return_value=""): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_input_no_ask(): @@ -1491,7 +1491,7 @@ def test_parse_args_in_yunohost_format_number_input_no_ask(): expected_result = OrderedDict({"some_number": (1337, "number")}) with patch.object(Moulinette.interface, "prompt", return_value="1337"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_no_input_optional(): @@ -1504,7 +1504,7 @@ def test_parse_args_in_yunohost_format_number_no_input_optional(): ] answers = {} expected_result = OrderedDict({"some_number": (0, "number")}) # default to 0 - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_optional_with_input(): @@ -1520,7 +1520,7 @@ def test_parse_args_in_yunohost_format_number_optional_with_input(): expected_result = OrderedDict({"some_number": (1337, "number")}) with patch.object(Moulinette.interface, "prompt", return_value="1337"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_optional_with_input_without_ask(): @@ -1535,7 +1535,7 @@ def test_parse_args_in_yunohost_format_number_optional_with_input_without_ask(): expected_result = OrderedDict({"some_number": (0, "number")}) with patch.object(Moulinette.interface, "prompt", return_value="0"): - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_no_input_default(): @@ -1549,7 +1549,7 @@ def test_parse_args_in_yunohost_format_number_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_number": (1337, "number")}) - assert _parse_args_in_yunohost_format(answers, questions) == expected_result + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_parse_args_in_yunohost_format_number_bad_default(): @@ -1563,7 +1563,7 @@ def test_parse_args_in_yunohost_format_number_bad_default(): ] answers = {} with pytest.raises(YunohostError): - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) def test_parse_args_in_yunohost_format_number_input_test_ask(): @@ -1578,7 +1578,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask(): answers = {} with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with("%s (default: 0)" % (ask_text), False) @@ -1596,7 +1596,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask_with_default(): answers = {} with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with("%s (default: %s)" % (ask_text, default_value), False) @@ -1615,7 +1615,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask_with_example(): answers = {} with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert example_value in prompt.call_args[0][0] @@ -1635,7 +1635,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask_with_help(): answers = {} with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert help_value in prompt.call_args[0][0] @@ -1645,5 +1645,5 @@ def test_parse_args_in_yunohost_format_display_text(): answers = {} with patch.object(sys, "stdout", new_callable=StringIO) as stdout: - _parse_args_in_yunohost_format(answers, questions) + parse_args_in_yunohost_format(answers, questions) assert "foobar" in stdout.getvalue() diff --git a/src/yunohost/tests/test_service.py b/src/yunohost/tests/test_service.py index 1007419a1..88013a3fe 100644 --- a/src/yunohost/tests/test_service.py +++ b/src/yunohost/tests/test_service.py @@ -132,7 +132,7 @@ def test_service_conf_broken(): status = service_status("nginx") assert status["status"] == "running" assert status["configuration"] == "broken" - assert "broken.conf" in status["configuration-details"] + assert "broken.conf" in status["configuration-details"][0] # Service reload-or-restart should check that the conf ain't valid # before reload-or-restart, hence the service should still be running From 1207b54de6c12f64735ffda21401b41e9861a862 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 5 Sep 2021 16:08:23 +0200 Subject: [PATCH 065/119] [fix] ynh_read_var_in_file with endline --- data/helpers.d/utils | 48 ++++++++++++++---- tests/test_helpers.d/ynhtest_config.sh | 67 +++++++++++++++----------- 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 3389101a6..0a820505c 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -507,6 +507,7 @@ ynh_replace_vars () { # # Requires YunoHost version 4.3 or higher. ynh_read_var_in_file() { + set +o xtrace # Declare an array to define the options of this helper. local legacy_args=fk local -A args_array=( [f]=file= [k]=key= ) @@ -514,18 +515,43 @@ ynh_read_var_in_file() { local key # Manage arguments with getopts ynh_handle_getopts_args "$@" + set +o xtrace + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "yaml yml toml ini env" =~ *"$ext"* ]]; then + endline='#' + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*(?:(const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="(?:$assign)" + var_part+='\s*' - local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' + # Extract the part after assignation sign + local expression_with_comment="$(grep -i -o -P $var_part'\K.*$' ${file} || echo YNH_NULL | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + echo YNH_NULL + return 0 + fi - local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL | head -n1)" + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@$comments[^$string]*\$@@" | sed "s@\s*[$endline]*\s*]*\$@@")" - local first_char="${crazy_value:0:1}" + local first_char="${expression:0:1}" if [[ "$first_char" == '"' ]] ; then - echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' elif [[ "$first_char" == "'" ]] ; then - echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" else - echo "$crazy_value" + echo "$expression" fi } @@ -538,6 +564,7 @@ ynh_read_var_in_file() { # # Requires YunoHost version 4.3 or higher. ynh_write_var_in_file() { + set +o xtrace # Declare an array to define the options of this helper. local legacy_args=fkv local -A args_array=( [f]=file= [k]=key= [v]=value=) @@ -546,9 +573,10 @@ ynh_write_var_in_file() { local value # Manage arguments with getopts ynh_handle_getopts_args "$@" - local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*' + set +o xtrace + local var_part='\s*\$?([\w.]*\[)?\s*["'"']?${key}['"'"]?\s*\]?\s*[:=]>?\s*' - local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)" + local crazy_value="$(grep -i -o -P '^\s*\$?([\w.]*\[)?\s*["'"']?${key}['"'"]?\s*\]?\s*[:=]>?\s*\K.*(?=[\s,;]*$)' ${file} | head -n1)" # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" local first_char="${crazy_value:0:1}" delimiter=$'\001' @@ -556,13 +584,13 @@ ynh_write_var_in_file() { # \ and sed is quite complex you need 2 \\ to get one in a sed # So we need \\\\ to go through 2 sed value="$(echo "$value" | sed 's/"/\\\\"/g')" - sed -ri s$delimiter'^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} + sed -ri s$delimiter'^('"${var_part}"'")([^"]|\\")*("[\s;,]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} elif [[ "$first_char" == "'" ]] ; then # \ and sed is quite complex you need 2 \\ to get one in a sed # However double quotes implies to double \\ to # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" - sed -ri "s$delimiter^(${var_part}')([^']|\\')*('"'[ \t,;]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} + sed -ri "s$delimiter^(${var_part}')([^']|\\')*('"'[\s,;]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} else if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' diff --git a/tests/test_helpers.d/ynhtest_config.sh b/tests/test_helpers.d/ynhtest_config.sh index 7b749adf5..36165e3ac 100644 --- a/tests/test_helpers.d/ynhtest_config.sh +++ b/tests/test_helpers.d/ynhtest_config.sh @@ -27,11 +27,13 @@ ENABLED = False # TITLE = "Old title" TITLE = "Lorem Ipsum" THEME = "colib'ris" -EMAIL = "root@example.com" -PORT = 1234 +EMAIL = "root@example.com" # This is a comment without quotes +PORT = 1234 # This is a comment without quotes URL = 'https://yunohost.org' DICT = {} DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" +DICT['ldap_conf'] = {} +DICT['ldap_conf']['user'] = "camille" EOF test "$(_read_py "$file" "FOO")" == "None" @@ -56,6 +58,8 @@ EOF test "$(ynh_read_var_in_file "$file" "URL")" == "https://yunohost.org" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + test "$(ynh_read_var_in_file "$file" "user")" == "camille" ! _read_py "$file" "NONEXISTENT" test "$(ynh_read_var_in_file "$file" "NONEXISTENT")" == "YNH_NULL" @@ -64,7 +68,7 @@ EOF test "$(ynh_read_var_in_file "$file" "ENABLE")" == "YNH_NULL" } -ynhtest_config_write_py() { +nhtest_config_write_py() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.py" @@ -75,8 +79,8 @@ ENABLED = False # TITLE = "Old title" TITLE = "Lorem Ipsum" THEME = "colib'ris" -EMAIL = "root@example.com" -PORT = 1234 +EMAIL = "root@example.com" // This is a comment without quotes +PORT = 1234 // This is a comment without quotes URL = 'https://yunohost.org' DICT = {} DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" @@ -141,7 +145,7 @@ _read_ini() { ynhtest_config_read_ini() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" - file="$dummy_dir/dummy.yml" + file="$dummy_dir/dummy.ini" cat << EOF > $file # Some comment @@ -152,8 +156,8 @@ enabled = False # title = Old title title = Lorem Ipsum theme = colib'ris -email = root@example.com -port = 1234 +email = root@example.com # This is a comment without quotes +port = 1234 # This is a comment without quotes url = https://yunohost.org [dict] ldap_base = ou=users,dc=yunohost,dc=org @@ -190,7 +194,7 @@ EOF } -ynhtest_config_write_ini() { +nhtest_config_write_ini() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.ini" @@ -203,8 +207,8 @@ enabled = False # title = Old title title = Lorem Ipsum theme = colib'ris -email = root@example.com -port = 1234 +email = root@example.com # This is a comment without quotes +port = 1234 # This is a comment without quotes url = https://yunohost.org [dict] ldap_base = ou=users,dc=yunohost,dc=org @@ -280,8 +284,8 @@ enabled: false # title: old title title: Lorem Ipsum theme: colib'ris -email: root@example.com -port: 1234 +email: root@example.com # This is a comment without quotes +port: 1234 # This is a comment without quotes url: https://yunohost.org dict: ldap_base: ou=users,dc=yunohost,dc=org @@ -318,7 +322,7 @@ EOF } -ynhtest_config_write_yaml() { +nhtest_config_write_yaml() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.yml" @@ -329,8 +333,8 @@ enabled: false # title: old title title: Lorem Ipsum theme: colib'ris -email: root@example.com -port: 1234 +email: root@example.com # This is a comment without quotes +port: 1234 # This is a comment without quotes url: https://yunohost.org dict: ldap_base: ou=users,dc=yunohost,dc=org @@ -415,10 +419,10 @@ EOF test "$(_read_json "$file" "foo")" == "None" - test "$(ynh_read_var_in_file "$file" "foo")" == "null," # FIXME FIXME FIXME trailing , + test "$(ynh_read_var_in_file "$file" "foo")" == "null" test "$(_read_json "$file" "enabled")" == "False" - test "$(ynh_read_var_in_file "$file" "enabled")" == "false," # FIXME FIXME FIXME trailing , + test "$(ynh_read_var_in_file "$file" "enabled")" == "false" test "$(_read_json "$file" "title")" == "Lorem Ipsum" test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" @@ -430,7 +434,7 @@ EOF test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" test "$(_read_json "$file" "port")" == "1234" - test "$(ynh_read_var_in_file "$file" "port")" == "1234," # FIXME FIXME FIXME trailing , + test "$(ynh_read_var_in_file "$file" "port")" == "1234" test "$(_read_json "$file" "url")" == "https://yunohost.org" test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" @@ -445,7 +449,7 @@ EOF } -ynhtest_config_write_json() { +nhtest_config_write_json() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.json" @@ -538,20 +542,23 @@ ynhtest_config_read_php() { // \$title = "old title"; \$title = "Lorem Ipsum"; \$theme = "colib'ris"; - \$email = "root@example.com"; - \$port = 1234; + \$email = "root@example.com"; // This is a comment without quotes + \$port = 1234; // This is a second comment without quotes \$url = "https://yunohost.org"; \$dict = [ 'ldap_base' => "ou=users,dc=yunohost,dc=org", + 'ldap_conf' => [] ]; + \$dict['ldap_conf']['user'] = 'camille'; + const DB_HOST = 'localhost'; ?> EOF test "$(_read_php "$file" "foo")" == "NULL" - test "$(ynh_read_var_in_file "$file" "foo")" == "NULL;" # FIXME FIXME FIXME trailing ; + test "$(ynh_read_var_in_file "$file" "foo")" == "NULL" test "$(_read_php "$file" "enabled")" == "false" - test "$(ynh_read_var_in_file "$file" "enabled")" == "false;" # FIXME FIXME FIXME trailing ; + test "$(ynh_read_var_in_file "$file" "enabled")" == "false" test "$(_read_php "$file" "title")" == "Lorem Ipsum" test "$(ynh_read_var_in_file "$file" "title")" == "Lorem Ipsum" @@ -563,12 +570,16 @@ EOF test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" test "$(_read_php "$file" "port")" == "1234" - test "$(ynh_read_var_in_file "$file" "port")" == "1234;" # FIXME FIXME FIXME trailing ; + test "$(ynh_read_var_in_file "$file" "port")" == "1234" test "$(_read_php "$file" "url")" == "https://yunohost.org" test "$(ynh_read_var_in_file "$file" "url")" == "https://yunohost.org" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + test "$(ynh_read_var_in_file "$file" "user")" == "camille" + + test "$(ynh_read_var_in_file "$file" "DB_HOST")" == "localhost" ! _read_php "$file" "nonexistent" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" @@ -578,7 +589,7 @@ EOF } -ynhtest_config_write_php() { +nhtest_config_write_php() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.php" @@ -590,8 +601,8 @@ ynhtest_config_write_php() { // \$title = "old title"; \$title = "Lorem Ipsum"; \$theme = "colib'ris"; - \$email = "root@example.com"; - \$port = 1234; + \$email = "root@example.com"; // This is a comment without quotes + \$port = 1234; // This is a comment without quotes \$url = "https://yunohost.org"; \$dict = [ 'ldap_base' => "ou=users,dc=yunohost,dc=org", From 3b0bf74274a07d26e3da6da9d77b52f9c475f272 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 5 Sep 2021 16:19:35 +0200 Subject: [PATCH 066/119] [fix] Unbound var in config panel --- src/yunohost/utils/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 5c81f5007..a50e460bb 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -599,7 +599,7 @@ class BooleanQuestion(Question): raise YunohostValidationError( "app_argument_choice_invalid", - name=self.name, # FIXME ... + name=option.get("name", ""), value=value, choices="yes, no, y, n, 1, 0", ) @@ -619,7 +619,7 @@ class BooleanQuestion(Question): return None raise YunohostValidationError( "app_argument_choice_invalid", - name=self.name, # FIXME.... + name=option.get("name", ""), value=value, choices="yes, no, y, n, 1, 0", ) From 4332278f07146aba49ef644f4d5a482f7886e44c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 17:08:18 +0200 Subject: [PATCH 067/119] service test conf: gotta decode the output --- src/yunohost/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 5f9f3e60a..2cebc5a0a 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -286,7 +286,7 @@ def service_reload_or_restart(names, test_conf=True): out, _ = p.communicate() if p.returncode != 0: - errors = out.strip().split("\n") + errors = out.decode().strip().split("\n") logger.error( m18n.n("service_not_reloading_because_conf_broken", errors=errors) ) From 1b99bb8b0f40678c1e930f0536197d664c76e177 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 17:11:00 +0200 Subject: [PATCH 068/119] app_argument_invalid: dunno why name was changed to field but this was breaking i18n consistency --- locales/en.json | 2 +- src/yunohost/utils/config.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/locales/en.json b/locales/en.json index 2bc0516e7..2ac7aa42a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -14,7 +14,7 @@ "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.", "app_already_up_to_date": "{app} is already up-to-date", "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'", - "app_argument_invalid": "Pick a valid value for the argument '{field}': {error}", + "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason", "app_argument_required": "Argument '{name}' is required", "app_change_url_failed_nginx_reload": "Could not reload NGINX. Here is the output of 'nginx -t':\n{nginx_errors}", diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index a50e460bb..26e6e3ebb 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -664,7 +664,7 @@ class DomainQuestion(Question): def _raise_invalid_answer(self): raise YunohostValidationError( - "app_argument_invalid", field=self.name, error=m18n.n("domain_unknown") + "app_argument_invalid", name=self.name, error=m18n.n("domain_unknown") ) @@ -687,7 +687,7 @@ class UserQuestion(Question): def _raise_invalid_answer(self): raise YunohostValidationError( "app_argument_invalid", - field=self.name, + name=self.name, error=m18n.n("user_unknown", user=self.value), ) @@ -713,21 +713,21 @@ class NumberQuestion(Question): ): raise YunohostValidationError( "app_argument_invalid", - field=self.name, + name=self.name, error=m18n.n("invalid_number"), ) if self.min is not None and int(self.value) < self.min: raise YunohostValidationError( "app_argument_invalid", - field=self.name, + name=self.name, error=m18n.n("invalid_number"), ) if self.max is not None and int(self.value) > self.max: raise YunohostValidationError( "app_argument_invalid", - field=self.name, + name=self.name, error=m18n.n("invalid_number"), ) @@ -739,7 +739,7 @@ class NumberQuestion(Question): return int(self.value) raise YunohostValidationError( - "app_argument_invalid", field=self.name, error=m18n.n("invalid_number") + "app_argument_invalid", name=self.name, error=m18n.n("invalid_number") ) @@ -805,7 +805,7 @@ class FileQuestion(Question): ): raise YunohostValidationError( "app_argument_invalid", - field=self.name, + name=self.name, error=m18n.n("file_does_not_exist", path=self.value), ) if self.value in [None, ""] or not self.accept: @@ -815,7 +815,7 @@ class FileQuestion(Question): if "." not in filename or "." + filename.split(".")[-1] not in self.accept: raise YunohostValidationError( "app_argument_invalid", - field=self.name, + name=self.name, error=m18n.n( "file_extension_not_accepted", file=filename, accept=self.accept ), From 2a8e4030925aaefd20df8c0b6fae47a27be73aaa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 17:34:03 +0200 Subject: [PATCH 069/119] test_parse_args_in_yunohost_format -> test_question --- .gitlab/ci/test.gitlab-ci.yml | 4 +- .../{test_config.py => test_questions.py} | 184 +++++++++--------- src/yunohost/utils/config.py | 2 +- 3 files changed, 95 insertions(+), 95 deletions(-) rename src/yunohost/tests/{test_config.py => test_questions.py} (87%) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 31f5aef17..0b35d3447 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -112,11 +112,11 @@ test-appurl: changes: - src/yunohost/app.py -test-config: +test-questions: extends: .test-stage script: - cd src/yunohost - - python3 -m pytest tests/test_config.py + - python3 -m pytest tests/test_questions.py only: changes: - src/yunohost/utils/config.py diff --git a/src/yunohost/tests/test_config.py b/src/yunohost/tests/test_questions.py similarity index 87% rename from src/yunohost/tests/test_config.py rename to src/yunohost/tests/test_questions.py index 2e52c4086..f41c3c0cd 100644 --- a/src/yunohost/tests/test_config.py +++ b/src/yunohost/tests/test_questions.py @@ -35,11 +35,11 @@ User answers: """ -def test_parse_args_in_yunohost_format_empty(): +def test_question_empty(): assert parse_args_in_yunohost_format({}, []) == {} -def test_parse_args_in_yunohost_format_string(): +def test_question_string(): questions = [ { "name": "some_string", @@ -51,7 +51,7 @@ def test_parse_args_in_yunohost_format_string(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_default_type(): +def test_question_string_default_type(): questions = [ { "name": "some_string", @@ -62,7 +62,7 @@ def test_parse_args_in_yunohost_format_string_default_type(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_no_input(): +def test_question_string_no_input(): questions = [ { "name": "some_string", @@ -74,7 +74,7 @@ def test_parse_args_in_yunohost_format_string_no_input(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_string_input(): +def test_question_string_input(): questions = [ { "name": "some_string", @@ -88,7 +88,7 @@ def test_parse_args_in_yunohost_format_string_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_input_no_ask(): +def test_question_string_input_no_ask(): questions = [ { "name": "some_string", @@ -101,7 +101,7 @@ def test_parse_args_in_yunohost_format_string_input_no_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_no_input_optional(): +def test_question_string_no_input_optional(): questions = [ { "name": "some_string", @@ -113,7 +113,7 @@ def test_parse_args_in_yunohost_format_string_no_input_optional(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_optional_with_input(): +def test_question_string_optional_with_input(): questions = [ { "name": "some_string", @@ -128,7 +128,7 @@ def test_parse_args_in_yunohost_format_string_optional_with_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_optional_with_empty_input(): +def test_question_string_optional_with_empty_input(): questions = [ { "name": "some_string", @@ -143,7 +143,7 @@ def test_parse_args_in_yunohost_format_string_optional_with_empty_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_optional_with_input_without_ask(): +def test_question_string_optional_with_input_without_ask(): questions = [ { "name": "some_string", @@ -157,7 +157,7 @@ def test_parse_args_in_yunohost_format_string_optional_with_input_without_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_no_input_default(): +def test_question_string_no_input_default(): questions = [ { "name": "some_string", @@ -170,7 +170,7 @@ def test_parse_args_in_yunohost_format_string_no_input_default(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_input_test_ask(): +def test_question_string_input_test_ask(): ask_text = "some question" questions = [ { @@ -187,7 +187,7 @@ def test_parse_args_in_yunohost_format_string_input_test_ask(): prompt.assert_called_with(ask_text, False) -def test_parse_args_in_yunohost_format_string_input_test_ask_with_default(): +def test_question_string_input_test_ask_with_default(): ask_text = "some question" default_text = "some example" questions = [ @@ -207,7 +207,7 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_default(): @pytest.mark.skip # we should do something with this example -def test_parse_args_in_yunohost_format_string_input_test_ask_with_example(): +def test_question_string_input_test_ask_with_example(): ask_text = "some question" example_text = "some example" questions = [ @@ -228,7 +228,7 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_example(): @pytest.mark.skip # we should do something with this help -def test_parse_args_in_yunohost_format_string_input_test_ask_with_help(): +def test_question_string_input_test_ask_with_help(): ask_text = "some question" help_text = "some_help" questions = [ @@ -248,14 +248,14 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_help(): assert help_text in prompt.call_args[0][0] -def test_parse_args_in_yunohost_format_string_with_choice(): +def test_question_string_with_choice(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "fr"} expected_result = OrderedDict({"some_string": ("fr", "string")}) assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_with_choice_prompt(): +def test_question_string_with_choice_prompt(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "fr"} expected_result = OrderedDict({"some_string": ("fr", "string")}) @@ -263,7 +263,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_prompt(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_string_with_choice_bad(): +def test_question_string_with_choice_bad(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "bad"} @@ -271,7 +271,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_bad(): assert parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_string_with_choice_ask(): +def test_question_string_with_choice_ask(): ask_text = "some question" choices = ["fr", "en", "es", "it", "ru"] questions = [ @@ -291,7 +291,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_ask(): assert choice in prompt.call_args[0][0] -def test_parse_args_in_yunohost_format_string_with_choice_default(): +def test_question_string_with_choice_default(): questions = [ { "name": "some_string", @@ -305,7 +305,7 @@ def test_parse_args_in_yunohost_format_string_with_choice_default(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password(): +def test_question_password(): questions = [ { "name": "some_password", @@ -317,7 +317,7 @@ def test_parse_args_in_yunohost_format_password(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password_no_input(): +def test_question_password_no_input(): questions = [ { "name": "some_password", @@ -330,7 +330,7 @@ def test_parse_args_in_yunohost_format_password_no_input(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_password_input(): +def test_question_password_input(): questions = [ { "name": "some_password", @@ -345,7 +345,7 @@ def test_parse_args_in_yunohost_format_password_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password_input_no_ask(): +def test_question_password_input_no_ask(): questions = [ { "name": "some_password", @@ -359,7 +359,7 @@ def test_parse_args_in_yunohost_format_password_input_no_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password_no_input_optional(): +def test_question_password_no_input_optional(): questions = [ { "name": "some_password", @@ -379,7 +379,7 @@ def test_parse_args_in_yunohost_format_password_no_input_optional(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password_optional_with_input(): +def test_question_password_optional_with_input(): questions = [ { "name": "some_password", @@ -395,7 +395,7 @@ def test_parse_args_in_yunohost_format_password_optional_with_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password_optional_with_empty_input(): +def test_question_password_optional_with_empty_input(): questions = [ { "name": "some_password", @@ -411,7 +411,7 @@ def test_parse_args_in_yunohost_format_password_optional_with_empty_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password_optional_with_input_without_ask(): +def test_question_password_optional_with_input_without_ask(): questions = [ { "name": "some_password", @@ -426,7 +426,7 @@ def test_parse_args_in_yunohost_format_password_optional_with_input_without_ask( assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_password_no_input_default(): +def test_question_password_no_input_default(): questions = [ { "name": "some_password", @@ -443,7 +443,7 @@ def test_parse_args_in_yunohost_format_password_no_input_default(): @pytest.mark.skip # this should raises -def test_parse_args_in_yunohost_format_password_no_input_example(): +def test_question_password_no_input_example(): questions = [ { "name": "some_password", @@ -459,7 +459,7 @@ def test_parse_args_in_yunohost_format_password_no_input_example(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_password_input_test_ask(): +def test_question_password_input_test_ask(): ask_text = "some question" questions = [ { @@ -478,7 +478,7 @@ def test_parse_args_in_yunohost_format_password_input_test_ask(): @pytest.mark.skip # we should do something with this example -def test_parse_args_in_yunohost_format_password_input_test_ask_with_example(): +def test_question_password_input_test_ask_with_example(): ask_text = "some question" example_text = "some example" questions = [ @@ -500,7 +500,7 @@ def test_parse_args_in_yunohost_format_password_input_test_ask_with_example(): @pytest.mark.skip # we should do something with this help -def test_parse_args_in_yunohost_format_password_input_test_ask_with_help(): +def test_question_password_input_test_ask_with_help(): ask_text = "some question" help_text = "some_help" questions = [ @@ -521,7 +521,7 @@ def test_parse_args_in_yunohost_format_password_input_test_ask_with_help(): assert help_text in prompt.call_args[0][0] -def test_parse_args_in_yunohost_format_password_bad_chars(): +def test_question_password_bad_chars(): questions = [ { "name": "some_password", @@ -536,7 +536,7 @@ def test_parse_args_in_yunohost_format_password_bad_chars(): parse_args_in_yunohost_format({"some_password": i * 8}, questions) -def test_parse_args_in_yunohost_format_password_strong_enough(): +def test_question_password_strong_enough(): questions = [ { "name": "some_password", @@ -554,7 +554,7 @@ def test_parse_args_in_yunohost_format_password_strong_enough(): parse_args_in_yunohost_format({"some_password": "password"}, questions) -def test_parse_args_in_yunohost_format_password_optional_strong_enough(): +def test_question_password_optional_strong_enough(): questions = [ { "name": "some_password", @@ -572,7 +572,7 @@ def test_parse_args_in_yunohost_format_password_optional_strong_enough(): parse_args_in_yunohost_format({"some_password": "password"}, questions) -def test_parse_args_in_yunohost_format_path(): +def test_question_path(): questions = [ { "name": "some_path", @@ -584,7 +584,7 @@ def test_parse_args_in_yunohost_format_path(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_no_input(): +def test_question_path_no_input(): questions = [ { "name": "some_path", @@ -597,7 +597,7 @@ def test_parse_args_in_yunohost_format_path_no_input(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_path_input(): +def test_question_path_input(): questions = [ { "name": "some_path", @@ -612,7 +612,7 @@ def test_parse_args_in_yunohost_format_path_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_input_no_ask(): +def test_question_path_input_no_ask(): questions = [ { "name": "some_path", @@ -626,7 +626,7 @@ def test_parse_args_in_yunohost_format_path_input_no_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_no_input_optional(): +def test_question_path_no_input_optional(): questions = [ { "name": "some_path", @@ -639,7 +639,7 @@ def test_parse_args_in_yunohost_format_path_no_input_optional(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_optional_with_input(): +def test_question_path_optional_with_input(): questions = [ { "name": "some_path", @@ -655,7 +655,7 @@ def test_parse_args_in_yunohost_format_path_optional_with_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_optional_with_empty_input(): +def test_question_path_optional_with_empty_input(): questions = [ { "name": "some_path", @@ -671,7 +671,7 @@ def test_parse_args_in_yunohost_format_path_optional_with_empty_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_optional_with_input_without_ask(): +def test_question_path_optional_with_input_without_ask(): questions = [ { "name": "some_path", @@ -686,7 +686,7 @@ def test_parse_args_in_yunohost_format_path_optional_with_input_without_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_no_input_default(): +def test_question_path_no_input_default(): questions = [ { "name": "some_path", @@ -700,7 +700,7 @@ def test_parse_args_in_yunohost_format_path_no_input_default(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_path_input_test_ask(): +def test_question_path_input_test_ask(): ask_text = "some question" questions = [ { @@ -718,7 +718,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask(): prompt.assert_called_with(ask_text, False) -def test_parse_args_in_yunohost_format_path_input_test_ask_with_default(): +def test_question_path_input_test_ask_with_default(): ask_text = "some question" default_text = "some example" questions = [ @@ -739,7 +739,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_default(): @pytest.mark.skip # we should do something with this example -def test_parse_args_in_yunohost_format_path_input_test_ask_with_example(): +def test_question_path_input_test_ask_with_example(): ask_text = "some question" example_text = "some example" questions = [ @@ -761,7 +761,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_example(): @pytest.mark.skip # we should do something with this help -def test_parse_args_in_yunohost_format_path_input_test_ask_with_help(): +def test_question_path_input_test_ask_with_help(): ask_text = "some question" help_text = "some_help" questions = [ @@ -782,7 +782,7 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_help(): assert help_text in prompt.call_args[0][0] -def test_parse_args_in_yunohost_format_boolean(): +def test_question_boolean(): questions = [ { "name": "some_boolean", @@ -794,7 +794,7 @@ def test_parse_args_in_yunohost_format_boolean(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_all_yes(): +def test_question_boolean_all_yes(): questions = [ { "name": "some_boolean", @@ -848,7 +848,7 @@ def test_parse_args_in_yunohost_format_boolean_all_yes(): ) -def test_parse_args_in_yunohost_format_boolean_all_no(): +def test_question_boolean_all_no(): questions = [ { "name": "some_boolean", @@ -903,7 +903,7 @@ def test_parse_args_in_yunohost_format_boolean_all_no(): # XXX apparently boolean are always False (0) by default, I'm not sure what to think about that -def test_parse_args_in_yunohost_format_boolean_no_input(): +def test_question_boolean_no_input(): questions = [ { "name": "some_boolean", @@ -916,7 +916,7 @@ def test_parse_args_in_yunohost_format_boolean_no_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_bad_input(): +def test_question_boolean_bad_input(): questions = [ { "name": "some_boolean", @@ -929,7 +929,7 @@ def test_parse_args_in_yunohost_format_boolean_bad_input(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_boolean_input(): +def test_question_boolean_input(): questions = [ { "name": "some_boolean", @@ -948,7 +948,7 @@ def test_parse_args_in_yunohost_format_boolean_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_input_no_ask(): +def test_question_boolean_input_no_ask(): questions = [ { "name": "some_boolean", @@ -962,7 +962,7 @@ def test_parse_args_in_yunohost_format_boolean_input_no_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_no_input_optional(): +def test_question_boolean_no_input_optional(): questions = [ { "name": "some_boolean", @@ -975,7 +975,7 @@ def test_parse_args_in_yunohost_format_boolean_no_input_optional(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_optional_with_input(): +def test_question_boolean_optional_with_input(): questions = [ { "name": "some_boolean", @@ -991,7 +991,7 @@ def test_parse_args_in_yunohost_format_boolean_optional_with_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_optional_with_empty_input(): +def test_question_boolean_optional_with_empty_input(): questions = [ { "name": "some_boolean", @@ -1007,7 +1007,7 @@ def test_parse_args_in_yunohost_format_boolean_optional_with_empty_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_optional_with_input_without_ask(): +def test_question_boolean_optional_with_input_without_ask(): questions = [ { "name": "some_boolean", @@ -1022,7 +1022,7 @@ def test_parse_args_in_yunohost_format_boolean_optional_with_input_without_ask() assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_no_input_default(): +def test_question_boolean_no_input_default(): questions = [ { "name": "some_boolean", @@ -1036,7 +1036,7 @@ def test_parse_args_in_yunohost_format_boolean_no_input_default(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_boolean_bad_default(): +def test_question_boolean_bad_default(): questions = [ { "name": "some_boolean", @@ -1050,7 +1050,7 @@ def test_parse_args_in_yunohost_format_boolean_bad_default(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_boolean_input_test_ask(): +def test_question_boolean_input_test_ask(): ask_text = "some question" questions = [ { @@ -1066,7 +1066,7 @@ def test_parse_args_in_yunohost_format_boolean_input_test_ask(): prompt.assert_called_with(ask_text + " [yes | no] (default: no)", False) -def test_parse_args_in_yunohost_format_boolean_input_test_ask_with_default(): +def test_question_boolean_input_test_ask_with_default(): ask_text = "some question" default_text = 1 questions = [ @@ -1084,7 +1084,7 @@ def test_parse_args_in_yunohost_format_boolean_input_test_ask_with_default(): prompt.assert_called_with("%s [yes | no] (default: yes)" % ask_text, False) -def test_parse_args_in_yunohost_format_domain_empty(): +def test_question_domain_empty(): questions = [ { "name": "some_domain", @@ -1101,7 +1101,7 @@ def test_parse_args_in_yunohost_format_domain_empty(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_domain(): +def test_question_domain(): main_domain = "my_main_domain.com" domains = [main_domain] questions = [ @@ -1120,7 +1120,7 @@ def test_parse_args_in_yunohost_format_domain(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_domain_two_domains(): +def test_question_domain_two_domains(): main_domain = "my_main_domain.com" other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] @@ -1148,7 +1148,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_domain_two_domains_wrong_answer(): +def test_question_domain_two_domains_wrong_answer(): main_domain = "my_main_domain.com" other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] @@ -1168,7 +1168,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains_wrong_answer(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_domain_two_domains_default_no_ask(): +def test_question_domain_two_domains_default_no_ask(): main_domain = "my_main_domain.com" other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] @@ -1188,7 +1188,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains_default_no_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_domain_two_domains_default(): +def test_question_domain_two_domains_default(): main_domain = "my_main_domain.com" other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] @@ -1203,7 +1203,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains_default(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_domain_two_domains_default_input(): +def test_question_domain_two_domains_default_input(): main_domain = "my_main_domain.com" other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] @@ -1223,7 +1223,7 @@ def test_parse_args_in_yunohost_format_domain_two_domains_default_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_user_empty(): +def test_question_user_empty(): users = { "some_user": { "ssh_allowed": False, @@ -1247,7 +1247,7 @@ def test_parse_args_in_yunohost_format_user_empty(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_user(): +def test_question_user(): username = "some_user" users = { username: { @@ -1274,7 +1274,7 @@ def test_parse_args_in_yunohost_format_user(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_user_two_users(): +def test_question_user_two_users(): username = "some_user" other_user = "some_other_user" users = { @@ -1315,7 +1315,7 @@ def test_parse_args_in_yunohost_format_user_two_users(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_user_two_users_wrong_answer(): +def test_question_user_two_users_wrong_answer(): username = "my_username.com" other_user = "some_other_user" users = { @@ -1348,7 +1348,7 @@ def test_parse_args_in_yunohost_format_user_two_users_wrong_answer(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_user_two_users_no_default(): +def test_question_user_two_users_no_default(): username = "my_username.com" other_user = "some_other_user.tld" users = { @@ -1376,7 +1376,7 @@ def test_parse_args_in_yunohost_format_user_two_users_no_default(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_user_two_users_default_input(): +def test_question_user_two_users_default_input(): username = "my_username.com" other_user = "some_other_user.tld" users = { @@ -1416,7 +1416,7 @@ def test_parse_args_in_yunohost_format_user_two_users_default_input(): ) -def test_parse_args_in_yunohost_format_number(): +def test_question_number(): questions = [ { "name": "some_number", @@ -1428,7 +1428,7 @@ def test_parse_args_in_yunohost_format_number(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_no_input(): +def test_question_number_no_input(): questions = [ { "name": "some_number", @@ -1441,7 +1441,7 @@ def test_parse_args_in_yunohost_format_number_no_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_bad_input(): +def test_question_number_bad_input(): questions = [ { "name": "some_number", @@ -1458,7 +1458,7 @@ def test_parse_args_in_yunohost_format_number_bad_input(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_number_input(): +def test_question_number_input(): questions = [ { "name": "some_number", @@ -1480,7 +1480,7 @@ def test_parse_args_in_yunohost_format_number_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_input_no_ask(): +def test_question_number_input_no_ask(): questions = [ { "name": "some_number", @@ -1494,7 +1494,7 @@ def test_parse_args_in_yunohost_format_number_input_no_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_no_input_optional(): +def test_question_number_no_input_optional(): questions = [ { "name": "some_number", @@ -1507,7 +1507,7 @@ def test_parse_args_in_yunohost_format_number_no_input_optional(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_optional_with_input(): +def test_question_number_optional_with_input(): questions = [ { "name": "some_number", @@ -1523,7 +1523,7 @@ def test_parse_args_in_yunohost_format_number_optional_with_input(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_optional_with_input_without_ask(): +def test_question_number_optional_with_input_without_ask(): questions = [ { "name": "some_number", @@ -1538,7 +1538,7 @@ def test_parse_args_in_yunohost_format_number_optional_with_input_without_ask(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_no_input_default(): +def test_question_number_no_input_default(): questions = [ { "name": "some_number", @@ -1552,7 +1552,7 @@ def test_parse_args_in_yunohost_format_number_no_input_default(): assert parse_args_in_yunohost_format(answers, questions) == expected_result -def test_parse_args_in_yunohost_format_number_bad_default(): +def test_question_number_bad_default(): questions = [ { "name": "some_number", @@ -1566,7 +1566,7 @@ def test_parse_args_in_yunohost_format_number_bad_default(): parse_args_in_yunohost_format(answers, questions) -def test_parse_args_in_yunohost_format_number_input_test_ask(): +def test_question_number_input_test_ask(): ask_text = "some question" questions = [ { @@ -1582,7 +1582,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask(): prompt.assert_called_with("%s (default: 0)" % (ask_text), False) -def test_parse_args_in_yunohost_format_number_input_test_ask_with_default(): +def test_question_number_input_test_ask_with_default(): ask_text = "some question" default_value = 1337 questions = [ @@ -1601,7 +1601,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask_with_default(): @pytest.mark.skip # we should do something with this example -def test_parse_args_in_yunohost_format_number_input_test_ask_with_example(): +def test_question_number_input_test_ask_with_example(): ask_text = "some question" example_value = 1337 questions = [ @@ -1621,7 +1621,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask_with_example(): @pytest.mark.skip # we should do something with this help -def test_parse_args_in_yunohost_format_number_input_test_ask_with_help(): +def test_question_number_input_test_ask_with_help(): ask_text = "some question" help_value = 1337 questions = [ @@ -1640,7 +1640,7 @@ def test_parse_args_in_yunohost_format_number_input_test_ask_with_help(): assert help_value in prompt.call_args[0][0] -def test_parse_args_in_yunohost_format_display_text(): +def test_question_display_text(): questions = [{"name": "some_app", "type": "display_text", "ask": "foobar"}] answers = {} diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 26e6e3ebb..b6149ad96 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -110,7 +110,7 @@ class ConfigPanel: if (args is not None or args_file is not None) and value is not None: raise YunohostError("config_args_value") - if self.filter_key.count(".") != 2 and not value is None: + if self.filter_key.count(".") != 2 and value is not None: raise YunohostError("config_set_value_on_section") # Import and parse pre-answered options From afcf125d12dd3c0b933b1262124de8b7a8eb1fd0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 17:39:39 +0200 Subject: [PATCH 070/119] i18n fix --- src/yunohost/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 2cebc5a0a..6e3b2d7a6 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -288,7 +288,7 @@ def service_reload_or_restart(names, test_conf=True): if p.returncode != 0: errors = out.decode().strip().split("\n") logger.error( - m18n.n("service_not_reloading_because_conf_broken", errors=errors) + m18n.n("service_not_reloading_because_conf_broken", name=name, errors=errors) ) continue From da44224794652e4077d660c160f0cf624e398923 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 5 Sep 2021 17:46:06 +0200 Subject: [PATCH 071/119] [enh] Add warning if bad keys in toml format --- src/yunohost/utils/config.py | 62 +++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index a50e460bb..9e694596c 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -166,7 +166,7 @@ class ConfigPanel: def _get_config_panel(self): # Split filter_key - filter_key = self.filter_key.split(".") + filter_key = self.filter_key.split(".") if self.filter_key != "" else [] if len(filter_key) > 3: raise YunohostError("config_too_many_sub_keys", key=self.filter_key) @@ -181,21 +181,34 @@ class ConfigPanel: ) # Transform toml format into internal format - defaults = { - "toml": {"version": 1.0}, + format_description = { + "toml": { + "properties": ["version", "i18n"], + "default": {"version": 1.0}, + }, "panels": { - "name": "", - "services": [], - "actions": {"apply": {"en": "Apply"}}, - }, # help + "properties": ["name", "services", "actions", "help"], + "default": { + "name": "", + "services": [], + "actions": {"apply": {"en": "Apply"}} + }, + }, "sections": { - "name": "", - "services": [], - "optional": True, - }, # visibleIf help - "options": {} - # ask type source help helpLink example style icon placeholder visibleIf - # optional choices pattern limit min max step accept redact + "properties": ["name", "services", "optional", "help", "visible"], + "default": { + "name": "", + "services": [], + "optional": True, + } + }, + "options": { + "properties": ["ask", "type", "source", "help", "example", + "style", "icon", "placeholder", "visible", + "optional", "choices", "yes", "no", "pattern", + "limit", "min", "max", "step", "accept", "redact"], + "default": {} + } } # @@ -214,19 +227,21 @@ class ConfigPanel: This function detects all children nodes and put them in a list """ # Prefill the node default keys if needed - default = defaults[node_type] + default = format_description[node_type]['default'] node = {key: toml_node.get(key, value) for key, value in default.items()} + properties = format_description[node_type]['properties'] + # Define the filter_key part to use and the children type - i = list(defaults).index(node_type) - search_key = filter_key.get(i) - subnode_type = list(defaults)[i + 1] if node_type != "options" else None + i = list(format_description).index(node_type) + subnode_type = list(format_description)[i + 1] if node_type != "options" else None + search_key = filter_key[i] if len(filter_key) > i else False for key, value in toml_node.items(): # Key/value are a child node if ( isinstance(value, OrderedDict) - and key not in default + and key not in properties and subnode_type ): # We exclude all nodes not referenced by the filter_key @@ -240,6 +255,8 @@ class ConfigPanel: node.setdefault(subnode_type, []).append(subnode) # Key/value are a property else: + if key not in properties: + logger.warning(f"Unknown key '{key}' found in config toml") # Todo search all i18n keys node[key] = ( value if key not in ["ask", "help", "name"] else {"en": value} @@ -378,7 +395,6 @@ class Question(object): self.pattern = question.get("pattern") self.ask = question.get("ask", {"en": self.name}) self.help = question.get("help") - self.helpLink = question.get("helpLink") self.value = user_answers.get(self.name) self.redact = question.get("redact", False) @@ -471,15 +487,11 @@ class Question(object): if self.choices: text_for_user_input_in_cli += " [{0}]".format(" | ".join(self.choices)) - if self.help or self.helpLink: + if self.help: text_for_user_input_in_cli += ":\033[m" if self.help: text_for_user_input_in_cli += "\n - " text_for_user_input_in_cli += _value_for_locale(self.help) - if self.helpLink: - if not isinstance(self.helpLink, dict): - self.helpLink = {"href": self.helpLink} - text_for_user_input_in_cli += f"\n - See {self.helpLink['href']}" return text_for_user_input_in_cli def _post_parse_value(self): From 62eecb28db3abc9d29541a8fd8d72dcecf227e2f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 18:53:39 +0200 Subject: [PATCH 072/119] --mode full/export -> --full / --export --- data/actionsmap/yunohost.yml | 16 ++++++++-------- src/yunohost/app.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index f694d4361..cbb580029 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -863,14 +863,14 @@ app: key: help: A specific panel, section or a question identifier nargs: '?' - -m: - full: --mode - help: Display mode to use - choices: - - classic - - full - - export - default: classic + -f: + full: --full + help: Display all details (meant to be used by the API) + action: store_true + -e: + full: --export + help: Only export key/values, meant to be reimported using "config set --args-file" + action: store_true ### app_config_set() set: diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d97c2824c..45386c129 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1751,10 +1751,20 @@ def app_action_run(operation_logger, app, action, args=None): return logger.success("Action successed!") -def app_config_get(app, key="", mode="classic"): +def app_config_get(app, key="", full=False, export=False): """ Display an app configuration in classic, full or export mode """ + if full and export: + raise YunohostValidationError("You can't use --full and --export together.", raw_msg=True) + + if full: + mode = "full" + elif export: + mode = "export" + else: + mode = "classic" + config_ = AppConfigPanel(app) return config_.get(key, mode) From ee55b9bf42055627880949ed656a9597cc15dc72 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 19:15:52 +0200 Subject: [PATCH 073/119] config helpers: Semantics / comments on the validation-apply workflow --- data/helpers.d/config | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 6223a17b2..3a2c55444 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -160,7 +160,8 @@ _ynh_app_config_show() { _ynh_app_config_validate() { # Change detection ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1 - local is_error=true + local nothing_changed=true + local changes_validated=true #for changed_status in "${!changed[@]}" for short_setting in "${!old[@]}" do @@ -178,7 +179,7 @@ _ynh_app_config_validate() { file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) if [ -z "${!short_setting}" ] ; then changed[$short_setting]=true - is_error=false + nothing_changed=false fi fi if [ -f "${!short_setting}" ] ; then @@ -186,18 +187,18 @@ _ynh_app_config_validate() { if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] then changed[$short_setting]=true - is_error=false + nothing_changed=false fi fi else if [[ "${!short_setting}" != "${old[$short_setting]}" ]] then changed[$short_setting]=true - is_error=false + nothing_changed=false fi fi done - if [[ "$is_error" == "true" ]] + if [[ "$nothing_changed" == "true" ]] then ynh_print_info "Nothing has changed" exit 0 @@ -217,11 +218,13 @@ _ynh_app_config_validate() { then local key="YNH_ERROR_${short_setting}" ynh_return "$key: \"$result\"" - is_error=true + changes_validated=false fi done - - if [[ "$is_error" == "true" ]] + + # If validation failed, exit the script right now (instead of going into apply) + # Yunohost core will pick up the errors returned via ynh_return previously + if [[ "$changes_validated" == "false" ]] then exit 0 fi From c5de8035312773b3b291abddf8ef2ad8966e9ac3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 19:17:17 +0200 Subject: [PATCH 074/119] config panels: try to improve the log and error handling: separate ask vs. actual apply --- src/yunohost/app.py | 6 ++---- src/yunohost/utils/config.py | 16 ++++++---------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 45386c129..9c40ef18d 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1780,11 +1780,9 @@ def app_config_set( config_ = AppConfigPanel(app) Question.operation_logger = operation_logger - operation_logger.start() - result = config_.set(key, value, args, args_file) - if "errors" not in result: - operation_logger.success() + result = config_.set(key, value, args, args_file, operation_logger=operation_logger) + return result diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 4f69729f7..6432856b0 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -98,7 +98,7 @@ class ConfigPanel: return result - def set(self, key=None, value=None, args=None, args_file=None): + def set(self, key=None, value=None, args=None, args_file=None, operation_logger=None): self.filter_key = key or "" # Read config panel toml @@ -128,11 +128,13 @@ class ConfigPanel: # Read or get values and hydrate the config self._load_current_values() self._hydrate() + self._ask() + + if operation_logger: + operation_logger.start() try: - self._ask() self._apply() - # Script got manually interrupted ... # N.B. : KeyboardInterrupt does not inherit from Exception except (KeyboardInterrupt, EOFError): @@ -158,6 +160,7 @@ class ConfigPanel: self._reload_services() logger.success("Config updated as expected") + operation_logger.success() return {} def _get_toml(self): @@ -211,13 +214,6 @@ class ConfigPanel: } } - # - # FIXME : this is hella confusing ... - # from what I understand, the purpose is to have some sort of "deep_update" - # to apply the defaults onto the loaded toml ... - # in that case we probably want to get inspiration from - # https://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth - # def convert(toml_node, node_type): """Convert TOML in internal format ('full' mode used by webadmin) Here are some properties of 1.0 config panel in toml: From 6f7485bf3eb259152977960a2fa3db98a50b3d09 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 19:31:07 +0200 Subject: [PATCH 075/119] config tests: Add a basic tests for app config panel --- .gitlab/ci/test.gitlab-ci.yml | 10 ++ src/yunohost/tests/test_app_config.py | 144 ++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 src/yunohost/tests/test_app_config.py diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 0b35d3447..2dc45171b 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -121,6 +121,16 @@ test-questions: changes: - src/yunohost/utils/config.py +test-app-config: + extends: .test-stage + script: + - cd src/yunohost + - python3 -m pytest tests/test_app_config.py + only: + changes: + - src/yunohost/app.py + - src/yunohost/utils/config.py + test-changeurl: extends: .test-stage script: diff --git a/src/yunohost/tests/test_app_config.py b/src/yunohost/tests/test_app_config.py new file mode 100644 index 000000000..d8dc5849f --- /dev/null +++ b/src/yunohost/tests/test_app_config.py @@ -0,0 +1,144 @@ +import glob +import os +import shutil +import pytest + +from .conftest import get_test_apps_dir + +from yunohost.domain import _get_maindomain +from yunohost.app import ( + app_install, + app_remove, + _is_installed, + app_config_get, + app_config_set, + app_ssowatconf, +) + +from yunohost.utils.errors import YunohostValidationError + + +def setup_function(function): + + clean() + + +def teardown_function(function): + + clean() + + +def clean(): + + # Make sure we have a ssowat + os.system("mkdir -p /etc/ssowat/") + app_ssowatconf() + + test_apps = ["config_app", "legacy_app"] + + for test_app in test_apps: + + if _is_installed(test_app): + app_remove(test_app) + + for filepath in glob.glob("/etc/nginx/conf.d/*.d/*%s*" % test_app): + os.remove(filepath) + for folderpath in glob.glob("/etc/yunohost/apps/*%s*" % test_app): + shutil.rmtree(folderpath, ignore_errors=True) + for folderpath in glob.glob("/var/www/*%s*" % test_app): + shutil.rmtree(folderpath, ignore_errors=True) + + os.system("bash -c \"mysql -B 2>/dev/null <<< 'DROP DATABASE %s' \"" % test_app) + os.system( + "bash -c \"mysql -B 2>/dev/null <<< 'DROP USER %s@localhost'\"" % test_app + ) + + # Reset failed quota for service to avoid running into start-limit rate ? + os.system("systemctl reset-failed nginx") + os.system("systemctl start nginx") + + +@pytest.fixture(scope="module") +def legacy_app(request): + + main_domain = _get_maindomain() + + app_install( + os.path.join(get_test_apps_dir(), "legacy_app_ynh"), + args="domain=%s&path=%s&is_public=%s" % (main_domain, "/", 1), + force=True, + ) + + def remove_app(): + app_remove("legacy_app") + + request.addfinalizer(remove_app) + + return "legacy_app" + + + +@pytest.fixture(scope="module") +def config_app(request): + + app_install( + os.path.join(get_test_apps_dir(), "config_app_ynh"), + args="", + force=True, + ) + + def remove_app(): + app_remove("config_app") + + request.addfinalizer(remove_app) + + return "config_app" + + +def test_app_config_get(config_app): + + assert isinstance(app_config_get(config_app), dict) + assert isinstance(app_config_get(config_app, full=True), dict) + assert isinstance(app_config_get(config_app, export=True), dict) + assert isinstance(app_config_get(config_app, "main"), dict) + assert isinstance(app_config_get(config_app, "main.components"), dict) + # Is it expected that we return None if no value defined yet ? + # c.f. the whole discussion about "should we have defaults" etc. + assert app_config_get(config_app, "main.components.boolean") is None + + +def test_app_config_nopanel(legacy_app): + + with pytest.raises(YunohostValidationError): + app_config_get(legacy_app) + + +def test_app_config_get_nonexistentstuff(config_app): + + with pytest.raises(YunohostValidationError): + app_config_get("nonexistent") + + with pytest.raises(YunohostValidationError): + app_config_get(config_app, "nonexistent") + + with pytest.raises(YunohostValidationError): + app_config_get(config_app, "main.nonexistent") + + with pytest.raises(YunohostValidationError): + app_config_get(config_app, "main.components.nonexistent") + + +def test_app_config_set_boolean(config_app): + + assert app_config_get(config_app, "main.components.boolean") is None + + app_config_set(config_app, "main.components.boolean", "no") + + assert app_config_get(config_app, "main.components.boolean") == "0" + + app_config_set(config_app, "main.components.boolean", "yes") + + assert app_config_get(config_app, "main.components.boolean") == "1" + + with pytest.raises(YunohostValidationError): + app_config_set(config_app, "main.components.boolean", "pwet") From 56f525cf80b32f5c397398065267cbd27bd61a19 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 19:42:46 +0200 Subject: [PATCH 076/119] Typo --- src/yunohost/tests/test_app_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tests/test_app_config.py b/src/yunohost/tests/test_app_config.py index d8dc5849f..003d9f3b2 100644 --- a/src/yunohost/tests/test_app_config.py +++ b/src/yunohost/tests/test_app_config.py @@ -15,7 +15,7 @@ from yunohost.app import ( app_ssowatconf, ) -from yunohost.utils.errors import YunohostValidationError +from yunohost.utils.error import YunohostValidationError def setup_function(function): From 3936589d3bd4b866a5213e047e922b1f030209b0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 20:24:36 +0200 Subject: [PATCH 077/119] Yolofactorize install/upgrade/restore error handling into a smart 'hook_exec_with_script_debug_if_failure' --- src/yunohost/app.py | 118 ++++++----------------------------------- src/yunohost/backup.py | 33 ++---------- src/yunohost/hook.py | 34 ++++++++++++ src/yunohost/log.py | 46 ++++++++++++++++ 4 files changed, 101 insertions(+), 130 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 9c40ef18d..e9712edb4 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -511,7 +511,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False """ from packaging import version - from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback + from yunohost.hook import hook_add, hook_remove, hook_callback, hook_exec_with_script_debug_if_failure from yunohost.permission import permission_sync_to_user from yunohost.regenconf import manually_modified_files @@ -633,36 +633,13 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False # Execute the app upgrade script upgrade_failed = True try: - upgrade_retcode = hook_exec( - extracted_app_folder + "/scripts/upgrade", env=env_dict - )[0] - - upgrade_failed = True if upgrade_retcode != 0 else False - if upgrade_failed: - error = m18n.n("app_upgrade_script_failed") - logger.error( - m18n.n("app_upgrade_failed", app=app_instance_name, error=error) - ) - failure_message_with_debug_instructions = operation_logger.error(error) - if Moulinette.interface.type != "api": - dump_app_log_extract_for_debugging(operation_logger) - # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception - except (KeyboardInterrupt, EOFError): - upgrade_retcode = -1 - error = m18n.n("operation_interrupted") - logger.error( - m18n.n("app_upgrade_failed", app=app_instance_name, error=error) + upgrade_failed, failure_message_with_debug_instructions = hook_exec_with_script_debug_if_failure( + extracted_app_folder + "/scripts/upgrade", + env=env_dict, + operation_logger=operation_logger, + error_message_if_script_failed=m18n.n("app_upgrade_script_failed"), + error_message_if_failed=lambda e: m18n.n("app_upgrade_failed", app=app_instance_name, error=e) ) - failure_message_with_debug_instructions = operation_logger.error(error) - # Something wrong happened in Yunohost's code (most probably hook_exec) - except Exception: - import traceback - - error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error( - m18n.n("app_install_failed", app=app_instance_name, error=error) - ) - failure_message_with_debug_instructions = operation_logger.error(error) finally: # Whatever happened (install success or failure) we check if it broke the system # and warn the user about it @@ -692,7 +669,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if upgrade_failed or broke_the_system: # display this if there are remaining apps - if apps[number + 1 :]: + if apps[number + 1:]: not_upgraded_apps = apps[number:] logger.error( m18n.n( @@ -808,7 +785,7 @@ def app_install( force -- Do not ask for confirmation when installing experimental / low-quality apps """ - from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback + from yunohost.hook import hook_add, hook_remove, hook_callback, hook_exec, hook_exec_with_script_debug_if_failure from yunohost.log import OperationLogger from yunohost.permission import ( user_permission_list, @@ -999,29 +976,13 @@ def app_install( # Execute the app install script install_failed = True try: - install_retcode = hook_exec( - os.path.join(extracted_app_folder, "scripts/install"), env=env_dict - )[0] - # "Common" app install failure : the script failed and returned exit code != 0 - install_failed = True if install_retcode != 0 else False - if install_failed: - error = m18n.n("app_install_script_failed") - logger.error(m18n.n("app_install_failed", app=app_id, error=error)) - failure_message_with_debug_instructions = operation_logger.error(error) - if Moulinette.interface.type != "api": - dump_app_log_extract_for_debugging(operation_logger) - # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception - except (KeyboardInterrupt, EOFError): - error = m18n.n("operation_interrupted") - logger.error(m18n.n("app_install_failed", app=app_id, error=error)) - failure_message_with_debug_instructions = operation_logger.error(error) - # Something wrong happened in Yunohost's code (most probably hook_exec) - except Exception: - import traceback - - error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("app_install_failed", app=app_id, error=error)) - failure_message_with_debug_instructions = operation_logger.error(error) + install_failed, failure_message_with_debug_instructions = hook_exec_with_script_debug_if_failure( + os.path.join(extracted_app_folder, "scripts/install"), + env=env_dict, + operation_logger=operation_logger, + error_message_if_script_failed=m18n.n("app_install_script_failed"), + error_message_if_failed=lambda e: m18n.n("app_install_failed", app=app_id, error=e) + ) finally: # If success so far, validate that app didn't break important stuff if not install_failed: @@ -1134,53 +1095,6 @@ def app_install( hook_callback("post_app_install", env=env_dict) -def dump_app_log_extract_for_debugging(operation_logger): - - with open(operation_logger.log_path, "r") as f: - lines = f.readlines() - - filters = [ - r"set [+-]x$", - r"set [+-]o xtrace$", - r"local \w+$", - r"local legacy_args=.*$", - r".*Helper used in legacy mode.*", - r"args_array=.*$", - r"local -A args_array$", - r"ynh_handle_getopts_args", - r"ynh_script_progression", - ] - - filters = [re.compile(f_) for f_ in filters] - - lines_to_display = [] - for line in lines: - - if ": " not in line.strip(): - continue - - # A line typically looks like - # 2019-10-19 16:10:27,611: DEBUG - + mysql -u piwigo --password=********** -B piwigo - # And we just want the part starting by "DEBUG - " - line = line.strip().split(": ", 1)[1] - - if any(filter_.search(line) for filter_ in filters): - continue - - lines_to_display.append(line) - - if line.endswith("+ ynh_exit_properly") or " + ynh_die " in line: - break - elif len(lines_to_display) > 20: - lines_to_display.pop(0) - - logger.warning( - "Here's an extract of the logs before the crash. It might help debugging the error:" - ) - for line in lines_to_display: - logger.info(line) - - @is_unit_operation() def app_remove(operation_logger, app, purge=False): """ diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 09b35cb67..c39bf656c 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -48,7 +48,6 @@ from yunohost.app import ( app_info, _is_installed, _make_environment_for_app_script, - dump_app_log_extract_for_debugging, _patch_legacy_helpers, _patch_legacy_php_versions, _patch_legacy_php_versions_in_settings, @@ -60,6 +59,7 @@ from yunohost.hook import ( hook_info, hook_callback, hook_exec, + hook_exec_with_script_debug_if_failure, CUSTOM_HOOK_FOLDER, ) from yunohost.tools import ( @@ -1496,37 +1496,14 @@ class RestoreManager: # Execute the app install script restore_failed = True try: - restore_retcode = hook_exec( + restore_failed, failure_message_with_debug_instructions = hook_exec_with_script_debug_if_failure( restore_script, chdir=app_backup_in_archive, env=env_dict, - )[0] - # "Common" app restore failure : the script failed and returned exit code != 0 - restore_failed = True if restore_retcode != 0 else False - if restore_failed: - error = m18n.n("app_restore_script_failed") - logger.error( - m18n.n("app_restore_failed", app=app_instance_name, error=error) - ) - failure_message_with_debug_instructions = operation_logger.error(error) - if Moulinette.interface.type != "api": - dump_app_log_extract_for_debugging(operation_logger) - # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception - except (KeyboardInterrupt, EOFError): - error = m18n.n("operation_interrupted") - logger.error( - m18n.n("app_restore_failed", app=app_instance_name, error=error) + operation_logger=operation_logger, + error_message_if_script_failed=m18n.n("app_restore_script_failed"), + error_message_if_failed=lambda e: m18n.n("app_restore_failed", app=app_instance_name, error=e) ) - failure_message_with_debug_instructions = operation_logger.error(error) - # Something wrong happened in Yunohost's code (most probably hook_exec) - except Exception: - import traceback - - error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error( - m18n.n("app_restore_failed", app=app_instance_name, error=error) - ) - failure_message_with_debug_instructions = operation_logger.error(error) finally: # Cleaning temporary scripts directory shutil.rmtree(tmp_workdir_for_app, ignore_errors=True) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index b589c27ea..c55809fce 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -498,6 +498,40 @@ def _hook_exec_python(path, args, env, loggers): return ret +def hook_exec_with_script_debug_if_failure(*args, **kwargs): + + operation_logger = kwargs.pop("operation_logger") + error_message_if_failed = kwargs.pop("error_message_if_failed") + error_message_if_script_failed = kwargs.pop("error_message_if_script_failed") + + failed = True + failure_message_with_debug_instructions = None + try: + retcode, retpayload = hook_exec(*args, **kwargs) + failed = True if retcode != 0 else False + if failed: + error = error_message_if_script_failed + logger.error(error_message_if_failed(error)) + failure_message_with_debug_instructions = operation_logger.error(error) + if Moulinette.interface.type != "api": + operation_logger.dump_script_log_extract_for_debugging() + # Script got manually interrupted ... + # N.B. : KeyboardInterrupt does not inherit from Exception + except (KeyboardInterrupt, EOFError): + error = m18n.n("operation_interrupted") + logger.error(error_message_if_failed(error)) + failure_message_with_debug_instructions = operation_logger.error(error) + # Something wrong happened in Yunohost's code (most probably hook_exec) + except Exception: + import traceback + + error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) + logger.error(error_message_if_failed(error)) + failure_message_with_debug_instructions = operation_logger.error(error) + + return failed, failure_message_with_debug_instructions + + def _extract_filename_parts(filename): """Extract hook parts from filename""" if "-" in filename: diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 3f6382af2..edb87af71 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -707,6 +707,52 @@ class OperationLogger(object): else: self.error(m18n.n("log_operation_unit_unclosed_properly")) + def dump_script_log_extract_for_debugging(self): + + with open(self.log_path, "r") as f: + lines = f.readlines() + + filters = [ + r"set [+-]x$", + r"set [+-]o xtrace$", + r"local \w+$", + r"local legacy_args=.*$", + r".*Helper used in legacy mode.*", + r"args_array=.*$", + r"local -A args_array$", + r"ynh_handle_getopts_args", + r"ynh_script_progression", + ] + + filters = [re.compile(f_) for f_ in filters] + + lines_to_display = [] + for line in lines: + + if ": " not in line.strip(): + continue + + # A line typically looks like + # 2019-10-19 16:10:27,611: DEBUG - + mysql -u piwigo --password=********** -B piwigo + # And we just want the part starting by "DEBUG - " + line = line.strip().split(": ", 1)[1] + + if any(filter_.search(line) for filter_ in filters): + continue + + lines_to_display.append(line) + + if line.endswith("+ ynh_exit_properly") or " + ynh_die " in line: + break + elif len(lines_to_display) > 20: + lines_to_display.pop(0) + + logger.warning( + "Here's an extract of the logs before the crash. It might help debugging the error:" + ) + for line in lines_to_display: + logger.info(line) + def _get_datetime_from_name(name): From 39006dbf134f61c242968aa3d60d52f0dea40322 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 20:55:17 +0200 Subject: [PATCH 078/119] tests: dunno what i'm doing but that scope=module is no good --- src/yunohost/tests/test_app_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/tests/test_app_config.py b/src/yunohost/tests/test_app_config.py index 003d9f3b2..af792c431 100644 --- a/src/yunohost/tests/test_app_config.py +++ b/src/yunohost/tests/test_app_config.py @@ -58,7 +58,7 @@ def clean(): os.system("systemctl start nginx") -@pytest.fixture(scope="module") +@pytest.fixture() def legacy_app(request): main_domain = _get_maindomain() @@ -78,7 +78,7 @@ def legacy_app(request): -@pytest.fixture(scope="module") +@pytest.fixture() def config_app(request): app_install( From 4f0df2bcfe31fbbb2f93a3987c1c59b273adb64f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 22:25:06 +0200 Subject: [PATCH 079/119] Define missing i18n keys --- locales/en.json | 8 ++++++-- src/yunohost/utils/config.py | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/locales/en.json b/locales/en.json index 2ac7aa42a..f1110df7b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -142,6 +142,10 @@ "certmanager_unable_to_parse_self_CA_name": "Could not parse name of self-signing authority (file: {file})", "certmanager_warning_subdomain_dns_record": "Subdomain '{subdomain}' does not resolve to the same IP address as '{domain}'. Some features will not be available until you fix this and regenerate the certificate.", "config_apply_failed": "Applying the new configuration failed: {error}", + "config_cant_set_value_on_section": "You can't set a single value on an entire config section.", + "config_no_panel": "No config panel found.", + "config_unknown_filter_key": "The filter key '{filter_key}' is incorrect.", + "config_version_not_supported": "Config panel versions '{version}' are not supported.", "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_warning": "Warning: This app may work, but is not well-integrated in YunoHost. Some features such as single sign-on and backup/restore might not be available. Install anyway? [{answers}] ", @@ -342,8 +346,8 @@ "global_settings_setting_backup_compress_tar_archives": "When creating new backups, compress the archives (.tar.gz) instead of uncompressed archives (.tar). N.B. : enabling this option means create lighter backup archives, but the initial backup procedure will be significantly longer and heavy on CPU.", "global_settings_setting_pop3_enabled": "Enable the POP3 protocol for the mail server", "global_settings_setting_security_experimental_enabled": "Enable experimental security features (don't enable this if you don't know what you're doing!)", - "global_settings_setting_security_nginx_redirect_to_https": "Redirect HTTP requests to HTTPs by default (DO NOT TURN OFF unless you really know what you're doing!)", "global_settings_setting_security_nginx_compatibility": "Compatibility vs. security tradeoff for the web server NGINX. Affects the ciphers (and other security-related aspects)", + "global_settings_setting_security_nginx_redirect_to_https": "Redirect HTTP requests to HTTPs by default (DO NOT TURN OFF unless you really know what you're doing!)", "global_settings_setting_security_password_admin_strength": "Admin password strength", "global_settings_setting_security_password_user_strength": "User password strength", "global_settings_setting_security_postfix_compatibility": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)", @@ -668,4 +672,4 @@ "yunohost_installing": "Installing YunoHost...", "yunohost_not_installed": "YunoHost is not correctly installed. Please run 'yunohost tools postinstall'", "yunohost_postinstall_end_tip": "The post-install completed! To finalize your setup, please consider:\n - adding a first user through the 'Users' section of the webadmin (or 'yunohost user create ' in command-line);\n - diagnose potential issues through the 'Diagnosis' section of the webadmin (or 'yunohost diagnosis run' in command-line);\n - reading the 'Finalizing your setup' and 'Getting to know YunoHost' parts in the admin documentation: https://yunohost.org/admindoc." -} +} \ No newline at end of file diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 6432856b0..b5b5fa6ed 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -105,13 +105,13 @@ class ConfigPanel: self._get_config_panel() if not self.config: - raise YunohostError("config_no_panel") + raise YunohostValidationError("config_no_panel") if (args is not None or args_file is not None) and value is not None: - raise YunohostError("config_args_value") + raise YunohostValidationError("You should either provide a value, or a serie of args/args_file, but not both at the same time", raw_msg=True) if self.filter_key.count(".") != 2 and value is not None: - raise YunohostError("config_set_value_on_section") + raise YunohostValidationError("config_cant_set_value_on_section") # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") @@ -171,7 +171,7 @@ class ConfigPanel: # Split filter_key filter_key = self.filter_key.split(".") if self.filter_key != "" else [] if len(filter_key) > 3: - raise YunohostError("config_too_many_sub_keys", key=self.filter_key) + raise YunohostError(f"The filter key {filter_key} has too many sub-levels, the max is 3.", raw_msg=True) if not os.path.exists(self.config_path): return None @@ -265,7 +265,7 @@ class ConfigPanel: self.config["panels"][0]["sections"][0]["options"][0] except (KeyError, IndexError): raise YunohostError( - "config_empty_or_bad_filter_key", filter_key=self.filter_key + "config_unknown_filter_key", filter_key=self.filter_key ) return self.config From c55b96b94b57a9b82bb55d3369dddb1c848ed040 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 23:08:48 +0200 Subject: [PATCH 080/119] config panel: rename source into bind --- data/helpers.d/config | 85 ++++++++++++++++++------------------ src/yunohost/utils/config.py | 2 +- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 3a2c55444..fe3a488de 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -21,15 +21,16 @@ for panel_name, panel in loaded_toml.items(): print(';'.join([ name, param.get('type', 'string'), - param.get('source', 'settings' if param.get('type', 'string') != 'file' else '') + param.get('bind', 'settings' if param.get('type', 'string') != 'file' else '') ])) EOL ` for line in $lines do - IFS=';' read short_setting type source <<< "$line" + # Split line into short_setting, type and bind + IFS=';' read short_setting type bind <<< "$line" local getter="get__${short_setting}" - sources[${short_setting}]="$source" + binds[${short_setting}]="$bind" types[${short_setting}]="$type" file_hash[${short_setting}]="" formats[${short_setting}]="" @@ -38,36 +39,36 @@ EOL old[$short_setting]="$($getter)" formats[${short_setting}]="yaml" - elif [[ "$source" == "" ]] ; then + elif [[ "$bind" == "" ]] ; then old[$short_setting]="YNH_NULL" # Get value from app settings or from another file elif [[ "$type" == "file" ]] ; then - if [[ "$source" == "settings" ]] ; then + if [[ "$bind" == "settings" ]] ; then ynh_die "File '${short_setting}' can't be stored in settings" fi - old[$short_setting]="$(ls $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" + old[$short_setting]="$(ls $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" # Get multiline text from settings or from a full file elif [[ "$type" == "text" ]] ; then - if [[ "$source" == "settings" ]] ; then + if [[ "$bind" == "settings" ]] ; then old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" - elif [[ "$source" == *":"* ]] ; then + elif [[ "$bind" == *":"* ]] ; then ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else - old[$short_setting]="$(cat $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" + old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" fi # Get value from a kind of key/value file else - if [[ "$source" == "settings" ]] ; then - source=":/etc/yunohost/apps/$app/settings.yml" + if [[ "$bind" == "settings" ]] ; then + bind=":/etc/yunohost/apps/$app/settings.yml" fi - local source_key="$(echo "$source" | cut -d: -f1)" - source_key=${source_key:-$short_setting} - local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - old[$short_setting]="$(ynh_read_var_in_file --file="${source_file}" --key="${source_key}")" + local bind_key="$(echo "$bind" | cut -d: -f1)" + bind_key=${bind_key:-$short_setting} + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}")" fi done @@ -79,63 +80,63 @@ _ynh_app_config_apply() { for short_setting in "${!old[@]}" do local setter="set__${short_setting}" - local source="${sources[$short_setting]}" + local bind="${binds[$short_setting]}" local type="${types[$short_setting]}" if [ "${changed[$short_setting]}" == "true" ] ; then # Apply setter if exists if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then $setter - elif [[ "$source" == "" ]] ; then + elif [[ "$bind" == "" ]] ; then continue # Save in a file elif [[ "$type" == "file" ]] ; then - if [[ "$source" == "settings" ]] ; then + if [[ "$bind" == "settings" ]] ; then ynh_die "File '${short_setting}' can't be stored in settings" fi - local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]] ; then - ynh_backup_if_checksum_is_different --file="$source_file" - rm -f "$source_file" - ynh_delete_file_checksum --file="$source_file" --update_only - ynh_print_info "File '$source_file' removed" + ynh_backup_if_checksum_is_different --file="$bind_file" + rm -f "$bind_file" + ynh_delete_file_checksum --file="$bind_file" --update_only + ynh_print_info "File '$bind_file' removed" else - ynh_backup_if_checksum_is_different --file="$source_file" - cp "${!short_setting}" "$source_file" - ynh_store_file_checksum --file="$source_file" --update_only - ynh_print_info "File '$source_file' overwrited with ${!short_setting}" + ynh_backup_if_checksum_is_different --file="$bind_file" + cp "${!short_setting}" "$bind_file" + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info "File '$bind_file' overwrited with ${!short_setting}" fi # Save value in app settings - elif [[ "$source" == "settings" ]] ; then + elif [[ "$bind" == "settings" ]] ; then ynh_app_setting_set $app $short_setting "${!short_setting}" ynh_print_info "Configuration key '$short_setting' edited in app settings" # Save multiline text in a file elif [[ "$type" == "text" ]] ; then - if [[ "$source" == *":"* ]] ; then + if [[ "$bind" == *":"* ]] ; then ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi - local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - ynh_backup_if_checksum_is_different --file="$source_file" - echo "${!short_setting}" > "$source_file" - ynh_store_file_checksum --file="$source_file" --update_only - ynh_print_info "File '$source_file' overwrited with the content you provieded in '${short_setting}' question" + local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + ynh_backup_if_checksum_is_different --file="$bind_file" + echo "${!short_setting}" > "$bind_file" + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info "File '$bind_file' overwrited with the content you provieded in '${short_setting}' question" # Set value into a kind of key/value file else - local source_key="$(echo "$source" | cut -d: -f1)" - source_key=${source_key:-$short_setting} - local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_key="$(echo "$bind" | cut -d: -f1)" + bind_key=${bind_key:-$short_setting} + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - ynh_backup_if_checksum_is_different --file="$source_file" - ynh_write_var_in_file --file="${source_file}" --key="${source_key}" --value="${!short_setting}" - ynh_store_file_checksum --file="$source_file" --update_only + ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" + ynh_store_file_checksum --file="$bind_file" --update_only # We stored the info in settings in order to be able to upgrade the app ynh_app_setting_set $app $short_setting "${!short_setting}" - ynh_print_info "Configuration key '$source_key' edited into $source_file" + ynh_print_info "Configuration key '$bind_key' edited into $bind_file" fi fi @@ -251,7 +252,7 @@ ynh_app_config_run() { declare -Ag old=() declare -Ag changed=() declare -Ag file_hash=() - declare -Ag sources=() + declare -Ag binds=() declare -Ag types=() declare -Ag formats=() diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index b5b5fa6ed..f078dda82 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -206,7 +206,7 @@ class ConfigPanel: } }, "options": { - "properties": ["ask", "type", "source", "help", "example", + "properties": ["ask", "type", "bind", "help", "example", "style", "icon", "placeholder", "visible", "optional", "choices", "yes", "no", "pattern", "limit", "min", "max", "step", "accept", "redact"], From 74714d0a6228c8881893266970693e17047684f4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 23:59:41 +0200 Subject: [PATCH 081/119] config panels: bind='' -> bind='null' --- data/helpers.d/config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index fe3a488de..049770651 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -39,7 +39,7 @@ EOL old[$short_setting]="$($getter)" formats[${short_setting}]="yaml" - elif [[ "$bind" == "" ]] ; then + elif [[ "$bind" == "null" ]] ; then old[$short_setting]="YNH_NULL" # Get value from app settings or from another file @@ -87,7 +87,7 @@ _ynh_app_config_apply() { if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then $setter - elif [[ "$bind" == "" ]] ; then + elif [[ "$bind" == "null" ]] ; then continue # Save in a file From 552db2e21db0049a7ee59a81f1de03c2b388ff36 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 00:01:14 +0200 Subject: [PATCH 082/119] config helpers: misc style + check if file exists --- data/helpers.d/config | 113 ++++++++++++++++++++++++++---------------- data/helpers.d/utils | 9 +++- 2 files changed, 76 insertions(+), 46 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 049770651..5d024442a 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -19,7 +19,7 @@ for panel_name, panel in loaded_toml.items(): if not isinstance(param, dict): continue print(';'.join([ - name, + name, param.get('type', 'string'), param.get('bind', 'settings' if param.get('type', 'string') != 'file' else '') ])) @@ -35,45 +35,53 @@ EOL file_hash[${short_setting}]="" formats[${short_setting}]="" # Get value from getter if exists - if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; + then old[$short_setting]="$($getter)" formats[${short_setting}]="yaml" - - elif [[ "$bind" == "null" ]] ; then + + elif [[ "$bind" == "null" ]]; + then old[$short_setting]="YNH_NULL" # Get value from app settings or from another file - elif [[ "$type" == "file" ]] ; then - if [[ "$bind" == "settings" ]] ; then + elif [[ "$type" == "file" ]]; + then + if [[ "$bind" == "settings" ]]; + then ynh_die "File '${short_setting}' can't be stored in settings" fi old[$short_setting]="$(ls $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" - + # Get multiline text from settings or from a full file - elif [[ "$type" == "text" ]] ; then - if [[ "$bind" == "settings" ]] ; then + elif [[ "$type" == "text" ]]; + then + if [[ "$bind" == "settings" ]]; + then old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" - elif [[ "$bind" == *":"* ]] ; then + elif [[ "$bind" == *":"* ]]; + then ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" fi - # Get value from a kind of key/value file + # Get value from a kind of key/value file else - if [[ "$bind" == "settings" ]] ; then + if [[ "$bind" == "settings" ]]; + then bind=":/etc/yunohost/apps/$app/settings.yml" fi local bind_key="$(echo "$bind" | cut -d: -f1)" bind_key=${bind_key:-$short_setting} local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}")" - + fi done - - + + } _ynh_app_config_apply() { @@ -82,21 +90,27 @@ _ynh_app_config_apply() { local setter="set__${short_setting}" local bind="${binds[$short_setting]}" local type="${types[$short_setting]}" - if [ "${changed[$short_setting]}" == "true" ] ; then + if [ "${changed[$short_setting]}" == "true" ]; + then # Apply setter if exists - if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; + then $setter - elif [[ "$bind" == "null" ]] ; then + elif [[ "$bind" == "null" ]]; + then continue # Save in a file - elif [[ "$type" == "file" ]] ; then - if [[ "$bind" == "settings" ]] ; then + elif [[ "$type" == "file" ]]; + then + if [[ "$bind" == "settings" ]]; + then ynh_die "File '${short_setting}' can't be stored in settings" fi local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - if [[ "${!short_setting}" == "" ]] ; then + if [[ "${!short_setting}" == "" ]]; + then ynh_backup_if_checksum_is_different --file="$bind_file" rm -f "$bind_file" ynh_delete_file_checksum --file="$bind_file" --update_only @@ -107,15 +121,18 @@ _ynh_app_config_apply() { ynh_store_file_checksum --file="$bind_file" --update_only ynh_print_info "File '$bind_file' overwrited with ${!short_setting}" fi - - # Save value in app settings - elif [[ "$bind" == "settings" ]] ; then + + # Save value in app settings + elif [[ "$bind" == "settings" ]]; + i then ynh_app_setting_set $app $short_setting "${!short_setting}" ynh_print_info "Configuration key '$short_setting' edited in app settings" - + # Save multiline text in a file - elif [[ "$type" == "text" ]] ; then - if [[ "$bind" == *":"* ]] ; then + elif [[ "$type" == "text" ]]; + then + if [[ "$bind" == *":"* ]]; + then ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" @@ -133,7 +150,7 @@ _ynh_app_config_apply() { ynh_backup_if_checksum_is_different --file="$bind_file" ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" ynh_store_file_checksum --file="$bind_file" --update_only - + # We stored the info in settings in order to be able to upgrade the app ynh_app_setting_set $app $short_setting "${!short_setting}" ynh_print_info "Configuration key '$bind_key' edited into $bind_file" @@ -146,8 +163,10 @@ _ynh_app_config_apply() { _ynh_app_config_show() { for short_setting in "${!old[@]}" do - if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then - if [[ "${formats[$short_setting]}" == "yaml" ]] ; then + if [[ "${old[$short_setting]}" != YNH_NULL ]]; + then + if [[ "${formats[$short_setting]}" == "yaml" ]]; + then ynh_return "${short_setting}:" ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/ /g')" else @@ -167,23 +186,28 @@ _ynh_app_config_validate() { for short_setting in "${!old[@]}" do changed[$short_setting]=false - if [ -z ${!short_setting+x} ]; then + if [ -z ${!short_setting+x} ]; + then # Assign the var with the old value in order to allows multiple # args validation declare "$short_setting"="${old[$short_setting]}" continue fi - if [ ! -z "${file_hash[${short_setting}]}" ] ; then + if [ ! -z "${file_hash[${short_setting}]}" ]; + then file_hash[old__$short_setting]="" file_hash[new__$short_setting]="" - if [ -f "${old[$short_setting]}" ] ; then + if [ -f "${old[$short_setting]}" ]; + then file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) - if [ -z "${!short_setting}" ] ; then + if [ -z "${!short_setting}" ]; + then changed[$short_setting]=true nothing_changed=false fi fi - if [ -f "${!short_setting}" ] ; then + if [ -f "${!short_setting}" ]; + then file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1) if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] then @@ -203,8 +227,8 @@ _ynh_app_config_validate() { then ynh_print_info "Nothing has changed" exit 0 - fi - + fi + # Run validation if something is changed ynh_script_progression --message="Validating the new configuration..." --weight=1 @@ -212,7 +236,8 @@ _ynh_app_config_validate() { do [[ "${changed[$short_setting]}" == "false" ]] && continue local result="" - if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then + if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; + then result="$(validate__$short_setting)" fi if [ -n "$result" ] @@ -228,8 +253,8 @@ _ynh_app_config_validate() { if [[ "$changes_validated" == "false" ]] then exit 0 - fi - + fi + } ynh_app_config_get() { @@ -255,19 +280,19 @@ ynh_app_config_run() { declare -Ag binds=() declare -Ag types=() declare -Ag formats=() - + case $1 in show) ynh_app_config_get ynh_app_config_show ;; - apply) + apply) max_progression=4 ynh_script_progression --message="Reading config panel description and current configuration..." ynh_app_config_get - + ynh_app_config_validate - + ynh_script_progression --message="Applying the new configuration..." ynh_app_config_apply ynh_script_progression --message="Configuration of $app completed" --last diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 0a820505c..54f936c4d 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -481,9 +481,9 @@ ynh_replace_vars () { # # This helpers match several var affectation use case in several languages # We don't use jq or equivalent to keep comments and blank space in files -# This helpers work line by line, it is not able to work correctly +# This helpers work line by line, it is not able to work correctly # if you have several identical keys in your files -# +# # Example of line this helpers can managed correctly # .yml # title: YunoHost documentation @@ -517,6 +517,9 @@ ynh_read_var_in_file() { ynh_handle_getopts_args "$@" set +o xtrace local filename="$(basename -- "$file")" + + [[ -f $file ]] || ynh_die "File $file does not exists" + local ext="${filename##*.}" local endline=',;' local assign="=>|:|=" @@ -576,6 +579,8 @@ ynh_write_var_in_file() { set +o xtrace local var_part='\s*\$?([\w.]*\[)?\s*["'"']?${key}['"'"]?\s*\]?\s*[:=]>?\s*' + [[ -f $file ]] || ynh_die "File $file does not exists" + local crazy_value="$(grep -i -o -P '^\s*\$?([\w.]*\[)?\s*["'"']?${key}['"'"]?\s*\]?\s*[:=]>?\s*\K.*(?=[\s,;]*$)' ${file} | head -n1)" # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" local first_char="${crazy_value:0:1}" From acf4d1c82a9e8ffefb2adc7628b8bd7f07be73bc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 00:01:36 +0200 Subject: [PATCH 083/119] i18n: 'danger' key is only defined in yunohost, not moulinette --- src/yunohost/utils/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index f078dda82..a4c27c693 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -771,7 +771,8 @@ class DisplayTextQuestion(Question): "warning": "yellow", "danger": "red", } - return colorize(m18n.g(self.style), color[self.style]) + f" {text}" + text = m18n.g(self.style) if self.style != "danger" else m18n.n("danger") + return colorize(text, color[self.style]) + f" {text}" else: return text From a5bf5246c5854fe367a50803745c64889a7d7042 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 00:02:41 +0200 Subject: [PATCH 084/119] Remove i18n stale strings --- locales/en.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/locales/en.json b/locales/en.json index f1110df7b..588e37357 100644 --- a/locales/en.json +++ b/locales/en.json @@ -17,7 +17,6 @@ "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason", "app_argument_required": "Argument '{name}' is required", - "app_change_url_failed_nginx_reload": "Could not reload NGINX. Here is the output of 'nginx -t':\n{nginx_errors}", "app_change_url_identical_domains": "The old and new domain/url_path are identical ('{domain}{path}'), nothing to do.", "app_change_url_no_script": "The app '{app_name}' doesn't support URL modification yet. Maybe you should upgrade it.", "app_change_url_success": "{app} URL is now {domain}{path}", @@ -397,10 +396,7 @@ "ldap_server_is_down_restart_it": "The LDAP service is down, attempt to restart it...", "log_app_action_run": "Run action of the '{}' app", "log_app_change_url": "Change the URL of the '{}' app", - "log_app_config_apply": "Apply config to the '{}' app", - "log_app_config_get": "Get a specific setting from config panel of the '{}' app", "log_app_config_set": "Apply config to the '{}' app", - "log_app_config_show": "Show the config panel of the '{}' app", "log_app_install": "Install the '{}' app", "log_app_makedefault": "Make '{}' the default app", "log_app_remove": "Remove the '{}' app", @@ -672,4 +668,4 @@ "yunohost_installing": "Installing YunoHost...", "yunohost_not_installed": "YunoHost is not correctly installed. Please run 'yunohost tools postinstall'", "yunohost_postinstall_end_tip": "The post-install completed! To finalize your setup, please consider:\n - adding a first user through the 'Users' section of the webadmin (or 'yunohost user create ' in command-line);\n - diagnose potential issues through the 'Diagnosis' section of the webadmin (or 'yunohost diagnosis run' in command-line);\n - reading the 'Finalizing your setup' and 'Getting to know YunoHost' parts in the admin documentation: https://yunohost.org/admindoc." -} \ No newline at end of file +} From ba6f90d966b48d6e6bf947d7c9a5826954307ac4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 00:15:25 +0200 Subject: [PATCH 085/119] YunohostError -> YunohostValidationError for some stuff --- src/yunohost/utils/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index a4c27c693..744849199 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -59,7 +59,7 @@ class ConfigPanel: self._get_config_panel() if not self.config: - raise YunohostError("config_no_panel") + raise YunohostValidationError("config_no_panel") # Read or get values and hydrate the config self._load_current_values() @@ -264,7 +264,7 @@ class ConfigPanel: try: self.config["panels"][0]["sections"][0]["options"][0] except (KeyError, IndexError): - raise YunohostError( + raise YunohostValidationError( "config_unknown_filter_key", filter_key=self.filter_key ) From 4a3d6e53c67f65a2e34109e7c4b33a2a43511a62 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 6 Sep 2021 00:58:46 +0200 Subject: [PATCH 086/119] [fix] ynh_read_var_in_file with ini file --- data/helpers.d/utils | 11 +++++++---- tests/test_helpers.d/ynhtest_config.sh | 12 ++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 54f936c4d..97aae12ef 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -515,7 +515,7 @@ ynh_read_var_in_file() { local key # Manage arguments with getopts ynh_handle_getopts_args "$@" - set +o xtrace + #set +o xtrace local filename="$(basename -- "$file")" [[ -f $file ]] || ynh_die "File $file does not exists" @@ -525,9 +525,12 @@ ynh_read_var_in_file() { local assign="=>|:|=" local comments="#" local string="\"'" - if [[ "yaml yml toml ini env" =~ *"$ext"* ]]; then + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then endline='#' fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then comments="//" fi @@ -535,7 +538,7 @@ ynh_read_var_in_file() { local var_part='^\s*(?:(const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' var_part+="[$string]?${key}[$string]?" var_part+='\s*\]?\s*' - var_part+="(?:$assign)" + var_part+="($assign)" var_part+='\s*' # Extract the part after assignation sign @@ -546,7 +549,7 @@ ynh_read_var_in_file() { fi # Remove comments if needed - local expression="$(echo "$expression_with_comment" | sed "s@$comments[^$string]*\$@@" | sed "s@\s*[$endline]*\s*]*\$@@")" + local expression="$(echo "$expression_with_comment" | sed "s@$comments[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" local first_char="${expression:0:1}" if [[ "$first_char" == '"' ]] ; then diff --git a/tests/test_helpers.d/ynhtest_config.sh b/tests/test_helpers.d/ynhtest_config.sh index 36165e3ac..3be72d191 100644 --- a/tests/test_helpers.d/ynhtest_config.sh +++ b/tests/test_helpers.d/ynhtest_config.sh @@ -79,8 +79,8 @@ ENABLED = False # TITLE = "Old title" TITLE = "Lorem Ipsum" THEME = "colib'ris" -EMAIL = "root@example.com" // This is a comment without quotes -PORT = 1234 // This is a comment without quotes +EMAIL = "root@example.com" # This is a comment without quotes +PORT = 1234 # This is a comment without quotes URL = 'https://yunohost.org' DICT = {} DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" @@ -156,8 +156,8 @@ enabled = False # title = Old title title = Lorem Ipsum theme = colib'ris -email = root@example.com # This is a comment without quotes -port = 1234 # This is a comment without quotes +email = root@example.com ; This is a comment without quotes +port = 1234 ; This is a comment without quotes url = https://yunohost.org [dict] ldap_base = ou=users,dc=yunohost,dc=org @@ -175,10 +175,10 @@ EOF test "$(_read_ini "$file" "theme")" == "colib'ris" test "$(ynh_read_var_in_file "$file" "theme")" == "colib'ris" - test "$(_read_ini "$file" "email")" == "root@example.com" + #test "$(_read_ini "$file" "email")" == "root@example.com" test "$(ynh_read_var_in_file "$file" "email")" == "root@example.com" - test "$(_read_ini "$file" "port")" == "1234" + #test "$(_read_ini "$file" "port")" == "1234" test "$(ynh_read_var_in_file "$file" "port")" == "1234" test "$(_read_ini "$file" "url")" == "https://yunohost.org" From e60804a69f030ef06be1af68d3457a07981ef88e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 02:07:56 +0200 Subject: [PATCH 087/119] config helpers: misc syntax issues --- data/helpers.d/config | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 5d024442a..5970351f7 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -40,14 +40,14 @@ EOL old[$short_setting]="$($getter)" formats[${short_setting}]="yaml" - elif [[ "$bind" == "null" ]]; + elif [[ "$bind" == "null" ]] then old[$short_setting]="YNH_NULL" # Get value from app settings or from another file - elif [[ "$type" == "file" ]]; + elif [[ "$type" == "file" ]] then - if [[ "$bind" == "settings" ]]; + if [[ "$bind" == "settings" ]] then ynh_die "File '${short_setting}' can't be stored in settings" fi @@ -55,12 +55,12 @@ EOL file_hash[$short_setting]="true" # Get multiline text from settings or from a full file - elif [[ "$type" == "text" ]]; + elif [[ "$type" == "text" ]] then - if [[ "$bind" == "settings" ]]; + if [[ "$bind" == "settings" ]] then old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" - elif [[ "$bind" == *":"* ]]; + elif [[ "$bind" == *":"* ]] then ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else @@ -69,7 +69,7 @@ EOL # Get value from a kind of key/value file else - if [[ "$bind" == "settings" ]]; + if [[ "$bind" == "settings" ]] then bind=":/etc/yunohost/apps/$app/settings.yml" fi @@ -90,26 +90,26 @@ _ynh_app_config_apply() { local setter="set__${short_setting}" local bind="${binds[$short_setting]}" local type="${types[$short_setting]}" - if [ "${changed[$short_setting]}" == "true" ]; + if [ "${changed[$short_setting]}" == "true" ] then # Apply setter if exists if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then $setter - elif [[ "$bind" == "null" ]]; + elif [[ "$bind" == "null" ]] then continue # Save in a file - elif [[ "$type" == "file" ]]; + elif [[ "$type" == "file" ]] then - if [[ "$bind" == "settings" ]]; + if [[ "$bind" == "settings" ]] then ynh_die "File '${short_setting}' can't be stored in settings" fi local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - if [[ "${!short_setting}" == "" ]]; + if [[ "${!short_setting}" == "" ]] then ynh_backup_if_checksum_is_different --file="$bind_file" rm -f "$bind_file" @@ -123,15 +123,15 @@ _ynh_app_config_apply() { fi # Save value in app settings - elif [[ "$bind" == "settings" ]]; - i then + elif [[ "$bind" == "settings" ]] + then ynh_app_setting_set $app $short_setting "${!short_setting}" ynh_print_info "Configuration key '$short_setting' edited in app settings" # Save multiline text in a file - elif [[ "$type" == "text" ]]; + elif [[ "$type" == "text" ]] then - if [[ "$bind" == *":"* ]]; + if [[ "$bind" == *":"* ]] then ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi @@ -163,9 +163,9 @@ _ynh_app_config_apply() { _ynh_app_config_show() { for short_setting in "${!old[@]}" do - if [[ "${old[$short_setting]}" != YNH_NULL ]]; + if [[ "${old[$short_setting]}" != YNH_NULL ]] then - if [[ "${formats[$short_setting]}" == "yaml" ]]; + if [[ "${formats[$short_setting]}" == "yaml" ]] then ynh_return "${short_setting}:" ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/ /g')" @@ -186,27 +186,27 @@ _ynh_app_config_validate() { for short_setting in "${!old[@]}" do changed[$short_setting]=false - if [ -z ${!short_setting+x} ]; + if [ -z ${!short_setting+x} ] then # Assign the var with the old value in order to allows multiple # args validation declare "$short_setting"="${old[$short_setting]}" continue fi - if [ ! -z "${file_hash[${short_setting}]}" ]; + if [ ! -z "${file_hash[${short_setting}]}" ] then file_hash[old__$short_setting]="" file_hash[new__$short_setting]="" - if [ -f "${old[$short_setting]}" ]; + if [ -f "${old[$short_setting]}" ] then file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) - if [ -z "${!short_setting}" ]; + if [ -z "${!short_setting}" ] then changed[$short_setting]=true nothing_changed=false fi fi - if [ -f "${!short_setting}" ]; + if [ -f "${!short_setting}" ] then file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1) if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]] From 66fcea72e5fbc4ddef4f6e66d4b465c365b3924d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 02:10:15 +0200 Subject: [PATCH 088/119] config: Add more tests for regular setting / bind / custom function --- src/yunohost/tests/test_app_config.py | 58 ++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/src/yunohost/tests/test_app_config.py b/src/yunohost/tests/test_app_config.py index af792c431..4ace0aaf9 100644 --- a/src/yunohost/tests/test_app_config.py +++ b/src/yunohost/tests/test_app_config.py @@ -5,8 +5,11 @@ import pytest from .conftest import get_test_apps_dir +from moulinette.utils.filesystem import read_file + from yunohost.domain import _get_maindomain from yunohost.app import ( + app_setting, app_install, app_remove, _is_installed, @@ -15,7 +18,7 @@ from yunohost.app import ( app_ssowatconf, ) -from yunohost.utils.error import YunohostValidationError +from yunohost.utils.error import YunohostError, YunohostValidationError def setup_function(function): @@ -128,17 +131,68 @@ def test_app_config_get_nonexistentstuff(config_app): app_config_get(config_app, "main.components.nonexistent") -def test_app_config_set_boolean(config_app): +def test_app_config_regular_setting(config_app): assert app_config_get(config_app, "main.components.boolean") is None app_config_set(config_app, "main.components.boolean", "no") assert app_config_get(config_app, "main.components.boolean") == "0" + assert app_setting(config_app, "boolean") == "0" app_config_set(config_app, "main.components.boolean", "yes") assert app_config_get(config_app, "main.components.boolean") == "1" + assert app_setting(config_app, "boolean") == "1" with pytest.raises(YunohostValidationError): app_config_set(config_app, "main.components.boolean", "pwet") + + +def test_app_config_bind_on_file(config_app): + + # c.f. conf/test.php in the config app + assert '$arg5= "Arg5 value";' in read_file("/var/www/config_app/test.php") + assert app_config_get(config_app, "bind.variable.arg5") == "Arg5 value" + assert app_setting(config_app, "arg5") is None + + app_config_set(config_app, "bind.variable.arg5", "Foo Bar") + + assert '$arg5= "Foo Bar";' in read_file("/var/www/config_app/test.php") + assert app_config_get(config_app, "bind.variable.arg5") == "Foo Bar" + assert app_setting(config_app, "arg5") == "Foo Bar" + + +def test_app_config_custom_get(config_app): + + assert app_setting(config_app, "arg9") is None + assert "Files in /var/www" in app_config_get(config_app, "bind.function.arg9")["ask"]["en"] + assert app_setting(config_app, "arg9") is None + + +def test_app_config_custom_validator(config_app): + + # c.f. the config script + # arg8 is a password that must be at least 8 chars + assert not os.path.exists("/var/www/config_app/password") + assert app_setting(config_app, "arg8") is None + + with pytest.raises(YunohostValidationError): + app_config_set(config_app, "bind.function.arg8", "pZo6i7u91h") + + assert not os.path.exists("/var/www/config_app/password") + assert app_setting(config_app, "arg8") is None + + +def test_app_config_custom_set(config_app): + + assert not os.path.exists("/var/www/config_app/password") + assert app_setting(config_app, "arg8") is None + + app_config_set(config_app, "bind.function.arg8", "OneSuperStrongPassword") + + assert os.path.exists("/var/www/config_app/password") + content = read_file("/var/www/config_app/password") + assert "OneSuperStrongPassword" not in content + assert content.startswith("$6$saltsalt$") + assert app_setting(config_app, "arg8") is None From 0789eca4e06279d45a23d5eacb78eb3881801989 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 02:10:45 +0200 Subject: [PATCH 089/119] config: Tweak logic to return a validation error when custom validation fails --- data/helpers.d/config | 16 ++++++++++++++-- src/yunohost/app.py | 17 +++++++++++++---- src/yunohost/utils/config.py | 8 ++------ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 5970351f7..71d41fbe9 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -242,8 +242,20 @@ _ynh_app_config_validate() { fi if [ -n "$result" ] then - local key="YNH_ERROR_${short_setting}" - ynh_return "$key: \"$result\"" + # + # Return a yaml such as: + # + # validation_errors: + # some_key: "An error message" + # some_other_key: "Another error message" + # + # We use changes_validated to know if this is + # the first validation error + if [[ "$changes_validated" == true ]] + then + ynh_return "validation_errors:" + fi + ynh_return " ${short_setting}: \"$result\"" changes_validated=false fi done diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e9712edb4..4047369e0 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1695,9 +1695,7 @@ def app_config_set( Question.operation_logger = operation_logger - result = config_.set(key, value, args, args_file, operation_logger=operation_logger) - - return result + return config_.set(key, value, args, args_file, operation_logger=operation_logger) class AppConfigPanel(ConfigPanel): @@ -1715,7 +1713,18 @@ class AppConfigPanel(ConfigPanel): def _apply(self): env = {key: str(value) for key, value in self.new_values.items()} - self.errors = self._call_config_script("apply", env=env) + return_content = self._call_config_script("apply", env=env) + + # If the script returned validation error + # raise a ValidationError exception using + # the first key + if return_content: + for key, message in return_content.get("validation_errors").items(): + raise YunohostValidationError( + "app_argument_invalid", + name=key, + error=message, + ) def _call_config_script(self, action, env={}): from yunohost.hook import hook_exec diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 744849199..6b491386f 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -135,6 +135,8 @@ class ConfigPanel: try: self._apply() + except YunohostError: + raise # Script got manually interrupted ... # N.B. : KeyboardInterrupt does not inherit from Exception except (KeyboardInterrupt, EOFError): @@ -152,16 +154,10 @@ class ConfigPanel: # Delete files uploaded from API FileQuestion.clean_upload_dirs() - if self.errors: - return { - "errors": self.errors, - } - self._reload_services() logger.success("Config updated as expected") operation_logger.success() - return {} def _get_toml(self): return read_toml(self.config_path) From 060d5b6dc5118885f69032dd29463e961a5e1ab2 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 6 Sep 2021 04:20:35 +0200 Subject: [PATCH 090/119] [fix] ynh_write_var_in_file and after option --- data/helpers.d/config | 8 +- data/helpers.d/utils | 94 ++++++++++++++++++----- tests/test_helpers.d/ynhtest_config.sh | 102 +++++++++++++------------ 3 files changed, 136 insertions(+), 68 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 5d024442a..e834db041 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -124,7 +124,7 @@ _ynh_app_config_apply() { # Save value in app settings elif [[ "$bind" == "settings" ]]; - i then + then ynh_app_setting_set $app $short_setting "${!short_setting}" ynh_print_info "Configuration key '$short_setting' edited in app settings" @@ -143,8 +143,14 @@ _ynh_app_config_apply() { # Set value into a kind of key/value file else + local bind_after="" local bind_key="$(echo "$bind" | cut -d: -f1)" bind_key=${bind_key:-$short_setting} + if [[ "$bind_key" == *">"* ]]; + then + bind_after="$(echo "${bind_key}" | cut -d'>' -f1)" + bind_key="$(echo "${bind_key}" | cut -d'>' -f2)" + fi local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 97aae12ef..5ecc0cf0b 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -507,19 +507,30 @@ ynh_replace_vars () { # # Requires YunoHost version 4.3 or higher. ynh_read_var_in_file() { - set +o xtrace # Declare an array to define the options of this helper. - local legacy_args=fk - local -A args_array=( [f]=file= [k]=key= ) + local legacy_args=fka + local -A args_array=( [f]=file= [k]=key= [a]=after=) local file local key + local after # Manage arguments with getopts ynh_handle_getopts_args "$@" - #set +o xtrace - local filename="$(basename -- "$file")" + after="${after:-}" [[ -f $file ]] || ynh_die "File $file does not exists" + # Get the line number after which we search for the variable + local line_number=1 + if [[ -n "$after" ]]; + then + line_number=$(grep -n $after $file | cut -d: -f1) + if [[ -z "$line_number" ]]; + then + return 1 + fi + fi + + local filename="$(basename -- "$file")" local ext="${filename##*.}" local endline=',;' local assign="=>|:|=" @@ -535,14 +546,14 @@ ynh_read_var_in_file() { comments="//" fi local list='\[\s*['$string']?\w+['$string']?\]' - local var_part='^\s*(?:(const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' var_part+="[$string]?${key}[$string]?" var_part+='\s*\]?\s*' var_part+="($assign)" var_part+='\s*' # Extract the part after assignation sign - local expression_with_comment="$(grep -i -o -P $var_part'\K.*$' ${file} || echo YNH_NULL | head -n1)" + local expression_with_comment="$(tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL | head -n1)" if [[ "$expression_with_comment" == "YNH_NULL" ]]; then echo YNH_NULL return 0 @@ -570,40 +581,85 @@ ynh_read_var_in_file() { # # Requires YunoHost version 4.3 or higher. ynh_write_var_in_file() { - set +o xtrace # Declare an array to define the options of this helper. - local legacy_args=fkv - local -A args_array=( [f]=file= [k]=key= [v]=value=) + local legacy_args=fkva + local -A args_array=( [f]=file= [k]=key= [v]=value= [a]=after=) local file local key local value + local after # Manage arguments with getopts ynh_handle_getopts_args "$@" - set +o xtrace - local var_part='\s*\$?([\w.]*\[)?\s*["'"']?${key}['"'"]?\s*\]?\s*[:=]>?\s*' + after="${after:-}" [[ -f $file ]] || ynh_die "File $file does not exists" - local crazy_value="$(grep -i -o -P '^\s*\$?([\w.]*\[)?\s*["'"']?${key}['"'"]?\s*\]?\s*[:=]>?\s*\K.*(?=[\s,;]*$)' ${file} | head -n1)" - # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)" - local first_char="${crazy_value:0:1}" + # Get the line number after which we search for the variable + local line_number=1 + if [[ -n "$after" ]]; + then + line_number=$(grep -n $after $file | cut -d: -f1) + if [[ -z "$line_number" ]]; + then + return 1 + fi + fi + local range="${line_number},\$ " + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$(tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + return 1 + fi + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@$comments[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + endline=${expression_with_comment#"$expression"} + endline="$(echo "$endline" | sed 's/\\/\\\\/g')" + value="$(echo "$value" | sed 's/\\/\\\\/g')" + local first_char="${expression:0:1}" delimiter=$'\001' if [[ "$first_char" == '"' ]] ; then # \ and sed is quite complex you need 2 \\ to get one in a sed # So we need \\\\ to go through 2 sed value="$(echo "$value" | sed 's/"/\\\\"/g')" - sed -ri s$delimiter'^('"${var_part}"'")([^"]|\\")*("[\s;,]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} + sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file} elif [[ "$first_char" == "'" ]] ; then # \ and sed is quite complex you need 2 \\ to get one in a sed # However double quotes implies to double \\ to # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" - sed -ri "s$delimiter^(${var_part}')([^']|\\')*('"'[\s,;]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file} + sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file} else - if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]] ; then value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' fi - sed -ri "s$delimiter^(${var_part}).*"'$'$delimiter'\1'"${value}"$delimiter'i' ${file} + if [[ "$ext" =~ ^yaml|yml$ ]] ; then + value=" $value" + fi + sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file} fi } diff --git a/tests/test_helpers.d/ynhtest_config.sh b/tests/test_helpers.d/ynhtest_config.sh index 3be72d191..b64943a48 100644 --- a/tests/test_helpers.d/ynhtest_config.sh +++ b/tests/test_helpers.d/ynhtest_config.sh @@ -34,6 +34,8 @@ DICT = {} DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" DICT['ldap_conf'] = {} DICT['ldap_conf']['user'] = "camille" +# YNH_ICI +DICT['TITLE'] = "Hello world" EOF test "$(_read_py "$file" "FOO")" == "None" @@ -60,6 +62,8 @@ EOF test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" test "$(ynh_read_var_in_file "$file" "user")" == "camille" + + test "$(ynh_read_var_in_file "$file" "TITLE" "YNH_ICI")" == "Hello world" ! _read_py "$file" "NONEXISTENT" test "$(ynh_read_var_in_file "$file" "NONEXISTENT")" == "YNH_NULL" @@ -68,7 +72,7 @@ EOF test "$(ynh_read_var_in_file "$file" "ENABLE")" == "YNH_NULL" } -nhtest_config_write_py() { +ynhtest_config_write_py() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.py" @@ -84,11 +88,13 @@ PORT = 1234 # This is a comment without quotes URL = 'https://yunohost.org' DICT = {} DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org" +# YNH_ICI +DICT['TITLE'] = "Hello world" EOF - #ynh_write_var_in_file "$file" "FOO" "bar" - #test "$(_read_py "$file" "FOO")" == "bar" # FIXME FIXME FIXME - #test "$(ynh_read_var_in_file "$file" "FOO")" == "bar" + ynh_write_var_in_file "$file" "FOO" "bar" + test "$(_read_py "$file" "FOO")" == "bar" + test "$(ynh_read_var_in_file "$file" "FOO")" == "bar" ynh_write_var_in_file "$file" "ENABLED" "True" test "$(_read_py "$file" "ENABLED")" == "True" @@ -116,12 +122,15 @@ EOF ynh_write_var_in_file "$file" "ldap_base" "ou=users,dc=yunohost,dc=org" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" + + ynh_write_var_in_file "$file" "TITLE" "YOLO" "YNH_ICI" + test "$(ynh_read_var_in_file "$file" "TITLE" "YNH_ICI")" == "YOLO" - ynh_write_var_in_file "$file" "NONEXISTENT" "foobar" + ! ynh_write_var_in_file "$file" "NONEXISTENT" "foobar" ! _read_py "$file" "NONEXISTENT" test "$(ynh_read_var_in_file "$file" "NONEXISTENT")" == "YNH_NULL" - ynh_write_var_in_file "$file" "ENABLE" "foobar" + ! ynh_write_var_in_file "$file" "ENABLE" "foobar" ! _read_py "$file" "ENABLE" test "$(ynh_read_var_in_file "$file" "ENABLE")" == "YNH_NULL" @@ -194,7 +203,7 @@ EOF } -nhtest_config_write_ini() { +ynhtest_config_write_ini() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.ini" @@ -231,11 +240,11 @@ EOF test "$(ynh_read_var_in_file "$file" "theme")" == "super-awesome-theme" ynh_write_var_in_file "$file" "email" "sam@domain.tld" - test "$(_read_ini "$file" "email")" == "sam@domain.tld" + test "$(_read_ini "$file" "email")" == "sam@domain.tld # This is a comment without quotes" test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" ynh_write_var_in_file "$file" "port" "5678" - test "$(_read_ini "$file" "port")" == "5678" + test "$(_read_ini "$file" "port")" == "5678 # This is a comment without quotes" test "$(ynh_read_var_in_file "$file" "port")" == "5678" ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" @@ -245,11 +254,11 @@ EOF ynh_write_var_in_file "$file" "ldap_base" "ou=users,dc=yunohost,dc=org" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org" - ynh_write_var_in_file "$file" "nonexistent" "foobar" + ! ynh_write_var_in_file "$file" "nonexistent" "foobar" ! _read_ini "$file" "nonexistent" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - ynh_write_var_in_file "$file" "enable" "foobar" + ! ynh_write_var_in_file "$file" "enable" "foobar" ! _read_ini "$file" "enable" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" @@ -322,7 +331,7 @@ EOF } -nhtest_config_write_yaml() { +ynhtest_config_write_yaml() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.yml" @@ -340,10 +349,10 @@ dict: ldap_base: ou=users,dc=yunohost,dc=org EOF - #ynh_write_var_in_file "$file" "foo" "bar" + ynh_write_var_in_file "$file" "foo" "bar" # cat $dummy_dir/dummy.yml # to debug - #! test "$(_read_yaml "$file" "foo")" == "bar" # FIXME FIXME FIXME : writing broke the yaml syntax... "foo:bar" (no space aftr :) - #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" + ! test "$(_read_yaml "$file" "foo")" == "bar" # writing broke the yaml syntax... "foo:bar" (no space aftr :) + test "$(ynh_read_var_in_file "$file" "foo")" == "bar" ynh_write_var_in_file "$file" "enabled" "true" test "$(_read_yaml "$file" "enabled")" == "True" @@ -372,10 +381,10 @@ EOF ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" - ynh_write_var_in_file "$file" "nonexistent" "foobar" + ! ynh_write_var_in_file "$file" "nonexistent" "foobar" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - ynh_write_var_in_file "$file" "enable" "foobar" + ! ynh_write_var_in_file "$file" "enable" "foobar" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" test "$(ynh_read_var_in_file "$file" "enabled")" == "true" } @@ -449,7 +458,7 @@ EOF } -nhtest_config_write_json() { +ynhtest_config_write_json() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.json" @@ -468,14 +477,15 @@ nhtest_config_write_json() { } EOF - #ynh_write_var_in_file "$file" "foo" "bar" - #cat $file - #test "$(_read_json "$file" "foo")" == "bar" # FIXME FIXME FIXME - #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" + ynh_write_var_in_file "$file" "foo" "bar" + cat $file + test "$(_read_json "$file" "foo")" == "bar" + test "$(ynh_read_var_in_file "$file" "foo")" == "bar" - #ynh_write_var_in_file "$file" "enabled" "true" - #test "$(_read_json "$file" "enabled")" == "True" # FIXME FIXME FIXME - #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" + ynh_write_var_in_file "$file" "enabled" "true" + cat $file + test "$(_read_json "$file" "enabled")" == "true" + test "$(ynh_read_var_in_file "$file" "enabled")" == "true" ynh_write_var_in_file "$file" "title" "Foo Bar" cat $file @@ -492,10 +502,9 @@ EOF test "$(_read_json "$file" "email")" == "sam@domain.tld" test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" - #ynh_write_var_in_file "$file" "port" "5678" - #cat $file - #test "$(_read_json "$file" "port")" == "5678" # FIXME FIXME FIXME - #test "$(ynh_read_var_in_file "$file" "port")" == "5678" + ynh_write_var_in_file "$file" "port" "5678" + test "$(_read_json "$file" "port")" == "5678" + test "$(ynh_read_var_in_file "$file" "port")" == "5678" ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" test "$(_read_json "$file" "url")" == "https://domain.tld/foobar" @@ -504,12 +513,12 @@ EOF ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" - ynh_write_var_in_file "$file" "nonexistent" "foobar" + ! ynh_write_var_in_file "$file" "nonexistent" "foobar" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - ynh_write_var_in_file "$file" "enable" "foobar" + ! ynh_write_var_in_file "$file" "enable" "foobar" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" - #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME + test "$(ynh_read_var_in_file "$file" "enabled")" == "true" } ####################### @@ -589,7 +598,7 @@ EOF } -nhtest_config_write_php() { +ynhtest_config_write_php() { local dummy_dir="$(mktemp -d -p $VAR_WWW)" file="$dummy_dir/dummy.php" @@ -610,15 +619,13 @@ nhtest_config_write_php() { ?> EOF - #ynh_write_var_in_file "$file" "foo" "bar" - #cat $file - #test "$(_read_php "$file" "foo")" == "bar" - #test "$(ynh_read_var_in_file "$file" "foo")" == "bar" # FIXME FIXME FIXME + ynh_write_var_in_file "$file" "foo" "bar" + test "$(_read_php "$file" "foo")" == "bar" + test "$(ynh_read_var_in_file "$file" "foo")" == "bar" - #ynh_write_var_in_file "$file" "enabled" "true" - #cat $file - #test "$(_read_php "$file" "enabled")" == "true" - #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME FIXME FIXME + ynh_write_var_in_file "$file" "enabled" "true" + test "$(_read_php "$file" "enabled")" == "true" + test "$(ynh_read_var_in_file "$file" "enabled")" == "true" ynh_write_var_in_file "$file" "title" "Foo Bar" cat $file @@ -635,10 +642,9 @@ EOF test "$(_read_php "$file" "email")" == "sam@domain.tld" test "$(ynh_read_var_in_file "$file" "email")" == "sam@domain.tld" - #ynh_write_var_in_file "$file" "port" "5678" - #cat $file - #test "$(_read_php "$file" "port")" == "5678" # FIXME FIXME FIXME - #test "$(ynh_read_var_in_file "$file" "port")" == "5678" + ynh_write_var_in_file "$file" "port" "5678" + test "$(_read_php "$file" "port")" == "5678" + test "$(ynh_read_var_in_file "$file" "port")" == "5678" ynh_write_var_in_file "$file" "url" "https://domain.tld/foobar" test "$(_read_php "$file" "url")" == "https://domain.tld/foobar" @@ -647,10 +653,10 @@ EOF ynh_write_var_in_file "$file" "ldap_base" "ou=foobar,dc=domain,dc=tld" test "$(ynh_read_var_in_file "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld" - ynh_write_var_in_file "$file" "nonexistent" "foobar" + ! ynh_write_var_in_file "$file" "nonexistent" "foobar" test "$(ynh_read_var_in_file "$file" "nonexistent")" == "YNH_NULL" - ynh_write_var_in_file "$file" "enable" "foobar" + ! ynh_write_var_in_file "$file" "enable" "foobar" test "$(ynh_read_var_in_file "$file" "enable")" == "YNH_NULL" - #test "$(ynh_read_var_in_file "$file" "enabled")" == "true" # FIXME + test "$(ynh_read_var_in_file "$file" "enabled")" == "true" } From f2d0732825abee1e94440d0988e04a03a5bd745a Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 6 Sep 2021 04:28:30 +0200 Subject: [PATCH 091/119] [fix] Missing call to --after args --- data/helpers.d/config | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 62ee228d9..6f04eaa11 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -69,14 +69,20 @@ EOL # Get value from a kind of key/value file else + local bind_after="" if [[ "$bind" == "settings" ]] then bind=":/etc/yunohost/apps/$app/settings.yml" fi local bind_key="$(echo "$bind" | cut -d: -f1)" bind_key=${bind_key:-$short_setting} + if [[ "$bind_key" == *">"* ]]; + then + bind_after="$(echo "${bind_key}" | cut -d'>' -f1)" + bind_key="$(echo "${bind_key}" | cut -d'>' -f2)" + fi local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" - old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}")" + old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}" --after="${bind_after}")" fi done @@ -154,7 +160,7 @@ _ynh_app_config_apply() { local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" - ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" + ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" --after="${bind_after}" ynh_store_file_checksum --file="$bind_file" --update_only # We stored the info in settings in order to be able to upgrade the app From 050185a0c280f615aec995613ad7735296c2510e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Sep 2021 21:32:52 +0200 Subject: [PATCH 092/119] config panel: fix file type returning weird value --- data/helpers.d/config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 6f04eaa11..4ad52e038 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -21,7 +21,7 @@ for panel_name, panel in loaded_toml.items(): print(';'.join([ name, param.get('type', 'string'), - param.get('bind', 'settings' if param.get('type', 'string') != 'file' else '') + param.get('bind', 'settings' if param.get('type', 'string') != 'file' else 'null') ])) EOL ` @@ -51,7 +51,7 @@ EOL then ynh_die "File '${short_setting}' can't be stored in settings" fi - old[$short_setting]="$(ls $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" + old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2> /dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" # Get multiline text from settings or from a full file From 6cfefd4b9a1a0d4b7a36471e6db6406b3a7ef98d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 7 Sep 2021 18:31:37 +0200 Subject: [PATCH 093/119] Attempt to fix backup test --- src/yunohost/tests/test_backuprestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 30204fa86..df84ee47f 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -344,7 +344,7 @@ def test_backup_script_failure_handling(monkeypatch, mocker): # Create a backup of this app and simulate a crash (patching the backup # call with monkeypatch). We also patch m18n to check later it's been called # with the expected error message key - monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) + monkeypatch.setattr("yunohost.hook.hook_exec", custom_hook_exec) with message(mocker, "backup_app_failed", app="backup_recommended_app"): with raiseYunohostError(mocker, "backup_nothings_done"): From 6b3af5fa3327d7538b4ff8bdec460fd5fc1a4bd8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 7 Sep 2021 19:39:10 +0200 Subject: [PATCH 094/119] Anotha shruberry --- src/yunohost/tests/test_backuprestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index df84ee47f..8db6982df 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -469,7 +469,7 @@ def test_restore_app_script_failure_handling(monkeypatch, mocker): monkeypatch.undo() return (1, None) - monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) + monkeypatch.setattr("yunohost.hook.hook_exec", custom_hook_exec) assert not _is_installed("wordpress") From b007102842f38156a1bb6d9b6c652b9f4c20cb66 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 7 Sep 2021 20:09:27 +0200 Subject: [PATCH 095/119] [fix] Question tests --- src/yunohost/tests/conftest.py | 7 +- src/yunohost/tests/test_questions.py | 432 +++++++++++++++++---------- src/yunohost/utils/config.py | 91 +++--- 3 files changed, 331 insertions(+), 199 deletions(-) diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py index 6b4e2c3fd..8c00693c0 100644 --- a/src/yunohost/tests/conftest.py +++ b/src/yunohost/tests/conftest.py @@ -84,9 +84,12 @@ def pytest_cmdline_main(config): class DummyInterface: - type = "test" + type = "cli" - def prompt(*args, **kwargs): + def prompt(self, *args, **kwargs): raise NotImplementedError + def display(self, message, *args, **kwargs): + print(message) + Moulinette._interface = DummyInterface() diff --git a/src/yunohost/tests/test_questions.py b/src/yunohost/tests/test_questions.py index f41c3c0cd..eaaad1791 100644 --- a/src/yunohost/tests/test_questions.py +++ b/src/yunohost/tests/test_questions.py @@ -1,14 +1,19 @@ import sys import pytest +import os -from mock import patch +from mock import patch, MagicMock from io import StringIO from collections import OrderedDict from moulinette import Moulinette from yunohost import domain, user -from yunohost.utils.config import parse_args_in_yunohost_format, PasswordQuestion +from yunohost.utils.config import ( + parse_args_in_yunohost_format, + PasswordQuestion, + Question +) from yunohost.utils.error import YunohostError @@ -70,7 +75,8 @@ def test_question_string_no_input(): ] answers = {} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -84,7 +90,8 @@ def test_question_string_input(): answers = {} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -97,7 +104,8 @@ def test_question_string_input_no_ask(): answers = {} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -110,7 +118,8 @@ def test_question_string_no_input_optional(): ] answers = {} expected_result = OrderedDict({"some_string": ("", "string")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_string_optional_with_input(): @@ -124,7 +133,8 @@ def test_question_string_optional_with_input(): answers = {} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -139,7 +149,8 @@ def test_question_string_optional_with_empty_input(): answers = {} expected_result = OrderedDict({"some_string": ("", "string")}) - with patch.object(Moulinette.interface, "prompt", return_value=""): + with patch.object(Moulinette, "prompt", return_value=""), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -153,7 +164,8 @@ def test_question_string_optional_with_input_without_ask(): answers = {} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -167,7 +179,8 @@ def test_question_string_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_string": ("some_value", "string")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_string_input_test_ask(): @@ -181,10 +194,13 @@ def test_question_string_input_test_ask(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with(ask_text, False) + prompt.assert_called_with( + message=ask_text, is_password=False, confirm=False, + prefill='', is_multiline=False + ) def test_question_string_input_test_ask_with_default(): @@ -200,10 +216,14 @@ def test_question_string_input_test_ask_with_default(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with("%s (default: %s)" % (ask_text, default_text), False) + prompt.assert_called_with( + message=ask_text, + is_password=False, confirm=False, + prefill=default_text, is_multiline=False + ) @pytest.mark.skip # we should do something with this example @@ -220,11 +240,11 @@ def test_question_string_input_test_ask_with_example(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert example_text in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert example_text in prompt.call_args[1]['message'] @pytest.mark.skip # we should do something with this help @@ -241,11 +261,11 @@ def test_question_string_input_test_ask_with_help(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert help_text in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert help_text in prompt.call_args[1]['message'] def test_question_string_with_choice(): @@ -259,7 +279,8 @@ def test_question_string_with_choice_prompt(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "fr"} expected_result = OrderedDict({"some_string": ("fr", "string")}) - with patch.object(Moulinette.interface, "prompt", return_value="fr"): + with patch.object(Moulinette, "prompt", return_value="fr"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -267,7 +288,8 @@ def test_question_string_with_choice_bad(): questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] answers = {"some_string": "bad"} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): assert parse_args_in_yunohost_format(answers, questions) @@ -283,12 +305,13 @@ def test_question_string_with_choice_ask(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="ru") as prompt: + with patch.object(Moulinette, "prompt", return_value="ru") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] for choice in choices: - assert choice in prompt.call_args[0][0] + assert choice in prompt.call_args[1]['message'] def test_question_string_with_choice_default(): @@ -302,7 +325,8 @@ def test_question_string_with_choice_default(): ] answers = {} expected_result = OrderedDict({"some_string": ("en", "string")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_password(): @@ -314,7 +338,9 @@ def test_question_password(): ] answers = {"some_password": "some_value"} expected_result = OrderedDict({"some_password": ("some_value", "password")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_password_no_input(): @@ -326,7 +352,8 @@ def test_question_password_no_input(): ] answers = {} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -339,10 +366,14 @@ def test_question_password_input(): } ] answers = {} + Question.operation_logger = { 'data_to_redact': [] } expected_result = OrderedDict({"some_password": ("some_value", "password")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): - assert parse_args_in_yunohost_format(answers, questions) == expected_result + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_password_input_no_ask(): @@ -355,7 +386,10 @@ def test_question_password_input_no_ask(): answers = {} expected_result = OrderedDict({"some_password": ("some_value", "password")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -370,13 +404,19 @@ def test_question_password_no_input_optional(): answers = {} expected_result = OrderedDict({"some_password": ("", "password")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result questions = [ {"name": "some_password", "type": "password", "optional": True, "default": ""} ] - assert parse_args_in_yunohost_format(answers, questions) == expected_result + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_password_optional_with_input(): @@ -391,7 +431,10 @@ def test_question_password_optional_with_input(): answers = {} expected_result = OrderedDict({"some_password": ("some_value", "password")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -407,7 +450,10 @@ def test_question_password_optional_with_empty_input(): answers = {} expected_result = OrderedDict({"some_password": ("", "password")}) - with patch.object(Moulinette.interface, "prompt", return_value=""): + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value=""), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -422,7 +468,10 @@ def test_question_password_optional_with_input_without_ask(): answers = {} expected_result = OrderedDict({"some_password": ("some_value", "password")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -438,7 +487,8 @@ def test_question_password_no_input_default(): answers = {} # no default for password! - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -455,7 +505,8 @@ def test_question_password_no_input_example(): answers = {"some_password": "some_value"} # no example for password! - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -470,11 +521,16 @@ def test_question_password_input_test_ask(): ] answers = {} - with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value="some_value") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with(ask_text, True) + prompt.assert_called_with( + message=ask_text, + is_password=True, confirm=True, + prefill='', is_multiline=False + ) @pytest.mark.skip # we should do something with this example @@ -491,12 +547,13 @@ def test_question_password_input_test_ask_with_example(): ] answers = {} - with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value="some_value") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert example_text in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert example_text in prompt.call_args[1]['message'] @pytest.mark.skip # we should do something with this help @@ -513,12 +570,13 @@ def test_question_password_input_test_ask_with_help(): ] answers = {} - with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Question.operation_logger = MagicMock() + with patch.object(Question.operation_logger, "data_to_redact", create=True), \ + patch.object(Moulinette, "prompt", return_value="some_value") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert help_text in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert help_text in prompt.call_args[1]['message'] def test_question_password_bad_chars(): @@ -532,7 +590,8 @@ def test_question_password_bad_chars(): ] for i in PasswordQuestion.forbidden_chars: - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format({"some_password": i * 8}, questions) @@ -546,11 +605,13 @@ def test_question_password_strong_enough(): } ] - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): # too short parse_args_in_yunohost_format({"some_password": "a"}, questions) - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format({"some_password": "password"}, questions) @@ -564,11 +625,13 @@ def test_question_password_optional_strong_enough(): } ] - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): # too short parse_args_in_yunohost_format({"some_password": "a"}, questions) - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format({"some_password": "password"}, questions) @@ -593,7 +656,8 @@ def test_question_path_no_input(): ] answers = {} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -608,7 +672,8 @@ def test_question_path_input(): answers = {} expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -622,7 +687,8 @@ def test_question_path_input_no_ask(): answers = {} expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -636,7 +702,8 @@ def test_question_path_no_input_optional(): ] answers = {} expected_result = OrderedDict({"some_path": ("", "path")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_path_optional_with_input(): @@ -651,7 +718,8 @@ def test_question_path_optional_with_input(): answers = {} expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -667,7 +735,8 @@ def test_question_path_optional_with_empty_input(): answers = {} expected_result = OrderedDict({"some_path": ("", "path")}) - with patch.object(Moulinette.interface, "prompt", return_value=""): + with patch.object(Moulinette, "prompt", return_value=""), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -682,7 +751,8 @@ def test_question_path_optional_with_input_without_ask(): answers = {} expected_result = OrderedDict({"some_path": ("some_value", "path")}) - with patch.object(Moulinette.interface, "prompt", return_value="some_value"): + with patch.object(Moulinette, "prompt", return_value="some_value"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -697,7 +767,8 @@ def test_question_path_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_path": ("some_value", "path")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_path_input_test_ask(): @@ -712,10 +783,14 @@ def test_question_path_input_test_ask(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with(ask_text, False) + prompt.assert_called_with( + message=ask_text, + is_password=False, confirm=False, + prefill='', is_multiline=False + ) def test_question_path_input_test_ask_with_default(): @@ -732,10 +807,14 @@ def test_question_path_input_test_ask_with_default(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with("%s (default: %s)" % (ask_text, default_text), False) + prompt.assert_called_with( + message=ask_text, + is_password=False, confirm=False, + prefill=default_text, is_multiline=False + ) @pytest.mark.skip # we should do something with this example @@ -753,11 +832,11 @@ def test_question_path_input_test_ask_with_example(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert example_text in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert example_text in prompt.call_args[1]['message'] @pytest.mark.skip # we should do something with this help @@ -775,11 +854,11 @@ def test_question_path_input_test_ask_with_help(): answers = {} with patch.object( - Moulinette.interface, "prompt", return_value="some_value" - ) as prompt: + Moulinette, "prompt", return_value="some_value" + ) as prompt, patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert help_text in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert help_text in prompt.call_args[1]['message'] def test_question_boolean(): @@ -913,7 +992,8 @@ def test_question_boolean_no_input(): answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_boolean_bad_input(): @@ -925,7 +1005,8 @@ def test_question_boolean_bad_input(): ] answers = {"some_boolean": "stuff"} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -940,11 +1021,13 @@ def test_question_boolean_input(): answers = {} expected_result = OrderedDict({"some_boolean": (1, "boolean")}) - with patch.object(Moulinette.interface, "prompt", return_value="y"): + with patch.object(Moulinette, "prompt", return_value="y"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result expected_result = OrderedDict({"some_boolean": (0, "boolean")}) - with patch.object(Moulinette.interface, "prompt", return_value="n"): + with patch.object(Moulinette, "prompt", return_value="n"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -958,7 +1041,8 @@ def test_question_boolean_input_no_ask(): answers = {} expected_result = OrderedDict({"some_boolean": (1, "boolean")}) - with patch.object(Moulinette.interface, "prompt", return_value="y"): + with patch.object(Moulinette, "prompt", return_value="y"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -972,7 +1056,8 @@ def test_question_boolean_no_input_optional(): ] answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) # default to false - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_boolean_optional_with_input(): @@ -987,7 +1072,8 @@ def test_question_boolean_optional_with_input(): answers = {} expected_result = OrderedDict({"some_boolean": (1, "boolean")}) - with patch.object(Moulinette.interface, "prompt", return_value="y"): + with patch.object(Moulinette, "prompt", return_value="y"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1003,7 +1089,8 @@ def test_question_boolean_optional_with_empty_input(): answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) # default to false - with patch.object(Moulinette.interface, "prompt", return_value=""): + with patch.object(Moulinette, "prompt", return_value=""), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1018,7 +1105,8 @@ def test_question_boolean_optional_with_input_without_ask(): answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) - with patch.object(Moulinette.interface, "prompt", return_value="n"): + with patch.object(Moulinette, "prompt", return_value="n"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1033,7 +1121,8 @@ def test_question_boolean_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_boolean": (0, "boolean")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_boolean_bad_default(): @@ -1061,9 +1150,14 @@ def test_question_boolean_input_test_ask(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value=0) as prompt: + with patch.object(Moulinette, "prompt", return_value=0) as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with(ask_text + " [yes | no] (default: no)", False) + prompt.assert_called_with( + message=ask_text + " [yes | no]", + is_password=False, confirm=False, + prefill='no', is_multiline=False + ) def test_question_boolean_input_test_ask_with_default(): @@ -1079,9 +1173,14 @@ def test_question_boolean_input_test_ask_with_default(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value=1) as prompt: + with patch.object(Moulinette, "prompt", return_value=1) as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with("%s [yes | no] (default: yes)" % ask_text, False) + prompt.assert_called_with( + message=ask_text + " [yes | no]", + is_password=False, confirm=False, + prefill='yes', is_multiline=False + ) def test_question_domain_empty(): @@ -1095,9 +1194,9 @@ def test_question_domain_empty(): expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) answers = {} - with patch.object( - domain, "_get_maindomain", return_value="my_main_domain.com" - ), patch.object(domain, "domain_list", return_value={"domains": [main_domain]}): + with patch.object(domain, "_get_maindomain", return_value="my_main_domain.com"),\ + patch.object(domain, "domain_list", return_value={"domains": [main_domain]}), \ + patch.object(os, "isatty", return_value=False): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1115,7 +1214,7 @@ def test_question_domain(): expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( - domain, "_get_maindomain", return_value=main_domain + domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1135,7 +1234,7 @@ def test_question_domain_two_domains(): expected_result = OrderedDict({"some_domain": (other_domain, "domain")}) with patch.object( - domain, "_get_maindomain", return_value=main_domain + domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1143,7 +1242,7 @@ def test_question_domain_two_domains(): expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( - domain, "_get_maindomain", return_value=main_domain + domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1162,9 +1261,10 @@ def test_question_domain_two_domains_wrong_answer(): answers = {"some_domain": "doesnt_exist.pouet"} with patch.object( - domain, "_get_maindomain", return_value=main_domain + domain, "_get_maindomain", return_value=main_domain ), patch.object(domain, "domain_list", return_value={"domains": domains}): - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -1183,8 +1283,9 @@ def test_question_domain_two_domains_default_no_ask(): expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object(domain, "domain_list", return_value={"domains": domains}): + domain, "_get_maindomain", return_value=main_domain + ), patch.object(domain, "domain_list", return_value={"domains": domains}), \ + patch.object(os, "isatty", return_value=False): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1198,8 +1299,9 @@ def test_question_domain_two_domains_default(): expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object(domain, "domain_list", return_value={"domains": domains}): + domain, "_get_maindomain", return_value=main_domain + ), patch.object(domain, "domain_list", return_value={"domains": domains}), \ + patch.object(os, "isatty", return_value=False): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1212,14 +1314,15 @@ def test_question_domain_two_domains_default_input(): answers = {} with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object(domain, "domain_list", return_value={"domains": domains}): + domain, "_get_maindomain", return_value=main_domain + ), patch.object(domain, "domain_list", return_value={"domains": domains}), \ + patch.object(os, "isatty", return_value=True): expected_result = OrderedDict({"some_domain": (main_domain, "domain")}) - with patch.object(Moulinette.interface, "prompt", return_value=main_domain): + with patch.object(Moulinette, "prompt", return_value=main_domain): assert parse_args_in_yunohost_format(answers, questions) == expected_result expected_result = OrderedDict({"some_domain": (other_domain, "domain")}) - with patch.object(Moulinette.interface, "prompt", return_value=other_domain): + with patch.object(Moulinette, "prompt", return_value=other_domain): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1243,7 +1346,8 @@ def test_question_user_empty(): answers = {} with patch.object(user, "user_list", return_value={"users": users}): - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -1269,8 +1373,8 @@ def test_question_user(): expected_result = OrderedDict({"some_user": (username, "user")}) - with patch.object(user, "user_list", return_value={"users": users}): - with patch.object(user, "user_info", return_value={}): + with patch.object(user, "user_list", return_value={"users": users}), \ + patch.object(user, "user_info", return_value={}): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1303,15 +1407,15 @@ def test_question_user_two_users(): answers = {"some_user": other_user} expected_result = OrderedDict({"some_user": (other_user, "user")}) - with patch.object(user, "user_list", return_value={"users": users}): - with patch.object(user, "user_info", return_value={}): + with patch.object(user, "user_list", return_value={"users": users}), \ + patch.object(user, "user_info", return_value={}): assert parse_args_in_yunohost_format(answers, questions) == expected_result answers = {"some_user": username} expected_result = OrderedDict({"some_user": (username, "user")}) - with patch.object(user, "user_list", return_value={"users": users}): - with patch.object(user, "user_info", return_value={}): + with patch.object(user, "user_list", return_value={"users": users}), \ + patch.object(user, "user_info", return_value={}): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1344,7 +1448,8 @@ def test_question_user_two_users_wrong_answer(): answers = {"some_user": "doesnt_exist.pouet"} with patch.object(user, "user_list", return_value={"users": users}): - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -1372,7 +1477,8 @@ def test_question_user_two_users_no_default(): answers = {} with patch.object(user, "user_list", return_value={"users": users}): - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -1399,17 +1505,18 @@ def test_question_user_two_users_default_input(): questions = [{"name": "some_user", "type": "user", "ask": "choose a user"}] answers = {} - with patch.object(user, "user_list", return_value={"users": users}): + with patch.object(user, "user_list", return_value={"users": users}), \ + patch.object(os, "isatty", return_value=True): with patch.object(user, "user_info", return_value={}): expected_result = OrderedDict({"some_user": (username, "user")}) - with patch.object(Moulinette.interface, "prompt", return_value=username): + with patch.object(Moulinette, "prompt", return_value=username): assert ( parse_args_in_yunohost_format(answers, questions) == expected_result ) expected_result = OrderedDict({"some_user": (other_user, "user")}) - with patch.object(Moulinette.interface, "prompt", return_value=other_user): + with patch.object(Moulinette, "prompt", return_value=other_user): assert ( parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1437,8 +1544,9 @@ def test_question_number_no_input(): ] answers = {} - expected_result = OrderedDict({"some_number": (0, "number")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): + parse_args_in_yunohost_format(answers, questions) def test_question_number_bad_input(): @@ -1450,11 +1558,13 @@ def test_question_number_bad_input(): ] answers = {"some_number": "stuff"} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) answers = {"some_number": 1.5} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -1469,14 +1579,17 @@ def test_question_number_input(): answers = {} expected_result = OrderedDict({"some_number": (1337, "number")}) - with patch.object(Moulinette.interface, "prompt", return_value="1337"): + with patch.object(Moulinette, "prompt", return_value="1337"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result - with patch.object(Moulinette.interface, "prompt", return_value=1337): + with patch.object(Moulinette, "prompt", return_value=1337), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result expected_result = OrderedDict({"some_number": (0, "number")}) - with patch.object(Moulinette.interface, "prompt", return_value=""): + with patch.object(Moulinette, "prompt", return_value="0"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1490,7 +1603,8 @@ def test_question_number_input_no_ask(): answers = {} expected_result = OrderedDict({"some_number": (1337, "number")}) - with patch.object(Moulinette.interface, "prompt", return_value="1337"): + with patch.object(Moulinette, "prompt", return_value="1337"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1503,8 +1617,9 @@ def test_question_number_no_input_optional(): } ] answers = {} - expected_result = OrderedDict({"some_number": (0, "number")}) # default to 0 - assert parse_args_in_yunohost_format(answers, questions) == expected_result + expected_result = OrderedDict({"some_number": (None, "number")}) # default to 0 + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_number_optional_with_input(): @@ -1519,7 +1634,8 @@ def test_question_number_optional_with_input(): answers = {} expected_result = OrderedDict({"some_number": (1337, "number")}) - with patch.object(Moulinette.interface, "prompt", return_value="1337"): + with patch.object(Moulinette, "prompt", return_value="1337"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1534,7 +1650,8 @@ def test_question_number_optional_with_input_without_ask(): answers = {} expected_result = OrderedDict({"some_number": (0, "number")}) - with patch.object(Moulinette.interface, "prompt", return_value="0"): + with patch.object(Moulinette, "prompt", return_value="0"), \ + patch.object(os, "isatty", return_value=True): assert parse_args_in_yunohost_format(answers, questions) == expected_result @@ -1549,7 +1666,8 @@ def test_question_number_no_input_default(): ] answers = {} expected_result = OrderedDict({"some_number": (1337, "number")}) - assert parse_args_in_yunohost_format(answers, questions) == expected_result + with patch.object(os, "isatty", return_value=False): + assert parse_args_in_yunohost_format(answers, questions) == expected_result def test_question_number_bad_default(): @@ -1562,7 +1680,8 @@ def test_question_number_bad_default(): } ] answers = {} - with pytest.raises(YunohostError): + with pytest.raises(YunohostError), \ + patch.object(os, "isatty", return_value=False): parse_args_in_yunohost_format(answers, questions) @@ -1577,9 +1696,14 @@ def test_question_number_input_test_ask(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: + with patch.object(Moulinette, "prompt", return_value="1111") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with("%s (default: 0)" % (ask_text), False) + prompt.assert_called_with( + message=ask_text, + is_password=False, confirm=False, + prefill='', is_multiline=False + ) def test_question_number_input_test_ask_with_default(): @@ -1595,9 +1719,14 @@ def test_question_number_input_test_ask_with_default(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: + with patch.object(Moulinette, "prompt", return_value="1111") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - prompt.assert_called_with("%s (default: %s)" % (ask_text, default_value), False) + prompt.assert_called_with( + message=ask_text, + is_password=False, confirm=False, + prefill=str(default_value), is_multiline=False + ) @pytest.mark.skip # we should do something with this example @@ -1614,10 +1743,11 @@ def test_question_number_input_test_ask_with_example(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: + with patch.object(Moulinette, "prompt", return_value="1111") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert example_value in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert example_value in prompt.call_args[1]['message'] @pytest.mark.skip # we should do something with this help @@ -1634,16 +1764,18 @@ def test_question_number_input_test_ask_with_help(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="1111") as prompt: + with patch.object(Moulinette, "prompt", return_value="1111") as prompt, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) - assert ask_text in prompt.call_args[0][0] - assert help_value in prompt.call_args[0][0] + assert ask_text in prompt.call_args[1]['message'] + assert help_value in prompt.call_args[1]['message'] def test_question_display_text(): questions = [{"name": "some_app", "type": "display_text", "ask": "foobar"}] answers = {} - with patch.object(sys, "stdout", new_callable=StringIO) as stdout: + with patch.object(sys, "stdout", new_callable=StringIO) as stdout, \ + patch.object(os, "isatty", return_value=True): parse_args_in_yunohost_format(answers, questions) assert "foobar" in stdout.getvalue() diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 6b491386f..6d3c322f2 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -403,9 +403,9 @@ class Question(object): return value def ask_if_needed(self): - while True: + for i in range(5): # Display question if no value filled or if it's a readonly message - if Moulinette.interface.type == "cli": + if Moulinette.interface.type == "cli" and os.isatty(1): text_for_user_input_in_cli = self._format_text_for_user_input_in_cli() if getattr(self, "readonly", False): Moulinette.display(text_for_user_input_in_cli) @@ -415,7 +415,7 @@ class Question(object): if self.current_value is not None: prefill = self.humanize(self.current_value, self) elif self.default is not None: - prefill = self.default + prefill = self.humanize(self.default, self) self.value = Moulinette.prompt( message=text_for_user_input_in_cli, is_password=self.hide_user_input_in_prompt, @@ -424,27 +424,33 @@ class Question(object): is_multiline=(self.type == "text"), ) - # Normalization - # This is done to enforce a certain formating like for boolean - self.value = self.normalize(self.value, self) - # Apply default value - if self.value in [None, ""] and self.default is not None: + class_default= getattr(self, "default_value", None) + if self.value in [None, ""] and \ + (self.default is not None or class_default is not None): self.value = ( - getattr(self, "default_value", None) + class_default if self.default is None else self.default ) + # Normalization + # This is done to enforce a certain formating like for boolean + self.value = self.normalize(self.value, self) + # Prevalidation try: self._prevalidate() except YunohostValidationError as e: - if Moulinette.interface.type == "api": - raise - Moulinette.display(str(e), "error") - self.value = None - continue + # If in interactive cli, re-ask the current question + if i < 4 and Moulinette.interface.type == "cli" and os.isatty(1): + logger.error(str(e)) + self.value = None + continue + + # Otherwise raise the ValidationError + raise + break self.value = self._post_parse_value() @@ -561,7 +567,7 @@ class PasswordQuestion(Question): def _prevalidate(self): super()._prevalidate() - if self.value is not None: + if self.value not in [None, ""]: if any(char in self.value for char in self.forbidden_chars): raise YunohostValidationError( "pattern_password_app", forbidden_chars=self.forbidden_chars @@ -580,7 +586,7 @@ class PathQuestion(Question): class BooleanQuestion(Question): argument_type = "boolean" - default_value = False + default_value = 0 yes_answers = ["1", "yes", "y", "true", "t", "on"] no_answers = ["0", "no", "n", "false", "f", "off"] @@ -633,17 +639,13 @@ class BooleanQuestion(Question): self.yes = question.get("yes", 1) self.no = question.get("no", 0) if self.default is None: - self.default = False + self.default = self.no def _format_text_for_user_input_in_cli(self): - text_for_user_input_in_cli = _value_for_locale(self.ask) + text_for_user_input_in_cli = super()._format_text_for_user_input_in_cli() text_for_user_input_in_cli += " [yes | no]" - if self.default is not None: - formatted_default = self.humanize(self.default) - text_for_user_input_in_cli += " (default: {0})".format(formatted_default) - return text_for_user_input_in_cli def get(self, key, default=None): @@ -698,11 +700,7 @@ class UserQuestion(Question): class NumberQuestion(Question): argument_type = "number" - default_value = "" - - @staticmethod - def humanize(value, option={}): - return str(value) + default_value = None def __init__(self, question, user_answers): super().__init__(question, user_answers) @@ -710,16 +708,25 @@ class NumberQuestion(Question): self.max = question.get("max", None) self.step = question.get("step", None) + @staticmethod + def normalize(value, option={}): + if isinstance(value, int): + return value + + if isinstance(value, str) and value.isdigit(): + return int(value) + + if value in [None, ""]: + return value + + raise YunohostValidationError( + "app_argument_invalid", name=option.name, error=m18n.n("invalid_number") + ) + def _prevalidate(self): super()._prevalidate() - if not isinstance(self.value, int) and not ( - isinstance(self.value, str) and self.value.isdigit() - ): - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n("invalid_number"), - ) + if self.value in [None, ""]: + return if self.min is not None and int(self.value) < self.min: raise YunohostValidationError( @@ -735,16 +742,6 @@ class NumberQuestion(Question): error=m18n.n("invalid_number"), ) - def _post_parse_value(self): - if isinstance(self.value, int): - return super()._post_parse_value() - - if isinstance(self.value, str) and self.value.isdigit(): - return int(self.value) - - raise YunohostValidationError( - "app_argument_invalid", name=self.name, error=m18n.n("invalid_number") - ) class DisplayTextQuestion(Question): @@ -755,10 +752,10 @@ class DisplayTextQuestion(Question): super().__init__(question, user_answers) self.optional = True - self.style = question.get("style", "info") + self.style = question.get("style", "info" if question['type'] == 'alert' else '') def _format_text_for_user_input_in_cli(self): - text = self.ask["en"] + text = _value_for_locale(self.ask) if self.style in ["success", "info", "warning", "danger"]: color = { From 0844c747646a7427663fc71144ae3e6ec71ba4b0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 7 Sep 2021 23:47:06 +0200 Subject: [PATCH 096/119] =?UTF-8?q?Anotha=20shruberry=C2=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/yunohost/tests/test_backuprestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 8db6982df..b24d3442d 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -344,7 +344,7 @@ def test_backup_script_failure_handling(monkeypatch, mocker): # Create a backup of this app and simulate a crash (patching the backup # call with monkeypatch). We also patch m18n to check later it's been called # with the expected error message key - monkeypatch.setattr("yunohost.hook.hook_exec", custom_hook_exec) + monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) with message(mocker, "backup_app_failed", app="backup_recommended_app"): with raiseYunohostError(mocker, "backup_nothings_done"): From 85b7239a4b662a40386f139b381feb023f59b62c Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:42:45 +0200 Subject: [PATCH 097/119] [enh] Use --message Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 4ad52e038..dba79295c 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -62,7 +62,7 @@ EOL old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" elif [[ "$bind" == *":"* ]] then - ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)" fi From 8efbc736d44c68089ef9ca696a53bd91a8cfdf16 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:42:56 +0200 Subject: [PATCH 098/119] [enh] Use --message Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index dba79295c..796016ef7 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -49,7 +49,7 @@ EOL then if [[ "$bind" == "settings" ]] then - ynh_die "File '${short_setting}' can't be stored in settings" + ynh_die --message="File '${short_setting}' can't be stored in settings" fi old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2> /dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" From 33505bff5f5013d8c93a09a7317158739c163ae9 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:43:15 +0200 Subject: [PATCH 099/119] [enh] Use --message Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 796016ef7..c1bef8f51 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -112,7 +112,7 @@ _ynh_app_config_apply() { then if [[ "$bind" == "settings" ]] then - ynh_die "File '${short_setting}' can't be stored in settings" + ynh_die --message="File '${short_setting}' can't be stored in settings" fi local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]] From df20a540133625bbf32714537c3a5e52f623767f Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:43:50 +0200 Subject: [PATCH 100/119] [enh] Avoid bad deletion Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index c1bef8f51..4f28f95a8 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -118,7 +118,7 @@ _ynh_app_config_apply() { if [[ "${!short_setting}" == "" ]] then ynh_backup_if_checksum_is_different --file="$bind_file" - rm -f "$bind_file" + ynh_secure_remove --file="$bind_file" ynh_delete_file_checksum --file="$bind_file" --update_only ynh_print_info "File '$bind_file' removed" else From 62875c30da2fce1c91a0ed3d3425c3334187f9fd Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:43:59 +0200 Subject: [PATCH 101/119] [enh] Use --message Co-authored-by: Kayou --- data/helpers.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 5ecc0cf0b..9938da771 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -592,7 +592,7 @@ ynh_write_var_in_file() { ynh_handle_getopts_args "$@" after="${after:-}" - [[ -f $file ]] || ynh_die "File $file does not exists" + [[ -f $file ]] || ynh_die --message="File $file does not exists" # Get the line number after which we search for the variable local line_number=1 From d3603632517dddbb8417816e2b64260f3621ba2d Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:44:35 +0200 Subject: [PATCH 102/119] [enh] Use --message Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 4f28f95a8..b1db5f362 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -120,7 +120,7 @@ _ynh_app_config_apply() { ynh_backup_if_checksum_is_different --file="$bind_file" ynh_secure_remove --file="$bind_file" ynh_delete_file_checksum --file="$bind_file" --update_only - ynh_print_info "File '$bind_file' removed" + ynh_print_info --message="File '$bind_file' removed" else ynh_backup_if_checksum_is_different --file="$bind_file" cp "${!short_setting}" "$bind_file" From a205015f0d47fb72a248adcd8d3433e27919139d Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:44:47 +0200 Subject: [PATCH 103/119] [enh] Use --message Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index b1db5f362..4da754036 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -125,7 +125,7 @@ _ynh_app_config_apply() { ynh_backup_if_checksum_is_different --file="$bind_file" cp "${!short_setting}" "$bind_file" ynh_store_file_checksum --file="$bind_file" --update_only - ynh_print_info "File '$bind_file' overwrited with ${!short_setting}" + ynh_print_info --message="File '$bind_file' overwrited with ${!short_setting}" fi # Save value in app settings From b487fbbe0069eb69bd0636df77ca9a2fcb045a24 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:45:24 +0200 Subject: [PATCH 104/119] [enh] Use named args in helpers calls Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 4da754036..6b77fb12e 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -131,7 +131,7 @@ _ynh_app_config_apply() { # Save value in app settings elif [[ "$bind" == "settings" ]] then - ynh_app_setting_set $app $short_setting "${!short_setting}" + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" ynh_print_info "Configuration key '$short_setting' edited in app settings" # Save multiline text in a file From f2b779c962f4c0b3b192766d8f952fede53d86c4 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 15:45:39 +0200 Subject: [PATCH 105/119] [enh] Use named args in helpers calls Co-authored-by: Kayou --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 6b77fb12e..56ad9857b 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -132,7 +132,7 @@ _ynh_app_config_apply() { elif [[ "$bind" == "settings" ]] then ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" - ynh_print_info "Configuration key '$short_setting' edited in app settings" + ynh_print_info --message="Configuration key '$short_setting' edited in app settings" # Save multiline text in a file elif [[ "$type" == "text" ]] From 924df9733e1f2d4e4621e46146e618f18b04fe6c Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 8 Sep 2021 16:49:42 +0200 Subject: [PATCH 106/119] [enh] Use named args in helpers calls Co-authored-by: Kayou --- data/helpers.d/config | 9 ++++----- data/helpers.d/utils | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 56ad9857b..b9471cf51 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -145,7 +145,7 @@ _ynh_app_config_apply() { ynh_backup_if_checksum_is_different --file="$bind_file" echo "${!short_setting}" > "$bind_file" ynh_store_file_checksum --file="$bind_file" --update_only - ynh_print_info "File '$bind_file' overwrited with the content you provieded in '${short_setting}' question" + ynh_print_info --message="File '$bind_file' overwrited with the content you provieded in '${short_setting}' question" # Set value into a kind of key/value file else @@ -164,8 +164,8 @@ _ynh_app_config_apply() { ynh_store_file_checksum --file="$bind_file" --update_only # We stored the info in settings in order to be able to upgrade the app - ynh_app_setting_set $app $short_setting "${!short_setting}" - ynh_print_info "Configuration key '$bind_key' edited into $bind_file" + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_print_info --message="Configuration key '$bind_key' edited into $bind_file" fi fi @@ -194,7 +194,6 @@ _ynh_app_config_validate() { ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1 local nothing_changed=true local changes_validated=true - #for changed_status in "${!changed[@]}" for short_setting in "${!old[@]}" do changed[$short_setting]=false @@ -237,7 +236,7 @@ _ynh_app_config_validate() { done if [[ "$nothing_changed" == "true" ]] then - ynh_print_info "Nothing has changed" + ynh_print_info --message="Nothing has changed" exit 0 fi diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 9938da771..511fa52fb 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -517,7 +517,7 @@ ynh_read_var_in_file() { ynh_handle_getopts_args "$@" after="${after:-}" - [[ -f $file ]] || ynh_die "File $file does not exists" + [[ -f $file ]] || ynh_die --message="File $file does not exists" # Get the line number after which we search for the variable local line_number=1 From 6d1e392634a8e613cd651fb67aa330e84d18ca01 Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 9 Sep 2021 10:50:49 +0200 Subject: [PATCH 107/119] Update data/helpers.d/config --- data/helpers.d/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index b9471cf51..8f3248949 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -139,7 +139,7 @@ _ynh_app_config_apply() { then if [[ "$bind" == *":"* ]] then - ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" From 89e49007a667db4c05c653fc8b918c8b0ee5270e Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 11 Sep 2021 16:55:59 +0200 Subject: [PATCH 108/119] [fix) Tags empty question --- src/yunohost/utils/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 6d3c322f2..4636eaf48 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -538,6 +538,8 @@ class TagsQuestion(Question): values = self.value if isinstance(values, str): values = values.split(",") + elif value is None: + values = [] for value in values: self.value = value super()._prevalidate() From 3695be8d934f31ebc7c37fcce0cfe59812e7c46e Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 11 Sep 2021 17:48:52 +0200 Subject: [PATCH 109/119] [fix) Tags empty question --- src/yunohost/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 4636eaf48..d13038b2b 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -538,7 +538,7 @@ class TagsQuestion(Question): values = self.value if isinstance(values, str): values = values.split(",") - elif value is None: + elif values is None: values = [] for value in values: self.value = value From b042b549e4d990d5f21bb0304aa0b80afcf1df99 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 12 Sep 2021 15:50:23 +0200 Subject: [PATCH 110/119] Let's not define a duplicate string for domain unknown... --- locales/en.json | 1 - src/yunohost/utils/config.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index 588e37357..89d520112 100644 --- a/locales/en.json +++ b/locales/en.json @@ -309,7 +309,6 @@ "domain_name_unknown": "Domain '{domain}' unknown", "domain_remove_confirm_apps_removal": "Removing this domain will remove those applications:\n{apps}\n\nAre you sure you want to do that? [{answers}]", "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal", - "domain_unknown": "Unknown domain", "domains_available": "Available domains:", "done": "Done", "downloading": "Downloading...", diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index d13038b2b..a02097d48 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -672,7 +672,7 @@ class DomainQuestion(Question): def _raise_invalid_answer(self): raise YunohostValidationError( - "app_argument_invalid", name=self.name, error=m18n.n("domain_unknown") + "app_argument_invalid", name=self.name, error=m18n.n("domain_name_unknown", domain=self.value) ) From e133b163dfc6b29fb7f181b74645c9c3d44a012f Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 12 Sep 2021 17:34:05 +0200 Subject: [PATCH 111/119] [wip] Check question are initialize --- locales/en.json | 2 ++ src/yunohost/utils/config.py | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/locales/en.json b/locales/en.json index 588e37357..481e7c644 100644 --- a/locales/en.json +++ b/locales/en.json @@ -142,6 +142,8 @@ "certmanager_warning_subdomain_dns_record": "Subdomain '{subdomain}' does not resolve to the same IP address as '{domain}'. Some features will not be available until you fix this and regenerate the certificate.", "config_apply_failed": "Applying the new configuration failed: {error}", "config_cant_set_value_on_section": "You can't set a single value on an entire config section.", + "config_forbidden_keyword": "The keyword '{keyword}' is reserved, you can't create or use a config panel with a question with this id.", + "config_missing_init_value": "Config panel question '{question}' should be initialize with a value during install or upgrade.", "config_no_panel": "No config panel found.", "config_unknown_filter_key": "The filter key '{filter_key}' is incorrect.", "config_version_not_supported": "Config panel versions '{version}' are not supported.", diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index d13038b2b..429606dfe 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -209,7 +209,6 @@ class ConfigPanel: "default": {} } } - def convert(toml_node, node_type): """Convert TOML in internal format ('full' mode used by webadmin) Here are some properties of 1.0 config panel in toml: @@ -264,6 +263,16 @@ class ConfigPanel: "config_unknown_filter_key", filter_key=self.filter_key ) + # List forbidden keywords from helpers and sections toml (to avoid conflict) + forbidden_keywords = ["old", "app", "changed", "file_hash", "binds", "types", + "formats", "getter", "setter", "short_setting", "type", + "bind", "nothing_changed", "changes_validated", "result", + "max_progression"] + forbidden_keywords += format_description["sections"] + + for _, _, option in self._iterate(): + if option["id"] in forbidden_keywords: + raise YunohostError("config_forbidden_keyword", keyword=option["id"]) return self.config def _hydrate(self): @@ -787,9 +796,9 @@ class FileQuestion(Question): def __init__(self, question, user_answers): super().__init__(question, user_answers) if question.get("accept"): - self.accept = question.get("accept").replace(" ", "").split(",") + self.accept = question.get("accept") else: - self.accept = [] + self.accept = "" if Moulinette.interface.type == "api": if user_answers.get(f"{self.name}[name]"): self.value = { @@ -816,7 +825,7 @@ class FileQuestion(Question): return filename = self.value if isinstance(self.value, str) else self.value["filename"] - if "." not in filename or "." + filename.split(".")[-1] not in self.accept: + if "." not in filename or "." + filename.split(".")[-1] not in self.accept.replace(" ", "").split(","): raise YunohostValidationError( "app_argument_invalid", name=self.name, From f8fed701b95d607a842ef1357aacc7f418502cdc Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 12 Sep 2021 17:43:03 +0200 Subject: [PATCH 112/119] [fix] Raise an error if question has not been initialize --- src/yunohost/utils/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 9701bf966..488cb54a3 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -279,8 +279,11 @@ class ConfigPanel: # Hydrating config panel with current value logger.debug("Hydrating config with current values") for _, _, option in self._iterate(): - if option["name"] not in self.values: - continue + if option["id"] not in self.values: + if option["type"] in ["alert", "display_text", "markdown", "file"]: + continue + else: + raise YunohostError("config_missing_init_value", question=option["id"]) value = self.values[option["name"]] # In general, the value is just a simple value. # Sometimes it could be a dict used to overwrite the option itself From ca5d7b32dc332fbe4b5550413318ad39e84bd81e Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 12 Sep 2021 19:06:25 +0200 Subject: [PATCH 113/119] [enh] Validate server side specific input html5 field --- locales/en.json | 5 ++++ src/yunohost/utils/config.py | 53 ++++++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/locales/en.json b/locales/en.json index a0c805b75..eb131fe43 100644 --- a/locales/en.json +++ b/locales/en.json @@ -146,6 +146,11 @@ "config_missing_init_value": "Config panel question '{question}' should be initialize with a value during install or upgrade.", "config_no_panel": "No config panel found.", "config_unknown_filter_key": "The filter key '{filter_key}' is incorrect.", + "config_validate_color": "Should be a valid RGB hexadecimal color", + "config_validate_date": "Should be a valid date like in the format YYYY-MM-DD", + "config_validate_email": "Should be a valid email", + "config_validate_time": "Should be a valid time like XX:YY", + "config_validate_url": "Should be a valid web URL", "config_version_not_supported": "Config panel versions '{version}' are not supported.", "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 488cb54a3..270503f8f 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -388,6 +388,7 @@ class ConfigPanel: class Question(object): hide_user_input_in_prompt = False operation_logger = None + pattern = None def __init__(self, question, user_answers): self.name = question["name"] @@ -396,7 +397,7 @@ class Question(object): self.current_value = question.get("current_value") self.optional = question.get("optional", False) self.choices = question.get("choices", []) - self.pattern = question.get("pattern") + self.pattern = question.get("pattern", self.pattern) self.ask = question.get("ask", {"en": self.name}) self.help = question.get("help") self.value = user_answers.get(self.name) @@ -536,6 +537,46 @@ class StringQuestion(Question): argument_type = "string" default_value = "" +class EmailQuestion(StringQuestion): + pattern = { + "regexp": "^.+@.+", + "error": "config_validate_email" + } + +class URLQuestion(StringQuestion): + pattern = { + "regexp": "^https?://.*$", + "error": "config_validate_url" + } + +class DateQuestion(StringQuestion): + pattern = { + "regexp": "^\d{4}-\d\d-\d\d$", + "error": "config_validate_date" + } + + def _prevalidate(self): + from datetime import datetime + super()._prevalidate() + + if self.value not in [None, ""]: + try: + datetime.strptime(self.value, '%Y-%m-%d') + except ValueError: + raise YunohostValidationError("config_validate_date") + +class TimeQuestion(StringQuestion): + pattern = { + "regexp": "^(1[12]|0?\d):[0-5]\d$", + "error": "config_validate_time" + } + +class ColorQuestion(StringQuestion): + pattern = { + "regexp": "^#[ABCDEFabcdef\d]{3,6}$", + "error": "config_validate_color" + } + class TagsQuestion(Question): argument_type = "tags" @@ -880,11 +921,11 @@ ARGUMENTS_TYPE_PARSERS = { "text": StringQuestion, "select": StringQuestion, "tags": TagsQuestion, - "email": StringQuestion, - "url": StringQuestion, - "date": StringQuestion, - "time": StringQuestion, - "color": StringQuestion, + "email": EmailQuestion, + "url": URLQuestion, + "date": DateQuestion, + "time": TimeQuestion, + "color": ColorQuestion, "password": PasswordQuestion, "path": PathQuestion, "boolean": BooleanQuestion, From 2710ca72719b9b90c798bbc066b40682098290bd Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 13 Sep 2021 00:25:55 +0200 Subject: [PATCH 114/119] [fix] Missing default property --- locales/en.json | 2 ++ src/yunohost/utils/config.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/locales/en.json b/locales/en.json index eb131fe43..fe340fff2 100644 --- a/locales/en.json +++ b/locales/en.json @@ -394,6 +394,8 @@ "hook_name_unknown": "Unknown hook name '{name}'", "installation_complete": "Installation completed", "invalid_number": "Must be a number", + "invalid_number_min": "Must be greater than {min}", + "invalid_number_max": "Must be lesser than {max}", "invalid_password": "Invalid password", "invalid_regex": "Invalid regex:'{regex}'", "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it", diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 270503f8f..e0b893356 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -202,7 +202,7 @@ class ConfigPanel: } }, "options": { - "properties": ["ask", "type", "bind", "help", "example", + "properties": ["ask", "type", "bind", "help", "example", "default", "style", "icon", "placeholder", "visible", "optional", "choices", "yes", "no", "pattern", "limit", "min", "max", "step", "accept", "redact"], @@ -787,14 +787,14 @@ class NumberQuestion(Question): raise YunohostValidationError( "app_argument_invalid", name=self.name, - error=m18n.n("invalid_number"), + error=m18n.n("invalid_number_min", min=self.min), ) if self.max is not None and int(self.value) > self.max: raise YunohostValidationError( "app_argument_invalid", name=self.name, - error=m18n.n("invalid_number"), + error=m18n.n("invalid_number_max", max=self.max), ) From ce34bb75c49fbe9ded178c94a63e1715800277c2 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 13 Sep 2021 00:47:21 +0200 Subject: [PATCH 115/119] [enh] Avoid to raise error with bin null and empty value --- src/yunohost/utils/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index e0b893356..6cc693740 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -280,7 +280,8 @@ class ConfigPanel: logger.debug("Hydrating config with current values") for _, _, option in self._iterate(): if option["id"] not in self.values: - if option["type"] in ["alert", "display_text", "markdown", "file"]: + allowed_empty_types = ["alert", "display_text", "markdown", "file"] + if option["type"] in allowed_empty_type or option["bind"] == "null": continue else: raise YunohostError("config_missing_init_value", question=option["id"]) From 02814833d5a84d6cd241918ab9fb6b564e43c3e1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 00:50:36 +0200 Subject: [PATCH 116/119] Misc fixes, try to fix tests --- locales/en.json | 1 - src/yunohost/tests/test_app_config.py | 10 +++++---- src/yunohost/utils/config.py | 29 ++++++++++++++++----------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/locales/en.json b/locales/en.json index fe340fff2..f3b1fc906 100644 --- a/locales/en.json +++ b/locales/en.json @@ -143,7 +143,6 @@ "config_apply_failed": "Applying the new configuration failed: {error}", "config_cant_set_value_on_section": "You can't set a single value on an entire config section.", "config_forbidden_keyword": "The keyword '{keyword}' is reserved, you can't create or use a config panel with a question with this id.", - "config_missing_init_value": "Config panel question '{question}' should be initialize with a value during install or upgrade.", "config_no_panel": "No config panel found.", "config_unknown_filter_key": "The filter key '{filter_key}' is incorrect.", "config_validate_color": "Should be a valid RGB hexadecimal color", diff --git a/src/yunohost/tests/test_app_config.py b/src/yunohost/tests/test_app_config.py index 4ace0aaf9..52f458b55 100644 --- a/src/yunohost/tests/test_app_config.py +++ b/src/yunohost/tests/test_app_config.py @@ -105,9 +105,7 @@ def test_app_config_get(config_app): assert isinstance(app_config_get(config_app, export=True), dict) assert isinstance(app_config_get(config_app, "main"), dict) assert isinstance(app_config_get(config_app, "main.components"), dict) - # Is it expected that we return None if no value defined yet ? - # c.f. the whole discussion about "should we have defaults" etc. - assert app_config_get(config_app, "main.components.boolean") is None + assert app_config_get(config_app, "main.components.boolean") == "0" def test_app_config_nopanel(legacy_app): @@ -130,10 +128,14 @@ def test_app_config_get_nonexistentstuff(config_app): with pytest.raises(YunohostValidationError): app_config_get(config_app, "main.components.nonexistent") + app_setting(config_app, "boolean", delete=True) + with pytest.raises(YunohostValidationError): + app_config_get(config_app, "main.components.boolean") + def test_app_config_regular_setting(config_app): - assert app_config_get(config_app, "main.components.boolean") is None + assert app_config_get(config_app, "main.components.boolean") == "0" app_config_set(config_app, "main.components.boolean", "no") diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 6cc693740..2ee01f97f 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -209,6 +209,7 @@ class ConfigPanel: "default": {} } } + def convert(toml_node, node_type): """Convert TOML in internal format ('full' mode used by webadmin) Here are some properties of 1.0 config panel in toml: @@ -284,7 +285,7 @@ class ConfigPanel: if option["type"] in allowed_empty_type or option["bind"] == "null": continue else: - raise YunohostError("config_missing_init_value", question=option["id"]) + raise YunohostError(f"Config panel question '{option['id']}' should be initialized with a value during install or upgrade.") value = self.values[option["name"]] # In general, the value is just a simple value. # Sometimes it could be a dict used to overwrite the option itself @@ -538,22 +539,25 @@ class StringQuestion(Question): argument_type = "string" default_value = "" + class EmailQuestion(StringQuestion): pattern = { - "regexp": "^.+@.+", - "error": "config_validate_email" + "regexp": r"^.+@.+", + "error": "config_validate_email" # i18n: config_validate_email } + class URLQuestion(StringQuestion): pattern = { - "regexp": "^https?://.*$", - "error": "config_validate_url" + "regexp": r"^https?://.*$", + "error": "config_validate_url" # i18n: config_validate_url } + class DateQuestion(StringQuestion): pattern = { - "regexp": "^\d{4}-\d\d-\d\d$", - "error": "config_validate_date" + "regexp": r"^\d{4}-\d\d-\d\d$", + "error": "config_validate_date" # i18n: config_validate_date } def _prevalidate(self): @@ -566,16 +570,18 @@ class DateQuestion(StringQuestion): except ValueError: raise YunohostValidationError("config_validate_date") + class TimeQuestion(StringQuestion): pattern = { - "regexp": "^(1[12]|0?\d):[0-5]\d$", - "error": "config_validate_time" + "regexp": r"^(1[12]|0?\d):[0-5]\d$", + "error": "config_validate_time" # i18n: config_validate_time } + class ColorQuestion(StringQuestion): pattern = { - "regexp": "^#[ABCDEFabcdef\d]{3,6}$", - "error": "config_validate_color" + "regexp": r"^#[ABCDEFabcdef\d]{3,6}$", + "error": "config_validate_color" # i18n: config_validate_color } @@ -799,7 +805,6 @@ class NumberQuestion(Question): ) - class DisplayTextQuestion(Question): argument_type = "display_text" readonly = True From 1d704e7b91f3006250d09868486127f9af9b59ab Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 13 Sep 2021 01:24:10 +0200 Subject: [PATCH 117/119] [enh] Avoid to raise error with bin null and empty value --- src/yunohost/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 6cc693740..4788a199a 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -281,7 +281,7 @@ class ConfigPanel: for _, _, option in self._iterate(): if option["id"] not in self.values: allowed_empty_types = ["alert", "display_text", "markdown", "file"] - if option["type"] in allowed_empty_type or option["bind"] == "null": + if option["type"] in allowed_empty_types or option["bind"] == "null": continue else: raise YunohostError("config_missing_init_value", question=option["id"]) From 380321a6eb0566b7ad83d5015b164d6be7a20438 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 01:50:04 +0200 Subject: [PATCH 118/119] config: bind key may not exist in option --- src/yunohost/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index d5dc9f598..fa461d43b 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -282,7 +282,7 @@ class ConfigPanel: for _, _, option in self._iterate(): if option["id"] not in self.values: allowed_empty_types = ["alert", "display_text", "markdown", "file"] - if option["type"] in allowed_empty_types or option["bind"] == "null": + if option["type"] in allowed_empty_types or option.get("bind") == "null": continue else: raise YunohostError(f"Config panel question '{option['id']}' should be initialized with a value during install or upgrade.") From 968cac32d12f950013ab466e95b23805069f44c9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 01:55:57 +0200 Subject: [PATCH 119/119] new config helpers: use $() instead of backticks Co-authored-by: Kayou --- data/helpers.d/config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/config b/data/helpers.d/config index 8f3248949..d12065220 100644 --- a/data/helpers.d/config +++ b/data/helpers.d/config @@ -4,7 +4,7 @@ _ynh_app_config_get() { # From settings local lines - lines=`python3 << EOL + lines=$(python3 << EOL import toml from collections import OrderedDict with open("../config_panel.toml", "r") as f: @@ -24,7 +24,7 @@ for panel_name, panel in loaded_toml.items(): param.get('bind', 'settings' if param.get('type', 'string') != 'file' else 'null') ])) EOL -` +) for line in $lines do # Split line into short_setting, type and bind