Merge branch 'dev' into enh-dns-autoconf

data/helpers.d/config

@@ -123,7 +123,10 @@ _ynh_app_config_apply() {
                     ynh_print_info --message="File '$bind_file' removed"
                 else
                     ynh_backup_if_checksum_is_different --file="$bind_file"
-                    cp "${!short_setting}" "$bind_file"
+                    if [[ "${!short_setting}" != "$bind_file" ]]
+                    then
+                        cp "${!short_setting}" "$bind_file"
+                    fi
                     ynh_store_file_checksum --file="$bind_file" --update_only
                     ynh_print_info --message="File '$bind_file' overwrited with ${!short_setting}"
                 fi

data/helpers.d/utils

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-YNH_APP_BASEDIR=$(realpath $([[ "$(basename $0)" =~ ^backup|restore$ ]] && echo '../settings' || [[ -n "${YNH_ACTION:-}" ]] && echo '.' || echo '..' ))
+YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)}
 
 # Handle script crashes / failures
 #

data/hooks/backup/50-conf_manually_modified_files

@@ -12,7 +12,7 @@ ynh_backup --src_path="./manually_modified_files_list"
 
 for file in $(cat ./manually_modified_files_list)
 do
-    ynh_backup --src_path="$file"
+    [[ -e $file ]] && ynh_backup --src_path="$file"
 done
     
 ynh_backup --src_path="/etc/ssowat/conf.json.persistent"

doc/helper_doc_template.md

@@ -10,11 +10,10 @@ routes:
 Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/doc/generate_helper_doc.py) on {{data.date}} (YunoHost version {{data.version}})
 
 {% for category, helpers in data.helpers %}
-### {{ category.upper() }}
+## {{ category.upper() }}
 {% for h in helpers %}
-**{{ h.name }}**<br/>
+#### {{ h.name }}
 [details summary="<i>{{ h.brief }}</i>" class="helper-card-subtitle text-muted"]
-<p></p>
 
 **Usage**: `{{ h.usage }}`
 {%- if h.args %}

locales/en.json

@@ -150,7 +150,7 @@
     "config_validate_color": "Should be a valid RGB hexadecimal color",
     "config_validate_date": "Should be a valid date like in the format YYYY-MM-DD",
     "config_validate_email": "Should be a valid email",
-    "config_validate_time": "Should be a valid time like XX:YY",
+    "config_validate_time": "Should be a valid time like HH:MM",
     "config_validate_url": "Should be a valid web URL",
     "config_version_not_supported": "Config panel versions '{version}' are not supported.",
     "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'",

src/yunohost/app.py

@@ -56,7 +56,6 @@ from yunohost.utils import packages
 from yunohost.utils.config import (
     ConfigPanel,
     parse_args_in_yunohost_format,
-    Question,
 )
 from yunohost.utils.i18n import _value_for_locale
 from yunohost.utils.error import YunohostError, YunohostValidationError
@@ -460,19 +459,21 @@ def app_change_url(operation_logger, app, domain, path):
     # TODO: Allow to specify arguments
     args_odict = _parse_args_from_manifest(manifest, "change_url")
 
+    tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
+
     # Prepare env. var. to pass to script
     env_dict = _make_environment_for_app_script(app, args=args_odict)
     env_dict["YNH_APP_OLD_DOMAIN"] = old_domain
     env_dict["YNH_APP_OLD_PATH"] = old_path
     env_dict["YNH_APP_NEW_DOMAIN"] = domain
     env_dict["YNH_APP_NEW_PATH"] = path
+    env_dict["YNH_APP_BASEDIR"] = tmp_workdir_for_app
 
     if domain != old_domain:
         operation_logger.related_to.append(("domain", old_domain))
     operation_logger.extra.update({"env": env_dict})
     operation_logger.start()
 
-    tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
     change_url_script = os.path.join(tmp_workdir_for_app, "scripts/change_url")
 
     # Execute App change_url script
@@ -623,6 +624,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
         env_dict["YNH_APP_MANIFEST_VERSION"] = str(app_new_version)
         env_dict["YNH_APP_CURRENT_VERSION"] = str(app_current_version)
         env_dict["NO_BACKUP_UPGRADE"] = "1" if no_safety_backup else "0"
+        env_dict["YNH_APP_BASEDIR"] = extracted_app_folder
 
         # We'll check that the app didn't brutally edit some system configuration
         manually_modified_files_before_install = manually_modified_files()
@@ -920,19 +922,6 @@ def app_install(
     # We'll check that the app didn't brutally edit some system configuration
     manually_modified_files_before_install = manually_modified_files()
 
-    # Tell the operation_logger to redact all password-type args
-    # Also redact the % escaped version of the password that might appear in
-    # the 'args' section of metadata (relevant for password with non-alphanumeric char)
-    data_to_redact = [
-        value[0] for value in args_odict.values() if value[1] == "password"
-    ]
-    data_to_redact += [
-        urllib.parse.quote(data)
-        for data in data_to_redact
-        if urllib.parse.quote(data) != data
-    ]
-    operation_logger.data_to_redact.extend(data_to_redact)
-
     operation_logger.related_to = [
         s for s in operation_logger.related_to if s[0] != "app"
     ]
@@ -988,6 +977,7 @@ def app_install(
 
     # Prepare env. var. to pass to script
     env_dict = _make_environment_for_app_script(app_instance_name, args=args_odict)
+    env_dict["YNH_APP_BASEDIR"] = extracted_app_folder
 
     env_dict_for_logging = env_dict.copy()
     for arg_name, arg_value_and_type in args_odict.items():
@@ -1052,6 +1042,7 @@ def app_install(
             env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name
             env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(instance_number)
             env_dict_remove["YNH_APP_MANIFEST_VERSION"] = manifest.get("version", "?")
+            env_dict_remove["YNH_APP_BASEDIR"] = extracted_app_folder
 
             # Execute remove script
             operation_logger_remove = OperationLogger(
@@ -1169,6 +1160,8 @@ def app_remove(operation_logger, app, purge=False):
     env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
     env_dict["YNH_APP_MANIFEST_VERSION"] = manifest.get("version", "?")
     env_dict["YNH_APP_PURGE"] = str(purge)
+    env_dict["YNH_APP_BASEDIR"] = tmp_workdir_for_app
+
     operation_logger.extra.update({"env": env_dict})
     operation_logger.flush()
 
@@ -1654,12 +1647,14 @@ def app_action_run(operation_logger, app, action, args=None):
     )
     args_odict = _parse_args_for_action(actions[action], args=args_dict)
 
+    tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
+
     env_dict = _make_environment_for_app_script(
         app, args=args_odict, args_prefix="ACTION_"
     )
     env_dict["YNH_ACTION"] = action
+    env_dict["YNH_APP_BASEDIR"] = tmp_workdir_for_app
 
-    tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
     _, action_script = tempfile.mkstemp(dir=tmp_workdir_for_app)
 
     with open(action_script, "w") as script:
@@ -1731,8 +1726,6 @@ def app_config_set(
 
     config_ = AppConfigPanel(app)
 
-    Question.operation_logger = operation_logger
-
     return config_.set(key, value, args, args_file, operation_logger=operation_logger)
 
 
@@ -1787,7 +1780,8 @@ ynh_app_config_run $1
                 "app_id": app_id,
                 "app": self.app,
                 "app_instance_nb": str(app_instance_nb),
-                "final_path": settings.get("final_path", "")
+                "final_path": settings.get("final_path", ""),
+                "YNH_APP_BASEDIR": os.path.join(APPS_SETTING_PATH, self.app),
             }
         )
 

src/yunohost/backup.py

@@ -707,6 +707,7 @@ class BackupManager:
 
         # Prepare environment
         env_dict = self._get_env_var(app)
+        env_dict["YNH_APP_BASEDIR"] = os.path.join(self.work_dir, "apps", app, "settings")
         tmp_app_bkp_dir = env_dict["YNH_APP_BACKUP_DIR"]
         settings_dir = os.path.join(self.work_dir, "apps", app, "settings")
 
@@ -1487,6 +1488,7 @@ class RestoreManager:
                 "YNH_APP_BACKUP_DIR": os.path.join(
                     self.work_dir, "apps", app_instance_name, "backup"
                 ),
+                "YNH_APP_BASEDIR": os.path.join(self.work_dir, "apps", app_instance_name, "settings"),
             }
         )
 
@@ -1524,6 +1526,7 @@ class RestoreManager:
 
                 # Setup environment for remove script
                 env_dict_remove = _make_environment_for_app_script(app_instance_name)
+                env_dict_remove["YNH_APP_BASEDIR"] = os.path.join(self.work_dir, "apps", app_instance_name, "settings")
 
                 remove_operation_logger = OperationLogger(
                     "remove_on_failed_restore",

src/yunohost/log.py

@@ -75,7 +75,7 @@ def log_list(limit=None, with_details=False, with_suboperations=False):
             # If we displaying only parent, we are still gonna load up to limit * 5 logs
             # because many of them are suboperations which are not gonna be kept
             # Yet we still want to obtain ~limit number of logs
-            logs = logs[:limit * 5]
+            logs = logs[: limit * 5]
 
     for log in logs:
 
@@ -186,12 +186,17 @@ def log_show(
                 r"DEBUG - \+ exit (1|0)$",
             ]
             filters = [re.compile(f) for f in filters]
-            return [line for line in lines if not any(f.search(line.strip()) for f in filters)]
+            return [
+                line
+                for line in lines
+                if not any(f.search(line.strip()) for f in filters)
+            ]
+
     else:
+
         def _filter(lines):
             return lines
 
-
     # Normalize log/metadata paths and filenames
     abs_path = path
     log_path = None

src/yunohost/permission.py

@@ -457,22 +457,26 @@ def permission_create(
             "permission_creation_failed", permission=permission, error=e
         )
 
-    permission_url(
-        permission,
-        url=url,
-        add_url=additional_urls,
-        auth_header=auth_header,
-        sync_perm=False,
-    )
+    try:
+        permission_url(
+            permission,
+            url=url,
+            add_url=additional_urls,
+            auth_header=auth_header,
+            sync_perm=False,
+        )
 
-    new_permission = _update_ldap_group_permission(
-        permission=permission,
-        allowed=allowed,
-        label=label,
-        show_tile=show_tile,
-        protected=protected,
-        sync_perm=sync_perm,
-    )
+        new_permission = _update_ldap_group_permission(
+            permission=permission,
+            allowed=allowed,
+            label=label,
+            show_tile=show_tile,
+            protected=protected,
+            sync_perm=sync_perm,
+        )
+    except:
+        permission_delete(permission, force=True)
+        raise
 
     logger.debug(m18n.n("permission_created", permission=permission))
     return new_permission

src/yunohost/tests/test_questions.py

@@ -348,9 +348,7 @@ def test_question_password():
     ]
     answers = {"some_password": "some_value"}
     expected_result = OrderedDict({"some_password": ("some_value", "password")})
-    Question.operation_logger = MagicMock()
-    with patch.object(Question.operation_logger, "data_to_redact", create=True):
-        assert parse_args_in_yunohost_format(answers, questions) == expected_result
+    assert parse_args_in_yunohost_format(answers, questions) == expected_result
 
 
 def test_question_password_no_input():
@@ -375,13 +373,9 @@ def test_question_password_input():
         }
     ]
     answers = {}
-    Question.operation_logger = {"data_to_redact": []}
     expected_result = OrderedDict({"some_password": ("some_value", "password")})
 
-    Question.operation_logger = MagicMock()
-    with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
+    with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
         os, "isatty", return_value=True
     ):
         assert parse_args_in_yunohost_format(answers, questions) == expected_result
@@ -397,10 +391,7 @@ def test_question_password_input_no_ask():
     answers = {}
     expected_result = OrderedDict({"some_password": ("some_value", "password")})
 
-    Question.operation_logger = MagicMock()
-    with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
+    with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
         os, "isatty", return_value=True
     ):
         assert parse_args_in_yunohost_format(answers, questions) == expected_result
@@ -417,20 +408,14 @@ def test_question_password_no_input_optional():
     answers = {}
     expected_result = OrderedDict({"some_password": ("", "password")})
 
-    Question.operation_logger = MagicMock()
-    with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(os, "isatty", return_value=False):
+    with patch.object(os, "isatty", return_value=False):
         assert parse_args_in_yunohost_format(answers, questions) == expected_result
 
     questions = [
         {"name": "some_password", "type": "password", "optional": True, "default": ""}
     ]
 
-    Question.operation_logger = MagicMock()
-    with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(os, "isatty", return_value=False):
+    with patch.object(os, "isatty", return_value=False):
         assert parse_args_in_yunohost_format(answers, questions) == expected_result
 
 
@@ -446,10 +431,7 @@ def test_question_password_optional_with_input():
     answers = {}
     expected_result = OrderedDict({"some_password": ("some_value", "password")})
 
-    Question.operation_logger = MagicMock()
-    with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
+    with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
         os, "isatty", return_value=True
     ):
         assert parse_args_in_yunohost_format(answers, questions) == expected_result
@@ -467,10 +449,7 @@ def test_question_password_optional_with_empty_input():
     answers = {}
     expected_result = OrderedDict({"some_password": ("", "password")})
 
-    Question.operation_logger = MagicMock()
-    with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(Moulinette, "prompt", return_value=""), patch.object(
+    with patch.object(Moulinette, "prompt", return_value=""), patch.object(
         os, "isatty", return_value=True
     ):
         assert parse_args_in_yunohost_format(answers, questions) == expected_result
@@ -487,10 +466,7 @@ def test_question_password_optional_with_input_without_ask():
     answers = {}
     expected_result = OrderedDict({"some_password": ("some_value", "password")})
 
-    Question.operation_logger = MagicMock()
-    with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
+    with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object(
         os, "isatty", return_value=True
     ):
         assert parse_args_in_yunohost_format(answers, questions) == expected_result
@@ -540,10 +516,7 @@ def test_question_password_input_test_ask():
     ]
     answers = {}
 
-    Question.operation_logger = MagicMock()
     with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(
         Moulinette, "prompt", return_value="some_value"
     ) as prompt, patch.object(
         os, "isatty", return_value=True
@@ -572,10 +545,7 @@ def test_question_password_input_test_ask_with_example():
     ]
     answers = {}
 
-    Question.operation_logger = MagicMock()
     with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(
         Moulinette, "prompt", return_value="some_value"
     ) as prompt, patch.object(
         os, "isatty", return_value=True
@@ -599,10 +569,7 @@ def test_question_password_input_test_ask_with_help():
     ]
     answers = {}
 
-    Question.operation_logger = MagicMock()
     with patch.object(
-        Question.operation_logger, "data_to_redact", create=True
-    ), patch.object(
         Moulinette, "prompt", return_value="some_value"
     ) as prompt, patch.object(
         os, "isatty", return_value=True

src/yunohost/utils/config.py

@@ -39,6 +39,7 @@ from moulinette.utils.filesystem import (
 
 from yunohost.utils.i18n import _value_for_locale
 from yunohost.utils.error import YunohostError, YunohostValidationError
+from yunohost.log import OperationLogger
 
 logger = getActionLogger("yunohost.config")
 CONFIG_PANEL_VERSION_SUPPORTED = 1.0
@@ -453,7 +454,6 @@ class ConfigPanel:
 
 class Question(object):
     hide_user_input_in_prompt = False
-    operation_logger = None
     pattern = None
 
     def __init__(self, question, user_answers):
@@ -490,7 +490,7 @@ class Question(object):
         self.value = Moulinette.prompt(
             message=text,
             is_password=self.hide_user_input_in_prompt,
-            confirm=False, # We doesn't want to confirm this kind of password like in webadmin
+            confirm=False,  # We doesn't want to confirm this kind of password like in webadmin
             prefill=prefill,
             is_multiline=(self.type == "text"),
         )
@@ -587,13 +587,9 @@ class Question(object):
             for data in data_to_redact
             if urllib.parse.quote(data) != data
         ]
-        if self.operation_logger:
-            self.operation_logger.data_to_redact.extend(data_to_redact)
-        elif data_to_redact:
-            raise YunohostError(
-                f"Can't redact {self.name} because no operation logger available in the context",
-                raw_msg=True,
-            )
+
+        for operation_logger in OperationLogger._instances:
+            operation_logger.data_to_redact.extend(data_to_redact)
 
         return self.value
 
@@ -658,6 +654,12 @@ class TagsQuestion(Question):
             return ",".join(value)
         return value
 
+    @staticmethod
+    def normalize(value, option={}):
+        if isinstance(value, list):
+            return ",".join(value)
+        return value
+
     def _prevalidate(self):
         values = self.value
         if isinstance(values, str):
@@ -669,6 +671,11 @@ class TagsQuestion(Question):
             super()._prevalidate()
         self.value = values
 
+    def _post_parse_value(self):
+        if isinstance(self.value, list):
+            self.value = ",".join(self.value)
+        return super()._post_parse_value()
+
 
 class PasswordQuestion(Question):
     hide_user_input_in_prompt = True
@@ -706,11 +713,17 @@ class PasswordQuestion(Question):
 
     def _format_text_for_user_input_in_cli(self):
         need_column = self.current_value or self.optional
-        text_for_user_input_in_cli = super()._format_text_for_user_input_in_cli(need_column)
+        text_for_user_input_in_cli = super()._format_text_for_user_input_in_cli(
+            need_column
+        )
         if self.current_value:
-            text_for_user_input_in_cli += "\n - " + m18n.n("app_argument_password_help_keep")
+            text_for_user_input_in_cli += "\n - " + m18n.n(
+                "app_argument_password_help_keep"
+            )
         if self.optional:
-            text_for_user_input_in_cli += "\n - " + m18n.n("app_argument_password_help_optional")
+            text_for_user_input_in_cli += "\n - " + m18n.n(
+                "app_argument_password_help_optional"
+            )
 
         return text_for_user_input_in_cli
 
@@ -834,7 +847,7 @@ class UserQuestion(Question):
             raise YunohostValidationError(
                 "app_argument_invalid",
                 name=self.name,
-                error="You should create a YunoHost user first."
+                error="You should create a YunoHost user first.",
             )
 
         if self.default is None:
@@ -949,11 +962,11 @@ class FileQuestion(Question):
                     "content": self.value,
                     "filename": user_answers.get(f"{self.name}[name]", self.name),
                 }
-        # If path file are the same
-        if self.value and str(self.value) == self.current_value:
-            self.value = None
 
     def _prevalidate(self):
+        if self.value is None:
+            self.value = self.current_value
+
         super()._prevalidate()
         if (
             isinstance(self.value, str)
@@ -988,7 +1001,7 @@ class FileQuestion(Question):
         if not self.value:
             return self.value
 
-        if Moulinette.interface.type == "api":
+        if Moulinette.interface.type == "api" and isinstance(self.value, dict):
 
             upload_dir = tempfile.mkdtemp(prefix="tmp_configpanel_")
             FileQuestion.upload_dirs += [upload_dir]