Merge branch 'dev' into enh-apps-v2

.gitlab/ci/test.gitlab-ci.yml

@@ -1,7 +1,7 @@
 .install_debs: &install_debs
     - apt-get update -o Acquire::Retries=3
     - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
-    - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2
+    - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22"
 
 .test-stage:
   stage: test

conf/fail2ban/postfix-sasl.conf

@@ -0,0 +1,6 @@
+# Fail2Ban filter for postfix authentication failures
+[INCLUDES]
+before = common.conf
+[Definition]
+_daemon = postfix/smtpd
+failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$

conf/fail2ban/yunohost-jails.conf

@@ -8,6 +8,13 @@ enabled = true
 [postfix]
 enabled = true
 
+[sasl]
+enabled  = true
+port     = smtp
+filter   = postfix-sasl
+logpath  = /var/log/mail.log
+maxretry = 5
+
 [dovecot]
 enabled = true
 

helpers/utils

@@ -957,3 +957,7 @@ _ynh_apply_default_permissions() {
         chown root:root $target
     fi
 }
+
+int_to_bool() {
+    sed -e 's/^1$/True/g' -e 's/^0$/False/g'
+}

hooks/conf_regen/03-ssh

@@ -17,7 +17,7 @@ do_pre_regen() {
     # Support different strategy for security configurations
     export compatibility="$(yunohost settings get 'security.ssh.ssh_compatibility')"
     export port="$(yunohost settings get 'security.ssh.ssh_port')"
-    export password_authentication="$(yunohost settings get 'security.ssh.ssh_password_authentication')"
+    export password_authentication="$(yunohost settings get 'security.ssh.ssh_password_authentication' | int_to_bool)"
     export ssh_keys
     export ipv6_enabled
     ynh_render_template "sshd_config" "${pending_dir}/etc/ssh/sshd_config"

hooks/conf_regen/06-slapd

@@ -123,6 +123,10 @@ do_post_regen() {
     chown -R openldap:openldap /etc/ldap/schema/
     chown -R openldap:openldap /etc/ldap/slapd.d/
 
+    # Fix weird scenarios where /etc/sudo-ldap.conf doesn't exists (yet is supposed to be
+    # created by the sudo-ldap package) : https://github.com/YunoHost/issues/issues/2091
+    [ -e /etc/sudo-ldap.conf ] || ln -s /etc/ldap/ldap.conf /etc/sudo-ldap.conf
+
     # If we changed the systemd ynh-override conf
     if echo "$regen_conf_files" | sed 's/,/\n/g' | grep -q "^/etc/systemd/system/slapd.service.d/ynh-override.conf$"; then
         systemctl daemon-reload

hooks/conf_regen/15-nginx

@@ -56,8 +56,8 @@ do_pre_regen() {
     # install / update plain conf files
     cp plain/* "$nginx_conf_dir"
     # remove the panel overlay if this is specified in settings
-    panel_overlay=$(yunohost settings get 'misc.portal.ssowat_panel_overlay_enabled')
-    if [ "$panel_overlay" == "false" ] || [ "$panel_overlay" == "False" ]; then
+    panel_overlay=$(yunohost settings get 'misc.portal.ssowat_panel_overlay_enabled' | int_to_bool)
+    if [ "$panel_overlay" == "False" ]; then
         echo "#" >"${nginx_conf_dir}/yunohost_panel.conf.inc"
     fi
 
@@ -65,9 +65,9 @@ do_pre_regen() {
     main_domain=$(cat /etc/yunohost/current_host)
 
     # Support different strategy for security configurations
-    export redirect_to_https="$(yunohost settings get 'security.nginx.nginx_redirect_to_https')"
+    export redirect_to_https="$(yunohost settings get 'security.nginx.nginx_redirect_to_https' | int_to_bool)"
     export compatibility="$(yunohost settings get 'security.nginx.nginx_compatibility')"
-    export experimental="$(yunohost settings get 'security.experimental.security_experimental_enabled')"
+    export experimental="$(yunohost settings get 'security.experimental.security_experimental_enabled' | int_to_bool)"
     ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc"
 
     cert_status=$(yunohost domain cert status --json)
@@ -109,7 +109,7 @@ do_pre_regen() {
 
     done
 
-    export webadmin_allowlist_enabled=$(yunohost settings get security.webadmin.webadmin_allowlist_enabled)
+    export webadmin_allowlist_enabled=$(yunohost settings get security.webadmin.webadmin_allowlist_enabled | int_to_bool)
     if [ "$webadmin_allowlist_enabled" == "True" ]; then
         export webadmin_allowlist=$(yunohost settings get security.webadmin.webadmin_allowlist)
     fi

hooks/conf_regen/19-postfix

@@ -29,8 +29,8 @@ do_pre_regen() {
     export relay_port=""
     export relay_user=""
     export relay_host=""
-    export relay_enabled="$(yunohost settings get 'email.smtp.smtp_relay_enabled')"
-    if [ "${relay_enabled}" == "1" ]; then
+    export relay_enabled="$(yunohost settings get 'email.smtp.smtp_relay_enabled' | int_to_bool)"
+    if [ "${relay_enabled}" == "True" ]; then
         relay_host="$(yunohost settings get 'email.smtp.smtp_relay_host')"
         relay_port="$(yunohost settings get 'email.smtp.smtp_relay_port')"
         relay_user="$(yunohost settings get 'email.smtp.smtp_relay_user')"
@@ -56,7 +56,7 @@ do_pre_regen() {
             >"${default_dir}/postsrsd"
 
     # adapt it for IPv4-only hosts
-    ipv6="$(yunohost settings get 'email.smtp.smtp_allow_ipv6')"
+    ipv6="$(yunohost settings get 'email.smtp.smtp_allow_ipv6' | int_to_bool)"
     if [ "$ipv6" == "False" ] || [ ! -f /proc/net/if_inet6 ]; then
         sed -i \
             's/ \[::ffff:127.0.0.0\]\/104 \[::1\]\/128//g' \

hooks/conf_regen/52-fail2ban

@@ -14,6 +14,7 @@ do_pre_regen() {
     mkdir -p "${fail2ban_dir}/jail.d"
 
     cp yunohost.conf "${fail2ban_dir}/filter.d/yunohost.conf"
+    cp postfix-sasl.conf "${fail2ban_dir}/filter.d/postfix-sasl.conf"
     cp jail.conf "${fail2ban_dir}/jail.conf"
 
     export ssh_port="$(yunohost settings get 'security.ssh.ssh_port')"

share/config_domain.toml

@@ -9,6 +9,8 @@ name = "Features"
         type = "app"
         filter = "is_webapp"
         default = "_none"
+        # FIXME: i18n
+        help = "People will automatically be redirected to this app when opening this domain. If no app is specified, people are redirected to the user portal login form."
 
     [feature.mail]
 
@@ -25,6 +27,7 @@ name = "Features"
         [feature.xmpp.xmpp]
         type = "boolean"
         default = 0
+        # FIXME: i18n
         help = "NB: some XMPP features will require that you update your DNS records and regenerate your Lets Encrypt certificate to be enabled"
 
 [dns]

src/certificate.py

@@ -624,8 +624,6 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder):
 
 def _get_status(domain):
 
-    import yunohost.domain
-
     cert_file = os.path.join(CERT_FOLDER, domain, "crt.pem")
 
     if not os.path.isfile(cert_file):
@@ -654,21 +652,9 @@ def _get_status(domain):
     )
     days_remaining = (valid_up_to - datetime.utcnow()).days
 
-    self_signed_issuers = ["yunohost.org"] + yunohost.domain.domain_list()["domains"]
-
-    # FIXME: is the .ca.cnf one actually used anywhere ? x_x
-    conf = os.path.join(SSL_DIR, "openssl.ca.cnf")
-    if os.path.exists(conf):
-        self_signed_issuers.append(
-            check_output(f"grep commonName_default {conf}").split()[-1]
-        )
-    conf = os.path.join(SSL_DIR, "openssl.cnf")
-    if os.path.exists(conf):
-        self_signed_issuers.append(
-            check_output(f"grep commonName_default {conf}").split()[-1]
-        )
-
-    if cert_issuer in self_signed_issuers:
+    # Identify that a domain's cert is self-signed if the cert dir
+    # is actually a symlink to a dir ending with -selfsigned
+    if os.path.realpath(os.path.join(CERT_FOLDER, domain)).endswith("-selfsigned"):
         CA_type = "selfsigned"
     elif organization_name == "Let's Encrypt":
         CA_type = "letsencrypt"
@@ -752,7 +738,7 @@ def _enable_certificate(domain, new_cert_folder):
 
     logger.debug("Restarting services...")
 
-    for service in ("postfix", "dovecot", "metronome"):
+    for service in ("dovecot", "metronome"):
         # Ugly trick to not restart metronome if it's not installed
         if (
             service == "metronome"
@@ -764,7 +750,8 @@ def _enable_certificate(domain, new_cert_folder):
     if os.path.isfile("/etc/yunohost/installed"):
         # regen nginx conf to be sure it integrates OCSP Stapling
         # (We don't do this yet if postinstall is not finished yet)
-        regen_conf(names=["nginx"])
+        # We also regenconf for postfix to propagate the SNI hash map thingy
+        regen_conf(names=["nginx", "postfix"])
 
     _run_service_command("reload", "nginx")
 

src/settings.py

@@ -131,8 +131,12 @@ class SettingsConfigPanel(ConfigPanel):
         root_password_confirm = self.new_values.pop("root_password_confirm", None)
         passwordless_sudo = self.new_values.pop("passwordless_sudo", None)
 
-        self.values = {k: v for k, v in self.values.items() if k not in self.virtual_settings}
-        self.new_values = {k: v for k, v in self.new_values.items() if k not in self.virtual_settings}
+        self.values = {
+            k: v for k, v in self.values.items() if k not in self.virtual_settings
+        }
+        self.new_values = {
+            k: v for k, v in self.new_values.items() if k not in self.virtual_settings
+        }
 
         assert all(v not in self.future_values for v in self.virtual_settings)
 
@@ -147,8 +151,12 @@ class SettingsConfigPanel(ConfigPanel):
 
         if passwordless_sudo is not None:
             from yunohost.utils.ldap import _get_ldap_interface
+
             ldap = _get_ldap_interface()
-            ldap.update("cn=admins,ou=sudo", {"sudoOption": ["!authenticate"] if passwordless_sudo else []})
+            ldap.update(
+                "cn=admins,ou=sudo",
+                {"sudoOption": ["!authenticate"] if passwordless_sudo else []},
+            )
 
         super()._apply()
 
@@ -173,7 +181,7 @@ class SettingsConfigPanel(ConfigPanel):
         try:
             themes = [d for d in os.listdir(THEMEDIR) if os.path.isdir(THEMEDIR + d)]
         except Exception:
-            themes = ['unsplash', 'vapor', 'light', 'default', 'clouds']
+            themes = ["unsplash", "vapor", "light", "default", "clouds"]
         toml["misc"]["portal"]["portal_theme"]["choices"] = themes
 
         return toml
@@ -190,8 +198,11 @@ class SettingsConfigPanel(ConfigPanel):
         # Specific logic for virtual setting "passwordless_sudo"
         try:
             from yunohost.utils.ldap import _get_ldap_interface
+
             ldap = _get_ldap_interface()
-            self.values["passwordless_sudo"] = "!authenticate" in ldap.search("ou=sudo", "cn=admins", ["sudoOption"])[0].get("sudoOption", [])
+            self.values["passwordless_sudo"] = "!authenticate" in ldap.search(
+                "ou=sudo", "cn=admins", ["sudoOption"]
+            )[0].get("sudoOption", [])
         except:
             self.values["passwordless_sudo"] = False
 
@@ -285,12 +296,15 @@ def trigger_post_change_hook(setting_name, old_value, new_value):
 #
 # ===========================================
 
+
 @post_change_hook("portal_theme")
 def regen_ssowatconf(setting_name, old_value, new_value):
     if old_value != new_value:
         from yunohost.app import app_ssowatconf
+
         app_ssowatconf()
 
+
 @post_change_hook("ssowat_panel_overlay_enabled")
 @post_change_hook("nginx_redirect_to_https")
 @post_change_hook("nginx_compatibility")

src/tests/test_app_config.py

@@ -109,7 +109,7 @@ def test_app_config_get(config_app):
     assert isinstance(app_config_get(config_app, export=True), dict)
     assert isinstance(app_config_get(config_app, "main"), dict)
     assert isinstance(app_config_get(config_app, "main.components"), dict)
-    assert app_config_get(config_app, "main.components.boolean") == "0"
+    assert app_config_get(config_app, "main.components.boolean") == 0
 
     user_delete("alice")
 
@@ -141,16 +141,16 @@ def test_app_config_get_nonexistentstuff(config_app):
 
 def test_app_config_regular_setting(config_app):
 
-    assert app_config_get(config_app, "main.components.boolean") == "0"
+    assert app_config_get(config_app, "main.components.boolean") == 0
 
     app_config_set(config_app, "main.components.boolean", "no")
 
-    assert app_config_get(config_app, "main.components.boolean") == "0"
+    assert app_config_get(config_app, "main.components.boolean") == 0
     assert app_setting(config_app, "boolean") == "0"
 
     app_config_set(config_app, "main.components.boolean", "yes")
 
-    assert app_config_get(config_app, "main.components.boolean") == "1"
+    assert app_config_get(config_app, "main.components.boolean") == 1
     assert app_setting(config_app, "boolean") == "1"
 
     with pytest.raises(YunohostValidationError), patch.object(

src/tests/test_permission.py

@@ -258,7 +258,7 @@ def check_LDAP_db_integrity():
 
     for user in user_search:
         user_dn = "uid=" + user["uid"][0] + ",ou=users,dc=yunohost,dc=org"
-        group_list = [_ldap_path_extract(m, "cn") for m in user["memberOf"]]
+        group_list = [_ldap_path_extract(m, "cn") for m in user.get("memberOf", [])]
         permission_list = [
             _ldap_path_extract(m, "cn") for m in user.get("permission", [])
         ]

src/tests/test_settings.py

@@ -1,6 +1,7 @@
 import os
 import pytest
 import yaml
+from mock import patch
 
 import moulinette
 from yunohost.utils.error import YunohostError, YunohostValidationError
@@ -152,10 +153,10 @@ def test_settings_get_doesnt_exists():
 
 def test_settings_set():
     settings_set("example.example.boolean", False)
-    assert settings_get("example.example.boolean") is False
+    assert settings_get("example.example.boolean") == 0
 
     settings_set("example.example.boolean", "on")
-    assert settings_get("example.example.boolean") is True
+    assert settings_get("example.example.boolean") == 1
 
 
 def test_settings_set_int():
@@ -174,35 +175,39 @@ def test_settings_set_doesexit():
 
 
 def test_settings_set_bad_type_bool():
-    with pytest.raises(YunohostError):
-        settings_set("example.example.boolean", 42)
-    with pytest.raises(YunohostError):
-        settings_set("example.example.boolean", "pouet")
+
+    with patch.object(os, "isatty", return_value=False):
+        with pytest.raises(YunohostError):
+            settings_set("example.example.boolean", 42)
+        with pytest.raises(YunohostError):
+            settings_set("example.example.boolean", "pouet")
 
 
 def test_settings_set_bad_type_int():
     #    with pytest.raises(YunohostError):
     #        settings_set("example.example.number", True)
-    with pytest.raises(YunohostError):
-        settings_set("example.example.number", "pouet")
+    with patch.object(os, "isatty", return_value=False):
+        with pytest.raises(YunohostError):
+            settings_set("example.example.number", "pouet")
 
 
 # def test_settings_set_bad_type_string():
 #    with pytest.raises(YunohostError):
-#        settings_set("example.example.string", True)
+#        settings_set(eexample.example.string", True)
 #    with pytest.raises(YunohostError):
 #        settings_set("example.example.string", 42)
 
 
 def test_settings_set_bad_value_select():
-    with pytest.raises(YunohostError):
-        settings_set("example.example.select", True)
-    with pytest.raises(YunohostError):
-        settings_set("example.example.select", "e")
-    with pytest.raises(YunohostError):
-        settings_set("example.example.select", 42)
-    with pytest.raises(YunohostError):
-        settings_set("example.example.select", "pouet")
+    with patch.object(os, "isatty", return_value=False):
+        with pytest.raises(YunohostError):
+            settings_set("example.example.select", True)
+        with pytest.raises(YunohostError):
+            settings_set("example.example.select", "e")
+        with pytest.raises(YunohostError):
+            settings_set("example.example.select", 42)
+        with pytest.raises(YunohostError):
+            settings_set("example.example.select", "pouet")
 
 
 def test_settings_list_modified():

src/utils/config.py

@@ -264,8 +264,17 @@ class ConfigPanel:
 
         # In 'classic' mode, we display the current value if key refer to an option
         if self.filter_key.count(".") == 2 and mode == "classic":
+
             option = self.filter_key.split(".")[-1]
-            return self.values.get(option, None)
+            value = self.values.get(option, None)
+
+            option_type = None
+            for _, _, option_ in self._iterate():
+                if option_["id"] == option:
+                    option_type = ARGUMENTS_TYPE_PARSERS[option_["type"]]
+                    break
+
+            return option_type.normalize(value) if option_type else value
 
         # Format result in 'classic' or 'export' mode
         logger.debug(f"Formating result in '{mode}' mode")