Merge branch 'dev' into bullseye

conf/ssh/sshd_config

@@ -2,6 +2,8 @@
 # by YunoHost
 
 Protocol 2
+# PLEASE: if you wish to change the ssh port properly in YunoHost, use this command:
+# yunohost settings set security.ssh.port -v <port>
 Port {{ port }}
 
 {% if ipv6_enabled == "true" %}ListenAddress ::{% endif %}
@@ -53,9 +55,13 @@ PermitEmptyPasswords no
 ChallengeResponseAuthentication no
 UsePAM yes
 
-# Change to no to disable tunnelled clear text passwords
+# PLEASE: if you wish to force everybody to authenticate using ssh keys, run this command:
-# (i.e. everybody will need to authenticate using ssh keys)
+# yunohost settings set security.ssh.password_authentication -v no
+{% if password_authentication == "False" %}
+PasswordAuthentication no
+{% else %}
 #PasswordAuthentication yes
+{% endif %}
 
 # Post-login stuff
 Banner /etc/issue.net

helpers/utils

@@ -918,4 +918,11 @@ _ynh_apply_default_permissions() {
             chown $app:$app $target
         fi
     fi
+
+    # Crons should be owned by root otherwise they probably don't run
+    if echo "$target" | grep -q '^/etc/cron'
+    then
+        chmod 400 $target
+        chown root:root $target
+    fi
 }

hooks/conf_regen/03-ssh

@@ -22,6 +22,7 @@ do_pre_regen() {
     # Support different strategy for security configurations
     export compatibility="$(yunohost settings get 'security.ssh.compatibility')"
     export port="$(yunohost settings get 'security.ssh.port')"
+    export password_authentication="$(yunohost settings get 'security.ssh.password_authentication')"
     export ssh_keys
     export ipv6_enabled
     ynh_render_template "sshd_config" "${pending_dir}/etc/ssh/sshd_config"

hooks/conf_regen/10-apt

@@ -12,7 +12,7 @@ do_pre_regen() {
     echo "deb https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list"
 
     # Ban some packages from sury
-    packages_to_refuse_from_sury="php php-fpm php-mysql php-xml php-zip php-mbstring php-ldap php-gd php-curl php-bz2 php-json php-sqlite3 php-intl openssl libssl1.1 libssl-dev"
+    packages_to_refuse_from_sury="php php-* openssl libssl1.1 libssl-dev"
     for package in $packages_to_refuse_from_sury; do
         echo "
 Package: $package
@@ -20,6 +20,11 @@ Pin: origin \"packages.sury.org\"
 Pin-Priority: -1" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version"
     done
 
+    echo "
+Package: php-common
+Pin: origin \"packages.sury.org\"
+Pin-Priority: 500" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version"
+
     # Ban some packages that users may inadvertendly try to install such as apache2 ...
     echo "
 

locales/en.json

@@ -382,6 +382,7 @@
     "global_settings_setting_security_password_user_strength": "User password strength",
     "global_settings_setting_security_postfix_compatibility": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)",
     "global_settings_setting_security_ssh_compatibility": "Compatibility vs. security tradeoff for the SSH server. Affects the ciphers (and other security-related aspects)",
+    "global_settings_setting_security_ssh_password_authentication": "Allow password authentication for SSH",
     "global_settings_setting_security_ssh_port": "SSH port",
     "global_settings_setting_security_webadmin_allowlist": "IP adresses allowed to access the webadmin. Comma-separated.",
     "global_settings_setting_security_webadmin_allowlist_enabled": "Allow only some IPs to access the webadmin.",

src/app.py

@@ -609,7 +609,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
             if upgrade_failed or broke_the_system:
 
                 # display this if there are remaining apps
-                if apps[number + 1:]:
+                if apps[number + 1 :]:
                     not_upgraded_apps = apps[number:]
                     logger.error(
                         m18n.n(

src/domain.py

@@ -187,7 +187,7 @@ def domain_add(operation_logger, domain, dyndns=False):
         # Actually subscribe
         dyndns_subscribe(domain=domain)
 
-    _certificate_install_selfsigned([domain], False)
+    _certificate_install_selfsigned([domain], True)
 
     try:
         attr_dict = {
@@ -517,9 +517,7 @@ def domain_cert_install(
 ):
     from yunohost.certificate import certificate_install
 
-    return certificate_install(
+    return certificate_install(domain_list, force, no_checks, self_signed, staging)
-        domain_list, force, no_checks, self_signed, staging
-    )
 
 
 def domain_cert_renew(
@@ -527,9 +525,7 @@ def domain_cert_renew(
 ):
     from yunohost.certificate import certificate_renew
 
-    return certificate_renew(
+    return certificate_renew(domain_list, force, no_checks, email, staging)
-        domain_list, force, no_checks, email, staging
-    )
 
 
 def domain_dns_conf(domain):

src/log.py

@@ -659,6 +659,11 @@ class OperationLogger:
                 data["error"] = self._error
         # TODO: detect if 'extra' erase some key of 'data'
         data.update(self.extra)
+        # Remove the 'args' arg from args (yodawg). It corresponds to url-encoded args for app install, config panel set, etc
+        # Because the data are url encoded, it's hell to properly redact secrets inside it,
+        # and the useful info is usually already available in `env` too
+        if "args" in data and isinstance(data["args"], dict) and "args" in data["args"]:
+            data["args"].pop("args")
         return data
 
     def success(self):

src/migrations/0021_migrate_to_bullseye.py

@@ -7,7 +7,7 @@ from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import check_output, call_async_output
 from moulinette.utils.filesystem import read_file, rm
 
-from yunohost.tools import Migration, tools_update, tools_upgrade
+from yunohost.tools import Migration, tools_update, tools_upgrade, _apt_log_line_is_relevant
 from yunohost.app import unstable_apps
 from yunohost.regenconf import manually_modified_files, _force_clear_hashes
 from yunohost.utils.filesystem import free_space_in_directory
@@ -50,6 +50,25 @@ class MyMigration(Migration):
         #
         logger.info(m18n.n("migration_0021_patching_sources_list"))
         self.patch_apt_sources_list()
+
+        # Force add sury if it's not there yet
+        # This is to solve some weird issue with php-common breaking php7.3-common,
+        # hence breaking many php7.3-deps
+        # hence triggering some dependency conflict (or foobar-ynh-deps uninstall)
+        # Adding it there shouldnt be a big deal - Yunohost 11.x does add it
+        # through its regen conf anyway.
+        if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"):
+            open("/etc/apt/sources.list.d/extra_php_version.list", "w").write(
+                "deb https://packages.sury.org/php/ bullseye main"
+            )
+            os.system(
+                'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"'
+            )
+
+        #
+        # Run apt update
+        #
+
         tools_update(target="system")
 
         # Tell libc6 it's okay to restart system stuff during the upgrade
@@ -77,9 +96,12 @@ class MyMigration(Migration):
         _force_clear_hashes(["/etc/mysql/my.cnf"])
         rm("/etc/mysql/mariadb.cnf", force=True)
         rm("/etc/mysql/my.cnf", force=True)
-        self.apt_install(
+        ret = self.apt_install(
             "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'"
         )
+        if ret != 0:
+            # FIXME: i18n once this is stable?
+            raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True)
 
         #
         # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl
@@ -97,6 +119,13 @@ class MyMigration(Migration):
             os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf")
             rm("/home/yunohost.conf", recursive=True, force=True)
 
+        # Remove legacy postgresql service record added by helpers,
+        # will now be dynamically handled by the core in bullseye
+        services = _get_services()
+        if "postgresql" in services:
+            del services["postgresql"]
+            _save_services(services)
+
         #
         # Main upgrade
         #
@@ -109,43 +138,105 @@ class MyMigration(Migration):
         if self.debian_major_version() == N_CURRENT_DEBIAN:
             raise YunohostError("migration_0021_still_on_buster_after_main_upgrade")
 
+        # Force explicit install of php7.4-fpm and other old 'default' dependencies
+        # that are now only in Recommends
+        #
+        # Also, we need to install php7.4 equivalents of other php7.3 dependencies.
+        # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration
+        # to autoupgrade Nextcloud to 7.4, it will need the php7.4-zip to work.
+        # The following list is based on an ad-hoc analysis of php deps found in the
+        # app ecosystem, with a known equivalent on php7.4.
+        #
+        # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages
+        # with the proper list of dependencies, and the dependencies install this way
+        # will get flagged as 'manually installed'.
+        #
+        # We'll probably want to do something during the Bullseye->Bookworm migration to re-flag
+        # these as 'auto' so they get autoremoved if not needed anymore.
+        # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use
+        # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;)
+
+        php73packages_suffixes = [
+            "apcu",
+            "bcmath",
+            "bz2",
+            "dom",
+            "gmp",
+            "igbinary",
+            "imagick",
+            "imap",
+            "mbstring",
+            "memcached",
+            "mysqli",
+            "mysqlnd",
+            "pgsql",
+            "redis",
+            "simplexml",
+            "soap",
+            "sqlite3",
+            "ssh2",
+            "tidy",
+            "xml",
+            "xmlrpc",
+            "xsl",
+            "zip",
+        ]
+
+        cmd = "apt show '*-ynh-deps' 2>/dev/null" \
+              "  | grep Depends" \
+              f" | grep -o -E \"php7.3-({'|'.join(php73packages_suffixes)})\"" \
+              "  | sort | uniq" \
+              "  | sed 's/php7.3/php7.4/g'" \
+              "  || true"
+
+        php74packages_to_install = [
+            "php7.4-fpm",
+            "php7.4-common",
+            "php7.4-ldap",
+            "php7.4-intl",
+            "php7.4-mysql",
+            "php7.4-gd",
+            "php7.4-curl",
+            "php-php-gettext",
+        ]
+
+        php74packages_to_install += [
+            f.strip() for f in check_output(cmd).split("\n") if f.strip()
+        ]
+
+        ret = self.apt_install(
+            f"{' '.join(php74packages_to_install)} "
+            "$(dpkg --list | grep ynh-deps | awk '{print $2}') "
+            "-o Dpkg::Options::='--force-confmiss'"
+        )
+        if ret != 0:
+            # FIXME: i18n once this is stable?
+            raise YunohostError("Failed to force the install of php dependencies ?", raw_msg=True)
+
         # Clean the mess
         logger.info(m18n.n("migration_0021_cleaning_up"))
         os.system("apt autoremove --assume-yes")
         os.system("apt clean --assume-yes")
 
-        # Force add sury if it's not there yet
-        # This is to solve some weird issue with php-common breaking php7.3-common,
-        # hence breaking many php7.3-deps
-        # hence triggering some dependency conflict (or foobar-ynh-deps uninstall)
-        # Adding it there shouldnt be a big deal - Yunohost 11.x does add it
-        # through its regen conf anyway.
-        if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"):
-            open("/etc/apt/sources.list.d/extra_php_version.list", "w").write(
-                "deb https://packages.sury.org/php/ bullseye main"
-            )
-            os.system(
-                'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"'
-            )
-
-        os.system("apt update")
-
-        # Force explicit install of php7.4-fpm to make sure it's ll be there
-        # during 0022_php73_to_php74_pools migration
-        self.apt_install("php7.4-fpm -o Dpkg::Options::='--force-confmiss'")
-
-        # Remove legacy postgresql service record added by helpers,
-        # will now be dynamically handled by the core in bullseye
-        services = _get_services()
-        if "postgresql" in services:
-            del services["postgresql"]
-            _save_services(services)
-
         #
         # Yunohost upgrade
         #
         logger.info(m18n.n("migration_0021_yunohost_upgrade"))
+
         self.unhold(apps_packages)
+
+        cmd = "LC_ALL=C"
+        cmd += " DEBIAN_FRONTEND=noninteractive"
+        cmd += " APT_LISTCHANGES_FRONTEND=none"
+        cmd += " apt dist-upgrade "
+        cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run"
+        cmd += " | grep -q '-ynh-deps'"
+
+        logger.info("Simulating upgrade...")
+        if os.system(cmd) == 0:
+            # FIXME: i18n once this is stable?
+            raise YunohostError("The upgrade cannot be completed, because some app dependencies would need to be removed?", raw_msg=True)
+
         tools_upgrade(target="system")
 
     def debian_major_version(self):
@@ -280,9 +371,11 @@ class MyMigration(Migration):
 
         callbacks = (
             lambda l: logger.info("+ " + l.rstrip() + "\r")
-            if is_relevant(l)
+            if _apt_log_line_is_relevant(l)
             else logger.debug(l.rstrip() + "\r"),
-            lambda l: logger.warning(l.rstrip()),
+            lambda l: logger.warning(l.rstrip())
+            if _apt_log_line_is_relevant(l)
+            else logger.debug(l.rstrip()),
         )
 
         cmd = (
@@ -292,7 +385,7 @@ class MyMigration(Migration):
 
         logger.debug("Running: %s" % cmd)
 
-        call_async_output(cmd, callbacks, shell=True)
+        return call_async_output(cmd, callbacks, shell=True)
 
     def patch_yunohost_conflicts(self):
         #

src/settings.py

@@ -81,6 +81,10 @@ DEFAULTS = OrderedDict(
             "security.ssh.port",
             {"type": "int", "default": 22},
         ),
+        (
+            "security.ssh.password_authentication",
+            {"type": "bool", "default": True},
+        ),
         (
             "security.nginx.redirect_to_https",
             {
@@ -420,6 +424,7 @@ def reconfigure_nginx_and_yunohost(setting_name, old_value, new_value):
 
 
 @post_change_hook("security.ssh.compatibility")
+@post_change_hook("security.ssh.password_authentication")
 def reconfigure_ssh(setting_name, old_value, new_value):
     if old_value != new_value:
         regen_conf(names=["ssh"])

src/tools.py

@@ -244,9 +244,7 @@ def tools_postinstall(
         # and inform the user that we could not contact the dyndns host server.
         except Exception:
             logger.warning(
-                m18n.n(
+                m18n.n("dyndns_provider_unreachable", provider="dyndns.yunohost.org")
-                    "dyndns_provider_unreachable", provider="dyndns.yunohost.org"
-                )
             )
 
         if available:
@@ -526,10 +524,10 @@ def tools_upgrade(
 
         callbacks = (
             lambda l: logger.info("+ " + l.rstrip() + "\r")
-            if is_relevant(l)
+            if _apt_log_line_is_relevant(l)
             else logger.debug(l.rstrip() + "\r"),
             lambda l: logger.warning(l.rstrip())
-            if is_relevant(l)
+            if _apt_log_line_is_relevant(l)
             else logger.debug(l.rstrip()),
         )
         returncode = call_async_output(dist_upgrade, callbacks, shell=True)
@@ -563,6 +561,24 @@ def tools_upgrade(
         operation_logger.success()
 
 
+def _apt_log_line_is_relevant(line):
+    irrelevants = [
+        "service sudo-ldap already provided",
+        "Reading database ...",
+        "Preparing to unpack",
+        "Selecting previously unselected package",
+        "Created symlink /etc/systemd",
+        "Replacing config file",
+        "Creating config file",
+        "Installing new version of config file",
+        "Installing new config file as you requested",
+        ", does not exist on system.",
+        "unable to delete old directory",
+        "update-alternatives:",
+    ]
+    return line.rstrip() and all(i not in line.rstrip() for i in irrelevants)
+
+
 @is_unit_operation()
 def tools_shutdown(operation_logger, force=False):
     shutdown = force
@@ -916,6 +932,10 @@ def _skip_all_migrations():
     all_migrations = _get_migrations_list()
     new_states = {"migrations": {}}
     for migration in all_migrations:
+        # Don't skip bullseye migration while we're
+        # still on buster
+        if "migrate_to_bullseye" in migration.id:
+            continue
         new_states["migrations"][migration.id] = "skipped"
     write_to_yaml(MIGRATIONS_STATE_PATH, new_states)
 

src/user.py

@@ -1258,9 +1258,7 @@ def user_group_remove(groupname, usernames, force=False, sync_perm=True):
 def user_permission_list(short=False, full=False, apps=[]):
     from yunohost.permission import user_permission_list
 
-    return user_permission_list(
+    return user_permission_list(short, full, absolute_urls=True, apps=apps)
-        short, full, absolute_urls=True, apps=apps
-    )
 
 
 def user_permission_update(permission, label=None, show_tile=None, sync_perm=True):