From 4106ed669e5a53652b46d9e7804dbdfa14d699fc Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 6 Dec 2021 23:01:06 +0100 Subject: [PATCH 01/17] [enh] Manage SSH PasswordAuthentication setting --- data/hooks/conf_regen/03-ssh | 1 + data/templates/ssh/sshd_config | 10 ++++++++-- src/yunohost/settings.py | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/data/hooks/conf_regen/03-ssh b/data/hooks/conf_regen/03-ssh index f10dbb653..3f04acd0c 100755 --- a/data/hooks/conf_regen/03-ssh +++ b/data/hooks/conf_regen/03-ssh @@ -26,6 +26,7 @@ do_pre_regen() { # Support different strategy for security configurations export compatibility="$(yunohost settings get 'security.ssh.compatibility')" export port="$(yunohost settings get 'security.ssh.port')" + export password_authentication="$(yunohost settings get 'security.ssh.password_authentication')" export ssh_keys export ipv6_enabled ynh_render_template "sshd_config" "${pending_dir}/etc/ssh/sshd_config" diff --git a/data/templates/ssh/sshd_config b/data/templates/ssh/sshd_config index 1c2854f73..22f0697d9 100644 --- a/data/templates/ssh/sshd_config +++ b/data/templates/ssh/sshd_config @@ -2,6 +2,8 @@ # by YunoHost Protocol 2 +# PLEASE: to change ssh port properly in YunoHost, use this command +# yunohost settings set security.ssh.port -v Port {{ port }} {% if ipv6_enabled == "true" %}ListenAddress ::{% endif %} @@ -53,9 +55,13 @@ PermitEmptyPasswords no ChallengeResponseAuthentication no UsePAM yes -# Change to no to disable tunnelled clear text passwords -# (i.e. everybody will need to authenticate using ssh keys) +# PLEASE: to force everybody to authenticate using ssh keys, run this command: +# yunohost settings set security.ssh.password_authentication -v no +{% if password_authentication == "True" %} #PasswordAuthentication yes +{% else %} +PasswordAuthentication no +{% endif %} # Post-login stuff Banner /etc/issue.net diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index d59b41a58..26b8c48b2 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -81,6 +81,10 @@ DEFAULTS = OrderedDict( "security.ssh.port", {"type": "int", "default": 22}, ), + ( + "security.ssh.password_authentication", + {"type": "bool", "default": True}, + ), ( "security.nginx.redirect_to_https", { @@ -420,6 +424,7 @@ def reconfigure_nginx_and_yunohost(setting_name, old_value, new_value): @post_change_hook("security.ssh.compatibility") +@post_change_hook("security.ssh.password_authentication") def reconfigure_ssh(setting_name, old_value, new_value): if old_value != new_value: regen_conf(names=["ssh"]) From 1b198e12f69badf99791cd88a037d96dca9ff039 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 7 Dec 2021 00:14:15 +0100 Subject: [PATCH 02/17] [fix] Missing locale key --- locales/en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/locales/en.json b/locales/en.json index 81e75eb32..66f42df58 100644 --- a/locales/en.json +++ b/locales/en.json @@ -382,6 +382,7 @@ "global_settings_setting_security_password_user_strength": "User password strength", "global_settings_setting_security_postfix_compatibility": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)", "global_settings_setting_security_ssh_compatibility": "Compatibility vs. security tradeoff for the SSH server. Affects the ciphers (and other security-related aspects)", + "global_settings_setting_security_ssh_password_authentication": "Password authentication allowed", "global_settings_setting_security_ssh_port": "SSH port", "global_settings_setting_security_webadmin_allowlist": "IP adresses allowed to access the webadmin. Comma-separated.", "global_settings_setting_security_webadmin_allowlist_enabled": "Allow only some IPs to access the webadmin.", From a063b63d1c86a1f30dac670cea17f2f24e9c944e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Dec 2021 15:35:02 +0100 Subject: [PATCH 03/17] ssh config : Invert the password_authentication value check to be more resilient in case something goes wrong while fetching / parsing the value --- data/templates/ssh/sshd_config | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data/templates/ssh/sshd_config b/data/templates/ssh/sshd_config index 22f0697d9..b6d4111ee 100644 --- a/data/templates/ssh/sshd_config +++ b/data/templates/ssh/sshd_config @@ -2,7 +2,7 @@ # by YunoHost Protocol 2 -# PLEASE: to change ssh port properly in YunoHost, use this command +# PLEASE: if you wish to change the ssh port properly in YunoHost, use this command: # yunohost settings set security.ssh.port -v Port {{ port }} @@ -55,12 +55,12 @@ PermitEmptyPasswords no ChallengeResponseAuthentication no UsePAM yes -# PLEASE: to force everybody to authenticate using ssh keys, run this command: +# PLEASE: if you wish to force everybody to authenticate using ssh keys, run this command: # yunohost settings set security.ssh.password_authentication -v no -{% if password_authentication == "True" %} -#PasswordAuthentication yes -{% else %} +{% if password_authentication == "False" %} PasswordAuthentication no +{% else %} +#PasswordAuthentication yes {% endif %} # Post-login stuff From dce411e0e62193094f6613dfdca081a7ca46a04c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Dec 2021 15:36:38 +0100 Subject: [PATCH 04/17] Improve setting description --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 66f42df58..bfd410740 100644 --- a/locales/en.json +++ b/locales/en.json @@ -382,7 +382,7 @@ "global_settings_setting_security_password_user_strength": "User password strength", "global_settings_setting_security_postfix_compatibility": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)", "global_settings_setting_security_ssh_compatibility": "Compatibility vs. security tradeoff for the SSH server. Affects the ciphers (and other security-related aspects)", - "global_settings_setting_security_ssh_password_authentication": "Password authentication allowed", + "global_settings_setting_security_ssh_password_authentication": "Allow password authentication for SSH", "global_settings_setting_security_ssh_port": "SSH port", "global_settings_setting_security_webadmin_allowlist": "IP adresses allowed to access the webadmin. Comma-separated.", "global_settings_setting_security_webadmin_allowlist_enabled": "Allow only some IPs to access the webadmin.", From fa9365d5fa50aff9b94b3356492679fa19ad8dca Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 3 Jan 2022 16:28:35 +0100 Subject: [PATCH 05/17] Yolotweaking for the bullseye migration mecanism to also install, for example, php7.4-zip if an app is gonna need it --- .../0021_migrate_to_bullseye.py | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index f97ab16da..10f385c23 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -130,9 +130,37 @@ class MyMigration(Migration): os.system("apt update") - # Force explicit install of php7.4-fpm to make sure it's ll be there - # during 0022_php73_to_php74_pools migration - self.apt_install("php7.4-fpm -o Dpkg::Options::='--force-confmiss'") + # Force explicit install of php7.4-fpm and other old 'default' dependencies + # that are now only in Recommends + # + # Also, we need to install php7.4 equivalents of other php7.3 dependencies. + # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration + # to autoupgrade Nextcloud to 7.4, it will need the php7.4-zip to work. + # The following list is based on an ad-hoc analysis of php deps found in the + # app ecosystem, with a known equivalent on php7.4. + # + # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages + # with the proper list of dependencies, and the dependencies install this way + # will get flagged as 'manually installed'. + # + # We'll probably want to do something during the Bullseye->Bookworm migration to re-flag + # these as 'auto' so they get autoremoved if not needed anymore. + # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use + # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) + + php73packages_suffixes = ['apcu', 'bcmath', 'bz2', 'dom', 'gmp', 'igbinary', 'imagick', 'imap', 'mbstring', 'memcached', 'mysqli', 'mysqlnd', 'pgsql', 'redis', 'simplexml', 'soap', 'sqlite3', 'ssh2', 'tidy', 'xml', 'xmlrpc', 'xsl', 'zip'] + + cmd = f""" + apt show '*-ynh-deps' 2>/dev/null + | grep Depends + | grep -o -E "php7.3-({'|'.join(php73packages_suffixes)})"; + | sort | uniq + | sed 's/php7.3/php7.4/g' + """ + php74packages_to_install = ["php7.4-fpm", "php7.4-common", "php7.4-ldap", "php7.4-intl", "php7.4-mysql", "php7.4-gd", "php7.4-curl", "php-php-gettext"] + php74packages_to_install += [f.strip() for f in check_output(cmd).split("\n") if f.strip()] + + self.apt_install("{' '.join(php74packages_to_install)} -o Dpkg::Options::='--force-confmiss'") # Remove legacy postgresql service record added by helpers, # will now be dynamically handled by the core in bullseye From e98ba46e4b740ee383a24653ed6db08b92eff104 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 3 Jan 2022 16:32:34 +0100 Subject: [PATCH 06/17] postinstall: don't skip migrate_to_bullseye migration when installing a fresh buster --- src/yunohost/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 79c558b2d..0ed88d871 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1045,6 +1045,10 @@ def _skip_all_migrations(): all_migrations = _get_migrations_list() new_states = {"migrations": {}} for migration in all_migrations: + # Don't skip bullseye migration while we're + # still on buster + if "migrate_to_bullseye" in migration.id: + continue new_states["migrations"][migration.id] = "skipped" write_to_yaml(MIGRATIONS_STATE_PATH, new_states) From 05b93aaaf1a9be736eae843cb60ca41258bb07af Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Mon, 3 Jan 2022 16:01:49 +0000 Subject: [PATCH 07/17] [CI] Format code with Black --- src/yunohost/app.py | 2 +- .../data_migrations/0018_xtable_to_nftable.py | 8 ++-- .../0021_migrate_to_bullseye.py | 45 +++++++++++++++++-- src/yunohost/domain.py | 8 +--- src/yunohost/tools.py | 4 +- src/yunohost/user.py | 4 +- 6 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index ca56be232..d6bb5d92f 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -627,7 +627,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if upgrade_failed or broke_the_system: # display this if there are remaining apps - if apps[number + 1:]: + if apps[number + 1 :]: not_upgraded_apps = apps[number:] logger.error( m18n.n( diff --git a/src/yunohost/data_migrations/0018_xtable_to_nftable.py b/src/yunohost/data_migrations/0018_xtable_to_nftable.py index 374620f2f..ae20354d7 100644 --- a/src/yunohost/data_migrations/0018_xtable_to_nftable.py +++ b/src/yunohost/data_migrations/0018_xtable_to_nftable.py @@ -41,8 +41,8 @@ class MyMigration(Migration): ) # For some reason if we don't do this, iptables-legacy-save is empty ? self.runcmd("iptables-legacy-save > %s" % self.backup_rules_ipv4) assert ( - os.path.exists(self.backup_rules_ipv4) and - os.stat(self.backup_rules_ipv4).st_size > 0 + os.path.exists(self.backup_rules_ipv4) + and os.stat(self.backup_rules_ipv4).st_size > 0 ), "Uhoh backup of legacy ipv4 rules is empty !?" if self.do_ipv6 and not os.path.exists(self.backup_rules_ipv6): self.runcmd( @@ -50,8 +50,8 @@ class MyMigration(Migration): ) # For some reason if we don't do this, iptables-legacy-save is empty ? self.runcmd("ip6tables-legacy-save > %s" % self.backup_rules_ipv6) assert ( - os.path.exists(self.backup_rules_ipv6) and - os.stat(self.backup_rules_ipv6).st_size > 0 + os.path.exists(self.backup_rules_ipv6) + and os.stat(self.backup_rules_ipv6).st_size > 0 ), "Uhoh backup of legacy ipv6 rules is empty !?" # We inject the legacy rules (iptables-legacy) into the new iptable (just "iptables") diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 10f385c23..717be4e15 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -148,7 +148,31 @@ class MyMigration(Migration): # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) - php73packages_suffixes = ['apcu', 'bcmath', 'bz2', 'dom', 'gmp', 'igbinary', 'imagick', 'imap', 'mbstring', 'memcached', 'mysqli', 'mysqlnd', 'pgsql', 'redis', 'simplexml', 'soap', 'sqlite3', 'ssh2', 'tidy', 'xml', 'xmlrpc', 'xsl', 'zip'] + php73packages_suffixes = [ + "apcu", + "bcmath", + "bz2", + "dom", + "gmp", + "igbinary", + "imagick", + "imap", + "mbstring", + "memcached", + "mysqli", + "mysqlnd", + "pgsql", + "redis", + "simplexml", + "soap", + "sqlite3", + "ssh2", + "tidy", + "xml", + "xmlrpc", + "xsl", + "zip", + ] cmd = f""" apt show '*-ynh-deps' 2>/dev/null @@ -157,10 +181,23 @@ class MyMigration(Migration): | sort | uniq | sed 's/php7.3/php7.4/g' """ - php74packages_to_install = ["php7.4-fpm", "php7.4-common", "php7.4-ldap", "php7.4-intl", "php7.4-mysql", "php7.4-gd", "php7.4-curl", "php-php-gettext"] - php74packages_to_install += [f.strip() for f in check_output(cmd).split("\n") if f.strip()] + php74packages_to_install = [ + "php7.4-fpm", + "php7.4-common", + "php7.4-ldap", + "php7.4-intl", + "php7.4-mysql", + "php7.4-gd", + "php7.4-curl", + "php-php-gettext", + ] + php74packages_to_install += [ + f.strip() for f in check_output(cmd).split("\n") if f.strip() + ] - self.apt_install("{' '.join(php74packages_to_install)} -o Dpkg::Options::='--force-confmiss'") + self.apt_install( + "{' '.join(php74packages_to_install)} -o Dpkg::Options::='--force-confmiss'" + ) # Remove legacy postgresql service record added by helpers, # will now be dynamically handled by the core in bullseye diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index d1ea45a08..7c512106a 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -517,9 +517,7 @@ def domain_cert_install( ): from yunohost.certificate import certificate_install - return certificate_install( - domain_list, force, no_checks, self_signed, staging - ) + return certificate_install(domain_list, force, no_checks, self_signed, staging) def domain_cert_renew( @@ -527,9 +525,7 @@ def domain_cert_renew( ): from yunohost.certificate import certificate_renew - return certificate_renew( - domain_list, force, no_checks, email, staging - ) + return certificate_renew(domain_list, force, no_checks, email, staging) def domain_dns_conf(domain): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 0ed88d871..c2014b466 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -244,9 +244,7 @@ def tools_postinstall( # and inform the user that we could not contact the dyndns host server. except Exception: logger.warning( - m18n.n( - "dyndns_provider_unreachable", provider="dyndns.yunohost.org" - ) + m18n.n("dyndns_provider_unreachable", provider="dyndns.yunohost.org") ) if available: diff --git a/src/yunohost/user.py b/src/yunohost/user.py index a27fffbee..64249e9d0 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -1265,9 +1265,7 @@ def user_group_remove(groupname, usernames, force=False, sync_perm=True): def user_permission_list(short=False, full=False, apps=[]): from yunohost.permission import user_permission_list - return user_permission_list( - short, full, absolute_urls=True, apps=apps - ) + return user_permission_list(short, full, absolute_urls=True, apps=apps) def user_permission_update(permission, label=None, show_tile=None, sync_perm=True): From 0973301b0f5d072657969befbd4744e11c89c773 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Jan 2022 20:09:48 +0100 Subject: [PATCH 08/17] ynh_add_config: crons should be owned by root, otherwise they probably don't run? --- data/helpers.d/utils | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 8b7179289..6929e3f95 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -967,4 +967,11 @@ _ynh_apply_default_permissions() { chown $app:$app $target fi fi + + # Crons should be owned by root otherwise they probably don't run + if echo "$target" | grep -q '^/etc/cron' + then + chmod 400 $target + chown root:root $target + fi } From 3cc1a0a59df905cab69444f4635d0bf875c64bd2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Jan 2022 17:44:15 +0100 Subject: [PATCH 09/17] tools_upgrade: filter more boring apt messages --- src/yunohost/tools.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index c2014b466..c825ca2ef 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -571,8 +571,18 @@ def tools_upgrade( irrelevants = [ "service sudo-ldap already provided", "Reading database ...", + "Preparing to unpack", + "Selecting previously unselected package", + "Created symlink /etc/systemd", + "Replacing config file", + "Creating config file", + "Installing new version of config file", + "Installing new config file as you requested", + ", does not exist on system.", + "unable to delete old directory", + "update-alternatives:", ] - return all(i not in line.rstrip() for i in irrelevants) + return line.rstrip() and all(i not in line.rstrip() for i in irrelevants) callbacks = ( lambda l: logger.info("+ " + l.rstrip() + "\r") From 17ad3c8581e3d0612132eddf541b1fab3c046510 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Jan 2022 17:45:23 +0100 Subject: [PATCH 10/17] migrate_to_bullseye: fix typo + reorganize a couple steps --- .../0021_migrate_to_bullseye.py | 63 +++++++++---------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 717be4e15..d26eede88 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -97,23 +97,6 @@ class MyMigration(Migration): os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") rm("/home/yunohost.conf", recursive=True, force=True) - # - # Main upgrade - # - logger.info(m18n.n("migration_0021_main_upgrade")) - - apps_packages = self.get_apps_equivs_packages() - self.hold(apps_packages) - tools_upgrade(target="system", allow_yunohost_upgrade=False) - - if self.debian_major_version() == N_CURRENT_DEBIAN: - raise YunohostError("migration_0021_still_on_buster_after_main_upgrade") - - # Clean the mess - logger.info(m18n.n("migration_0021_cleaning_up")) - os.system("apt autoremove --assume-yes") - os.system("apt clean --assume-yes") - # Force add sury if it's not there yet # This is to solve some weird issue with php-common breaking php7.3-common, # hence breaking many php7.3-deps @@ -128,7 +111,24 @@ class MyMigration(Migration): 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' ) - os.system("apt update") + # Remove legacy postgresql service record added by helpers, + # will now be dynamically handled by the core in bullseye + services = _get_services() + if "postgresql" in services: + del services["postgresql"] + _save_services(services) + + # + # Main upgrade + # + logger.info(m18n.n("migration_0021_main_upgrade")) + + apps_packages = self.get_apps_equivs_packages() + self.hold(apps_packages) + tools_upgrade(target="system", allow_yunohost_upgrade=False) + + if self.debian_major_version() == N_CURRENT_DEBIAN: + raise YunohostError("migration_0021_still_on_buster_after_main_upgrade") # Force explicit install of php7.4-fpm and other old 'default' dependencies # that are now only in Recommends @@ -174,13 +174,13 @@ class MyMigration(Migration): "zip", ] - cmd = f""" - apt show '*-ynh-deps' 2>/dev/null - | grep Depends - | grep -o -E "php7.3-({'|'.join(php73packages_suffixes)})"; - | sort | uniq - | sed 's/php7.3/php7.4/g' - """ + cmd = "apt show '*-ynh-deps' 2>/dev/null" \ + " | grep Depends" \ + f" | grep -o -E \"php7.3-({'|'.join(php73packages_suffixes)})\"" \ + " | sort | uniq" \ + " | sed 's/php7.3/php7.4/g'" \ + " || true" + php74packages_to_install = [ "php7.4-fpm", "php7.4-common", @@ -191,20 +191,19 @@ class MyMigration(Migration): "php7.4-curl", "php-php-gettext", ] + php74packages_to_install += [ f.strip() for f in check_output(cmd).split("\n") if f.strip() ] self.apt_install( - "{' '.join(php74packages_to_install)} -o Dpkg::Options::='--force-confmiss'" + f"{' '.join(php74packages_to_install)} -o Dpkg::Options::='--force-confmiss'" ) - # Remove legacy postgresql service record added by helpers, - # will now be dynamically handled by the core in bullseye - services = _get_services() - if "postgresql" in services: - del services["postgresql"] - _save_services(services) + # Clean the mess + logger.info(m18n.n("migration_0021_cleaning_up")) + os.system("apt autoremove --assume-yes") + os.system("apt clean --assume-yes") # # Yunohost upgrade From fa163cbc2e44a5c54ac7d48adff7c573da6c2b30 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Jan 2022 19:50:50 +0100 Subject: [PATCH 11/17] migrate_to_bullseye: zblerg, more constrains to not yoloremove app-ynh-deps ... --- .../0021_migrate_to_bullseye.py | 34 +++++++++++++--- src/yunohost/tools.py | 39 ++++++++++--------- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index d26eede88..153081916 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -7,7 +7,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_file, rm -from yunohost.tools import Migration, tools_update, tools_upgrade +from yunohost.tools import Migration, tools_update, tools_upgrade, _apt_log_line_is_relevant from yunohost.app import unstable_apps from yunohost.regenconf import manually_modified_files, _force_clear_hashes from yunohost.utils.filesystem import free_space_in_directory @@ -77,9 +77,12 @@ class MyMigration(Migration): _force_clear_hashes(["/etc/mysql/my.cnf"]) rm("/etc/mysql/mariadb.cnf", force=True) rm("/etc/mysql/my.cnf", force=True) - self.apt_install( + ret = self.apt_install( "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" ) + if ret != 0: + # FIXME: i18n once this is stable? + raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) # # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl @@ -196,9 +199,12 @@ class MyMigration(Migration): f.strip() for f in check_output(cmd).split("\n") if f.strip() ] - self.apt_install( + ret = self.apt_install( f"{' '.join(php74packages_to_install)} -o Dpkg::Options::='--force-confmiss'" ) + if ret != 0: + # FIXME: i18n once this is stable? + raise YunohostError("Failed to force the install of php dependencies ?", raw_msg=True) # Clean the mess logger.info(m18n.n("migration_0021_cleaning_up")) @@ -209,7 +215,21 @@ class MyMigration(Migration): # Yunohost upgrade # logger.info(m18n.n("migration_0021_yunohost_upgrade")) + self.unhold(apps_packages) + + cmd = "LC_ALL=C" + cmd += " DEBIAN_FRONTEND=noninteractive" + cmd += " APT_LISTCHANGES_FRONTEND=none" + cmd += " apt dist-upgrade " + cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" + cmd += " | grep -q '-ynh-deps'" + + logger.info("Simulating upgrade...") + if os.system(cmd) == 0: + # FIXME: i18n once this is stable? + raise YunohostError("The upgrade cannot be completed, because some app dependencies would need to be removed?", raw_msg=True) + tools_upgrade(target="system") def debian_major_version(self): @@ -344,9 +364,11 @@ class MyMigration(Migration): callbacks = ( lambda l: logger.info("+ " + l.rstrip() + "\r") - if is_relevant(l) + if _apt_log_line_is_relevant(l) else logger.debug(l.rstrip() + "\r"), - lambda l: logger.warning(l.rstrip()), + lambda l: logger.warning(l.rstrip()) + if _apt_log_line_is_relevant(l) + else logger.debug(l.rstrip()), ) cmd = ( @@ -356,7 +378,7 @@ class MyMigration(Migration): logger.debug("Running: %s" % cmd) - call_async_output(cmd, callbacks, shell=True) + return call_async_output(cmd, callbacks, shell=True) def patch_yunohost_conflicts(self): # diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index c825ca2ef..021a8f3a1 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -567,29 +567,12 @@ def tools_upgrade( logger.debug("Running apt command :\n{}".format(dist_upgrade)) - def is_relevant(line): - irrelevants = [ - "service sudo-ldap already provided", - "Reading database ...", - "Preparing to unpack", - "Selecting previously unselected package", - "Created symlink /etc/systemd", - "Replacing config file", - "Creating config file", - "Installing new version of config file", - "Installing new config file as you requested", - ", does not exist on system.", - "unable to delete old directory", - "update-alternatives:", - ] - return line.rstrip() and all(i not in line.rstrip() for i in irrelevants) - callbacks = ( lambda l: logger.info("+ " + l.rstrip() + "\r") - if is_relevant(l) + if _apt_log_line_is_relevant(l) else logger.debug(l.rstrip() + "\r"), lambda l: logger.warning(l.rstrip()) - if is_relevant(l) + if _apt_log_line_is_relevant(l) else logger.debug(l.rstrip()), ) returncode = call_async_output(dist_upgrade, callbacks, shell=True) @@ -689,6 +672,24 @@ def tools_upgrade( operation_logger.success() +def _apt_log_line_is_relevant(line): + irrelevants = [ + "service sudo-ldap already provided", + "Reading database ...", + "Preparing to unpack", + "Selecting previously unselected package", + "Created symlink /etc/systemd", + "Replacing config file", + "Creating config file", + "Installing new version of config file", + "Installing new config file as you requested", + ", does not exist on system.", + "unable to delete old directory", + "update-alternatives:", + ] + return line.rstrip() and all(i not in line.rstrip() for i in irrelevants) + + @is_unit_operation() def tools_shutdown(operation_logger, force=False): shutdown = force From b5e04df39800a1c4754766439d0ebfa7a641ee36 Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Thu, 6 Jan 2022 11:50:10 +0100 Subject: [PATCH 12/17] [fix] Force install certificate on yunohost domain add --- src/yunohost/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 7c512106a..21787ea36 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -187,7 +187,7 @@ def domain_add(operation_logger, domain, dyndns=False): # Actually subscribe dyndns_subscribe(domain=domain) - _certificate_install_selfsigned([domain], False) + _certificate_install_selfsigned([domain], True) try: attr_dict = { From 2b3138ef8027ed13db301e88d3341fc162d7d80c Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 6 Jan 2022 17:37:05 +0100 Subject: [PATCH 13/17] remove args for metadata (#1405) * remove args for metadata * Add explanation for removing 'args' from log metadata Co-authored-by: Alexandre Aubin --- src/yunohost/log.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index d28a35e18..9f9e0b753 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -659,6 +659,11 @@ class OperationLogger: data["error"] = self._error # TODO: detect if 'extra' erase some key of 'data' data.update(self.extra) + # Remove the 'args' arg from args (yodawg). It corresponds to url-encoded args for app install, config panel set, etc + # Because the data are url encoded, it's hell to properly redact secrets inside it, + # and the useful info is usually already available in `env` too + if "args" in data and isinstance(data["args"], dict) and "args" in data["args"]: + data["args"].pop("args") return data def success(self): From 607fad210593e756f37de8ed053f4f4d65080485 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Jan 2022 00:54:57 +0100 Subject: [PATCH 14/17] Sury has a crazy new amount of php-* packages that depend on specific versions of php, we shall ban them or it creates a frakin mess --- data/hooks/conf_regen/10-apt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/10-apt b/data/hooks/conf_regen/10-apt index da0620e59..2b3ae006f 100755 --- a/data/hooks/conf_regen/10-apt +++ b/data/hooks/conf_regen/10-apt @@ -7,7 +7,7 @@ do_pre_regen() { mkdir --parents "${pending_dir}/etc/apt/preferences.d" - packages_to_refuse_from_sury="php php-fpm php-mysql php-xml php-zip php-mbstring php-ldap php-gd php-curl php-bz2 php-json php-sqlite3 php-intl openssl libssl1.1 libssl-dev" + packages_to_refuse_from_sury="php php-* openssl libssl1.1 libssl-dev" for package in $packages_to_refuse_from_sury; do echo " Package: $package From 79c70b76298688a404a3f89555a97abe5a36c04c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Jan 2022 01:01:19 +0100 Subject: [PATCH 15/17] migrate_to_bullseye: add the list of app-ynh-deps in the php7.4 install trick, because that seem to solve some stupid dependency blocage issues... --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 153081916..9ec8dc17d 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -200,7 +200,9 @@ class MyMigration(Migration): ] ret = self.apt_install( - f"{' '.join(php74packages_to_install)} -o Dpkg::Options::='--force-confmiss'" + f"{' '.join(php74packages_to_install)} " + "$(dpkg --list | grep ynh-deps | awk '{print $2}') " + "-o Dpkg::Options::='--force-confmiss'" ) if ret != 0: # FIXME: i18n once this is stable? From 9cfbbd122b341eb53dde43f90c2e877024fa3287 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Jan 2022 02:26:18 +0100 Subject: [PATCH 16/17] migrate_to_bullseye: add sury before running apt update otherwise we end up in an inconsistent state --- .../0021_migrate_to_bullseye.py | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 9ec8dc17d..8f15019e9 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -50,6 +50,25 @@ class MyMigration(Migration): # logger.info(m18n.n("migration_0021_patching_sources_list")) self.patch_apt_sources_list() + + # Force add sury if it's not there yet + # This is to solve some weird issue with php-common breaking php7.3-common, + # hence breaking many php7.3-deps + # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) + # Adding it there shouldnt be a big deal - Yunohost 11.x does add it + # through its regen conf anyway. + if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): + open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( + "deb https://packages.sury.org/php/ bullseye main" + ) + os.system( + 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' + ) + + # + # Run apt update + # + tools_update(target="system") # Tell libc6 it's okay to restart system stuff during the upgrade @@ -100,20 +119,6 @@ class MyMigration(Migration): os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") rm("/home/yunohost.conf", recursive=True, force=True) - # Force add sury if it's not there yet - # This is to solve some weird issue with php-common breaking php7.3-common, - # hence breaking many php7.3-deps - # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) - # Adding it there shouldnt be a big deal - Yunohost 11.x does add it - # through its regen conf anyway. - if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): - open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( - "deb https://packages.sury.org/php/ bullseye main" - ) - os.system( - 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' - ) - # Remove legacy postgresql service record added by helpers, # will now be dynamically handled by the core in bullseye services = _get_services() From b7121502d646796622fb4b1ade5859e9c6c18d78 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Jan 2022 02:48:02 +0100 Subject: [PATCH 17/17] More sury/php madness ... forbidding php-common doesnt seem to be good idea so let's have it --- data/hooks/conf_regen/10-apt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/hooks/conf_regen/10-apt b/data/hooks/conf_regen/10-apt index 2b3ae006f..9eadea90a 100755 --- a/data/hooks/conf_regen/10-apt +++ b/data/hooks/conf_regen/10-apt @@ -15,6 +15,11 @@ Pin: origin \"packages.sury.org\" Pin-Priority: -1" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" done + echo " +Package: php-common +Pin: origin \"packages.sury.org\" +Pin-Priority: 500" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" + echo " # PLEASE READ THIS WARNING AND DON'T EDIT THIS FILE