From 1bbe3441d863467215ea63ad9378c30a861b9deb Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 17 Jan 2022 04:07:48 +0000 Subject: [PATCH 1/9] Translated using Weblate (German) Currently translated at 100.0% (720 of 720 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/de/ --- locales/de.json | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/locales/de.json b/locales/de.json index 3aa07475e..9e1087316 100644 --- a/locales/de.json +++ b/locales/de.json @@ -350,7 +350,7 @@ "diagnosis_ram_low": "Das System hat nur {available} ({available_percent}%) RAM zur Verfügung! (von insgesamt {total}). Seien Sie vorsichtig.", "service_reload_or_restart_failed": "Der Dienst '{service}' konnte nicht erneut geladen oder gestartet werden.\n\nKürzlich erstellte Logs des Dienstes: {logs}", "diagnosis_domain_expiration_not_found_details": "Die WHOIS-Informationen für die Domäne {domain} scheinen keine Informationen über das Ablaufdatum zu enthalten. Stimmt das?", - "diagnosis_domain_expiration_warning": "Einige Domänen werden bald ablaufen.", + "diagnosis_domain_expiration_warning": "Einige Domänen werden bald ablaufen!", "diagnosis_diskusage_ok": "Der Speicher {mountpoint} (auf Gerät {device}) hat immer noch {free} ({free_percent}%) freien Speicherplatz übrig(von insgesamt {total})!", "diagnosis_ram_ok": "Das System hat immer noch {available} ({available_percent}%) RAM zu Verfügung von {total}.", "diagnosis_swap_none": "Das System hat gar keinen Swap. Sie sollten sich überlegen mindestens {recommended} an Swap einzurichten, um Situationen zu verhindern, in welchen der RAM des Systems knapp wird.", @@ -698,10 +698,25 @@ "ldap_attribute_already_exists": "LDAP-Attribut '{attribute}' existiert bereits mit dem Wert '{value}'", "user_import_success": "Benutzer:innen erfolgreich importiert", "domain_registrar_is_not_configured": "Der DNS-Registrar ist noch nicht für die Domäne '{domain}' konfiguriert.", - "domain_dns_push_not_applicable": "Die automatische DNS-Konfiguration ist nicht auf die Domäne {domain} anwendbar. Konfiguriere die DNS-Einträge manuell, wie unter https://yunohost.org/dns_config beschrieben.", + "domain_dns_push_not_applicable": "Die automatische DNS-Konfiguration ist nicht auf die Domäne {domain} anwendbar. Konfiguriere die DNS-Einträge manuell, wie unter https://yunohost.org/dns_config beschrieben.", "domain_dns_registrar_experimental": "Bislang wurde die Schnittstelle zur API von **{registrar}** noch nicht außreichend von der YunoHost-Community getestet und geprüft. Der Support ist **sehr experimentell** – sei vorsichtig!", "domain_dns_push_failed_to_authenticate": "Die Authentifizierung bei der API des Registrars für die Domäne '{domain}' ist fehlgeschlagen. Wahrscheinlich sind die Anmeldedaten falsch? (Fehler: {error})", "log_domain_config_set": "Konfiguration für die Domäne '{}' aktualisieren", "log_domain_dns_push": "DNS-Einträge für die Domäne '{}' übertragen", - "service_description_yunomdns": "Ermöglicht es dir, deinen Server über 'yunohost.local' in deinem lokalen Netzwerk zu erreichen" + "service_description_yunomdns": "Ermöglicht es dir, deinen Server über 'yunohost.local' in deinem lokalen Netzwerk zu erreichen", + "migration_0021_start": "Beginnen von Migration zu Bullseye", + "migration_0021_patching_sources_list": "Aktualisieren der sources.lists...", + "migration_0021_main_upgrade": "Starte Hauptupdate...", + "migration_0021_still_on_buster_after_main_upgrade": "Irgendetwas ist während des Haupt-Upgrades schief gelaufen, das System scheint immer noch auf Debian Buster zu laufen", + "migration_0021_yunohost_upgrade": "Start des YunoHost Kern-Upgrades...", + "migration_0021_not_buster": "Die aktuelle Debian-Distribution ist nicht Buster!", + "migration_0021_not_enough_free_space": "Der freie Speicherplatz in /var/ ist ziemlich gering! Sie sollten mindestens 1 GB frei haben, um diese Migration durchzuführen.", + "migration_0021_system_not_fully_up_to_date": "Ihr System ist nicht ganz aktuell. Bitte führen Sie ein reguläres Update durch, bevor Sie die Migration zu Bullseye durchführen.", + "migration_0021_problematic_apps_warning": "Bitte beachten Sie, dass die folgenden möglicherweise problematischen installierten Anwendungen erkannt wurden. Es sieht so aus, als ob diese nicht aus dem YunoHost-App-Katalog installiert wurden oder nicht als \"funktionierend\" gekennzeichnet sind. Es kann daher nicht garantiert werden, dass sie nach dem Update noch funktionieren werden: {problematic_apps}", + "migration_0021_modified_files": "Bitte beachten Sie, dass die folgenden Dateien manuell geändert wurden und nach dem Update möglicherweise überschrieben werden: {manually_modified_files}", + "migration_0021_cleaning_up": "Bereinigung von Cache und Paketen nicht mehr nötig...", + "migration_0021_patch_yunohost_conflicts": "Patch anwenden, um das Konfliktproblem zu umgehen...", + "global_settings_setting_security_ssh_password_authentication": "Passwort-Authentifizierung für SSH zulassen", + "migration_description_0021_migrate_to_bullseye": "Upgrade des Systems auf Debian Bullseye und YunoHost 11.x", + "migration_0021_general_warning": "Bitte beachten Sie, dass diese Migration ein heikler Vorgang ist. Das YunoHost-Team hat sein Bestes getan, um sie zu überprüfen und zu testen, aber die Migration könnte immer noch Teile des Systems oder seiner Anwendungen beschädigen.\n\nEs wird daher empfohlen,:\n - Führen Sie ein Backup aller kritischen Daten oder Anwendungen durch. Mehr Informationen unter https://yunohost.org/backup;\n - Haben Sie Geduld, nachdem Sie die Migration gestartet haben: Je nach Internetverbindung und Hardware kann es bis zu ein paar Stunden dauern, bis alles aktualisiert ist." } From 7373971573a13b72da9eef8b9c7f106d27dc5552 Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Mon, 17 Jan 2022 22:03:13 +0000 Subject: [PATCH 2/9] [CI] Format code with Black --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 5 ++--- src/yunohost/tools.py | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 58bb259f7..eace6ca2e 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -87,8 +87,8 @@ class MyMigration(Migration): # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... # and also invoke-rc.d ... write_to_file( - '/usr/sbin/policy-rc.d', - '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0' + "/usr/sbin/policy-rc.d", + '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', ) os.system("chmod +x /usr/sbin/policy-rc.d") @@ -260,7 +260,6 @@ class MyMigration(Migration): raw_msg=True, ) - postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" postupgradecmds += "echo 'Restarting nginx...' >&2\n" diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 1eaad28bf..dbd3af5f5 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -448,7 +448,12 @@ def _list_upgradable_apps(): @is_unit_operation() def tools_upgrade( - operation_logger, target=None, apps=False, system=False, allow_yunohost_upgrade=True, postupgradecmds="" + operation_logger, + target=None, + apps=False, + system=False, + allow_yunohost_upgrade=True, + postupgradecmds="", ): """ Update apps & package cache, then display changelog From 383e540ee0d95ed46cedb7bdc5dbb92d9d3c8a5c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jan 2022 20:08:34 +0100 Subject: [PATCH 3/9] Tmp remove bullseye migration for stable release --- .../0021_migrate_to_bullseye.py | 441 ------------------ 1 file changed, 441 deletions(-) delete mode 100644 src/yunohost/data_migrations/0021_migrate_to_bullseye.py diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py deleted file mode 100644 index eace6ca2e..000000000 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ /dev/null @@ -1,441 +0,0 @@ -import glob -import os - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_file, rm, write_to_file - -from yunohost.tools import ( - Migration, - tools_update, - tools_upgrade, - _apt_log_line_is_relevant, -) -from yunohost.app import unstable_apps -from yunohost.regenconf import manually_modified_files, _force_clear_hashes -from yunohost.utils.filesystem import free_space_in_directory -from yunohost.utils.packages import ( - get_ynh_package_version, - _list_upgradable_apt_packages, -) -from yunohost.service import _get_services, _save_services - -logger = getActionLogger("yunohost.migration") - -N_CURRENT_DEBIAN = 10 -N_CURRENT_YUNOHOST = 4 - -N_NEXT_DEBAN = 11 -N_NEXT_YUNOHOST = 11 - - -class MyMigration(Migration): - - "Upgrade the system to Debian Bullseye and Yunohost 11.x" - - mode = "manual" - - def run(self): - - self.check_assertions() - - logger.info(m18n.n("migration_0021_start")) - - # - # Add new apt .deb signing key - # - - new_apt_key = "https://forge.yunohost.org/yunohost_bullseye.asc" - check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -") - - # - # Patch sources.list - # - logger.info(m18n.n("migration_0021_patching_sources_list")) - self.patch_apt_sources_list() - - # Force add sury if it's not there yet - # This is to solve some weird issue with php-common breaking php7.3-common, - # hence breaking many php7.3-deps - # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) - # Adding it there shouldnt be a big deal - Yunohost 11.x does add it - # through its regen conf anyway. - if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): - open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( - "deb https://packages.sury.org/php/ bullseye main" - ) - os.system( - 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' - ) - - # - # Run apt update - # - - tools_update(target="system") - - # Tell libc6 it's okay to restart system stuff during the upgrade - os.system( - "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" - ) - - # Do not restart nginx during the upgrade of nginx-common and nginx-extras ... - # c.f. https://manpages.debian.org/bullseye/init-system-helpers/deb-systemd-invoke.1p.en.html - # and zcat /usr/share/doc/init-system-helpers/README.policy-rc.d.gz - # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... - # and also invoke-rc.d ... - write_to_file( - "/usr/sbin/policy-rc.d", - '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', - ) - os.system("chmod +x /usr/sbin/policy-rc.d") - - # Don't send an email to root about the postgresql migration. It should be handled automatically after. - os.system( - "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" - ) - - # - # Patch yunohost conflicts - # - logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) - - self.patch_yunohost_conflicts() - - # - # Specific tweaking to get rid of custom my.cnf and use debian's default one - # (my.cnf is actually a symlink to mariadb.cnf) - # - - _force_clear_hashes(["/etc/mysql/my.cnf"]) - rm("/etc/mysql/mariadb.cnf", force=True) - rm("/etc/mysql/my.cnf", force=True) - ret = self.apt_install( - "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - # FIXME: i18n once this is stable? - raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) - - # - # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl - # - if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"): - os.system( - "mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl" - ) - rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True) - - # - # /home/yunohost.conf -> /var/cache/yunohost/regenconf - # - if os.path.exists("/home/yunohost.conf"): - os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") - rm("/home/yunohost.conf", recursive=True, force=True) - - # Remove legacy postgresql service record added by helpers, - # will now be dynamically handled by the core in bullseye - services = _get_services() - if "postgresql" in services: - del services["postgresql"] - _save_services(services) - - # - # Main upgrade - # - logger.info(m18n.n("migration_0021_main_upgrade")) - - apps_packages = self.get_apps_equivs_packages() - self.hold(apps_packages) - tools_upgrade(target="system", allow_yunohost_upgrade=False) - - if self.debian_major_version() == N_CURRENT_DEBIAN: - raise YunohostError("migration_0021_still_on_buster_after_main_upgrade") - - # Force explicit install of php7.4-fpm and other old 'default' dependencies - # that are now only in Recommends - # - # Also, we need to install php7.4 equivalents of other php7.3 dependencies. - # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration - # to autoupgrade Nextcloud to 7.4, it will need the php7.4-zip to work. - # The following list is based on an ad-hoc analysis of php deps found in the - # app ecosystem, with a known equivalent on php7.4. - # - # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages - # with the proper list of dependencies, and the dependencies install this way - # will get flagged as 'manually installed'. - # - # We'll probably want to do something during the Bullseye->Bookworm migration to re-flag - # these as 'auto' so they get autoremoved if not needed anymore. - # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use - # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) - - php73packages_suffixes = [ - "apcu", - "bcmath", - "bz2", - "dom", - "gmp", - "igbinary", - "imagick", - "imap", - "mbstring", - "memcached", - "mysqli", - "mysqlnd", - "pgsql", - "redis", - "simplexml", - "soap", - "sqlite3", - "ssh2", - "tidy", - "xml", - "xmlrpc", - "xsl", - "zip", - ] - - cmd = ( - "apt show '*-ynh-deps' 2>/dev/null" - " | grep Depends" - f" | grep -o -E \"php7.3-({'|'.join(php73packages_suffixes)})\"" - " | sort | uniq" - " | sed 's/php7.3/php7.4/g'" - " || true" - ) - - basephp74packages_to_install = [ - "php7.4-fpm", - "php7.4-common", - "php7.4-ldap", - "php7.4-intl", - "php7.4-mysql", - "php7.4-gd", - "php7.4-curl", - "php-php-gettext", - ] - - php74packages_to_install = basephp74packages_to_install + [ - f.strip() for f in check_output(cmd).split("\n") if f.strip() - ] - - ret = self.apt_install( - f"{' '.join(php74packages_to_install)} " - "$(dpkg --list | grep ynh-deps | awk '{print $2}') " - "-o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - # FIXME: i18n once this is stable? - raise YunohostError( - "Failed to force the install of php dependencies ?", raw_msg=True - ) - - # Clean the mess - logger.info(m18n.n("migration_0021_cleaning_up")) - os.system("apt autoremove --assume-yes") - os.system("apt clean --assume-yes") - - # - # Yunohost upgrade - # - logger.info(m18n.n("migration_0021_yunohost_upgrade")) - - self.unhold(apps_packages) - - cmd = "LC_ALL=C" - cmd += " DEBIAN_FRONTEND=noninteractive" - cmd += " APT_LISTCHANGES_FRONTEND=none" - cmd += " apt dist-upgrade " - cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" - cmd += " | grep -q 'ynh-deps'" - - logger.info("Simulating upgrade...") - if os.system(cmd) == 0: - # FIXME: i18n once this is stable? - raise YunohostError( - "The upgrade cannot be completed, because some app dependencies would need to be removed?", - raw_msg=True, - ) - - postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" - postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" - postupgradecmds += "echo 'Restarting nginx...' >&2\n" - postupgradecmds += "systemctl restart nginx\n" - - tools_upgrade(target="system", postupgradecmds=postupgradecmds) - - def debian_major_version(self): - # The python module "platform" and lsb_release are not reliable because - # on some setup, they may still return Release=9 even after upgrading to - # buster ... (Apparently this is related to OVH overriding some stuff - # with /etc/lsb-release for instance -_-) - # Instead, we rely on /etc/os-release which should be the raw info from - # the distribution... - return int( - check_output( - "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" - ) - ) - - def yunohost_major_version(self): - return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) - - def check_assertions(self): - - # Be on buster (10.x) and yunohost 4.x - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be > 9.x but yunohost package - # would still be in 3.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - raise YunohostError("migration_0021_not_buster") - - # Have > 1 Go free space on /var/ ? - if free_space_in_directory("/var/") / (1024 ** 3) < 1.0: - raise YunohostError("migration_0021_not_enough_free_space") - - # Check system is up to date - # (but we don't if 'bullseye' is already in the sources.list ... - # which means maybe a previous upgrade crashed and we're re-running it) - if " bullseye " not in read_file("/etc/apt/sources.list"): - tools_update(target="system") - upgradable_system_packages = list(_list_upgradable_apt_packages()) - if upgradable_system_packages: - raise YunohostError("migration_0021_system_not_fully_up_to_date") - - @property - def disclaimer(self): - - # Avoid having a super long disclaimer + uncessary check if we ain't - # on buster / yunohost 4.x anymore - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be >= 10.x but yunohost package - # would still be in 4.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - return None - - # Get list of problematic apps ? I.e. not official or community+working - problematic_apps = unstable_apps() - problematic_apps = "".join(["\n - " + app for app in problematic_apps]) - - # Manually modified files ? (c.f. yunohost service regen-conf) - modified_files = manually_modified_files() - modified_files = "".join(["\n - " + f for f in modified_files]) - - message = m18n.n("migration_0021_general_warning") - - # FIXME: re-enable this message with updated topic link once we release the migration as stable - # message = ( - # "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n" - # + message - # ) - - if problematic_apps: - message += "\n\n" + m18n.n( - "migration_0021_problematic_apps_warning", - problematic_apps=problematic_apps, - ) - - if modified_files: - message += "\n\n" + m18n.n( - "migration_0021_modified_files", manually_modified_files=modified_files - ) - - return message - - def patch_apt_sources_list(self): - - sources_list = glob.glob("/etc/apt/sources.list.d/*.list") - sources_list.append("/etc/apt/sources.list") - - # This : - # - replace single 'buster' occurence by 'bulleye' - # - comments lines containing "backports" - # - replace 'buster/updates' by 'bullseye/updates' (or same with -) - # Special note about the security suite: - # https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#security-archive - for f in sources_list: - command = ( - f"sed -i {f} " - "-e 's@ buster @ bullseye @g' " - "-e '/backports/ s@^#*@#@' " - "-e 's@ buster/updates @ bullseye-security @g' " - "-e 's@ buster-@ bullseye-@g' " - ) - os.system(command) - - def get_apps_equivs_packages(self): - - command = ( - "dpkg --get-selections" - " | grep -v deinstall" - " | awk '{print $1}'" - " | { grep 'ynh-deps$' || true; }" - ) - - output = check_output(command) - - return output.split("\n") if output else [] - - def hold(self, packages): - for package in packages: - os.system(f"apt-mark hold {package}") - - def unhold(self, packages): - for package in packages: - os.system(f"apt-mark unhold {package}") - - def apt_install(self, cmd): - def is_relevant(line): - return "Reading database ..." not in line.rstrip() - - callbacks = ( - lambda l: logger.info("+ " + l.rstrip() + "\r") - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip() + "\r"), - lambda l: logger.warning(l.rstrip()) - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip()), - ) - - cmd = ( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " - + cmd - ) - - logger.debug("Running: %s" % cmd) - - return call_async_output(cmd, callbacks, shell=True) - - def patch_yunohost_conflicts(self): - # - # This is a super dirty hack to remove the conflicts from yunohost's debian/control file - # Those conflicts are there to prevent mistakenly upgrading critical packages - # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly - # using backports etc. - # - # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status - # - - # We only patch the conflict if we're on yunohost 4.x - if self.yunohost_major_version() != N_CURRENT_YUNOHOST: - return - - conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() - if conflicts: - # We want to keep conflicting with apache/bind9 tho - new_conflicts = "Conflicts: apache2, bind9" - - command = ( - f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" - ) - logger.debug(f"Running: {command}") - os.system(command) From 1725da45d43226c7d008fdaa5230e3f1ebe0f368 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jan 2022 20:08:50 +0100 Subject: [PATCH 4/9] Update changelog for 4.3.6.1 --- debian/changelog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/debian/changelog b/debian/changelog index a59dfe2b0..520f6485e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +yunohost (4.3.6.1) stable; urgency=low + + - [fix] dnsmasq: ensure interface is up ([#1410](https://github.com/YunoHost/yunohost/pull/1410)) + - [fix] apt helpers: fix ynh_install_app_dependencies when an app change his default phpversion (6ea32728) + - [fix] certificates: fix edge case where None is returned, triggering 'NoneType has no attribute get' (019839db) + - [i18n] Translations updated for German + + Thanks to all contributors <3 ! (Gregor, Kay0u) + + -- Alexandre Aubin Wed, 19 Jan 2022 20:05:13 +0100 + yunohost (4.3.6) stable; urgency=low - [enh] ssh: add a new setting to manage PasswordAuthentication in sshd_config ([#1388](https://github.com/YunoHost/yunohost/pull/1388)) From 7920cc6280ac8ca3f93a6fd3e418532a186bcea7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jan 2022 20:30:11 +0100 Subject: [PATCH 5/9] apt helpers: fix bug when var is empty... --- data/helpers.d/apt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/apt b/data/helpers.d/apt index b50f4a87b..490a59f24 100644 --- a/data/helpers.d/apt +++ b/data/helpers.d/apt @@ -443,7 +443,7 @@ ynh_install_extra_app_dependencies() { # Without doing apt install, an already installed dep is not upgraded local apps_auto_installed="$(apt-mark showauto $package)" ynh_package_install "$package" - apt-mark auto $apps_auto_installed + [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed # Remove this extra repository after packages are installed ynh_remove_extra_repo --name=$app From 981fca64e234848510a2daeafec1eda07516e0ac Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jan 2022 20:31:04 +0100 Subject: [PATCH 6/9] Update changelog for 4.3.6.2 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 520f6485e..066fcbe9d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (4.3.6.2) stable; urgency=low + + - [fix] apt helpers: fix bug when var is empty... (7920cc62) + + -- Alexandre Aubin Wed, 19 Jan 2022 20:30:25 +0100 + yunohost (4.3.6.1) stable; urgency=low - [fix] dnsmasq: ensure interface is up ([#1410](https://github.com/YunoHost/yunohost/pull/1410)) From 9c1f3c4be885abc3b39268f451cfe325cb6ff005 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jan 2022 20:34:59 +0100 Subject: [PATCH 7/9] Revert "Tmp remove bullseye migration for stable release" This reverts commit 383e540ee0d95ed46cedb7bdc5dbb92d9d3c8a5c. --- .../0021_migrate_to_bullseye.py | 441 ++++++++++++++++++ 1 file changed, 441 insertions(+) create mode 100644 src/yunohost/data_migrations/0021_migrate_to_bullseye.py diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py new file mode 100644 index 000000000..eace6ca2e --- /dev/null +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -0,0 +1,441 @@ +import glob +import os + +from moulinette import m18n +from yunohost.utils.error import YunohostError +from moulinette.utils.log import getActionLogger +from moulinette.utils.process import check_output, call_async_output +from moulinette.utils.filesystem import read_file, rm, write_to_file + +from yunohost.tools import ( + Migration, + tools_update, + tools_upgrade, + _apt_log_line_is_relevant, +) +from yunohost.app import unstable_apps +from yunohost.regenconf import manually_modified_files, _force_clear_hashes +from yunohost.utils.filesystem import free_space_in_directory +from yunohost.utils.packages import ( + get_ynh_package_version, + _list_upgradable_apt_packages, +) +from yunohost.service import _get_services, _save_services + +logger = getActionLogger("yunohost.migration") + +N_CURRENT_DEBIAN = 10 +N_CURRENT_YUNOHOST = 4 + +N_NEXT_DEBAN = 11 +N_NEXT_YUNOHOST = 11 + + +class MyMigration(Migration): + + "Upgrade the system to Debian Bullseye and Yunohost 11.x" + + mode = "manual" + + def run(self): + + self.check_assertions() + + logger.info(m18n.n("migration_0021_start")) + + # + # Add new apt .deb signing key + # + + new_apt_key = "https://forge.yunohost.org/yunohost_bullseye.asc" + check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -") + + # + # Patch sources.list + # + logger.info(m18n.n("migration_0021_patching_sources_list")) + self.patch_apt_sources_list() + + # Force add sury if it's not there yet + # This is to solve some weird issue with php-common breaking php7.3-common, + # hence breaking many php7.3-deps + # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) + # Adding it there shouldnt be a big deal - Yunohost 11.x does add it + # through its regen conf anyway. + if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): + open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( + "deb https://packages.sury.org/php/ bullseye main" + ) + os.system( + 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' + ) + + # + # Run apt update + # + + tools_update(target="system") + + # Tell libc6 it's okay to restart system stuff during the upgrade + os.system( + "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" + ) + + # Do not restart nginx during the upgrade of nginx-common and nginx-extras ... + # c.f. https://manpages.debian.org/bullseye/init-system-helpers/deb-systemd-invoke.1p.en.html + # and zcat /usr/share/doc/init-system-helpers/README.policy-rc.d.gz + # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... + # and also invoke-rc.d ... + write_to_file( + "/usr/sbin/policy-rc.d", + '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', + ) + os.system("chmod +x /usr/sbin/policy-rc.d") + + # Don't send an email to root about the postgresql migration. It should be handled automatically after. + os.system( + "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" + ) + + # + # Patch yunohost conflicts + # + logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) + + self.patch_yunohost_conflicts() + + # + # Specific tweaking to get rid of custom my.cnf and use debian's default one + # (my.cnf is actually a symlink to mariadb.cnf) + # + + _force_clear_hashes(["/etc/mysql/my.cnf"]) + rm("/etc/mysql/mariadb.cnf", force=True) + rm("/etc/mysql/my.cnf", force=True) + ret = self.apt_install( + "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" + ) + if ret != 0: + # FIXME: i18n once this is stable? + raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) + + # + # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl + # + if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"): + os.system( + "mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl" + ) + rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True) + + # + # /home/yunohost.conf -> /var/cache/yunohost/regenconf + # + if os.path.exists("/home/yunohost.conf"): + os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") + rm("/home/yunohost.conf", recursive=True, force=True) + + # Remove legacy postgresql service record added by helpers, + # will now be dynamically handled by the core in bullseye + services = _get_services() + if "postgresql" in services: + del services["postgresql"] + _save_services(services) + + # + # Main upgrade + # + logger.info(m18n.n("migration_0021_main_upgrade")) + + apps_packages = self.get_apps_equivs_packages() + self.hold(apps_packages) + tools_upgrade(target="system", allow_yunohost_upgrade=False) + + if self.debian_major_version() == N_CURRENT_DEBIAN: + raise YunohostError("migration_0021_still_on_buster_after_main_upgrade") + + # Force explicit install of php7.4-fpm and other old 'default' dependencies + # that are now only in Recommends + # + # Also, we need to install php7.4 equivalents of other php7.3 dependencies. + # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration + # to autoupgrade Nextcloud to 7.4, it will need the php7.4-zip to work. + # The following list is based on an ad-hoc analysis of php deps found in the + # app ecosystem, with a known equivalent on php7.4. + # + # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages + # with the proper list of dependencies, and the dependencies install this way + # will get flagged as 'manually installed'. + # + # We'll probably want to do something during the Bullseye->Bookworm migration to re-flag + # these as 'auto' so they get autoremoved if not needed anymore. + # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use + # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) + + php73packages_suffixes = [ + "apcu", + "bcmath", + "bz2", + "dom", + "gmp", + "igbinary", + "imagick", + "imap", + "mbstring", + "memcached", + "mysqli", + "mysqlnd", + "pgsql", + "redis", + "simplexml", + "soap", + "sqlite3", + "ssh2", + "tidy", + "xml", + "xmlrpc", + "xsl", + "zip", + ] + + cmd = ( + "apt show '*-ynh-deps' 2>/dev/null" + " | grep Depends" + f" | grep -o -E \"php7.3-({'|'.join(php73packages_suffixes)})\"" + " | sort | uniq" + " | sed 's/php7.3/php7.4/g'" + " || true" + ) + + basephp74packages_to_install = [ + "php7.4-fpm", + "php7.4-common", + "php7.4-ldap", + "php7.4-intl", + "php7.4-mysql", + "php7.4-gd", + "php7.4-curl", + "php-php-gettext", + ] + + php74packages_to_install = basephp74packages_to_install + [ + f.strip() for f in check_output(cmd).split("\n") if f.strip() + ] + + ret = self.apt_install( + f"{' '.join(php74packages_to_install)} " + "$(dpkg --list | grep ynh-deps | awk '{print $2}') " + "-o Dpkg::Options::='--force-confmiss'" + ) + if ret != 0: + # FIXME: i18n once this is stable? + raise YunohostError( + "Failed to force the install of php dependencies ?", raw_msg=True + ) + + # Clean the mess + logger.info(m18n.n("migration_0021_cleaning_up")) + os.system("apt autoremove --assume-yes") + os.system("apt clean --assume-yes") + + # + # Yunohost upgrade + # + logger.info(m18n.n("migration_0021_yunohost_upgrade")) + + self.unhold(apps_packages) + + cmd = "LC_ALL=C" + cmd += " DEBIAN_FRONTEND=noninteractive" + cmd += " APT_LISTCHANGES_FRONTEND=none" + cmd += " apt dist-upgrade " + cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" + cmd += " | grep -q 'ynh-deps'" + + logger.info("Simulating upgrade...") + if os.system(cmd) == 0: + # FIXME: i18n once this is stable? + raise YunohostError( + "The upgrade cannot be completed, because some app dependencies would need to be removed?", + raw_msg=True, + ) + + postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" + postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" + postupgradecmds += "echo 'Restarting nginx...' >&2\n" + postupgradecmds += "systemctl restart nginx\n" + + tools_upgrade(target="system", postupgradecmds=postupgradecmds) + + def debian_major_version(self): + # The python module "platform" and lsb_release are not reliable because + # on some setup, they may still return Release=9 even after upgrading to + # buster ... (Apparently this is related to OVH overriding some stuff + # with /etc/lsb-release for instance -_-) + # Instead, we rely on /etc/os-release which should be the raw info from + # the distribution... + return int( + check_output( + "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" + ) + ) + + def yunohost_major_version(self): + return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) + + def check_assertions(self): + + # Be on buster (10.x) and yunohost 4.x + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be > 9.x but yunohost package + # would still be in 3.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + raise YunohostError("migration_0021_not_buster") + + # Have > 1 Go free space on /var/ ? + if free_space_in_directory("/var/") / (1024 ** 3) < 1.0: + raise YunohostError("migration_0021_not_enough_free_space") + + # Check system is up to date + # (but we don't if 'bullseye' is already in the sources.list ... + # which means maybe a previous upgrade crashed and we're re-running it) + if " bullseye " not in read_file("/etc/apt/sources.list"): + tools_update(target="system") + upgradable_system_packages = list(_list_upgradable_apt_packages()) + if upgradable_system_packages: + raise YunohostError("migration_0021_system_not_fully_up_to_date") + + @property + def disclaimer(self): + + # Avoid having a super long disclaimer + uncessary check if we ain't + # on buster / yunohost 4.x anymore + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be >= 10.x but yunohost package + # would still be in 4.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + return None + + # Get list of problematic apps ? I.e. not official or community+working + problematic_apps = unstable_apps() + problematic_apps = "".join(["\n - " + app for app in problematic_apps]) + + # Manually modified files ? (c.f. yunohost service regen-conf) + modified_files = manually_modified_files() + modified_files = "".join(["\n - " + f for f in modified_files]) + + message = m18n.n("migration_0021_general_warning") + + # FIXME: re-enable this message with updated topic link once we release the migration as stable + # message = ( + # "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n" + # + message + # ) + + if problematic_apps: + message += "\n\n" + m18n.n( + "migration_0021_problematic_apps_warning", + problematic_apps=problematic_apps, + ) + + if modified_files: + message += "\n\n" + m18n.n( + "migration_0021_modified_files", manually_modified_files=modified_files + ) + + return message + + def patch_apt_sources_list(self): + + sources_list = glob.glob("/etc/apt/sources.list.d/*.list") + sources_list.append("/etc/apt/sources.list") + + # This : + # - replace single 'buster' occurence by 'bulleye' + # - comments lines containing "backports" + # - replace 'buster/updates' by 'bullseye/updates' (or same with -) + # Special note about the security suite: + # https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#security-archive + for f in sources_list: + command = ( + f"sed -i {f} " + "-e 's@ buster @ bullseye @g' " + "-e '/backports/ s@^#*@#@' " + "-e 's@ buster/updates @ bullseye-security @g' " + "-e 's@ buster-@ bullseye-@g' " + ) + os.system(command) + + def get_apps_equivs_packages(self): + + command = ( + "dpkg --get-selections" + " | grep -v deinstall" + " | awk '{print $1}'" + " | { grep 'ynh-deps$' || true; }" + ) + + output = check_output(command) + + return output.split("\n") if output else [] + + def hold(self, packages): + for package in packages: + os.system(f"apt-mark hold {package}") + + def unhold(self, packages): + for package in packages: + os.system(f"apt-mark unhold {package}") + + def apt_install(self, cmd): + def is_relevant(line): + return "Reading database ..." not in line.rstrip() + + callbacks = ( + lambda l: logger.info("+ " + l.rstrip() + "\r") + if _apt_log_line_is_relevant(l) + else logger.debug(l.rstrip() + "\r"), + lambda l: logger.warning(l.rstrip()) + if _apt_log_line_is_relevant(l) + else logger.debug(l.rstrip()), + ) + + cmd = ( + "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " + + cmd + ) + + logger.debug("Running: %s" % cmd) + + return call_async_output(cmd, callbacks, shell=True) + + def patch_yunohost_conflicts(self): + # + # This is a super dirty hack to remove the conflicts from yunohost's debian/control file + # Those conflicts are there to prevent mistakenly upgrading critical packages + # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly + # using backports etc. + # + # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status + # + + # We only patch the conflict if we're on yunohost 4.x + if self.yunohost_major_version() != N_CURRENT_YUNOHOST: + return + + conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() + if conflicts: + # We want to keep conflicting with apache/bind9 tho + new_conflicts = "Conflicts: apache2, bind9" + + command = ( + f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" + ) + logger.debug(f"Running: {command}") + os.system(command) From d07cf83d93086d57060d4afb4284df45326f3bae Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jan 2022 20:44:51 +0100 Subject: [PATCH 8/9] migrate_to_bullseye: tweaks to point to the current beta-testing forum thread --- src/yunohost/app.py | 3 ++- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d6bb5d92f..997403a07 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2502,13 +2502,14 @@ def is_true(arg): def unstable_apps(): output = [] + deprecated_apps = ["mailman"] for infos in app_list(full=True)["apps"]: if not infos.get("from_catalog") or infos.get("from_catalog").get("state") in [ "inprogress", "notworking", - ]: + ] or infos["id"] in deprecated_apps: output.append(infos["id"]) return output diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index eace6ca2e..77797c63f 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -116,7 +116,6 @@ class MyMigration(Migration): "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" ) if ret != 0: - # FIXME: i18n once this is stable? raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) # @@ -228,7 +227,6 @@ class MyMigration(Migration): "-o Dpkg::Options::='--force-confmiss'" ) if ret != 0: - # FIXME: i18n once this is stable? raise YunohostError( "Failed to force the install of php dependencies ?", raw_msg=True ) @@ -254,7 +252,6 @@ class MyMigration(Migration): logger.info("Simulating upgrade...") if os.system(cmd) == 0: - # FIXME: i18n once this is stable? raise YunohostError( "The upgrade cannot be completed, because some app dependencies would need to be removed?", raw_msg=True, @@ -332,7 +329,11 @@ class MyMigration(Migration): message = m18n.n("migration_0021_general_warning") - # FIXME: re-enable this message with updated topic link once we release the migration as stable + # FIXME: update this message with updated topic link once we release the migration as stable + message = ( + "N.B.: **THIS MIGRATION IS STILL IN BETA-STAGE** ! If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read and share feedbacks on this forum thread: https://forum.yunohost.org/t/18531\n\n" + + message + ) # message = ( # "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n" # + message From 61a01f5c813f0bb6b1f8f00ad8fa584a966aa27b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jan 2022 20:47:41 +0100 Subject: [PATCH 9/9] Update changelog for 4.4.0 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 066fcbe9d..900ed7beb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (4.4.0) testing; urgency=low + + - [enh] Add buster->bullseye migration + + -- Alexandre Aubin Wed, 19 Jan 2022 20:45:22 +0100 + yunohost (4.3.6.2) stable; urgency=low - [fix] apt helpers: fix bug when var is empty... (7920cc62)