From 91d3d15c32a9d5739bad3596e426ebda3b19c010 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 May 2021 17:22:55 +0200 Subject: [PATCH 01/18] Bump conflict version for openssl because version in bullseye repo is now 1.1.1k --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 21485b114..f123239eb 100644 --- a/debian/control +++ b/debian/control @@ -41,7 +41,7 @@ Conflicts: iptables-persistent , apache2 , bind9 , nginx-extras (>= 1.19) - , openssl (>= 1.1.1j-2) + , openssl (>= 1.1.1l-1) , slapd (>= 2.4.58) , dovecot-core (>= 1:2.3.14) , redis-server (>= 5:6.0.12) From e2a3a0bedf4b0ca532578201727eca70075b81a6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 30 May 2021 17:55:39 +0200 Subject: [PATCH 02/18] Fix 'from_version' issue because of ~alpha in current bullseye version name --- src/yunohost/backup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 862c07690..b94eda683 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -860,9 +860,13 @@ class RestoreManager: # FIXME this way to get the info is not compatible with copy or custom # backup methods self.info = backup_info(name, with_details=True) - if not self.info["from_yunohost_version"] or version.parse( - self.info["from_yunohost_version"] - ) < version.parse("3.8.0"): + + from_version = self.info.get("from_yunohost_version", "") + # Remove any '~foobar' in the version ... c.f ~alpha, ~beta version during + # early dev for next debian version + from_version = re.sub(r'~\w+', '', from_version) + + if not from_version or version.parse(from_version) < version.parse("3.8.0"): raise YunohostValidationError("restore_backup_too_old") self.archive_path = self.info["path"] From cb129e2bf17de6997833555f09e5caa5d030b37e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 30 May 2021 18:48:26 +0200 Subject: [PATCH 03/18] Fix compat issue in equivs helper --- data/helpers.d/apt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/apt b/data/helpers.d/apt index c3439a583..da87e7bad 100644 --- a/data/helpers.d/apt +++ b/data/helpers.d/apt @@ -181,8 +181,9 @@ ynh_package_install_from_equivs () { # Build and install the package local TMPDIR=$(mktemp --directory) - # Force the compatibility level at 10, levels below are deprecated - echo 10 > /usr/share/equivs/template/debian/compat + # Make sure to delete the legacy compat file + # It's now handle somewhat magically through the control file + rm -f /usr/share/equivs/template/debian/compat # Note that the cd executes into a sub shell # Create a fake deb package with equivs-build and the given control file From 5046e49c8b8dd051319bd35f7f7b5b70c7d39c90 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 30 May 2021 19:03:44 +0200 Subject: [PATCH 04/18] Many app explicitly specify YNH_PHP_VERSION=7.3, then trying to install php7.3-foo which doesn't exist ... so gotta yolopatch that definition ... --- src/yunohost/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4e2cd5317..eafff1db3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -3475,6 +3475,7 @@ LEGACY_PHP_VERSION_REPLACEMENTS = [ ("php5", "php7.4"), ("php7.0", "php7.4"), ("php7.3", "php7.4"), + ('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="7.4"'), ( 'phpversion="${phpversion:-7.0}"', 'phpversion="${phpversion:-7.4}"', From ba2428d9b9db734e1fa394f973a0d2fec6bd9d34 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 30 May 2021 19:07:30 +0200 Subject: [PATCH 05/18] equivs-build now sends its logs to stderr but we don't really care about what it says, redirect to debug --- data/helpers.d/apt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/apt b/data/helpers.d/apt index da87e7bad..4f48c2172 100644 --- a/data/helpers.d/apt +++ b/data/helpers.d/apt @@ -192,7 +192,7 @@ ynh_package_install_from_equivs () { ynh_wait_dpkg_free cp "$controlfile" "${TMPDIR}/control" (cd "$TMPDIR" - LC_ALL=C equivs-build ./control 1> /dev/null + LC_ALL=C equivs-build ./control 2>&1 LC_ALL=C dpkg --force-depends --install "./${pkgname}_${pkgversion}_all.deb" 2>&1 | tee ./dpkg_log) ynh_package_install --fix-broken || \ From c794e63f67b06407c9e180f6cbce99d3a40ced42 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 31 May 2021 16:00:23 +0200 Subject: [PATCH 06/18] Re-add missing nginx dependency in debian/control ..\! --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index f123239eb..cc39a838c 100644 --- a/debian/control +++ b/debian/control @@ -15,6 +15,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 , python-is-python3 + , nginx, nginx-extras (>=1.18) , apt, apt-transport-https, apt-utils, dirmngr , php7.4-common, php7.4-fpm, php7.4-ldap, php7.4-intl , mariadb-server, php7.4-mysql From 473bff6e5dfeecd744d5fb88d686867669a1e722 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 31 May 2021 18:46:28 +0200 Subject: [PATCH 07/18] Draft for bullseye migration --- .../0021_migrate_to_bullseye.py | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 src/yunohost/data_migrations/0021_migrate_to_bullseye.py diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py new file mode 100644 index 000000000..289e9d765 --- /dev/null +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -0,0 +1,252 @@ +import glob +import os + +from moulinette import m18n +from yunohost.utils.error import YunohostError +from moulinette.utils.log import getActionLogger +from moulinette.utils.process import check_output, call_async_output +from moulinette.utils.filesystem import read_file + +from yunohost.tools import Migration, tools_update, tools_upgrade +from yunohost.app import unstable_apps +from yunohost.regenconf import manually_modified_files +from yunohost.utils.filesystem import free_space_in_directory +from yunohost.utils.packages import ( + get_ynh_package_version, + _list_upgradable_apt_packages, +) + +logger = getActionLogger("yunohost.migration") + +N_CURRENT_DEBIAN = 10 +N_CURRENT_YUNOHOST = 4 + +N_NEXT_DEBAN = 11 +N_NEXT_YUNOHOST == 11 + +class MyMigration(Migration): + + "Upgrade the system to Debian Bullseye and Yunohost 11.x" + + mode = "manual" + + def run(self): + + self.check_assertions() + + logger.info(m18n.n("migration_0021_start")) + + # + # Patch sources.list + # + logger.info(m18n.n("migration_0021_patching_sources_list")) + self.patch_apt_sources_list() + tools_update(target="system") + + # Tell libc6 it's okay to restart system stuff during the upgrade + os.system( + "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" + ) + + # Don't send an email to root about the postgresql migration. It should be handled automatically after. + os.system( + "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" + ) + + # + # Specific packages upgrades + # + logger.info(m18n.n("migration_0021_specific_upgrade")) + + # + # Main upgrade + # + logger.info(m18n.n("migration_0021_main_upgrade")) + + self.patch_yunohost_conflicts() + + apps_packages = self.get_apps_equivs_packages() + self.hold(apps_packages) + tools_upgrade(target="system", allow_yunohost_upgrade=False) + + if self.debian_major_version() == N_CURRENT_DEBIAN: + raise YunohostError("migration_0021_still_on_buster_after_main_upgrade") + + # Clean the mess + logger.info(m18n.n("migration_0021_cleaning_up")) + os.system("apt autoremove --assume-yes") + os.system("apt clean --assume-yes") + + # + # Yunohost upgrade + # + logger.info(m18n.n("migration_0021_yunohost_upgrade")) + self.unhold(apps_packages) + tools_upgrade(target="system") + + def debian_major_version(self): + # The python module "platform" and lsb_release are not reliable because + # on some setup, they may still return Release=9 even after upgrading to + # buster ... (Apparently this is related to OVH overriding some stuff + # with /etc/lsb-release for instance -_-) + # Instead, we rely on /etc/os-release which should be the raw info from + # the distribution... + return int( + check_output( + "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" + ) + ) + + def yunohost_major_version(self): + return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) + + def check_assertions(self): + + # Be on buster (10.x) and yunohost 4.x + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be > 9.x but yunohost package + # would still be in 3.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + raise YunohostError("migration_0021_not_buster") + + # Have > 1 Go free space on /var/ ? + if free_space_in_directory("/var/") / (1024 ** 3) < 1.0: + raise YunohostError("migration_0021_not_enough_free_space") + + # Check system is up to date + # (but we don't if 'bullseye' is already in the sources.list ... + # which means maybe a previous upgrade crashed and we're re-running it) + if " bullseye " not in read_file("/etc/apt/sources.list"): + tools_update(target="system") + upgradable_system_packages = list(_list_upgradable_apt_packages()) + if upgradable_system_packages: + raise YunohostError("migration_0021_system_not_fully_up_to_date") + + @property + def disclaimer(self): + + # Avoid having a super long disclaimer + uncessary check if we ain't + # on buster / yunohost 4.x anymore + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be >= 10.x but yunohost package + # would still be in 4.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + return None + + # Get list of problematic apps ? I.e. not official or community+working + problematic_apps = unstable_apps() + problematic_apps = "".join(["\n - " + app for app in problematic_apps]) + + # Manually modified files ? (c.f. yunohost service regen-conf) + modified_files = manually_modified_files() + modified_files = "".join(["\n - " + f for f in modified_files]) + + message = m18n.n("migration_0021_general_warning") + + # FIXME: re-enable this message with updated topic link once we release the migration as stable + #message = ( + # "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n" + # + message + #) + + if problematic_apps: + message += "\n\n" + m18n.n( + "migration_0021_problematic_apps_warning", + problematic_apps=problematic_apps, + ) + + if modified_files: + message += "\n\n" + m18n.n( + "migration_0021_modified_files", manually_modified_files=modified_files + ) + + return message + + def patch_apt_sources_list(self): + + sources_list = glob.glob("/etc/apt/sources.list.d/*.list") + sources_list.append("/etc/apt/sources.list") + + # This : + # - replace single 'buster' occurence by 'bulleye' + # - comments lines containing "backports" + # - replace 'buster/updates' by 'bullseye/updates' (or same with -) + for f in sources_list: + command = ( + f"sed -i {f} " + "-e 's@ buster @ bullseye @g' " + "-e '/backports/ s@^#*@#@' " + "-e 's@ buster/updates @ bullseye/updates @g' " + "-e 's@ buster-@ bullseye-@g' " + ) + os.system(command) + + def get_apps_equivs_packages(self): + + command = ( + "dpkg --get-selections" + " | grep -v deinstall" + " | awk '{print $1}'" + " | { grep 'ynh-deps$' || true; }" + ) + + output = check_output(command) + + return output.split("\n") if output else [] + + def hold(self, packages): + for package in packages: + os.system("apt-mark hold {}".format(package)) + + def unhold(self, packages): + for package in packages: + os.system("apt-mark unhold {}".format(package)) + + def apt_install(self, cmd): + def is_relevant(line): + return "Reading database ..." not in line.rstrip() + + callbacks = ( + lambda l: logger.info("+ " + l.rstrip() + "\r") + if is_relevant(l) + else logger.debug(l.rstrip() + "\r"), + lambda l: logger.warning(l.rstrip()), + ) + + cmd = ( + "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " + + cmd + ) + + logger.debug("Running: %s" % cmd) + + call_async_output(cmd, callbacks, shell=True) + + + def patch_yunohost_conflicts(self): + # + # This is a super dirty hack to remove the conflicts from yunohost's debian/control file + # Those conflicts are there to prevent mistakenly upgrading critical packages + # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly + # using backports etc. + # + # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status + # + + # We only patch the conflict if we're on yunohost 4.x + if self.yunohost_major_version() != N_CURRENT_YUNOHOST: + return + + conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() + if conflicts: + # We want to keep conflicting with apache/bind9 tho + new_conflicts = "Conflicts: apache2, bind9" + + command = f"sed -i /var/lib/dpkg/status 's@{conflicts}@{new_conflicts}@g'" + os.system(command) From d6baad6f45abf70b8f062fcd73f32a2cad975261 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 31 May 2021 19:02:05 +0200 Subject: [PATCH 08/18] Postgresql version is 13 in bullseye --- data/helpers.d/postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 12738a922..e190ce419 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -1,7 +1,7 @@ #!/bin/bash PSQL_ROOT_PWD_FILE=/etc/yunohost/psql -PSQL_VERSION=11 +PSQL_VERSION=13 # Open a connection as a user # From da57653a01770e130a53c6e732ee6dac61035790 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 31 May 2021 19:12:08 +0200 Subject: [PATCH 09/18] Misc fixes after tests on the battlefield --- .../data_migrations/0021_migrate_to_bullseye.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 289e9d765..a5bb1e523 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -22,7 +22,7 @@ N_CURRENT_DEBIAN = 10 N_CURRENT_YUNOHOST = 4 N_NEXT_DEBAN = 11 -N_NEXT_YUNOHOST == 11 +N_NEXT_YUNOHOST = 11 class MyMigration(Migration): @@ -54,17 +54,17 @@ class MyMigration(Migration): ) # - # Specific packages upgrades + # Patch yunohost conflicts # - logger.info(m18n.n("migration_0021_specific_upgrade")) + logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) + + self.patch_yunohost_conflicts() # # Main upgrade # logger.info(m18n.n("migration_0021_main_upgrade")) - self.patch_yunohost_conflicts() - apps_packages = self.get_apps_equivs_packages() self.hold(apps_packages) tools_upgrade(target="system", allow_yunohost_upgrade=False) @@ -248,5 +248,6 @@ class MyMigration(Migration): # We want to keep conflicting with apache/bind9 tho new_conflicts = "Conflicts: apache2, bind9" - command = f"sed -i /var/lib/dpkg/status 's@{conflicts}@{new_conflicts}@g'" + command = f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" + logger.debug(f"Running: {command}") os.system(command) From 24211a90bc6ea250c904454fbc6bf8ba1bc7287a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 4 Jun 2021 17:17:27 +0200 Subject: [PATCH 10/18] Yolo php7.3 -> php7.4 migration --- locales/en.json | 1 + src/yunohost/app.py | 16 +++- .../0022_php73_to_php74_pools.py | 83 +++++++++++++++++++ 3 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 src/yunohost/data_migrations/0022_php73_to_php74_pools.py diff --git a/locales/en.json b/locales/en.json index 813688a87..922ece34a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -426,6 +426,7 @@ "migration_description_0018_xtable_to_nftable": "Migrate old network traffic rules to the new nftable system", "migration_description_0019_extend_permissions_features": "Extend/rework the app permission management system", "migration_description_0020_ssh_sftp_permissions": "Add SSH and SFTP permissions support", + "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4", "migration_ldap_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.", "migration_ldap_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error:s}", "migration_ldap_migration_failed_trying_to_rollback": "Could not migrate... trying to roll back the system.", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index eafff1db3..98a8053e7 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -3480,10 +3480,18 @@ LEGACY_PHP_VERSION_REPLACEMENTS = [ 'phpversion="${phpversion:-7.0}"', 'phpversion="${phpversion:-7.4}"', ), # Many helpers like the composer ones use 7.0 by default ... + ( + 'phpversion="${phpversion:-7.3}"', + 'phpversion="${phpversion:-7.4}"', + ), # Many helpers like the composer ones use 7.0 by default ... ( '"$phpversion" == "7.0"', '$(bc <<< "$phpversion >= 7.4") -eq 1', ), # patch ynh_install_php to refuse installing/removing php <= 7.3 + ( + '"$phpversion" == "7.3"', + '$(bc <<< "$phpversion >= 7.4") -eq 1', + ), # patch ynh_install_php to refuse installing/removing php <= 7.3 ] @@ -3518,15 +3526,15 @@ def _patch_legacy_php_versions_in_settings(app_folder): settings = read_yaml(os.path.join(app_folder, "settings.yml")) - if settings.get("fpm_config_dir") == "/etc/php/7.0/fpm": + if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm"]: settings["fpm_config_dir"] = "/etc/php/7.4/fpm" - if settings.get("fpm_service") == "php7.0-fpm": + if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm"]: settings["fpm_service"] = "php7.4-fpm" - if settings.get("phpversion") == "7.0": + if settings.get("phpversion") in ["7.0", "7.3"]: settings["phpversion"] = "7.4" # We delete these checksums otherwise the file will appear as manually modified - list_to_remove = ["checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d"] + list_to_remove = ["checksum__etc_php_7.3_fpm_pool", "checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d"] settings = { k: v for k, v in settings.items() diff --git a/src/yunohost/data_migrations/0022_php73_to_php74_pools.py b/src/yunohost/data_migrations/0022_php73_to_php74_pools.py new file mode 100644 index 000000000..0157fa275 --- /dev/null +++ b/src/yunohost/data_migrations/0022_php73_to_php74_pools.py @@ -0,0 +1,83 @@ +import os +import glob +from shutil import copy2 + +from moulinette.utils.log import getActionLogger + +from yunohost.app import _is_installed, _patch_legacy_php_versions_in_settings +from yunohost.tools import Migration +from yunohost.service import _run_service_command + +logger = getActionLogger("yunohost.migration") + +OLDPHP_POOLS = "/etc/php/7.3/fpm/pool.d" +NEWPHP_POOLS = "/etc/php/7.4/fpm/pool.d" + +OLDPHP_SOCKETS_PREFIX = "/run/php/php7.3-fpm" +NEWPHP_SOCKETS_PREFIX = "/run/php/php7.4-fpm" + +MIGRATION_COMMENT = ( + "; YunoHost note : this file was automatically moved from {}".format(OLDPHP_POOLS) +) + + +class MyMigration(Migration): + + "Migrate php7.3-fpm 'pool' conf files to php7.4" + + dependencies = ["migrate_to_bullseye"] + + def run(self): + # Get list of php7.3 pool files + oldphp_pool_files = glob.glob("{}/*.conf".format(OLDPHP_POOLS)) + + # Keep only basenames + oldphp_pool_files = [os.path.basename(f) for f in oldphp_pool_files] + + # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) + oldphp_pool_files = [f for f in oldphp_pool_files if f != "www.conf"] + + for f in oldphp_pool_files: + + # Copy the files to the php7.4 pool + src = "{}/{}".format(OLDPHP_POOLS, f) + dest = "{}/{}".format(NEWPHP_POOLS, f) + copy2(src, dest) + + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format( + OLDPHP_SOCKETS_PREFIX, NEWPHP_SOCKETS_PREFIX, dest + ) + os.system(c) + + # Also add a comment that it was automatically moved from php7.3 + # (for human traceability and backward migration) + c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) + os.system(c) + + app_id = os.path.basename(f)[: -len(".conf")] + if _is_installed(app_id): + _patch_legacy_php_versions_in_settings( + "/etc/yunohost/apps/%s/" % app_id + ) + + nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/%s.conf" % app_id) + for f in nginx_conf_files: + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format( + OLDPHP_SOCKETS_PREFIX, NEWPHP_SOCKETS_PREFIX, f + ) + os.system(c) + + os.system( + "rm /etc/logrotate.d/php7.3-fpm" + ) # We remove this otherwise the logrotate cron will be unhappy + + # Reload/restart the php pools + _run_service_command("restart", "php7.4-fpm") + _run_service_command("enable", "php7.4-fpm") + os.system("systemctl stop php7.3-fpm") + os.system("systemctl disable php7.3-fpm") + + # Reload nginx + _run_service_command("reload", "nginx") From 4e2e54676a1c23475903823ad621f765d36c3b3a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 4 Jun 2021 17:23:09 +0200 Subject: [PATCH 11/18] Yolo postgresql 11 -> 13 cluster migration --- locales/en.json | 4 + .../0023_postgresql_11_to_13.py | 82 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/yunohost/data_migrations/0023_postgresql_11_to_13.py diff --git a/locales/en.json b/locales/en.json index 922ece34a..1ab813bc4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -427,6 +427,7 @@ "migration_description_0019_extend_permissions_features": "Extend/rework the app permission management system", "migration_description_0020_ssh_sftp_permissions": "Add SSH and SFTP permissions support", "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4", + "migration_description_0023_postgresql_11_to_13": "Migrate databases from PostgreSQL 11 to 13", "migration_ldap_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.", "migration_ldap_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error:s}", "migration_ldap_migration_failed_trying_to_rollback": "Could not migrate... trying to roll back the system.", @@ -453,6 +454,9 @@ "migration_0018_failed_to_reset_legacy_rules": "Failed to reset legacy iptables rules: {error}", "migration_0019_add_new_attributes_in_ldap": "Add new attributes for permissions in LDAP database", "migration_0019_slapd_config_will_be_overwritten": "It looks like you manually edited the slapd configuration. For this critical migration, YunoHost needs to force the update of the slapd configuration. The original files will be backuped in {conf_backup_folder}.", + "migration_0023_postgresql_11_not_installed": "PostgreSQL was not installed on your system. Nothing to do.", + "migration_0023_postgresql_13_not_installed": "PostgreSQL 11 is installed, but not postgresql 13!? Something weird might have happened on your system :(...", + "migration_0023_not_enough_space": "Make sufficient space available in {path} to run the migration.", "migrations_already_ran": "Those migrations are already done: {ids}", "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'", "migrations_dependencies_not_satisfied": "Run these migrations: '{dependencies_id}', before migration {id}.", diff --git a/src/yunohost/data_migrations/0023_postgresql_11_to_13.py b/src/yunohost/data_migrations/0023_postgresql_11_to_13.py new file mode 100644 index 000000000..30527f0c8 --- /dev/null +++ b/src/yunohost/data_migrations/0023_postgresql_11_to_13.py @@ -0,0 +1,82 @@ +import subprocess + +from moulinette import m18n +from yunohost.utils.error import YunohostError, YunohostValidationError +from moulinette.utils.log import getActionLogger + +from yunohost.tools import Migration +from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory + +logger = getActionLogger("yunohost.migration") + + +class MyMigration(Migration): + + "Migrate DBs from Postgresql 11 to 13 after migrating to Bullseye" + + dependencies = ["migrate_to_bullseye"] + + def run(self): + + if not self.package_is_installed("postgresql-11"): + logger.warning(m18n.n("migration_0023_postgresql_11_not_installed")) + return + + if not self.package_is_installed("postgresql-13"): + raise YunohostValidationError("migration_0023_postgresql_13_not_installed") + + # Make sure there's a 11 cluster + try: + self.runcmd("pg_lsclusters | grep -q '^11 '") + except Exception: + logger.warning( + "It looks like there's not active 11 cluster, so probably don't need to run this migration" + ) + return + + if not space_used_by_directory( + "/var/lib/postgresql/11" + ) > free_space_in_directory("/var/lib/postgresql"): + raise YunohostValidationError( + "migration_0023_not_enough_space", path="/var/lib/postgresql/" + ) + + self.runcmd("systemctl stop postgresql") + self.runcmd( + "LC_ALL=C pg_dropcluster --stop 13 main || true" + ) # We do not trigger an exception if the command fails because that probably means cluster 13 doesn't exists, which is fine because it's created during the pg_upgradecluster) + self.runcmd("LC_ALL=C pg_upgradecluster -m upgrade 11 main") + self.runcmd("LC_ALL=C pg_dropcluster --stop 11 main") + self.runcmd("systemctl start postgresql") + + def package_is_installed(self, package_name): + + (returncode, out, err) = self.runcmd( + "dpkg --list | grep '^ii ' | grep -q -w {}".format(package_name), + raise_on_errors=False, + ) + return returncode == 0 + + def runcmd(self, cmd, raise_on_errors=True): + + logger.debug("Running command: " + cmd) + + p = subprocess.Popen( + cmd, + shell=True, + executable="/bin/bash", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + out, err = p.communicate() + returncode = p.returncode + if raise_on_errors and returncode != 0: + raise YunohostError( + "Failed to run command '{}'.\nreturncode: {}\nstdout:\n{}\nstderr:\n{}\n".format( + cmd, returncode, out, err + ) + ) + + out = out.strip().split("\n") + return (returncode, out, err) From fd3a10489deec0c99f2566408c134f1bca78c4d6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 7 Jun 2021 16:17:58 +0200 Subject: [PATCH 12/18] Add i18n strings for bullseye migration --- locales/en.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/locales/en.json b/locales/en.json index 84a01dfaa..a479672d7 100644 --- a/locales/en.json +++ b/locales/en.json @@ -426,6 +426,7 @@ "migration_description_0018_xtable_to_nftable": "Migrate old network traffic rules to the new nftable system", "migration_description_0019_extend_permissions_features": "Extend/rework the app permission management system", "migration_description_0020_ssh_sftp_permissions": "Add SSH and SFTP permissions support", + "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x", "migration_ldap_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.", "migration_ldap_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error:s}", "migration_ldap_migration_failed_trying_to_rollback": "Could not migrate... trying to roll back the system.", @@ -452,6 +453,21 @@ "migration_0018_failed_to_reset_legacy_rules": "Failed to reset legacy iptables rules: {error}", "migration_0019_add_new_attributes_in_ldap": "Add new attributes for permissions in LDAP database", "migration_0019_slapd_config_will_be_overwritten": "It looks like you manually edited the slapd configuration. For this critical migration, YunoHost needs to force the update of the slapd configuration. The original files will be backuped in {conf_backup_folder}.", + "migration_0021_start" : "Starting migration to Bullseye", + "migration_0021_patching_sources_list": "Patching the sources.lists...", + "migration_0021_main_upgrade": "Starting main upgrade...", + "migration_0021_still_on_buster_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Buster", + "migration_0021_yunohost_upgrade" : "Starting YunoHost core upgrade...", + "migration_0021_not_buster" : "The current Debian distribution is not Buster!", + "migration_0021_not_enough_free_space" : "Free space is pretty low in /var/! You should have at least 1GB free to run this migration.", + "migration_0021_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Bullseye.", + "migration_0021_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.", + "migration_0021_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", + "migration_0021_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", + "migration_0021_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently...", + "migration_0021_cleaning_up": "Cleaning up cache and packages not useful anymore...", + "migration_0021_weak_certs": "The following certificates were found to still use weak signature algorithms and have to be upgraded to be compatible with the next version of nginx: {certs}", + "migration_0021_patch_yunohost_conflicts": "Applying patch to workaround conflict issue...", "migrations_already_ran": "Those migrations are already done: {ids}", "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'", "migrations_dependencies_not_satisfied": "Run these migrations: '{dependencies_id}', before migration {id}.", From 8e25895cbff9a2e374c47b9aa959574066d58248 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 7 Jun 2021 16:31:39 +0200 Subject: [PATCH 13/18] Fix helpers test: SimpleHTTPServer was python2 only, use http.server for python3/bullseye --- tests/test_helpers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_helpers.sh b/tests/test_helpers.sh index 55d26483e..153ce1386 100644 --- a/tests/test_helpers.sh +++ b/tests/test_helpers.sh @@ -35,7 +35,7 @@ trap cleanup EXIT SIGINT HTTPSERVER_DIR=$(mktemp -d) HTTPSERVER_PORT=1312 pushd "$HTTPSERVER_DIR" >/dev/null -python -m SimpleHTTPServer $HTTPSERVER_PORT &>/dev/null & +python3 -m http.server $HTTPSERVER_PORT --bind 127.0.0.1 &>/dev/null & HTTPSERVER="$!" popd >/dev/null From a24ea4dafa90932675765c56ffb0f822bf56b675 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 7 Jun 2021 16:39:06 +0200 Subject: [PATCH 14/18] Stupid python3 issue in helpers --- data/helpers.d/backup | 4 ++-- data/helpers.d/network | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/backup b/data/helpers.d/backup index 17da0fb2e..ae746a37b 100644 --- a/data/helpers.d/backup +++ b/data/helpers.d/backup @@ -207,14 +207,14 @@ ynh_restore () { # usage: _get_archive_path ORIGIN_PATH _get_archive_path () { # For security reasons we use csv python library to read the CSV - python -c " + python3 -c " import sys import csv with open(sys.argv[1], 'r') as backup_file: backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest']) for row in backup_csv: if row['source']==sys.argv[2].strip('\"'): - print row['dest'] + print(row['dest']) sys.exit(0) raise Exception('Original path for %s not found' % sys.argv[2]) " "${YNH_BACKUP_CSV}" "$1" diff --git a/data/helpers.d/network b/data/helpers.d/network index 2011d502b..4e536a8db 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -80,7 +80,7 @@ ynh_validate_ip() [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 - python /dev/stdin << EOF + python3 /dev/stdin << EOF import socket import sys family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 } From 43d924252fa0dc4d6ffdcfcbcb32a3a4f3ee18eb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 7 Jun 2021 17:41:57 +0200 Subject: [PATCH 15/18] Fix patch of security repo in sources.list --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index a5bb1e523..6678fa040 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -177,12 +177,14 @@ class MyMigration(Migration): # - replace single 'buster' occurence by 'bulleye' # - comments lines containing "backports" # - replace 'buster/updates' by 'bullseye/updates' (or same with -) + # Special note about the security suite: + # https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#security-archive for f in sources_list: command = ( f"sed -i {f} " "-e 's@ buster @ bullseye @g' " "-e '/backports/ s@^#*@#@' " - "-e 's@ buster/updates @ bullseye/updates @g' " + "-e 's@ buster/updates @ bullseye-security @g' " "-e 's@ buster-@ bullseye-@g' " ) os.system(command) From 43ef4f4a29e53f139439b4ab46004384abcfcf76 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 7 Jun 2021 17:44:24 +0200 Subject: [PATCH 16/18] Unecessary i18n strings --- locales/en.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index a479672d7..a739b93c4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -464,9 +464,7 @@ "migration_0021_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.", "migration_0021_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", "migration_0021_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", - "migration_0021_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently...", "migration_0021_cleaning_up": "Cleaning up cache and packages not useful anymore...", - "migration_0021_weak_certs": "The following certificates were found to still use weak signature algorithms and have to be upgraded to be compatible with the next version of nginx: {certs}", "migration_0021_patch_yunohost_conflicts": "Applying patch to workaround conflict issue...", "migrations_already_ran": "Those migrations are already done: {ids}", "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'", From e22c683c3449b4d659fac5376959c05c550cd4d0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 7 Jun 2021 19:17:31 +0200 Subject: [PATCH 17/18] Fix conflict issue on redis-server because bullseye version changed --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index cc39a838c..09926c961 100644 --- a/debian/control +++ b/debian/control @@ -45,7 +45,7 @@ Conflicts: iptables-persistent , openssl (>= 1.1.1l-1) , slapd (>= 2.4.58) , dovecot-core (>= 1:2.3.14) - , redis-server (>= 5:6.0.12) + , redis-server (>= 5:6.1) , fail2ban (>= 0.11.3) , iptables (>= 1.8.8) Description: manageable and configured self-hosting server From f6857625decd69f63bc9a783409003bb2922aeac Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 7 Jun 2021 22:06:54 +0200 Subject: [PATCH 18/18] Stupid encoding issue in migration 0023 --- src/yunohost/data_migrations/0023_postgresql_11_to_13.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0023_postgresql_11_to_13.py b/src/yunohost/data_migrations/0023_postgresql_11_to_13.py index 30527f0c8..145f15558 100644 --- a/src/yunohost/data_migrations/0023_postgresql_11_to_13.py +++ b/src/yunohost/data_migrations/0023_postgresql_11_to_13.py @@ -78,5 +78,5 @@ class MyMigration(Migration): ) ) - out = out.strip().split("\n") + out = out.strip().split(b"\n") return (returncode, out, err)