[wip] Draft of migration procedure for Stretch->Buster

locales/en.json

@@ -428,6 +428,7 @@
     "migration_description_0012_postgresql_password_to_md5_authentication": "Force PostgreSQL authentication to use MD5 for local connections",
     "migration_description_0013_futureproof_apps_catalog_system": "Migrate to the new future-proof apps catalog system",
     "migration_description_0014_remove_app_status_json": "Remove legacy status.json app files",
+    "migration_description_0015_migrate_to_buster": "Upgrade the system to Debian Buster and YunoHost 4.x",
     "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.",
     "migration_0003_patching_sources_list": "Patching the sources.lists…",
     "migration_0003_main_upgrade": "Starting main upgrade…",
@@ -465,6 +466,19 @@
     "migration_0011_update_LDAP_database": "Updating LDAP database…",
     "migration_0011_update_LDAP_schema": "Updating LDAP schema…",
     "migration_0011_failed_to_remove_stale_object": "Could not remove stale object {dn}: {error}",
+    "migration_0015_start" : "Starting migration to Buster",
+    "migration_0015_patching_sources_list": "Patching the sources.lists…",
+    "migration_0015_main_upgrade": "Starting main upgrade…",
+    "migration_0015_still_on_jessie_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Stretch",
+    "migration_0015_yunohost_upgrade" : "Starting YunoHost core upgrade…",
+    "migration_0015_not_stretch" : "The current Debian distribution is not Stretch!",
+    "migration_0015_not_enough_free_space" : "Free space is pretty low in /var/! You should have at least 1GB free to run this migration.",
+    "migration_0015_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Buster.",
+    "migration_0015_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n    - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n    - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.",
+    "migration_0015_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}",
+    "migration_0015_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}",
+    "migration_0015_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently…",
+    "migration_0015_cleaning_up": "Cleaning up cache and packages not useful anymore…",
     "migrations_already_ran": "Those migrations are already done: {ids}",
     "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'",
     "migrations_dependencies_not_satisfied": "Run these migrations: '{dependencies_id}', before migration {id}.",

src/yunohost/data_migrations/0015_migrate_to_buster.py

@@ -0,0 +1,194 @@
+
+import glob
+import os
+
+from moulinette import m18n
+from yunohost.utils.error import YunohostError
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.process import check_output, call_async_output
+from moulinette.utils.filesystem import read_file
+
+from yunohost.tools import Migration, tools_update, tools_upgrade
+from yunohost.app import unstable_apps
+from yunohost.regenconf import manually_modified_files
+from yunohost.utils.filesystem import free_space_in_directory
+from yunohost.utils.packages import get_ynh_package_version, _list_upgradable_apt_packages
+
+logger = getActionLogger('yunohost.migration')
+
+class MyMigration(Migration):
+
+    "Upgrade the system to Debian Buster and Yunohost 4.x"
+
+    mode = "manual"
+
+    def run(self):
+
+        self.check_assertions()
+
+        logger.info(m18n.n("migration_0015_start"))
+
+        #
+        # Patch sources.list
+        #
+        logger.info(m18n.n("migration_0015_patching_sources_list"))
+        self.patch_apt_sources_list()
+        tools_update(system=True)
+
+        # Tell libc6 it's okay to restart system stuff during the upgrade
+        os.system("echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections")
+
+         # Don't send an email to root about the postgresql migration. It should be handled automatically after.
+        os.system("echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections")
+
+        #
+        # Specific packages upgrades
+        #
+        logger.info(m18n.n("migration_0015_specific_upgrade"))
+
+        # Update unscd independently, was 0.53-1+yunohost on stretch (custom build of ours) but now it's 0.53-1+b1 on vanilla buster, which for apt appears as a lower version (hence the --allow-downgrades and the hardcoded version number)
+        self.apt_install('unscd=0.53-1+b1 --allow-downgrades')
+
+        # Upgrade libpam-modules independently, small issue related to willing to overwrite a file previously provided by Yunohost
+        self.apt_install('libpam-modules -o Dpkg::Options::="--force-overwrite"')
+
+        #
+        # Main upgrade
+        #
+        logger.info(m18n.n("migration_0015_main_upgrade"))
+
+        apps_packages = self.get_apps_equivs_packages()
+        self.hold(apps_packages)
+        tools_upgrade(system=True, allow_yunohost_upgrade=False)
+
+        if self.debian_major_version() == 9:
+            raise YunohostError("migration_0015_still_on_stretch_after_main_upgrade")
+
+        # Clean the mess
+        logger.info(m18n.n("migration_0015_cleaning_up"))
+        os.system("apt autoremove --assume-yes")
+        os.system("apt clean --assume-yes")
+
+        #
+        # Yunohost upgrade
+        #
+        logger.info(m18n.n("migration_0015_yunohost_upgrade"))
+        self.unhold(apps_packages)
+        tools_upgrade(system=True)
+
+    def debian_major_version(self):
+        # The python module "platform" and lsb_release are not reliable because
+        # on some setup, they may still return Release=9 even after upgrading to
+        # buster ... (Apparently this is related to OVH overriding some stuff
+        # with /etc/lsb-release for instance -_-)
+        # Instead, we rely on /etc/os-release which should be the raw info from
+        # the distribution...
+        return int(check_output("grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2"))
+
+    def yunohost_major_version(self):
+        return int(get_ynh_package_version("yunohost")["version"].split('.')[0])
+
+    def check_assertions(self):
+
+        # Be on jessie (8.x) and yunohost 2.x
+        # NB : we do both check to cover situations where the upgrade crashed
+        # in the middle and debian version could be >= 9.x but yunohost package
+        # would still be in 2.x...
+        if not self.debian_major_version() == 9 \
+           and not self.yunohost_major_version() == 3:
+            raise YunohostError("migration_0015_not_stretch")
+
+        # Have > 1 Go free space on /var/ ?
+        if free_space_in_directory("/var/") / (1024**3) < 1.0:
+            raise YunohostError("migration_0015_not_enough_free_space")
+
+        # Check system is up to date
+        # (but we don't if 'stretch' is already in the sources.list ...
+        # which means maybe a previous upgrade crashed and we're re-running it)
+        if " buster " not in read_file("/etc/apt/sources.list"):
+            tools_update(system=True)
+            upgradable_system_packages = list(_list_upgradable_apt_packages())
+            if upgradable_system_packages:
+                raise YunohostError("migration_0015_system_not_fully_up_to_date")
+
+    @property
+    def disclaimer(self):
+
+        # Avoid having a super long disclaimer + uncessary check if we ain't
+        # on stretch / yunohost 3.x anymore
+        # NB : we do both check to cover situations where the upgrade crashed
+        # in the middle and debian version could be >= 10.x but yunohost package
+        # would still be in 3.x...
+        if not self.debian_major_version() == 9 \
+           and not self.yunohost_major_version() == 3:
+            return None
+
+        # Get list of problematic apps ? I.e. not official or community+working
+        problematic_apps = unstable_apps()
+        problematic_apps = "".join(["\n    - " + app for app in problematic_apps])
+
+        # Manually modified files ? (c.f. yunohost service regen-conf)
+        modified_files = manually_modified_files()
+        modified_files = "".join(["\n    - " + f for f in modified_files])
+
+        message = m18n.n("migration_0015_general_warning")
+
+        if problematic_apps:
+            message += "\n\n" + m18n.n("migration_0015_problematic_apps_warning", problematic_apps=problematic_apps)
+
+        if modified_files:
+            message += "\n\n" + m18n.n("migration_0015_modified_files", manually_modified_files=modified_files)
+
+        return message
+
+    def patch_apt_sources_list(self):
+
+        sources_list = glob.glob("/etc/apt/sources.list.d/*.list")
+        sources_list.append("/etc/apt/sources.list")
+
+        # This :
+        # - replace single 'stretch' occurence by 'buster'
+        # - comments lines containing "backports"
+        # - comments lines containing "sury"
+        # - replace 'stretch/updates' by 'strech/updates' (or same with -)
+        for f in sources_list:
+            command = "sed -i -e 's@ stretch @ buster @g' " \
+                      "-e '/backports/ s@^#*@#@' " \
+                      "-e '/sury/ s@^#*@#@' " \
+                      "-e 's@ stretch/updates @ buster/updates @g' " \
+                      "-e 's@ stretch-updates @ buster-updates @g' " \
+                      "{}".format(f)
+            os.system(command)
+
+    def get_apps_equivs_packages(self):
+
+        command = "dpkg --get-selections" \
+                  " | grep -v deinstall" \
+                  " | awk '{print $1}'" \
+                  " | { grep 'ynh-deps$' || true; }"
+
+        output = check_output(command).strip()
+
+        return output.split('\n') if output else []
+
+    def hold(self, packages):
+        for package in packages:
+            os.system("apt-mark hold {}".format(package))
+
+    def unhold(self, packages):
+        for package in packages:
+            os.system("apt-mark unhold {}".format(package))
+
+    def apt_install(self, cmd):
+
+        def is_relevant(l):
+            return "Reading database ..." not in l.rstrip()
+
+        callbacks = (
+            lambda l: logger.info("+ " + l.rstrip() + "\r") if is_relevant(l) else logger.debug(l.rstrip() + "\r"),
+            lambda l: logger.warning(l.rstrip()),
+        )
+
+        cmd = "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --fix-broken --assume-yes " + cmd
+
+        call_async_output(cmd, callbacks, shell=True)

src/yunohost/tools.py

@@ -492,7 +492,7 @@ def _list_upgradable_apps():
 
 
 @is_unit_operation()
-def tools_upgrade(operation_logger, apps=None, system=False):
+def tools_upgrade(operation_logger, apps=None, system=False, allow_yunohost_upgrade=True):
     """
     Update apps & package cache, then display changelog
 
@@ -555,7 +555,7 @@ def tools_upgrade(operation_logger, apps=None, system=False):
 
         # Critical packages are packages that we can't just upgrade
         # randomly from yunohost itself... upgrading them is likely to
-        critical_packages = ("moulinette", "yunohost", "yunohost-admin", "ssowat", "python")
+        critical_packages = ["moulinette", "yunohost", "yunohost-admin", "ssowat"]
 
         critical_packages_upgradable = [p["name"] for p in upgradables if p["name"] in critical_packages]
         noncritical_packages_upgradable = [p["name"] for p in upgradables if p["name"] not in critical_packages]
@@ -608,7 +608,7 @@ def tools_upgrade(operation_logger, apps=None, system=False):
         #
         # Critical packages upgrade
         #
-        if critical_packages_upgradable:
+        if critical_packages_upgradable and allow_yunohost_upgrade:
 
             logger.info(m18n.n("tools_upgrade_special_packages"))