From 73e0d6c271e60fc609126f3c1e672bf4ad421a70 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 16 Jul 2024 23:26:35 +0200 Subject: [PATCH 01/10] remove pkg_resources from pip freeze --- src/migrations/0027_migrate_to_bookworm.py.disabled | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index d61ca2d3e..637b237f8 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -74,8 +74,9 @@ def _backup_pip_freeze_for_python_app_venvs(): venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") for venv in venvs: # Generate a requirements file from venv + # Remove pkg resources from the freeze to avoid an error during the python venv https://stackoverflow.com/a/40167445 os.system( - f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} 2>/dev/null" + f"{venv}/bin/pip freeze | grep -E -v 'pkg(-|_)resources==' > {venv}{VENV_REQUIREMENTS_SUFFIX} 2>/dev/null" ) From 4232fc7c4b44f600b0f0d148cb5ac6e231f55d5b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 15:04:52 +0200 Subject: [PATCH 02/10] bullseye->bookworm: explicitly install yunohost-portal --- src/migrations/0027_migrate_to_bookworm.py.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index 637b237f8..0f56b6f46 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -221,7 +221,7 @@ class MyMigration(Migration): aptitude_with_progress_bar(f"unhold yunohost moulinette ssowat yunohost-admin {' '.join(apps_packages)}") # FIXME : find a way to simulate and validate the upgrade first - aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin moulinette ssowat -o Dpkg::Options::='--force-confold'") + aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat -o Dpkg::Options::='--force-confold'") #cmd = "LC_ALL=C" #cmd += " DEBIAN_FRONTEND=noninteractive" From 079cdc2624683051a05203f21be9b9069cbbfd79 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 15:06:05 +0200 Subject: [PATCH 03/10] bullseye->bookworm: explicitly remove python3.9 and python3.9-venv which seems to confuse aptitude... --- src/migrations/0027_migrate_to_bookworm.py.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index 0f56b6f46..3ecc97b3b 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -221,7 +221,7 @@ class MyMigration(Migration): aptitude_with_progress_bar(f"unhold yunohost moulinette ssowat yunohost-admin {' '.join(apps_packages)}") # FIXME : find a way to simulate and validate the upgrade first - aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat -o Dpkg::Options::='--force-confold'") + aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat python3.9- python3.9-venv- -o Dpkg::Options::='--force-confold'") #cmd = "LC_ALL=C" #cmd += " DEBIAN_FRONTEND=noninteractive" From a5868733d7e0b4c278bc957623932fd77d50c472 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 15:18:46 +0200 Subject: [PATCH 04/10] bullseye->bookworm: uncessary comments / FIXME --- .../0027_migrate_to_bookworm.py.disabled | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index 3ecc97b3b..b2289ed6c 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -185,7 +185,6 @@ class MyMigration(Migration): aptitude_with_progress_bar("upgrade cron rspamd- libluajit-5.1-2- --show-why -o APT::Force-LoopBreak=1 -o Dpkg::Options::='--force-confold'") - # FIXME : find a way to simulate and validate the upgrade first aptitude_with_progress_bar("full-upgrade --show-why -o Dpkg::Options::='--force-confold'") # Force regenconf of nsswitch because for some reason @@ -220,28 +219,8 @@ class MyMigration(Migration): aptitude_with_progress_bar(f"unhold yunohost moulinette ssowat yunohost-admin {' '.join(apps_packages)}") - # FIXME : find a way to simulate and validate the upgrade first aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat python3.9- python3.9-venv- -o Dpkg::Options::='--force-confold'") - #cmd = "LC_ALL=C" - #cmd += " DEBIAN_FRONTEND=noninteractive" - #cmd += " APT_LISTCHANGES_FRONTEND=none" - #cmd += " apt dist-upgrade " - #cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" - #cmd += " | grep -q 'ynh-deps'" - - #logger.info("Simulating upgrade...") - #if os.system(cmd) == 0: - # raise YunohostError( - # "The upgrade cannot be completed, because some app dependencies would need to be removed?", - # raw_msg=True, - # ) - - # FIXME : - #postupgradecmds = "rm -f /usr/sbin/policy-rc.d\n" - #postupgradecmds += "echo 'Restarting nginx...' >&2\n" - #postupgradecmds += "systemctl restart nginx\n" - # If running from the webadmin, restart the API after a delay if Moulinette.interface.type == "api": logger.warning(m18n.n("migration_0027_delayed_api_restart")) From a8fd6afeee7efc89af286921a0bebc7f76cc8110 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 15:24:04 +0200 Subject: [PATCH 05/10] bullseye->bookworm: try the yunohost upgrade without unholding the app-ynh-deps virtual packages, then after unholding if it didnt work for some reason --- src/migrations/0027_migrate_to_bookworm.py.disabled | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index b2289ed6c..0416e8f05 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -216,10 +216,19 @@ class MyMigration(Migration): # Yunohost upgrade # logger.info(m18n.n("migration_0027_yunohost_upgrade")) - aptitude_with_progress_bar(f"unhold yunohost moulinette ssowat yunohost-admin {' '.join(apps_packages)}") - aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat python3.9- python3.9-venv- -o Dpkg::Options::='--force-confold'") + try: + aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat python3.9- python3.9-venv- -o Dpkg::Options::='--force-confold'") + except Exception as e: + # Retry after unholding the app packages, maybe it can unlock the situation idk + if apps_packages: + aptitude_with_progress_bar(f"unhold {' '.join(apps_packages)}") + aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat python3.9- python3.9-venv- -o Dpkg::Options::='--force-confold'") + else: + # If the upgrade was sucessful, we want to unhold the apps packages + if apps_packages: + aptitude_with_progress_bar(f"unhold {' '.join(apps_packages)}") # If running from the webadmin, restart the API after a delay if Moulinette.interface.type == "api": From ca59886303275e155b427fc3de9ba4d69f8dbdb3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 15:28:01 +0200 Subject: [PATCH 06/10] bullseye->bookworm: fix typo in message --- locales/en.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index ec871a649..b10293434 100644 --- a/locales/en.json +++ b/locales/en.json @@ -610,7 +610,7 @@ "migration_0027_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", "migration_0027_start": "Starting migration to Bookworm…", "migration_0027_still_on_bullseye_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Bullseye.", - "migration_0027_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Bullseye.", + "migration_0027_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Bookworm.", "migration_0027_yunohost_upgrade": "Starting YunoHost core upgrade…", "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x", "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4", @@ -804,4 +804,4 @@ "yunohost_installing": "Installing YunoHost…", "yunohost_not_installed": "YunoHost is not correctly installed. Please run 'yunohost tools postinstall'", "yunohost_postinstall_end_tip": "The post-install completed! To finalize your setup, please consider:\n - diagnose potential issues through the 'Diagnosis' section of the webadmin (or 'yunohost diagnosis run' in command-line);\n - reading the 'Finalizing your setup' and 'Getting to know YunoHost' parts in the admin documentation: https://yunohost.org/admindoc." -} \ No newline at end of file +} From 97bb6bde095909102c70666dc5f9b7290b998298 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 15:31:25 +0200 Subject: [PATCH 07/10] bullseye->bookworm: automatically add non-free-firmware if non-free is enabled --- src/migrations/0027_migrate_to_bookworm.py.disabled | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index 0416e8f05..4e22cbd75 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -371,6 +371,7 @@ class MyMigration(Migration): # - comments lines containing "backports" # - replace 'bullseye/updates' by 'bookworm/updates' (or same with -) # - make sure the yunohost line has the "signed-by" thingy + # - replace "non-free" with "non-free non-free-firmware" # Special note about the security suite: # https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#security-archive for f in sources_list: @@ -380,6 +381,7 @@ class MyMigration(Migration): "-e '/backports/ s@^#*@#@' " "-e 's@ bullseye/updates @ bookworm-security @g' " "-e 's@ bullseye-@ bookworm-@g' " + "-e 's@ non-free@ non-free non-free-firmware@g' " "-e 's@deb.*http://forge.yunohost.org@deb [signed-by=/usr/share/keyrings/yunohost-bookworm.gpg] http://forge.yunohost.org@g' " ) os.system(command) From 67d6baa15182fb29a27c61c874a1c843e55d5c36 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 16:17:46 +0200 Subject: [PATCH 08/10] bullseye->bookworm: forgot to remove the unhold for apps packages >_> --- src/migrations/0027_migrate_to_bookworm.py.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index 4e22cbd75..4d46459e9 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -216,7 +216,7 @@ class MyMigration(Migration): # Yunohost upgrade # logger.info(m18n.n("migration_0027_yunohost_upgrade")) - aptitude_with_progress_bar(f"unhold yunohost moulinette ssowat yunohost-admin {' '.join(apps_packages)}") + aptitude_with_progress_bar(f"unhold yunohost moulinette ssowat yunohost-admin") try: aptitude_with_progress_bar("full-upgrade --show-why yunohost yunohost-admin yunohost-portal moulinette ssowat python3.9- python3.9-venv- -o Dpkg::Options::='--force-confold'") From f11f11973b0c53f36c3f5f7d23027f0fd99740af Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 17 Jul 2024 16:36:46 +0200 Subject: [PATCH 09/10] bullseye->bookworm: trigger the 'new' migration from inside the bullseye->bookworm migration --- .../0027_migrate_to_bookworm.py.disabled | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/migrations/0027_migrate_to_bookworm.py.disabled b/src/migrations/0027_migrate_to_bookworm.py.disabled index 4d46459e9..30d36544a 100644 --- a/src/migrations/0027_migrate_to_bookworm.py.disabled +++ b/src/migrations/0027_migrate_to_bookworm.py.disabled @@ -4,7 +4,9 @@ import subprocess from time import sleep from moulinette import Moulinette, m18n +from moulinette.utils.process import call_async_output from yunohost.utils.error import YunohostError +from yunohost.tools import _write_migration_state from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file, write_to_file @@ -230,6 +232,18 @@ class MyMigration(Migration): if apps_packages: aptitude_with_progress_bar(f"unhold {' '.join(apps_packages)}") + # Mark this migration as completed before triggering the "new" migrations + _write_migration_state(self.id, "done") + + callbacks = ( + lambda l: logger.debug("+ " + l.rstrip() + "\r"), + lambda l: logger.warning(l.rstrip()), + ) + try: + call_async_output(["yunohost", "tools", "migrations", "run"], callbacks) + except Exception as e: + logger.error(e) + # If running from the webadmin, restart the API after a delay if Moulinette.interface.type == "api": logger.warning(m18n.n("migration_0027_delayed_api_restart")) From ebcf3c79ff4ca1f88a961252231ce6712664458f Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 17 Jul 2024 16:45:43 +0200 Subject: [PATCH 10/10] Fix "log list" : use root_dir for iglob / make sure we use absolute paths (#1913) * use root_dir for iglob, fix parent_symlink path and check if it exists * fix log path * do not try to read a yaml of a symlink to /dev/null * use hidden files, needs python 3.11 (bookworm) * don't worry, I'm an expert! * Update log.py: log_file -> log_md_fullpath (otherwise it feel like log_file refers to the .log) * Update log.py: remove debug statement * Update log.py: revert unecessary if change --------- Co-authored-by: Alexandre Aubin <4533074+alexAubin@users.noreply.github.com> --- src/log.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/log.py b/src/log.py index 1e116baf1..7e17d527e 100755 --- a/src/log.py +++ b/src/log.py @@ -87,9 +87,10 @@ def _update_log_parent_symlinks(): one_year_ago = time.time() - 365 * 24 * 3600 - logs = glob.iglob(OPERATIONS_PATH + "*" + METADATA_FILE_EXT) + logs = glob.iglob("*" + METADATA_FILE_EXT, root_dir=OPERATIONS_PATH) for log_md in logs: - if os.path.getctime(log_md) < one_year_ago: + log_md_fullpath = os.path.join(OPERATIONS_PATH, log_md) + if os.path.getctime(log_md_fullpath) < one_year_ago: # Let's ignore files older than one year because hmpf reading a shitload of yml is not free continue @@ -100,7 +101,7 @@ def _update_log_parent_symlinks(): try: metadata = ( - read_yaml(log_md) or {} + read_yaml(log_md_fullpath) or {} ) # Making sure this is a dict and not None..? except Exception as e: # If we can't read the yaml for some reason, report an error and ignore this entry... @@ -110,7 +111,6 @@ def _update_log_parent_symlinks(): parent = metadata.get("parent") parent = parent + METADATA_FILE_EXT if parent else "/dev/null" try: - print(parent, parent_symlink) os.symlink(parent, parent_symlink) except Exception as e: logger.warning(f"Failed to create symlink {parent_symlink} ? {e}")