From 2d223c9158e377b58002ffc2797a6c85de7f9d00 Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 10:03:29 +0200 Subject: [PATCH 01/15] Saving app's venv in a requirements file, in order to regenerate it post-update --- .../0021_migrate_to_bullseye.py | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 6b08f5792..0fb698c32 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -5,7 +5,7 @@ from moulinette import m18n from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_file, rm, write_to_file +from moulinette.utils.filesystem import read_file, rm, write_to_file, cp from yunohost.tools import ( Migration, @@ -30,6 +30,52 @@ N_CURRENT_YUNOHOST = 4 N_NEXT_DEBAN = 11 N_NEXT_YUNOHOST = 11 +VENV_BACKUP_PREFIX= "BACKUP_VENV_" +VENV_REQUIREMENTS_SUFFIX= "_req.txt" + +def _get_all_venvs(): + result = [] + exclude = glob.glob(f"/opt/{VENV_BACKUP_PREFIX}*") + for x in glob.glob('/opt/*'): + if x not in exclude and os.path.isdir(x) and os.path.isfile(f"{x}/bin/activate"): + content = read_file(f"{x}/bin/activate") + if "VIRTUAL_ENV" and "PYTHONHOME" in content: + result.append(x) + return result + +def _generate_requirements(): + + venvs = _get_all_venvs() + for venv in venvs: + # Generate a requirements file from venv + os.system(f"bash -c 'source {venv}/bin/activate && pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") + + +def _rebuild_venvs(): + + venvs = _get_all_venvs() + for venv in venvs: + venvdirname = venv.split("/")[-1] + # Create a backup of the venv, in case there's a problem + if os.path.isdir(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}"): + rm(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True) + backup = True + try: + cp(venv, f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True) + except: + backup = False + if backup and os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX): + # Recreate the venv + rm(venv, recursive=True) + os.system(f"python -m venv {venv}") + status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") + if status!=0: + logger.warning(m18n.n("venv_regen_failed", venv=venv)) + else: + rm(venv+VENV_REQUIREMENTS_SUFFIX, recursive=True) + else: + logger.warning(m18n.n("venv_regen_failed", venv=venv)) + class MyMigration(Migration): @@ -70,6 +116,12 @@ class MyMigration(Migration): 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' ) + # + # Get requirements of the different venvs from python apps + # + + _generate_requirements() + # # Run apt update # @@ -264,6 +316,12 @@ class MyMigration(Migration): tools_upgrade(target="system", postupgradecmds=postupgradecmds) + # + # Recreate the venvs + # + + _rebuild_venvs() + def debian_major_version(self): # The python module "platform" and lsb_release are not reliable because # on some setup, they may still return Release=9 even after upgrading to From 1ce13d631dd7463de65cf8348aca43930afd3645 Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 14:15:21 +0200 Subject: [PATCH 02/15] Detect venvs from /var/www/ and recursively --- .../0021_migrate_to_bullseye.py | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 0fb698c32..56044d48f 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -30,22 +30,30 @@ N_CURRENT_YUNOHOST = 4 N_NEXT_DEBAN = 11 N_NEXT_YUNOHOST = 11 -VENV_BACKUP_PREFIX= "BACKUP_VENV_" -VENV_REQUIREMENTS_SUFFIX= "_req.txt" +VENV_BACKUP_SUFFIX = "_BACKUP_VENV" +VENV_REQUIREMENTS_SUFFIX = "_req.txt" +VENV_IGNORE = "VENVNOREGEN" -def _get_all_venvs(): +def _get_all_venvs(dir,level=0,maxlevel=2): result = [] - exclude = glob.glob(f"/opt/{VENV_BACKUP_PREFIX}*") - for x in glob.glob('/opt/*'): - if x not in exclude and os.path.isdir(x) and os.path.isfile(f"{x}/bin/activate"): - content = read_file(f"{x}/bin/activate") - if "VIRTUAL_ENV" and "PYTHONHOME" in content: - result.append(x) + for file in os.listdir(dir): + path = os.path.join(dir,file) + if os.path.isdir(path): + if os.path.isfile(os.path.join(path,VENV_IGNORE)): + continue + activatepath = os.path.join(path,"bin","activate") + if os.path.isfile(activatepath): + content = read_file(activatepath) + if "VIRTUAL_ENV" and "PYTHONHOME" in content: + result.append(path) + continue + if level {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") @@ -53,15 +61,14 @@ def _generate_requirements(): def _rebuild_venvs(): - venvs = _get_all_venvs() + venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/") for venv in venvs: - venvdirname = venv.split("/")[-1] # Create a backup of the venv, in case there's a problem - if os.path.isdir(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}"): - rm(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True) + if os.path.isdir(venv+VENV_BACKUP_SUFFIX): + rm(venv+VENV_BACKUP_SUFFIX, recursive=True) backup = True try: - cp(venv, f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True) + cp(venv, venv+VENV_BACKUP_SUFFIX, recursive=True) except: backup = False if backup and os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX): From 2e9c4f991e6b620abca349e1d8ef8a7a85a3aa3c Mon Sep 17 00:00:00 2001 From: theo-is-taken <108329355+theo-is-taken@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:23:06 +0200 Subject: [PATCH 03/15] Update src/yunohost/data_migrations/0021_migrate_to_bullseye.py Co-authored-by: ljf (zamentur) --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 56044d48f..34e503fac 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -32,7 +32,7 @@ N_NEXT_YUNOHOST = 11 VENV_BACKUP_SUFFIX = "_BACKUP_VENV" VENV_REQUIREMENTS_SUFFIX = "_req.txt" -VENV_IGNORE = "VENVNOREGEN" +VENV_IGNORE = "ynh_migration_no_regen" def _get_all_venvs(dir,level=0,maxlevel=2): result = [] From a440afe8ebc8d9fe3ce02f3f6da40dfcae505027 Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 15:26:10 +0200 Subject: [PATCH 04/15] Clarification regarding the use of os functions instead of glob --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 56044d48f..59b6ba517 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -35,6 +35,7 @@ VENV_REQUIREMENTS_SUFFIX = "_req.txt" VENV_IGNORE = "VENVNOREGEN" def _get_all_venvs(dir,level=0,maxlevel=2): + # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth result = [] for file in os.listdir(dir): path = os.path.join(dir,file) From fc0266a62e8c1e9658b3829f15b3f5dbca03e06e Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 15:53:50 +0200 Subject: [PATCH 05/15] Removed "useless" venv backup --- .../data_migrations/0021_migrate_to_bullseye.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index a2511faf9..cff11b598 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -30,7 +30,6 @@ N_CURRENT_YUNOHOST = 4 N_NEXT_DEBAN = 11 N_NEXT_YUNOHOST = 11 -VENV_BACKUP_SUFFIX = "_BACKUP_VENV" VENV_REQUIREMENTS_SUFFIX = "_req.txt" VENV_IGNORE = "ynh_migration_no_regen" @@ -64,15 +63,7 @@ def _rebuild_venvs(): venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/") for venv in venvs: - # Create a backup of the venv, in case there's a problem - if os.path.isdir(venv+VENV_BACKUP_SUFFIX): - rm(venv+VENV_BACKUP_SUFFIX, recursive=True) - backup = True - try: - cp(venv, venv+VENV_BACKUP_SUFFIX, recursive=True) - except: - backup = False - if backup and os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX): + if os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX): # Recreate the venv rm(venv, recursive=True) os.system(f"python -m venv {venv}") From b7b4dbfcdff2264011e1b247991bf15a2587a362 Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 15:56:54 +0200 Subject: [PATCH 06/15] Increased depth of scan --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index cff11b598..58498ab56 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -33,7 +33,7 @@ N_NEXT_YUNOHOST = 11 VENV_REQUIREMENTS_SUFFIX = "_req.txt" VENV_IGNORE = "ynh_migration_no_regen" -def _get_all_venvs(dir,level=0,maxlevel=2): +def _get_all_venvs(dir,level=0,maxlevel=3): # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth result = [] for file in os.listdir(dir): From cdd579080833bc77e78246f78dbb16762c91fe14 Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 16:00:23 +0200 Subject: [PATCH 07/15] Fixed content condition --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 58498ab56..1c88a0e4e 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -44,7 +44,7 @@ def _get_all_venvs(dir,level=0,maxlevel=3): activatepath = os.path.join(path,"bin","activate") if os.path.isfile(activatepath): content = read_file(activatepath) - if "VIRTUAL_ENV" and "PYTHONHOME" in content: + if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content): result.append(path) continue if level Date: Fri, 22 Jul 2022 16:01:18 +0200 Subject: [PATCH 08/15] Update src/yunohost/data_migrations/0021_migrate_to_bullseye.py Co-authored-by: ljf (zamentur) --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 1c88a0e4e..7aa20d17d 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -71,7 +71,7 @@ def _rebuild_venvs(): if status!=0: logger.warning(m18n.n("venv_regen_failed", venv=venv)) else: - rm(venv+VENV_REQUIREMENTS_SUFFIX, recursive=True) + rm(venv+VENV_REQUIREMENTS_SUFFIX) else: logger.warning(m18n.n("venv_regen_failed", venv=venv)) From 22fc36e16e7af91607f5fea30a12b9ac0a59cfa0 Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 16:15:31 +0200 Subject: [PATCH 09/15] Doc-strings and formatting --- .../0021_migrate_to_bullseye.py | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 7aa20d17d..f83d79239 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -33,45 +33,61 @@ N_NEXT_YUNOHOST = 11 VENV_REQUIREMENTS_SUFFIX = "_req.txt" VENV_IGNORE = "ynh_migration_no_regen" -def _get_all_venvs(dir,level=0,maxlevel=3): + +def _get_all_venvs(dir, level=0, maxlevel=3): + """ + Returns the list of all python virtual env directories recursively + + Arguments: + dir - the directory to scan in + maxlevel - the depth of the recursion + level - do not edit this, used as an iterator + """ # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth result = [] for file in os.listdir(dir): - path = os.path.join(dir,file) + path = os.path.join(dir, file) if os.path.isdir(path): - if os.path.isfile(os.path.join(path,VENV_IGNORE)): + if os.path.isfile(os.path.join(path, VENV_IGNORE)): continue - activatepath = os.path.join(path,"bin","activate") + activatepath = os.path.join(path,"bin", "activate") if os.path.isfile(activatepath): content = read_file(activatepath) if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content): result.append(path) continue - if level {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") def _rebuild_venvs(): + """ + After the update, recreate a python virtual env based on the previously generated requirements file + """ - venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/") + venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") for venv in venvs: - if os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX): + if os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX): # Recreate the venv rm(venv, recursive=True) os.system(f"python -m venv {venv}") status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") - if status!=0: + if status != 0: logger.warning(m18n.n("venv_regen_failed", venv=venv)) else: - rm(venv+VENV_REQUIREMENTS_SUFFIX) + rm(venv + VENV_REQUIREMENTS_SUFFIX) else: logger.warning(m18n.n("venv_regen_failed", venv=venv)) From d531f8e0853c78d82a31db599f11870fa1f874f0 Mon Sep 17 00:00:00 2001 From: "theo@manjaro" Date: Fri, 22 Jul 2022 16:22:04 +0200 Subject: [PATCH 10/15] Proper locales --- locales/en.json | 1 + src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index ce36edaa4..cfb8a01f9 100644 --- a/locales/en.json +++ b/locales/en.json @@ -507,6 +507,7 @@ "migration_0018_failed_to_reset_legacy_rules": "Failed to reset legacy iptables rules: {error}", "migration_0019_add_new_attributes_in_ldap": "Add new attributes for permissions in LDAP database", "migration_0019_slapd_config_will_be_overwritten": "It looks like you manually edited the slapd configuration. For this critical migration, YunoHost needs to force the update of the slapd configuration. The original files will be backuped in {conf_backup_folder}.", + "migration_0021_venv_regen_failed": "The virtual environment '{venv}' failed to regenerate, you probably need to run the command `yunohost app upgrade --force`", "migration_0021_start" : "Starting migration to Bullseye", "migration_0021_patching_sources_list": "Patching the sources.lists...", "migration_0021_main_upgrade": "Starting main upgrade...", diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index f83d79239..4cdec3f24 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -85,11 +85,11 @@ def _rebuild_venvs(): os.system(f"python -m venv {venv}") status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") if status != 0: - logger.warning(m18n.n("venv_regen_failed", venv=venv)) + logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv)) else: rm(venv + VENV_REQUIREMENTS_SUFFIX) else: - logger.warning(m18n.n("venv_regen_failed", venv=venv)) + logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv)) class MyMigration(Migration): From bcdb36b7b2159b0e09c47d6f620f7259a6244308 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sun, 7 Aug 2022 15:03:27 +0200 Subject: [PATCH 11/15] [enh] Simplify the pip freeze call Co-authored-by: Alexandre Aubin --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 4cdec3f24..32a30bf2e 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -69,7 +69,7 @@ def _generate_requirements(): venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") for venv in venvs: # Generate a requirements file from venv - os.system(f"bash -c 'source {venv}/bin/activate && pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") + os.system(f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX}") def _rebuild_venvs(): From e94b7197f26ff4b455775221ebc6f7f771287834 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sun, 7 Aug 2022 15:04:55 +0200 Subject: [PATCH 12/15] [enh] Simplify the pip install call Co-authored-by: Alexandre Aubin --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 32a30bf2e..7dc0f39aa 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -83,7 +83,7 @@ def _rebuild_venvs(): # Recreate the venv rm(venv, recursive=True) os.system(f"python -m venv {venv}") - status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'") + status = os.system(f"{venv}/bin/pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX}") if status != 0: logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv)) else: From 5a3911c65f0f7bfd509a13d901502351cec71742 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 7 Aug 2022 15:26:54 +0200 Subject: [PATCH 13/15] [enh] Split bullseye migration and venv migrations --- .../0021_migrate_to_bullseye.py | 39 ++++--------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 7dc0f39aa..93ff2f930 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -5,7 +5,7 @@ from moulinette import m18n from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_file, rm, write_to_file, cp +from moulinette.utils.filesystem import read_file, rm, write_to_file from yunohost.tools import ( Migration, @@ -30,7 +30,7 @@ N_CURRENT_YUNOHOST = 4 N_NEXT_DEBAN = 11 N_NEXT_YUNOHOST = 11 -VENV_REQUIREMENTS_SUFFIX = "_req.txt" +VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt" VENV_IGNORE = "ynh_migration_no_regen" @@ -72,26 +72,6 @@ def _generate_requirements(): os.system(f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX}") -def _rebuild_venvs(): - """ - After the update, recreate a python virtual env based on the previously generated requirements file - """ - - venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") - for venv in venvs: - if os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX): - # Recreate the venv - rm(venv, recursive=True) - os.system(f"python -m venv {venv}") - status = os.system(f"{venv}/bin/pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX}") - if status != 0: - logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv)) - else: - rm(venv + VENV_REQUIREMENTS_SUFFIX) - else: - logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv)) - - class MyMigration(Migration): "Upgrade the system to Debian Bullseye and Yunohost 11.x" @@ -331,11 +311,6 @@ class MyMigration(Migration): tools_upgrade(target="system", postupgradecmds=postupgradecmds) - # - # Recreate the venvs - # - - _rebuild_venvs() def debian_major_version(self): # The python module "platform" and lsb_release are not reliable because @@ -380,11 +355,11 @@ class MyMigration(Migration): # Lime2 have hold packages to avoid ethernet instability # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df lime2_hold_packages = set([ - "armbian-firmware", - "armbian-bsp-cli-lime2", - "linux-dtb-current-sunxi", - "linux-image-current-sunxi", - "linux-u-boot-lime2-current", + "armbian-firmware", + "armbian-bsp-cli-lime2", + "linux-dtb-current-sunxi", + "linux-image-current-sunxi", + "linux-u-boot-lime2-current", "linux-image-next-sunxi" ]) if upgradable_system_packages - lime2_hold_packages: From f83a357d33e36c5a1724c4a6fcc6a97291f76665 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 7 Aug 2022 19:06:53 +0200 Subject: [PATCH 14/15] bullseye migration: in venv / pip freeze backup mechanism, remove the venv_ignore stuff because it's not relevant anymore --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 93ff2f930..31c07191f 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -31,7 +31,6 @@ N_NEXT_DEBAN = 11 N_NEXT_YUNOHOST = 11 VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt" -VENV_IGNORE = "ynh_migration_no_regen" def _get_all_venvs(dir, level=0, maxlevel=3): @@ -48,8 +47,6 @@ def _get_all_venvs(dir, level=0, maxlevel=3): for file in os.listdir(dir): path = os.path.join(dir, file) if os.path.isdir(path): - if os.path.isfile(os.path.join(path, VENV_IGNORE)): - continue activatepath = os.path.join(path,"bin", "activate") if os.path.isfile(activatepath): content = read_file(activatepath) From 2b63058b4599029d70b26670583c0d69e514ca0f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 7 Aug 2022 19:07:32 +0200 Subject: [PATCH 15/15] bullseye migration: _generate_requirements -> _backup_pip_freeze_for_python_app_venvs --- src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py index 31c07191f..9bd746276 100644 --- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py +++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py @@ -58,7 +58,7 @@ def _get_all_venvs(dir, level=0, maxlevel=3): return result -def _generate_requirements(): +def _backup_pip_freeze_for_python_app_venvs(): """ Generate a requirements file for all python virtual env located inside /opt/ and /var/www/ """ @@ -112,7 +112,7 @@ class MyMigration(Migration): # Get requirements of the different venvs from python apps # - _generate_requirements() + _backup_pip_freeze_for_python_app_venvs() # # Run apt update