From b928dd12227e18f4106b74ee36559db73e23d184 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Wed, 27 Apr 2022 16:25:14 +0200
Subject: [PATCH 01/34] migrate_to_bullseye: /etc/apt/sources.list may not
 exist

---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 77797c63f..e8ed6bc4d 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -299,7 +299,7 @@ class MyMigration(Migration):
         # Check system is up to date
         # (but we don't if 'bullseye' is already in the sources.list ...
         # which means maybe a previous upgrade crashed and we're re-running it)
-        if " bullseye " not in read_file("/etc/apt/sources.list"):
+        if os.path.exists("/etc/apt/sources.list") and " bullseye " not in read_file("/etc/apt/sources.list"):
             tools_update(target="system")
             upgradable_system_packages = list(_list_upgradable_apt_packages())
             if upgradable_system_packages:
@@ -355,7 +355,8 @@ class MyMigration(Migration):
     def patch_apt_sources_list(self):
 
         sources_list = glob.glob("/etc/apt/sources.list.d/*.list")
-        sources_list.append("/etc/apt/sources.list")
+        if os.path.exists("/etc/apt/sources.list"):
+            sources_list.append("/etc/apt/sources.list")
 
         # This :
         # - replace single 'buster' occurence by 'bulleye'

From cc5649cfb245fa08fc4611659137c3e5b0ab8098 Mon Sep 17 00:00:00 2001
From: "ljf (zamentur)" <zamentur@users.noreply.github.com>
Date: Fri, 6 May 2022 16:25:54 +0200
Subject: [PATCH 02/34] [fix] Allow lime2 to upgrade even if kernel is hold
 (#1452)

* [fix] Allow lime2 to upgrade even if kernel is hold

* [fix] Bad set operation
---
 .../data_migrations/0021_migrate_to_bullseye.py    | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index e8ed6bc4d..6b08f5792 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -302,7 +302,19 @@ class MyMigration(Migration):
         if os.path.exists("/etc/apt/sources.list") and " bullseye " not in read_file("/etc/apt/sources.list"):
             tools_update(target="system")
             upgradable_system_packages = list(_list_upgradable_apt_packages())
-            if upgradable_system_packages:
+            upgradable_system_packages = [package["name"] for package in upgradable_system_packages]
+            upgradable_system_packages = set(upgradable_system_packages)
+            # Lime2 have hold packages to avoid ethernet instability
+            # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df
+            lime2_hold_packages = set([
+                "armbian-firmware", 
+                "armbian-bsp-cli-lime2", 
+                "linux-dtb-current-sunxi", 
+                "linux-image-current-sunxi", 
+                "linux-u-boot-lime2-current", 
+                "linux-image-next-sunxi"
+            ])
+            if upgradable_system_packages - lime2_hold_packages:
                 raise YunohostError("migration_0021_system_not_fully_up_to_date")
 
     @property

From 31bbc026fde7c4f31f84afbd68b66391da7079d9 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Wed, 20 Jul 2022 11:24:42 +0200
Subject: [PATCH 03/34] Don't restrict choices if there's no choices specified

---
 src/utils/config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/utils/config.py b/src/utils/config.py
index 56f632b09..e4e353360 100644
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -703,7 +703,8 @@ class Question:
         self.default = question.get("default", None)
         self.optional = question.get("optional", False)
         self.visible = question.get("visible", None)
-        self.choices = question.get("choices", [])
+        # Don't restrict choices if there's none specified
+        self.choices = question.get("choices", None)
         self.pattern = question.get("pattern", self.pattern)
         self.ask = question.get("ask", {"en": self.name})
         self.help = question.get("help")

From 2d223c9158e377b58002ffc2797a6c85de7f9d00 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 10:03:29 +0200
Subject: [PATCH 04/34] Saving app's venv in a requirements file, in order to
 regenerate it post-update

---
 .../0021_migrate_to_bullseye.py               | 60 ++++++++++++++++++-
 1 file changed, 59 insertions(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 6b08f5792..0fb698c32 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -5,7 +5,7 @@ from moulinette import m18n
 from yunohost.utils.error import YunohostError
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import check_output, call_async_output
-from moulinette.utils.filesystem import read_file, rm, write_to_file
+from moulinette.utils.filesystem import read_file, rm, write_to_file, cp
 
 from yunohost.tools import (
     Migration,
@@ -30,6 +30,52 @@ N_CURRENT_YUNOHOST = 4
 N_NEXT_DEBAN = 11
 N_NEXT_YUNOHOST = 11
 
+VENV_BACKUP_PREFIX= "BACKUP_VENV_"
+VENV_REQUIREMENTS_SUFFIX= "_req.txt"
+
+def _get_all_venvs():
+    result = []
+    exclude = glob.glob(f"/opt/{VENV_BACKUP_PREFIX}*")
+    for x in glob.glob('/opt/*'):
+        if x not in exclude and os.path.isdir(x) and os.path.isfile(f"{x}/bin/activate"):
+            content = read_file(f"{x}/bin/activate")
+            if "VIRTUAL_ENV" and "PYTHONHOME" in content:
+                result.append(x)
+    return result
+
+def _generate_requirements():
+
+    venvs = _get_all_venvs()
+    for venv in venvs:
+        # Generate a requirements file from venv
+        os.system(f"bash -c 'source {venv}/bin/activate && pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
+
+
+def _rebuild_venvs():
+
+    venvs = _get_all_venvs()
+    for venv in venvs:
+        venvdirname = venv.split("/")[-1]
+        # Create a backup of the venv, in case there's a problem
+        if os.path.isdir(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}"):
+            rm(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True)
+        backup = True
+        try:
+            cp(venv, f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True)
+        except:
+            backup = False
+        if backup and os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX):
+            # Recreate the venv
+            rm(venv, recursive=True)
+            os.system(f"python -m venv {venv}")
+            status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
+            if status!=0:
+                logger.warning(m18n.n("venv_regen_failed", venv=venv))
+            else:
+                rm(venv+VENV_REQUIREMENTS_SUFFIX, recursive=True)
+        else:
+            logger.warning(m18n.n("venv_regen_failed", venv=venv))
+
 
 class MyMigration(Migration):
 
@@ -70,6 +116,12 @@ class MyMigration(Migration):
                 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"'
             )
 
+        #
+        # Get requirements of the different venvs from python apps
+        #
+
+        _generate_requirements()
+
         #
         # Run apt update
         #
@@ -264,6 +316,12 @@ class MyMigration(Migration):
 
         tools_upgrade(target="system", postupgradecmds=postupgradecmds)
 
+        #
+        # Recreate the venvs
+        #
+
+        _rebuild_venvs()
+
     def debian_major_version(self):
         # The python module "platform" and lsb_release are not reliable because
         # on some setup, they may still return Release=9 even after upgrading to

From 1ce13d631dd7463de65cf8348aca43930afd3645 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 14:15:21 +0200
Subject: [PATCH 05/34] Detect venvs from /var/www/ and recursively

---
 .../0021_migrate_to_bullseye.py               | 37 +++++++++++--------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 0fb698c32..56044d48f 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -30,22 +30,30 @@ N_CURRENT_YUNOHOST = 4
 N_NEXT_DEBAN = 11
 N_NEXT_YUNOHOST = 11
 
-VENV_BACKUP_PREFIX= "BACKUP_VENV_"
-VENV_REQUIREMENTS_SUFFIX= "_req.txt"
+VENV_BACKUP_SUFFIX = "_BACKUP_VENV"
+VENV_REQUIREMENTS_SUFFIX = "_req.txt"
+VENV_IGNORE = "VENVNOREGEN"
 
-def _get_all_venvs():
+def _get_all_venvs(dir,level=0,maxlevel=2):
     result = []
-    exclude = glob.glob(f"/opt/{VENV_BACKUP_PREFIX}*")
-    for x in glob.glob('/opt/*'):
-        if x not in exclude and os.path.isdir(x) and os.path.isfile(f"{x}/bin/activate"):
-            content = read_file(f"{x}/bin/activate")
-            if "VIRTUAL_ENV" and "PYTHONHOME" in content:
-                result.append(x)
+    for file in os.listdir(dir):
+        path = os.path.join(dir,file)
+        if os.path.isdir(path):
+            if os.path.isfile(os.path.join(path,VENV_IGNORE)):
+                continue
+            activatepath = os.path.join(path,"bin","activate")
+            if os.path.isfile(activatepath):
+                content = read_file(activatepath)
+                if "VIRTUAL_ENV" and "PYTHONHOME" in content:
+                    result.append(path)
+                    continue
+            if level<maxlevel:
+                result+=_get_all_venvs(path,level=level+1)
     return result
 
 def _generate_requirements():
 
-    venvs = _get_all_venvs()
+    venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/")
     for venv in venvs:
         # Generate a requirements file from venv
         os.system(f"bash -c 'source {venv}/bin/activate && pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
@@ -53,15 +61,14 @@ def _generate_requirements():
 
 def _rebuild_venvs():
 
-    venvs = _get_all_venvs()
+    venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/")
     for venv in venvs:
-        venvdirname = venv.split("/")[-1]
         # Create a backup of the venv, in case there's a problem
-        if os.path.isdir(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}"):
-            rm(f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True)
+        if os.path.isdir(venv+VENV_BACKUP_SUFFIX):
+            rm(venv+VENV_BACKUP_SUFFIX, recursive=True)
         backup = True
         try:
-            cp(venv, f"/opt/{VENV_BACKUP_PREFIX}{venvdirname}", recursive=True)
+            cp(venv, venv+VENV_BACKUP_SUFFIX, recursive=True)
         except:
             backup = False
         if backup and os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX):

From 2e9c4f991e6b620abca349e1d8ef8a7a85a3aa3c Mon Sep 17 00:00:00 2001
From: theo-is-taken <108329355+theo-is-taken@users.noreply.github.com>
Date: Fri, 22 Jul 2022 15:23:06 +0200
Subject: [PATCH 06/34] Update
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py

Co-authored-by: ljf (zamentur) <zamentur@users.noreply.github.com>
---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 56044d48f..34e503fac 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -32,7 +32,7 @@ N_NEXT_YUNOHOST = 11
 
 VENV_BACKUP_SUFFIX = "_BACKUP_VENV"
 VENV_REQUIREMENTS_SUFFIX = "_req.txt"
-VENV_IGNORE = "VENVNOREGEN"
+VENV_IGNORE = "ynh_migration_no_regen"
 
 def _get_all_venvs(dir,level=0,maxlevel=2):
     result = []

From a440afe8ebc8d9fe3ce02f3f6da40dfcae505027 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 15:26:10 +0200
Subject: [PATCH 07/34] Clarification regarding the use of os functions instead
 of glob

---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 56044d48f..59b6ba517 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -35,6 +35,7 @@ VENV_REQUIREMENTS_SUFFIX = "_req.txt"
 VENV_IGNORE = "VENVNOREGEN"
 
 def _get_all_venvs(dir,level=0,maxlevel=2):
+    # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth
     result = []
     for file in os.listdir(dir):
         path = os.path.join(dir,file)

From fc0266a62e8c1e9658b3829f15b3f5dbca03e06e Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 15:53:50 +0200
Subject: [PATCH 08/34] Removed "useless" venv backup

---
 .../data_migrations/0021_migrate_to_bullseye.py       | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index a2511faf9..cff11b598 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -30,7 +30,6 @@ N_CURRENT_YUNOHOST = 4
 N_NEXT_DEBAN = 11
 N_NEXT_YUNOHOST = 11
 
-VENV_BACKUP_SUFFIX = "_BACKUP_VENV"
 VENV_REQUIREMENTS_SUFFIX = "_req.txt"
 VENV_IGNORE = "ynh_migration_no_regen"
 
@@ -64,15 +63,7 @@ def _rebuild_venvs():
 
     venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/")
     for venv in venvs:
-        # Create a backup of the venv, in case there's a problem
-        if os.path.isdir(venv+VENV_BACKUP_SUFFIX):
-            rm(venv+VENV_BACKUP_SUFFIX, recursive=True)
-        backup = True
-        try:
-            cp(venv, venv+VENV_BACKUP_SUFFIX, recursive=True)
-        except:
-            backup = False
-        if backup and os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX):
+        if os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX):
             # Recreate the venv
             rm(venv, recursive=True)
             os.system(f"python -m venv {venv}")

From b7b4dbfcdff2264011e1b247991bf15a2587a362 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 15:56:54 +0200
Subject: [PATCH 09/34] Increased depth of scan

---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index cff11b598..58498ab56 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -33,7 +33,7 @@ N_NEXT_YUNOHOST = 11
 VENV_REQUIREMENTS_SUFFIX = "_req.txt"
 VENV_IGNORE = "ynh_migration_no_regen"
 
-def _get_all_venvs(dir,level=0,maxlevel=2):
+def _get_all_venvs(dir,level=0,maxlevel=3):
     # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth
     result = []
     for file in os.listdir(dir):

From cdd579080833bc77e78246f78dbb16762c91fe14 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 16:00:23 +0200
Subject: [PATCH 10/34] Fixed content condition

---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 58498ab56..1c88a0e4e 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -44,7 +44,7 @@ def _get_all_venvs(dir,level=0,maxlevel=3):
             activatepath = os.path.join(path,"bin","activate")
             if os.path.isfile(activatepath):
                 content = read_file(activatepath)
-                if "VIRTUAL_ENV" and "PYTHONHOME" in content:
+                if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content):
                     result.append(path)
                     continue
             if level<maxlevel:

From 3fea05c4d6e176d2f05bfc8f7e45462fc391a83c Mon Sep 17 00:00:00 2001
From: theo-is-taken <108329355+theo-is-taken@users.noreply.github.com>
Date: Fri, 22 Jul 2022 16:01:18 +0200
Subject: [PATCH 11/34] Update
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py

Co-authored-by: ljf (zamentur) <zamentur@users.noreply.github.com>
---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 1c88a0e4e..7aa20d17d 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -71,7 +71,7 @@ def _rebuild_venvs():
             if status!=0:
                 logger.warning(m18n.n("venv_regen_failed", venv=venv))
             else:
-                rm(venv+VENV_REQUIREMENTS_SUFFIX, recursive=True)
+                rm(venv+VENV_REQUIREMENTS_SUFFIX)
         else:
             logger.warning(m18n.n("venv_regen_failed", venv=venv))
 

From 22fc36e16e7af91607f5fea30a12b9ac0a59cfa0 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 16:15:31 +0200
Subject: [PATCH 12/34] Doc-strings and formatting

---
 .../0021_migrate_to_bullseye.py               | 40 +++++++++++++------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 7aa20d17d..f83d79239 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -33,45 +33,61 @@ N_NEXT_YUNOHOST = 11
 VENV_REQUIREMENTS_SUFFIX = "_req.txt"
 VENV_IGNORE = "ynh_migration_no_regen"
 
-def _get_all_venvs(dir,level=0,maxlevel=3):
+
+def _get_all_venvs(dir, level=0, maxlevel=3):
+    """
+        Returns the list of all python virtual env directories recursively
+
+        Arguments:
+            dir - the directory to scan in
+            maxlevel - the depth of the recursion
+            level - do not edit this, used as an iterator
+    """
     # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth
     result = []
     for file in os.listdir(dir):
-        path = os.path.join(dir,file)
+        path = os.path.join(dir, file)
         if os.path.isdir(path):
-            if os.path.isfile(os.path.join(path,VENV_IGNORE)):
+            if os.path.isfile(os.path.join(path, VENV_IGNORE)):
                 continue
-            activatepath = os.path.join(path,"bin","activate")
+            activatepath = os.path.join(path,"bin", "activate")
             if os.path.isfile(activatepath):
                 content = read_file(activatepath)
                 if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content):
                     result.append(path)
                     continue
-            if level<maxlevel:
-                result+=_get_all_venvs(path,level=level+1)
+            if level < maxlevel:
+                result += _get_all_venvs(path, level=level + 1)
     return result
 
-def _generate_requirements():
 
-    venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/")
+def _generate_requirements():
+    """
+        Generate a requirements file for all python virtual env located inside /opt/ and /var/www/
+    """
+
+    venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
     for venv in venvs:
         # Generate a requirements file from venv
         os.system(f"bash -c 'source {venv}/bin/activate && pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
 
 
 def _rebuild_venvs():
+    """
+        After the update, recreate a python virtual env based on the previously generated requirements file
+    """
 
-    venvs = _get_all_venvs("/opt/")+_get_all_venvs("/var/www/")
+    venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
     for venv in venvs:
-        if os.path.isfile(venv+VENV_REQUIREMENTS_SUFFIX):
+        if os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
             # Recreate the venv
             rm(venv, recursive=True)
             os.system(f"python -m venv {venv}")
             status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
-            if status!=0:
+            if status != 0:
                 logger.warning(m18n.n("venv_regen_failed", venv=venv))
             else:
-                rm(venv+VENV_REQUIREMENTS_SUFFIX)
+                rm(venv + VENV_REQUIREMENTS_SUFFIX)
         else:
             logger.warning(m18n.n("venv_regen_failed", venv=venv))
 

From d531f8e0853c78d82a31db599f11870fa1f874f0 Mon Sep 17 00:00:00 2001
From: "theo@manjaro" <ayte@tutanota.com>
Date: Fri, 22 Jul 2022 16:22:04 +0200
Subject: [PATCH 13/34] Proper locales

---
 locales/en.json                                          | 1 +
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index ce36edaa4..cfb8a01f9 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -507,6 +507,7 @@
     "migration_0018_failed_to_reset_legacy_rules": "Failed to reset legacy iptables rules: {error}",
     "migration_0019_add_new_attributes_in_ldap": "Add new attributes for permissions in LDAP database",
     "migration_0019_slapd_config_will_be_overwritten": "It looks like you manually edited the slapd configuration. For this critical migration, YunoHost needs to force the update of the slapd configuration. The original files will be backuped in {conf_backup_folder}.",
+	"migration_0021_venv_regen_failed": "The virtual environment '{venv}' failed to regenerate, you probably need to run the command `yunohost app upgrade --force`",
     "migration_0021_start" : "Starting migration to Bullseye",
     "migration_0021_patching_sources_list": "Patching the sources.lists...",
     "migration_0021_main_upgrade": "Starting main upgrade...",
diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index f83d79239..4cdec3f24 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -85,11 +85,11 @@ def _rebuild_venvs():
             os.system(f"python -m venv {venv}")
             status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
             if status != 0:
-                logger.warning(m18n.n("venv_regen_failed", venv=venv))
+                logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv))
             else:
                 rm(venv + VENV_REQUIREMENTS_SUFFIX)
         else:
-            logger.warning(m18n.n("venv_regen_failed", venv=venv))
+            logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv))
 
 
 class MyMigration(Migration):

From 5535896efab91af800248698f12dce312447b243 Mon Sep 17 00:00:00 2001
From: ButterflyOfFire <butterflyoffire@protonmail.com>
Date: Fri, 5 Aug 2022 11:05:20 +0000
Subject: [PATCH 14/34] Translated using Weblate (Arabic)

Currently translated at 13.1% (90 of 686 strings)

Translation: YunoHost/core
Translate-URL: https://translate.yunohost.org/projects/yunohost/core/ar/
---
 locales/ar.json | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/locales/ar.json b/locales/ar.json
index c440e442f..e37cdbcc7 100644
--- a/locales/ar.json
+++ b/locales/ar.json
@@ -15,7 +15,7 @@
     "app_requirements_checking": "جار فحص الحزم اللازمة لـ {app}…",
     "app_sources_fetch_failed": "تعذرت عملية جلب مصادر الملفات",
     "app_unknown": "برنامج مجهول",
-    "app_upgrade_app_name": "جارٍ تحديث تطبيق {app}…",
+    "app_upgrade_app_name": "جارٍ تحديث {app}…",
     "app_upgrade_failed": "تعذرت عملية ترقية {app}",
     "app_upgrade_some_app_failed": "تعذرت عملية ترقية بعض التطبيقات",
     "app_upgraded": "تم تحديث التطبيق {app}",
@@ -30,9 +30,9 @@
     "backup_method_copy_finished": "إنتهت عملية النسخ الإحتياطي",
     "backup_nothings_done": "ليس هناك أي شيء للحفظ",
     "backup_output_directory_required": "يتوجب عليك تحديد مجلد لتلقي النسخ الإحتياطية",
-    "certmanager_cert_install_success": "تمت عملية تنصيب شهادة Let's Encrypt  بنجاح على النطاق  {domain} !",
-    "certmanager_cert_install_success_selfsigned": "نجحت عملية تثبيت الشهادة الموقعة ذاتيا الخاصة بالنطاق {domain}",
-    "certmanager_cert_renew_success": "نجحت عملية تجديد شهادة Let's Encrypt الخاصة باسم النطاق {domain} !",
+    "certmanager_cert_install_success": "تمت عملية تنصيب شهادة Let's Encrypt  بنجاح على النطاق  {domain}",
+    "certmanager_cert_install_success_selfsigned": "نجحت عملية تثبيت الشهادة الموقعة ذاتيا الخاصة بالنطاق '{domain}'",
+    "certmanager_cert_renew_success": "نجحت عملية تجديد شهادة Let's Encrypt الخاصة باسم النطاق '{domain}'",
     "certmanager_cert_signing_failed": "فشل إجراء توقيع الشهادة الجديدة",
     "certmanager_no_cert_file": "تعذرت عملية قراءة شهادة نطاق {domain} (الملف : {file})",
     "domain_created": "تم إنشاء النطاق",
@@ -114,9 +114,9 @@
     "aborting": "إلغاء.",
     "admin_password_too_long": "يرجى اختيار كلمة سرية أقصر مِن 127 حرف",
     "app_not_upgraded": "",
-    "app_start_install": "جارٍ تثبيت التطبيق {app}…",
-    "app_start_remove": "جارٍ حذف التطبيق {app}…",
-    "app_start_restore": "جارٍ استرجاع التطبيق {app}…",
+    "app_start_install": "جارٍ تثبيت {app}…",
+    "app_start_remove": "جارٍ حذف {app}…",
+    "app_start_restore": "جارٍ استرجاع {app}…",
     "app_upgrade_several_apps": "سوف يتم تحديث التطبيقات التالية: {apps}",
     "ask_new_domain": "نطاق جديد",
     "ask_new_path": "مسار جديد",
@@ -127,7 +127,7 @@
     "service_description_slapd": "يخزّن المستخدمين والنطاقات والمعلومات المتعلقة بها",
     "service_reloaded": "تم إعادة تشغيل خدمة '{service}'",
     "service_restarted": "تم إعادة تشغيل خدمة  '{service}'",
-    "group_unknown": "الفريق {group} مجهول",
+    "group_unknown": "الفريق '{group}' مجهول",
     "group_deletion_failed": "فشلت عملية حذف الفريق '{group}': {error}",
     "group_deleted": "تم حذف الفريق '{group}'",
     "group_created": "تم إنشاء الفريق '{group}'",
@@ -145,7 +145,7 @@
     "diagnosis_ip_not_connected_at_all": "يبدو أنّ الخادم غير مُتّصل بتاتا بالإنترنت!؟",
     "app_install_failed": "لا يمكن تنصيب {app}: {error}",
     "apps_already_up_to_date": "كافة التطبيقات مُحدّثة",
-    "app_remove_after_failed_install": "جارٍ حذف التطبيق بعدما فشل تنصيبها…",
+    "app_remove_after_failed_install": "جارٍ حذف التطبيق بعدما فشل تنصيبه…",
     "apps_catalog_updating": "جارٍ تحديث فهرس التطبيقات…",
     "apps_catalog_update_success": "تم تحديث فهرس التطبيقات!",
     "diagnosis_domain_expiration_error": "ستنتهي مدة صلاحية بعض النطاقات في القريب العاجل!",
@@ -158,5 +158,6 @@
     "diagnosis_description_services": "حالة الخدمات",
     "diagnosis_description_dnsrecords": "تسجيلات خدمة DNS",
     "diagnosis_description_ip": "الإتصال بالإنترنت",
-    "diagnosis_description_basesystem": "النظام الأساسي"
+    "diagnosis_description_basesystem": "النظام الأساسي",
+    "field_invalid": "الحقل غير صحيح : '{}'"
 }

From 92467b5e59311df163cd6fb2279037815b456542 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 11:33:33 +0200
Subject: [PATCH 15/34] Update changelog for 11.0.8

---
 debian/changelog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 6a945f739..473fb1abc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,19 @@
+yunohost (11.0.8) testing; urgency=low
+
+  - [fix] helpers: escape username in ynh_user_exists ([#1469](https://github.com/YunoHost/yunohost/pull/1469))
+  - [fix] helpers: in nginx helpers, do not change the nginx template conf, replace #sub_path_only and #root_path_only after ynh_add_config, otherwise it breaks the change_url script (30e926f9)
+  - [fix] helpers: fix arg parsing in ynh_install_apps ([#1480](https://github.com/YunoHost/yunohost/pull/1480))
+  - [fix] postinstall: be able to redo postinstall when the 128+ chars
+  password error is raised ([#1476](https://github.com/YunoHost/yunohost/pull/1476))
+  - [fix] regenconf dhclient/resolvconf: fix weird typo, probably meant 'search' (like in our rpi-image tweaking) (9d39a2c0)
+  - [fix] configpanels: remove debug message because it floods the regenconf logs (f6cd35d9)
+  - [fix] configpanels: don't restrict choices if there's no choices specified ([#1478](https://github.com/YunoHost/yunohost/pull/1478)
+  - [i18n] Translations updated for Arabic, German, Slovak, Telugu
+
+  Thanks to all contributors <3 ! (Alice Kile, ButterflyOfFire, Éric Gaspar, Gregor, Jose Riha, Kay0u, ljf, Meta Meta, tituspijean, Valentin von Guttenberg, yalh76)
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Sun, 07 Aug 2022 11:26:54 +0200
+
 yunohost (11.0.7) testing; urgency=low
 
   - [fix] Allow lime2 to upgrade even if kernel is hold ([#1452](https://github.com/YunoHost/yunohost/pull/1452))

From 7fa67b2b229ffd02e4bd909099df892ff40245c9 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 12:41:18 +0200
Subject: [PATCH 16/34] Zbleuarg

---
 src/utils/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/utils/config.py b/src/utils/config.py
index c68d43aef..ec7faa719 100644
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -742,7 +742,7 @@ class Question:
             confirm=False,
             prefill=prefill,
             is_multiline=(self.type == "text"),
-            autocomplete=self.choices,
+            autocomplete=self.choices or [],
             help=_value_for_locale(self.help),
         )
 

From 4496996c340816c12f43627fe48e132527ff9b64 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 12:42:15 +0200
Subject: [PATCH 17/34] Update changelog for 11.0.8.1

---
 debian/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 473fb1abc..3a7d37aed 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+yunohost (11.0.8.1) testing; urgency=low
+
+  - Fix tests é_è (7fa67b2b)
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Sun, 07 Aug 2022 12:41:28 +0200
+
 yunohost (11.0.8) testing; urgency=low
 
   - [fix] helpers: escape username in ynh_user_exists ([#1469](https://github.com/YunoHost/yunohost/pull/1469))

From ebc24362b2783dff588b8cb3277693e0bd1a23b9 Mon Sep 17 00:00:00 2001
From: Kayou <pierre@kayou.io>
Date: Sun, 7 Aug 2022 14:10:59 +0200
Subject: [PATCH 18/34] Fixing path in the generating doc script

---
 .gitlab/ci/doc.gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/ci/doc.gitlab-ci.yml b/.gitlab/ci/doc.gitlab-ci.yml
index 59179f7a7..528d8f5aa 100644
--- a/.gitlab/ci/doc.gitlab-ci.yml
+++ b/.gitlab/ci/doc.gitlab-ci.yml
@@ -14,7 +14,7 @@ generate-helpers-doc:
     - cd doc
     - python3 generate_helper_doc.py
     - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo
-    - cp helpers.md doc_repo/pages/04.contribute/04.packaging_apps/11.helpers/packaging_apps_helpers.md
+    - cp helpers.md doc_repo/pages/06.contribute/10.packaging_apps/11.helpers/packaging_apps_helpers.md
     - cd doc_repo
     # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ?
     - hub checkout -b "${CI_COMMIT_REF_NAME}"

From bcdb36b7b2159b0e09c47d6f620f7259a6244308 Mon Sep 17 00:00:00 2001
From: "ljf (zamentur)" <zamentur@users.noreply.github.com>
Date: Sun, 7 Aug 2022 15:03:27 +0200
Subject: [PATCH 19/34] [enh] Simplify the pip freeze call

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>
---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 4cdec3f24..32a30bf2e 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -69,7 +69,7 @@ def _generate_requirements():
     venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
     for venv in venvs:
         # Generate a requirements file from venv
-        os.system(f"bash -c 'source {venv}/bin/activate && pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
+        os.system(f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX}")
 
 
 def _rebuild_venvs():

From e94b7197f26ff4b455775221ebc6f7f771287834 Mon Sep 17 00:00:00 2001
From: "ljf (zamentur)" <zamentur@users.noreply.github.com>
Date: Sun, 7 Aug 2022 15:04:55 +0200
Subject: [PATCH 20/34] [enh] Simplify the pip install call

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>
---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 32a30bf2e..7dc0f39aa 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -83,7 +83,7 @@ def _rebuild_venvs():
             # Recreate the venv
             rm(venv, recursive=True)
             os.system(f"python -m venv {venv}")
-            status = os.system(f"bash -c 'source {venv}/bin/activate && pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX} && deactivate'")
+            status = os.system(f"{venv}/bin/pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX}")
             if status != 0:
                 logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv))
             else:

From 5a3911c65f0f7bfd509a13d901502351cec71742 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Sun, 7 Aug 2022 15:26:54 +0200
Subject: [PATCH 21/34] [enh] Split bullseye migration and venv migrations

---
 .../0021_migrate_to_bullseye.py               | 39 ++++---------------
 1 file changed, 7 insertions(+), 32 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 7dc0f39aa..93ff2f930 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -5,7 +5,7 @@ from moulinette import m18n
 from yunohost.utils.error import YunohostError
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import check_output, call_async_output
-from moulinette.utils.filesystem import read_file, rm, write_to_file, cp
+from moulinette.utils.filesystem import read_file, rm, write_to_file
 
 from yunohost.tools import (
     Migration,
@@ -30,7 +30,7 @@ N_CURRENT_YUNOHOST = 4
 N_NEXT_DEBAN = 11
 N_NEXT_YUNOHOST = 11
 
-VENV_REQUIREMENTS_SUFFIX = "_req.txt"
+VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt"
 VENV_IGNORE = "ynh_migration_no_regen"
 
 
@@ -72,26 +72,6 @@ def _generate_requirements():
         os.system(f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX}")
 
 
-def _rebuild_venvs():
-    """
-        After the update, recreate a python virtual env based on the previously generated requirements file
-    """
-
-    venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
-    for venv in venvs:
-        if os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
-            # Recreate the venv
-            rm(venv, recursive=True)
-            os.system(f"python -m venv {venv}")
-            status = os.system(f"{venv}/bin/pip install -r {venv}{VENV_REQUIREMENTS_SUFFIX}")
-            if status != 0:
-                logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv))
-            else:
-                rm(venv + VENV_REQUIREMENTS_SUFFIX)
-        else:
-            logger.warning(m18n.n("migration_0021_venv_regen_failed", venv=venv))
-
-
 class MyMigration(Migration):
 
     "Upgrade the system to Debian Bullseye and Yunohost 11.x"
@@ -331,11 +311,6 @@ class MyMigration(Migration):
 
         tools_upgrade(target="system", postupgradecmds=postupgradecmds)
 
-        #
-        # Recreate the venvs
-        #
-
-        _rebuild_venvs()
 
     def debian_major_version(self):
         # The python module "platform" and lsb_release are not reliable because
@@ -380,11 +355,11 @@ class MyMigration(Migration):
             # Lime2 have hold packages to avoid ethernet instability
             # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df
             lime2_hold_packages = set([
-                "armbian-firmware", 
-                "armbian-bsp-cli-lime2", 
-                "linux-dtb-current-sunxi", 
-                "linux-image-current-sunxi", 
-                "linux-u-boot-lime2-current", 
+                "armbian-firmware",
+                "armbian-bsp-cli-lime2",
+                "linux-dtb-current-sunxi",
+                "linux-image-current-sunxi",
+                "linux-u-boot-lime2-current",
                 "linux-image-next-sunxi"
             ])
             if upgradable_system_packages - lime2_hold_packages:

From f83a357d33e36c5a1724c4a6fcc6a97291f76665 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 19:06:53 +0200
Subject: [PATCH 22/34] bullseye migration: in venv / pip freeze backup
 mechanism, remove the venv_ignore stuff because it's not relevant anymore

---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 93ff2f930..31c07191f 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -31,7 +31,6 @@ N_NEXT_DEBAN = 11
 N_NEXT_YUNOHOST = 11
 
 VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt"
-VENV_IGNORE = "ynh_migration_no_regen"
 
 
 def _get_all_venvs(dir, level=0, maxlevel=3):
@@ -48,8 +47,6 @@ def _get_all_venvs(dir, level=0, maxlevel=3):
     for file in os.listdir(dir):
         path = os.path.join(dir, file)
         if os.path.isdir(path):
-            if os.path.isfile(os.path.join(path, VENV_IGNORE)):
-                continue
             activatepath = os.path.join(path,"bin", "activate")
             if os.path.isfile(activatepath):
                 content = read_file(activatepath)

From 2b63058b4599029d70b26670583c0d69e514ca0f Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 19:07:32 +0200
Subject: [PATCH 23/34] bullseye migration: _generate_requirements ->
 _backup_pip_freeze_for_python_app_venvs

---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 31c07191f..9bd746276 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -58,7 +58,7 @@ def _get_all_venvs(dir, level=0, maxlevel=3):
     return result
 
 
-def _generate_requirements():
+def _backup_pip_freeze_for_python_app_venvs():
     """
         Generate a requirements file for all python virtual env located inside /opt/ and /var/www/
     """
@@ -112,7 +112,7 @@ class MyMigration(Migration):
         # Get requirements of the different venvs from python apps
         #
 
-        _generate_requirements()
+        _backup_pip_freeze_for_python_app_venvs()
 
         #
         # Run apt update

From 80015a728ce08d77b0d9e155ed5e0de319b96dc5 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 22:57:36 +0200
Subject: [PATCH 24/34] bullseye migration: tweak message to prepare for stable
 release

---
 src/yunohost/data_migrations/0021_migrate_to_bullseye.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
index 9bd746276..3be8c9add 100644
--- a/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
+++ b/src/yunohost/data_migrations/0021_migrate_to_bullseye.py
@@ -386,15 +386,10 @@ class MyMigration(Migration):
 
         message = m18n.n("migration_0021_general_warning")
 
-        # FIXME: update this message with updated topic link once we release the migration as stable
         message = (
-           "N.B.: **THIS MIGRATION IS STILL IN BETA-STAGE** ! If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read and share feedbacks on this forum thread: https://forum.yunohost.org/t/18531\n\n"
+           "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n"
            + message
         )
-        # message = (
-        #    "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n"
-        #    + message
-        # )
 
         if problematic_apps:
             message += "\n\n" + m18n.n(

From 51804925f6fd906efdd94066cd5b9fc8ac99449d Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 23:04:21 +0200
Subject: [PATCH 25/34] services: Skip php 7.3 which is most likely dead after
 buster->bullseye migration because users get spooked

---
 src/service.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/service.py b/src/service.py
index 506d3223e..5800f6e4d 100644
--- a/src/service.py
+++ b/src/service.py
@@ -710,6 +710,10 @@ def _get_services():
     )
     php_fpm_versions = [v for v in php_fpm_versions.split("\n") if v.strip()]
     for version in php_fpm_versions:
+        # Skip php 7.3 which is most likely dead after buster->bullseye migration
+        # because users get spooked
+        if version == "7.3":
+            continue
         services[f"php{version}-fpm"] = {
             "log": f"/var/log/php{version}-fpm.log",
             "test_conf": f"php-fpm{version} --test",  # ofc the service is phpx.y-fpm but the program is php-fpmx.y because why not ...

From 3b8e49dc64522da2787874762f69ea6dee64df70 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Sun, 7 Aug 2022 16:50:22 +0200
Subject: [PATCH 26/34] [enh] Split python rebuild migrations

---
 locales/en.json                            |  1 +
 src/migrations/0024_rebuild_python_venv.py | 74 ++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 src/migrations/0024_rebuild_python_venv.py

diff --git a/locales/en.json b/locales/en.json
index e48ba65ce..f4d4492ab 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -503,6 +503,7 @@
     "migration_0023_not_enough_space": "Make sufficient space available in {path} to run the migration.",
     "migration_0023_postgresql_11_not_installed": "PostgreSQL was not installed on your system. Nothing to do.",
     "migration_0023_postgresql_13_not_installed": "PostgreSQL 11 is installed, but not PostgreSQL 13!? Something weird might have happened on your system :(...",
+    "migration_0024_rebuild_python_venv_failed": "Unable to rebuild the python virtual env {venv}, this app is probably broken. If your app is broken, you probably should force the upgrade of this app thanks to `yunohost app upgrade --force APP`",
     "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x",
     "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4",
     "migration_description_0023_postgresql_11_to_13": "Migrate databases from PostgreSQL 11 to 13",
diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py
new file mode 100644
index 000000000..7d6530c8c
--- /dev/null
+++ b/src/migrations/0024_rebuild_python_venv.py
@@ -0,0 +1,74 @@
+import subprocess
+import os
+
+from moulinette import m18n
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.process import call_async_output
+
+from yunohost.tools import Migration
+from yunohost.utils.filesystem import read_file, rm
+
+logger = getActionLogger("yunohost.migration")
+
+VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt"
+VENV_IGNORE = "ynh_migration_no_regen"
+
+
+def _get_all_venvs(dir, level=0, maxlevel=3):
+    """
+        Returns the list of all python virtual env directories recursively
+
+        Arguments:
+            dir - the directory to scan in
+            maxlevel - the depth of the recursion
+            level - do not edit this, used as an iterator
+    """
+    # Using os functions instead of glob, because glob doesn't support hidden
+    # folders, and we need recursion with a fixed depth
+    result = []
+    for file in os.listdir(dir):
+        path = os.path.join(dir, file)
+        if os.path.isdir(path):
+            if os.path.isfile(os.path.join(path, VENV_IGNORE)):
+                continue
+            activatepath = os.path.join(path, "bin", "activate")
+            if os.path.isfile(activatepath):
+                content = read_file(activatepath)
+                if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content):
+                    result.append(path)
+                    continue
+            if level < maxlevel:
+                result += _get_all_venvs(path, level=level + 1)
+    return result
+
+
+class MyMigration(Migration):
+    """
+    After the update, recreate a python virtual env based on the previously
+    generated requirements file
+    """
+
+    dependencies = ["migrate_to_bullseye"]
+
+    def run(self):
+
+        venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
+        for venv in venvs:
+            if not os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
+                continue
+
+            # Recreate the venv
+            rm(venv, recursive=True)
+            callbacks = (
+                lambda l: logger.info("+ " + l.rstrip() + "\r"),
+                lambda l: logger.warning(l.rstrip())
+            )
+            call_async_output(["python", "-m", "venv", venv], callbacks)
+            status = call_async_output([
+                "{venv}/bin/pip", "install", "-r",
+                venv + VENV_REQUIREMENTS_SUFFIX], callbacks)
+            if status != 0:
+                logger.warning(m18n.n("migration_0024_rebuild_python_venv",
+                                      venv=venv))
+            else:
+                rm(venv + VENV_REQUIREMENTS_SUFFIX)

From 1d84e07988a59cc55348450860d2e91ff95bdd08 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Sun, 7 Aug 2022 18:59:39 +0200
Subject: [PATCH 27/34] [enh] Add disclaimer and manual or automatic mode on
 python migrations

---
 locales/en.json                            |  4 ++
 src/migrations/0024_rebuild_python_venv.py | 74 ++++++++++++++++++++--
 2 files changed, 73 insertions(+), 5 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index f4d4492ab..2d283d9fe 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -503,10 +503,14 @@
     "migration_0023_not_enough_space": "Make sufficient space available in {path} to run the migration.",
     "migration_0023_postgresql_11_not_installed": "PostgreSQL was not installed on your system. Nothing to do.",
     "migration_0023_postgresql_13_not_installed": "PostgreSQL 11 is installed, but not PostgreSQL 13!? Something weird might have happened on your system :(...",
+    "migration_0024_rebuild_python_venv_broken_app": "To upgrade your app {app} to bullseye, you probably should force the upgrade of this app thanks to `yunohost app upgrade --force {app}`",
+    "migration_0024_rebuild_python_venv_disclaimer": "Python applications doesn't support upgrade to bullseye and need an extra steps to rebuild their virtual environement. By running this migration, YunoHost will try to rebuild automaticcaly your Python apps except for : {apps}. For those apps, you should force the upgrade manually by running `yunohost app upgrade -f APP`.",
+    "migration_0024_rebuild_python_venv_in_progress": "Rebuild the python virtualenv `{venv}`",
     "migration_0024_rebuild_python_venv_failed": "Unable to rebuild the python virtual env {venv}, this app is probably broken. If your app is broken, you probably should force the upgrade of this app thanks to `yunohost app upgrade --force APP`",
     "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x",
     "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4",
     "migration_description_0023_postgresql_11_to_13": "Migrate databases from PostgreSQL 11 to 13",
+    "migration_description_0024_rebuild_python_venv": "Repair python app after bullseye migration",
     "migration_ldap_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.",
     "migration_ldap_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error}",
     "migration_ldap_migration_failed_trying_to_rollback": "Could not migrate... trying to roll back the system.",
diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py
index 7d6530c8c..e3ffa99ae 100644
--- a/src/migrations/0024_rebuild_python_venv.py
+++ b/src/migrations/0024_rebuild_python_venv.py
@@ -1,12 +1,12 @@
-import subprocess
 import os
 
 from moulinette import m18n
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import call_async_output
 
-from yunohost.tools import Migration
-from yunohost.utils.filesystem import read_file, rm
+from yunohost.tools import Migration, tools_migrations_state
+from moulinette.utils.filesystem import rm, read_file
+
 
 logger = getActionLogger("yunohost.migration")
 
@@ -47,8 +47,59 @@ class MyMigration(Migration):
     After the update, recreate a python virtual env based on the previously
     generated requirements file
     """
+    ignored_python_apps = [
+        "calibreweb",
+        "django-for-runners",
+        "ffsync",
+        "jupiterlab",
+        "librephotos",
+        "mautrix",
+        "mediadrop",
+        "mopidy",
+        "pgadmin",
+        "tracim",
+        "synapse",
+        "weblate"
+    ]
 
     dependencies = ["migrate_to_bullseye"]
+    state = None
+
+    def is_pending(self):
+        if not self.state:
+            self.state = tools_migrations_state()["migrations"].get("0024_rebuild_python_venv", "pending")
+        return self.state == "pending"
+
+    @property
+    def mode(self):
+        if not self.is_pending():
+            return "auto"
+
+        if _get_all_venvs("/opt/") + _get_all_venvs("/var/www/"):
+            return "manual"
+        else:
+            return "auto"
+
+    @property
+    def disclaimer(self):
+        # Avoid having a super long disclaimer to generate if migrations has
+        # been done
+        if not self.is_pending():
+            return None
+
+        apps = []
+        venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
+        for venv in venvs:
+            if not os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
+                continue
+
+            # Search for ignore apps
+            for app in self.ignored_python_apps:
+                if app in venv:
+                    apps.append(app)
+
+        return m18n.n("migration_0024_rebuild_python_venv_disclaimer",
+                      apps=", ".join(apps))
 
     def run(self):
 
@@ -57,15 +108,28 @@ class MyMigration(Migration):
             if not os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
                 continue
 
+            # Search for ignore apps
+            ignored_app = None
+            for app in self.ignored_python_apps:
+                if app in venv:
+                    ignored_app = app
+
+            if ignored_app:
+                rm(venv + VENV_REQUIREMENTS_SUFFIX)
+                logger.info(m18n.n("migration_0024_rebuild_python_venv_broken_app", app=ignored_app))
+                continue
+
+            logger.info(m18n.n("migration_0024_rebuild_python_venv_in_progress", venv=venv))
+
             # Recreate the venv
             rm(venv, recursive=True)
             callbacks = (
-                lambda l: logger.info("+ " + l.rstrip() + "\r"),
+                lambda l: logger.debug("+ " + l.rstrip() + "\r"),
                 lambda l: logger.warning(l.rstrip())
             )
             call_async_output(["python", "-m", "venv", venv], callbacks)
             status = call_async_output([
-                "{venv}/bin/pip", "install", "-r",
+                f"{venv}/bin/pip", "install", "-r",
                 venv + VENV_REQUIREMENTS_SUFFIX], callbacks)
             if status != 0:
                 logger.warning(m18n.n("migration_0024_rebuild_python_venv",

From 0a737c4e9b3a44811b20027b52c38cf12a3f60e4 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 22:24:50 +0200
Subject: [PATCH 28/34] venv rebuild: various improvements in the migration
 code

---
 locales/en.json                            | 10 ++--
 src/migrations/0024_rebuild_python_venv.py | 54 ++++++++++++++--------
 2 files changed, 40 insertions(+), 24 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index 2d283d9fe..8ede9f092 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -503,10 +503,12 @@
     "migration_0023_not_enough_space": "Make sufficient space available in {path} to run the migration.",
     "migration_0023_postgresql_11_not_installed": "PostgreSQL was not installed on your system. Nothing to do.",
     "migration_0023_postgresql_13_not_installed": "PostgreSQL 11 is installed, but not PostgreSQL 13!? Something weird might have happened on your system :(...",
-    "migration_0024_rebuild_python_venv_broken_app": "To upgrade your app {app} to bullseye, you probably should force the upgrade of this app thanks to `yunohost app upgrade --force {app}`",
-    "migration_0024_rebuild_python_venv_disclaimer": "Python applications doesn't support upgrade to bullseye and need an extra steps to rebuild their virtual environement. By running this migration, YunoHost will try to rebuild automaticcaly your Python apps except for : {apps}. For those apps, you should force the upgrade manually by running `yunohost app upgrade -f APP`.",
-    "migration_0024_rebuild_python_venv_in_progress": "Rebuild the python virtualenv `{venv}`",
-    "migration_0024_rebuild_python_venv_failed": "Unable to rebuild the python virtual env {venv}, this app is probably broken. If your app is broken, you probably should force the upgrade of this app thanks to `yunohost app upgrade --force APP`",
+    "migration_0024_rebuild_python_venv_broken_app": "Skipping {app} because virtualenv can't easily be rebuilt for this app. Instead, you should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.",
+     "migration_0024_rebuild_python_venv_disclaimer_base": "Following the upgrade to Debian Bullseye, some Python applications needs to be partially rebuilt to get converted to the new Python version shipped in Debian (in technical terms: what's called the 'virtualenv' needs to be recreated). In the meantime, those Python applications may not work. YunoHost can attempt to rebuild the virtualenv for some of those, as detailed below. For other apps, or if the rebuild attempt fails, you will need to manually force an upgrade for those apps.",
+     "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Rebuilding the virtualenv will be attempted for the following apps (NB: the operation may take some time!): {rebuid_apps}",
+     "migration_0024_rebuild_python_venv_disclaimer_ignored": "Virtualenvs can't be rebuilt automatically for those apps. You need to force an upgrade for those, which can be done from the command line with: `yunohost app upgrade -f APP`: {ignored_apps}",
+    "migration_0024_rebuild_python_venv_in_progress": "Now attempting to rebuild python virtualenv for `{app}`",
+    "migration_0024_rebuild_python_venv_failed": "Failed to rebuild the python virtual env for {app}. The app may not work as long as this is not resolved. You should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.",
     "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x",
     "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4",
     "migration_description_0023_postgresql_11_to_13": "Migrate databases from PostgreSQL 11 to 13",
diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py
index e3ffa99ae..7e659b3df 100644
--- a/src/migrations/0024_rebuild_python_venv.py
+++ b/src/migrations/0024_rebuild_python_venv.py
@@ -11,7 +11,14 @@ from moulinette.utils.filesystem import rm, read_file
 logger = getActionLogger("yunohost.migration")
 
 VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt"
-VENV_IGNORE = "ynh_migration_no_regen"
+
+
+def extract_app_from_venv_path(venv_path):
+
+    venv_path = venv_path.replace("/var/www/", "")
+    venv_path = venv_path.replace("/opt/yunohost/", "")
+    venv_path = venv_path.replace("/opt/", "")
+    return venv_path.split("/")[0]
 
 
 def _get_all_venvs(dir, level=0, maxlevel=3):
@@ -29,8 +36,6 @@ def _get_all_venvs(dir, level=0, maxlevel=3):
     for file in os.listdir(dir):
         path = os.path.join(dir, file)
         if os.path.isdir(path):
-            if os.path.isfile(os.path.join(path, VENV_IGNORE)):
-                continue
             activatepath = os.path.join(path, "bin", "activate")
             if os.path.isfile(activatepath):
                 content = read_file(activatepath)
@@ -87,19 +92,31 @@ class MyMigration(Migration):
         if not self.is_pending():
             return None
 
-        apps = []
+        ignored_apps = []
+        rebuild_apps = []
+
         venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
         for venv in venvs:
             if not os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
                 continue
 
-            # Search for ignore apps
-            for app in self.ignored_python_apps:
-                if app in venv:
-                    apps.append(app)
+            app_corresponding_to_venv = extract_app_from_venv_path(venv)
 
-        return m18n.n("migration_0024_rebuild_python_venv_disclaimer",
-                      apps=", ".join(apps))
+            # Search for ignore apps
+            if any(app_corresponding_to_venv.startswith(app) for app in self.ignored_python_apps):
+                ignored_apps.append(app_corresponding_to_venv)
+            else:
+                rebuild_apps.append(app_corresponding_to_venv)
+
+        msg = m18n.n("migration_0024_rebuild_python_venv_disclaimer_base",
+        if rebuild_apps:
+            msg += "\n\n" + m18n.n("migration_0024_rebuild_python_venv_disclaimer_rebuild",
+                                   rebuild_apps="\n    - " + "\n    - ".join(rebuild_apps))
+        if ignored_apps:
+            msg += "\n\n" + m18n.n("migration_0024_rebuild_python_venv_disclaimer_ignored",
+                                   ignored_apps="\n    - " + "\n    - ".join(ignored_apps))
+
+        return msg
 
     def run(self):
 
@@ -108,18 +125,15 @@ class MyMigration(Migration):
             if not os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
                 continue
 
-            # Search for ignore apps
-            ignored_app = None
-            for app in self.ignored_python_apps:
-                if app in venv:
-                    ignored_app = app
+            app_corresponding_to_venv = extract_app_from_venv_path(venv)
 
-            if ignored_app:
+            # Search for ignore apps
+            if any(app_corresponding_to_venv.startswith(app) for app in self.ignored_python_apps):
                 rm(venv + VENV_REQUIREMENTS_SUFFIX)
-                logger.info(m18n.n("migration_0024_rebuild_python_venv_broken_app", app=ignored_app))
+                logger.info(m18n.n("migration_0024_rebuild_python_venv_broken_app", app=app_corresponding_to_venv))
                 continue
 
-            logger.info(m18n.n("migration_0024_rebuild_python_venv_in_progress", venv=venv))
+            logger.info(m18n.n("migration_0024_rebuild_python_venv_in_progress", app=app_corresponding_to_venv))
 
             # Recreate the venv
             rm(venv, recursive=True)
@@ -132,7 +146,7 @@ class MyMigration(Migration):
                 f"{venv}/bin/pip", "install", "-r",
                 venv + VENV_REQUIREMENTS_SUFFIX], callbacks)
             if status != 0:
-                logger.warning(m18n.n("migration_0024_rebuild_python_venv",
-                                      venv=venv))
+                logger.error(m18n.n("migration_0024_rebuild_python_venv_failed",
+                                    app=app_corresponding_to_venv))
             else:
                 rm(venv + VENV_REQUIREMENTS_SUFFIX)

From 5f265d0c160eed83bf5e6b1107180c4dc61fb9e5 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 22:26:59 +0200
Subject: [PATCH 29/34] Typo :|

---
 src/migrations/0024_rebuild_python_venv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py
index 7e659b3df..0362c8576 100644
--- a/src/migrations/0024_rebuild_python_venv.py
+++ b/src/migrations/0024_rebuild_python_venv.py
@@ -108,7 +108,7 @@ class MyMigration(Migration):
             else:
                 rebuild_apps.append(app_corresponding_to_venv)
 
-        msg = m18n.n("migration_0024_rebuild_python_venv_disclaimer_base",
+        msg = m18n.n("migration_0024_rebuild_python_venv_disclaimer_base")
         if rebuild_apps:
             msg += "\n\n" + m18n.n("migration_0024_rebuild_python_venv_disclaimer_rebuild",
                                    rebuild_apps="\n    - " + "\n    - ".join(rebuild_apps))

From a4fca12ebfc4f2e30217acea96fee28d25c01f2f Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 22:33:53 +0200
Subject: [PATCH 30/34] venv rebuild: moar logic fixes

---
 locales/en.json                            |  2 +-
 src/migrations/0024_rebuild_python_venv.py | 10 +++-------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index 8ede9f092..c34cf0198 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -505,7 +505,7 @@
     "migration_0023_postgresql_13_not_installed": "PostgreSQL 11 is installed, but not PostgreSQL 13!? Something weird might have happened on your system :(...",
     "migration_0024_rebuild_python_venv_broken_app": "Skipping {app} because virtualenv can't easily be rebuilt for this app. Instead, you should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.",
      "migration_0024_rebuild_python_venv_disclaimer_base": "Following the upgrade to Debian Bullseye, some Python applications needs to be partially rebuilt to get converted to the new Python version shipped in Debian (in technical terms: what's called the 'virtualenv' needs to be recreated). In the meantime, those Python applications may not work. YunoHost can attempt to rebuild the virtualenv for some of those, as detailed below. For other apps, or if the rebuild attempt fails, you will need to manually force an upgrade for those apps.",
-     "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Rebuilding the virtualenv will be attempted for the following apps (NB: the operation may take some time!): {rebuid_apps}",
+     "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Rebuilding the virtualenv will be attempted for the following apps (NB: the operation may take some time!): {rebuild_apps}",
      "migration_0024_rebuild_python_venv_disclaimer_ignored": "Virtualenvs can't be rebuilt automatically for those apps. You need to force an upgrade for those, which can be done from the command line with: `yunohost app upgrade -f APP`: {ignored_apps}",
     "migration_0024_rebuild_python_venv_in_progress": "Now attempting to rebuild python virtualenv for `{app}`",
     "migration_0024_rebuild_python_venv_failed": "Failed to rebuild the python virtual env for {app}. The app may not work as long as this is not resolved. You should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.",
diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py
index 0362c8576..a2f3e57f9 100644
--- a/src/migrations/0024_rebuild_python_venv.py
+++ b/src/migrations/0024_rebuild_python_venv.py
@@ -37,11 +37,9 @@ def _get_all_venvs(dir, level=0, maxlevel=3):
         path = os.path.join(dir, file)
         if os.path.isdir(path):
             activatepath = os.path.join(path, "bin", "activate")
-            if os.path.isfile(activatepath):
-                content = read_file(activatepath)
-                if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content):
-                    result.append(path)
-                    continue
+            if os.path.isfile(activatepath) and os.path.isfile(path + VENV_REQUIREMENTS_SUFFIX):
+                result.append(path)
+                continue
             if level < maxlevel:
                 result += _get_all_venvs(path, level=level + 1)
     return result
@@ -122,8 +120,6 @@ class MyMigration(Migration):
 
         venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
         for venv in venvs:
-            if not os.path.isfile(venv + VENV_REQUIREMENTS_SUFFIX):
-                continue
 
             app_corresponding_to_venv = extract_app_from_venv_path(venv)
 

From 4251976c48fce21f2da8aa1ee42d653e613cf916 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 22:46:41 +0200
Subject: [PATCH 31/34] Unused import

---
 src/migrations/0024_rebuild_python_venv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py
index a2f3e57f9..b90a35f60 100644
--- a/src/migrations/0024_rebuild_python_venv.py
+++ b/src/migrations/0024_rebuild_python_venv.py
@@ -5,7 +5,7 @@ from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import call_async_output
 
 from yunohost.tools import Migration, tools_migrations_state
-from moulinette.utils.filesystem import rm, read_file
+from moulinette.utils.filesystem import rm
 
 
 logger = getActionLogger("yunohost.migration")

From bb87891cb6211d49162a16a755359473ebfa18fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89ric=20Gaspar?= <junk.eg@free.fr>
Date: Sun, 7 Aug 2022 13:27:51 +0000
Subject: [PATCH 32/34] Translated using Weblate (French)

Currently translated at 100.0% (686 of 686 strings)

Translation: YunoHost/core
Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/
---
 locales/fr.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/locales/fr.json b/locales/fr.json
index 2773d0bee..77f1ca9d1 100644
--- a/locales/fr.json
+++ b/locales/fr.json
@@ -151,7 +151,7 @@
     "certmanager_attempt_to_renew_nonLE_cert": "Le certificat pour le domaine {domain} n'est pas émis par Let's Encrypt. Impossible de le renouveler automatiquement !",
     "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain} n'est pas sur le point d'expirer ! (Vous pouvez utiliser --force si vous savez ce que vous faites)",
     "certmanager_domain_http_not_working": "Le domaine {domain} ne semble pas être accessible via HTTP. Merci de vérifier la catégorie 'Web' dans le diagnostic pour plus d'informations. (Ou si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver la vérification.)",
-    "certmanager_domain_dns_ip_differs_from_public_ip": "Les enregistrements DNS du domaine '{domain}' sont différents de l'adresse IP de ce serveur. Pour plus d'informations, veuillez consulter la catégorie \"Enregistrements DNS\" dans la section diagnostic. Si vous avez récemment modifié votre enregistrement A, veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver ces contrôles)",
+    "certmanager_domain_dns_ip_differs_from_public_ip": "Les enregistrements DNS du domaine '{domain}' sont différents de l'adresse IP de ce serveur. Pour plus d'informations, veuillez consulter la catégorie \"Enregistrements DNS\" dans la section Diagnostic. Si vous avez récemment modifié votre enregistrement A, veuillez attendre sa propagation (des vérificateurs de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez '--no-checks' pour désactiver ces contrôles)",
     "certmanager_cannot_read_cert": "Quelque chose s'est mal passé lors de la tentative d'ouverture du certificat actuel pour le domaine {domain} (fichier : {file}), la cause est : {reason}",
     "certmanager_cert_install_success_selfsigned": "Le certificat auto-signé est maintenant installé pour le domaine '{domain}'",
     "certmanager_cert_install_success": "Le certificat Let's Encrypt est maintenant installé pour le domaine '{domain}'",
@@ -546,7 +546,7 @@
     "app_manifest_install_ask_domain": "Choisissez le domaine sur lequel vous souhaitez installer cette application",
     "global_settings_setting_smtp_relay_user": "Compte utilisateur du relais SMTP",
     "global_settings_setting_smtp_relay_port": "Port du relais SMTP",
-    "global_settings_setting_smtp_relay_host": "Un relais SMTP permet d'envoyer du courrier à la place de cette instance YunoHost. Cela est utile si vous êtes dans l'une de ces situations : le port 25 est bloqué par votre FAI ou par votre fournisseur VPS, vous avez une IP résidentielle répertoriée sur DUHL, vous ne pouvez pas configurer de reverse DNS ou le serveur n'est pas directement accessible depuis Internet et que vous voulez en utiliser un autre pour envoyer des mails.",
+    "global_settings_setting_smtp_relay_host": "Un relais SMTP permet d'envoyer du courrier à la place de cette instance YunoHost. Cela est utile si vous êtes dans l'une de ces situations : le port 25 est bloqué par votre FAI ou par votre fournisseur VPS ; vous avez une IP résidentielle répertoriée sur DUHL ; vous ne pouvez pas configurer le DNS inversé ; ou le serveur n'est pas directement accessible depuis Internet et vous voulez en utiliser un autre pour envoyer des mails.",
     "diagnosis_package_installed_from_sury_details": "Certains paquets ont été installés par inadvertance à partir d'un dépôt tiers appelé Sury. L'équipe YunoHost a amélioré la stratégie de gestion de ces paquets, mais on s'attend à ce que certaines configurations qui ont installé des applications PHP7.3 tout en étant toujours sur Stretch présentent des incohérences. Pour résoudre cette situation, vous devez essayer d'exécuter la commande suivante : <cmd>{cmd_to_fix}</cmd>",
     "app_argument_password_no_default": "Erreur lors de l'analyse de l'argument de mot de passe '{name}' : l'argument de mot de passe ne peut pas avoir de valeur par défaut pour des raisons de sécurité",
     "pattern_email_forward": "L'adresse électronique doit être valide, le symbole '+' étant accepté (par exemple : johndoe+yunohost@exemple.com)",
@@ -571,7 +571,7 @@
     "diagnosis_rootfstotalspace_warning": "Le système de fichiers racine n'est que de {space}. Cela peut suffire, mais faites attention car vous risquez de les remplir rapidement... Il est recommandé d'avoir au moins 16 GB pour ce système de fichiers.",
     "app_restore_script_failed": "Une erreur s'est produite dans le script de restauration de l'application",
     "restore_backup_too_old": "Cette sauvegarde ne peut pas être restaurée car elle provient d'une version de YunoHost trop ancienne.",
-    "log_backup_create": "Création d'une archive de sauvegarde",
+    "log_backup_create": "Créer une archive de sauvegarde",
     "global_settings_setting_ssowat_panel_overlay_enabled": "Activer la superposition de la vignette SSOwat",
     "migration_ldap_rollback_success": "Système rétabli dans son état initial.",
     "permission_cant_add_to_all_users": "L'autorisation {permission} ne peut pas être ajoutée à tous les utilisateurs.",

From 8a9ed1ed7af94b3585ec2f2e43f49c93f2416110 Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Sun, 7 Aug 2022 23:29:44 +0200
Subject: [PATCH 33/34] Update changelog for 11.0.9

---
 debian/changelog | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 3a7d37aed..929d94c44 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+yunohost (11.0.9) stable; urgency=low
+
+  - [fix] services: Skip php 7.3 which is most likely dead after buster->bullseye migration because users get spooked (51804925)
+  - [enh] bullseye: add a migration process to automatically attempt to rebuild venvs (3b8e49dc)
+  - [i18n] Translations updated for French
+
+  Thanks to all contributors <3 ! (Éric Gaspar, Kayou, ljf, theo-is-taken)
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Sun, 07 Aug 2022 23:27:41 +0200
+
 yunohost (11.0.8.1) testing; urgency=low
 
   - Fix tests é_è (7fa67b2b)

From 8cef37d704aed37d74254ee63f3498a2f1a697c1 Mon Sep 17 00:00:00 2001
From: yunohost-bot <yunohost@yunohost.org>
Date: Mon, 8 Aug 2022 16:59:42 +0000
Subject: [PATCH 34/34] [CI] Format code with Black

---
 src/migrations/0021_migrate_to_bullseye.py | 23 +++----
 src/migrations/0024_rebuild_python_venv.py | 74 +++++++++++++++-------
 src/tools.py                               |  4 +-
 src/user.py                                |  4 +-
 4 files changed, 68 insertions(+), 37 deletions(-)

diff --git a/src/migrations/0021_migrate_to_bullseye.py b/src/migrations/0021_migrate_to_bullseye.py
index f5e7f518c..7577c852c 100644
--- a/src/migrations/0021_migrate_to_bullseye.py
+++ b/src/migrations/0021_migrate_to_bullseye.py
@@ -35,19 +35,19 @@ VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt"
 
 def _get_all_venvs(dir, level=0, maxlevel=3):
     """
-        Returns the list of all python virtual env directories recursively
+    Returns the list of all python virtual env directories recursively
 
-        Arguments:
-            dir - the directory to scan in
-            maxlevel - the depth of the recursion
-            level - do not edit this, used as an iterator
+    Arguments:
+        dir - the directory to scan in
+        maxlevel - the depth of the recursion
+        level - do not edit this, used as an iterator
     """
     # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth
     result = []
     for file in os.listdir(dir):
         path = os.path.join(dir, file)
         if os.path.isdir(path):
-            activatepath = os.path.join(path,"bin", "activate")
+            activatepath = os.path.join(path, "bin", "activate")
             if os.path.isfile(activatepath):
                 content = read_file(activatepath)
                 if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content):
@@ -60,7 +60,7 @@ def _get_all_venvs(dir, level=0, maxlevel=3):
 
 def _backup_pip_freeze_for_python_app_venvs():
     """
-        Generate a requirements file for all python virtual env located inside /opt/ and /var/www/
+    Generate a requirements file for all python virtual env located inside /opt/ and /var/www/
     """
 
     venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/")
@@ -308,7 +308,6 @@ class MyMigration(Migration):
 
         tools_upgrade(target="system", postupgradecmds=postupgradecmds)
 
-
     def debian_major_version(self):
         # The python module "platform" and lsb_release are not reliable because
         # on some setup, they may still return Release=9 even after upgrading to
@@ -344,7 +343,9 @@ class MyMigration(Migration):
         # Check system is up to date
         # (but we don't if 'bullseye' is already in the sources.list ...
         # which means maybe a previous upgrade crashed and we're re-running it)
-        if os.path.exists("/etc/apt/sources.list") and " bullseye " not in read_file("/etc/apt/sources.list"):
+        if os.path.exists("/etc/apt/sources.list") and " bullseye " not in read_file(
+            "/etc/apt/sources.list"
+        ):
             tools_update(target="system")
             upgradable_system_packages = list(_list_upgradable_apt_packages())
             upgradable_system_packages = [
@@ -391,8 +392,8 @@ class MyMigration(Migration):
         message = m18n.n("migration_0021_general_warning")
 
         message = (
-           "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n"
-           + message
+            "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n"
+            + message
         )
 
         if problematic_apps:
diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py
index b90a35f60..f39c27c49 100644
--- a/src/migrations/0024_rebuild_python_venv.py
+++ b/src/migrations/0024_rebuild_python_venv.py
@@ -23,12 +23,12 @@ def extract_app_from_venv_path(venv_path):
 
 def _get_all_venvs(dir, level=0, maxlevel=3):
     """
-        Returns the list of all python virtual env directories recursively
+    Returns the list of all python virtual env directories recursively
 
-        Arguments:
-            dir - the directory to scan in
-            maxlevel - the depth of the recursion
-            level - do not edit this, used as an iterator
+    Arguments:
+        dir - the directory to scan in
+        maxlevel - the depth of the recursion
+        level - do not edit this, used as an iterator
     """
     # Using os functions instead of glob, because glob doesn't support hidden
     # folders, and we need recursion with a fixed depth
@@ -37,7 +37,9 @@ def _get_all_venvs(dir, level=0, maxlevel=3):
         path = os.path.join(dir, file)
         if os.path.isdir(path):
             activatepath = os.path.join(path, "bin", "activate")
-            if os.path.isfile(activatepath) and os.path.isfile(path + VENV_REQUIREMENTS_SUFFIX):
+            if os.path.isfile(activatepath) and os.path.isfile(
+                path + VENV_REQUIREMENTS_SUFFIX
+            ):
                 result.append(path)
                 continue
             if level < maxlevel:
@@ -50,6 +52,7 @@ class MyMigration(Migration):
     After the update, recreate a python virtual env based on the previously
     generated requirements file
     """
+
     ignored_python_apps = [
         "calibreweb",
         "django-for-runners",
@@ -62,7 +65,7 @@ class MyMigration(Migration):
         "pgadmin",
         "tracim",
         "synapse",
-        "weblate"
+        "weblate",
     ]
 
     dependencies = ["migrate_to_bullseye"]
@@ -70,7 +73,9 @@ class MyMigration(Migration):
 
     def is_pending(self):
         if not self.state:
-            self.state = tools_migrations_state()["migrations"].get("0024_rebuild_python_venv", "pending")
+            self.state = tools_migrations_state()["migrations"].get(
+                "0024_rebuild_python_venv", "pending"
+            )
         return self.state == "pending"
 
     @property
@@ -101,18 +106,25 @@ class MyMigration(Migration):
             app_corresponding_to_venv = extract_app_from_venv_path(venv)
 
             # Search for ignore apps
-            if any(app_corresponding_to_venv.startswith(app) for app in self.ignored_python_apps):
+            if any(
+                app_corresponding_to_venv.startswith(app)
+                for app in self.ignored_python_apps
+            ):
                 ignored_apps.append(app_corresponding_to_venv)
             else:
                 rebuild_apps.append(app_corresponding_to_venv)
 
         msg = m18n.n("migration_0024_rebuild_python_venv_disclaimer_base")
         if rebuild_apps:
-            msg += "\n\n" + m18n.n("migration_0024_rebuild_python_venv_disclaimer_rebuild",
-                                   rebuild_apps="\n    - " + "\n    - ".join(rebuild_apps))
+            msg += "\n\n" + m18n.n(
+                "migration_0024_rebuild_python_venv_disclaimer_rebuild",
+                rebuild_apps="\n    - " + "\n    - ".join(rebuild_apps),
+            )
         if ignored_apps:
-            msg += "\n\n" + m18n.n("migration_0024_rebuild_python_venv_disclaimer_ignored",
-                                   ignored_apps="\n    - " + "\n    - ".join(ignored_apps))
+            msg += "\n\n" + m18n.n(
+                "migration_0024_rebuild_python_venv_disclaimer_ignored",
+                ignored_apps="\n    - " + "\n    - ".join(ignored_apps),
+            )
 
         return msg
 
@@ -124,25 +136,43 @@ class MyMigration(Migration):
             app_corresponding_to_venv = extract_app_from_venv_path(venv)
 
             # Search for ignore apps
-            if any(app_corresponding_to_venv.startswith(app) for app in self.ignored_python_apps):
+            if any(
+                app_corresponding_to_venv.startswith(app)
+                for app in self.ignored_python_apps
+            ):
                 rm(venv + VENV_REQUIREMENTS_SUFFIX)
-                logger.info(m18n.n("migration_0024_rebuild_python_venv_broken_app", app=app_corresponding_to_venv))
+                logger.info(
+                    m18n.n(
+                        "migration_0024_rebuild_python_venv_broken_app",
+                        app=app_corresponding_to_venv,
+                    )
+                )
                 continue
 
-            logger.info(m18n.n("migration_0024_rebuild_python_venv_in_progress", app=app_corresponding_to_venv))
+            logger.info(
+                m18n.n(
+                    "migration_0024_rebuild_python_venv_in_progress",
+                    app=app_corresponding_to_venv,
+                )
+            )
 
             # Recreate the venv
             rm(venv, recursive=True)
             callbacks = (
                 lambda l: logger.debug("+ " + l.rstrip() + "\r"),
-                lambda l: logger.warning(l.rstrip())
+                lambda l: logger.warning(l.rstrip()),
             )
             call_async_output(["python", "-m", "venv", venv], callbacks)
-            status = call_async_output([
-                f"{venv}/bin/pip", "install", "-r",
-                venv + VENV_REQUIREMENTS_SUFFIX], callbacks)
+            status = call_async_output(
+                [f"{venv}/bin/pip", "install", "-r", venv + VENV_REQUIREMENTS_SUFFIX],
+                callbacks,
+            )
             if status != 0:
-                logger.error(m18n.n("migration_0024_rebuild_python_venv_failed",
-                                    app=app_corresponding_to_venv))
+                logger.error(
+                    m18n.n(
+                        "migration_0024_rebuild_python_venv_failed",
+                        app=app_corresponding_to_venv,
+                    )
+                )
             else:
                 rm(venv + VENV_REQUIREMENTS_SUFFIX)
diff --git a/src/tools.py b/src/tools.py
index 844a2a3ba..abf224c1c 100644
--- a/src/tools.py
+++ b/src/tools.py
@@ -73,7 +73,7 @@ def tools_adminpw(new_password, check_strength=True):
     from yunohost.user import _hash_user_password
     from yunohost.utils.password import (
         assert_password_is_strong_enough,
-        assert_password_is_compatible
+        assert_password_is_compatible,
     )
     import spwd
 
@@ -203,7 +203,7 @@ def tools_postinstall(
     from yunohost.utils.dns import is_yunohost_dyndns_domain
     from yunohost.utils.password import (
         assert_password_is_strong_enough,
-        assert_password_is_compatible
+        assert_password_is_compatible,
     )
     from yunohost.domain import domain_main_domain
     import psutil
diff --git a/src/user.py b/src/user.py
index ce0cea60d..0c5a577d7 100644
--- a/src/user.py
+++ b/src/user.py
@@ -145,7 +145,7 @@ def user_create(
     from yunohost.hook import hook_callback
     from yunohost.utils.password import (
         assert_password_is_strong_enough,
-        assert_password_is_compatible
+        assert_password_is_compatible,
     )
     from yunohost.utils.ldap import _get_ldap_interface
 
@@ -371,7 +371,7 @@ def user_update(
     from yunohost.app import app_ssowatconf
     from yunohost.utils.password import (
         assert_password_is_strong_enough,
-        assert_password_is_compatible
+        assert_password_is_compatible,
     )
     from yunohost.utils.ldap import _get_ldap_interface
     from yunohost.hook import hook_callback