Merge branch 'dev' into weblate-yunohost-core

.gitlab/ci/lint.gitlab-ci.yml

@@ -42,6 +42,6 @@ black:
     - '[ $(git diff | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
     - git commit -am "[CI] Format code with Black" || true
     - git push -f origin "ci-format-${CI_COMMIT_REF_NAME}":"ci-format-${CI_COMMIT_REF_NAME}"
-    - hub pull-request -m "[CI] Format code with Black" -b Yunohost:$CI_COMMIT_REF_NAME -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
+    - hub pull-request -m "[CI] Format code with Black" -b Yunohost:dev -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
   only:
     - tags

helpers/apt

@@ -277,7 +277,10 @@ ynh_install_app_dependencies() {
         ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version
 
         # Set the default php version back as the default version for php-cli.
-        update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        then
+            update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        fi
     elif grep --quiet 'php' <<< "$dependencies"; then
         ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION
     fi

helpers/php

@@ -57,6 +57,7 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}
 #
 # Requires YunoHost version 4.1.0 or higher.
 ynh_add_fpm_config() {
+    local _globalphpversion=${phpversion-:}
     # Declare an array to define the options of this helper.
     local legacy_args=vtufpd
     local -A args_array=([v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service)
@@ -81,11 +82,16 @@ ynh_add_fpm_config() {
     dedicated_service=${dedicated_service:-0}
 
     # Set the default PHP-FPM version by default
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi
 
     local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
 
     # If the PHP version changed, remove the old fpm conf
+    # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper)
     if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then
         local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
         local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
@@ -100,6 +106,7 @@ ynh_add_fpm_config() {
     # Legacy args (packager should just list their php dependency as regular apt dependencies...
     if [ -n "$package" ]; then
         # Install the additionnal packages from the default repository
+        ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future"
         ynh_install_app_dependencies "$package"
     fi
 
@@ -481,6 +488,7 @@ YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION}
 #
 # Requires YunoHost version 4.2 or higher.
 ynh_composer_exec() {
+    local _globalphpversion=${phpversion-:}
     # Declare an array to define the options of this helper.
     local legacy_args=vwc
     declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=)
@@ -490,7 +498,12 @@ ynh_composer_exec() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     workdir="${workdir:-${install_dir:-$final_path}}"
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi
 
     COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \
         php${phpversion} "$workdir/composer.phar" $commands \
@@ -507,6 +520,7 @@ ynh_composer_exec() {
 #
 # Requires YunoHost version 4.2 or higher.
 ynh_install_composer() {
+    local _globalphpversion=${phpversion-:}
     # Declare an array to define the options of this helper.
     local legacy_args=vwac
     declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=)
@@ -521,7 +535,13 @@ ynh_install_composer() {
     else
         workdir="${workdir:-$install_dir}"
     fi
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi
+
     install_args="${install_args:-}"
     composerversion="${composerversion:-$YNH_COMPOSER_VERSION}"
 

locales/en.json

@@ -13,7 +13,7 @@
     "app_already_installed": "{app} is already installed",
     "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.",
     "app_already_up_to_date": "{app} is already up-to-date",
-    "app_arch_not_supported": "This app can only be installed on architectures {', '.join(required)} but your server architecture is {current}",
+    "app_arch_not_supported": "This app can only be installed on architectures {required} but your server architecture is {current}",
     "app_argument_choice_invalid": "Pick a valid value for argument '{name}': '{value}' is not among the available choices ({choices})",
     "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}",
     "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reasons",

locales/es.json

@@ -704,7 +704,7 @@
     "global_settings_setting_security_experimental_enabled": "Funciones de seguridad experimentales",
     "migration_0024_rebuild_python_venv_disclaimer_ignored": "Virtualenvs no puede reconstruirse automáticamente para esas aplicaciones. Necesitas forzar una actualización para ellas, lo que puede hacerse desde la línea de comandos con: `yunohost app upgrade --force APP`: {ignored_apps}",
     "migration_0024_rebuild_python_venv_failed": "Error al reconstruir el virtualenv de Python para {app}. La aplicación puede no funcionar mientras esto no se resuelva. Deberías arreglar la situación forzando la actualización de esta app usando `yunohost app upgrade --force {app}`.",
-    "app_arch_not_supported": "Esta aplicación sólo puede instalarse en arquitecturas {', '.join(required)} pero la arquitectura de su servidor es {current}",
+    "app_arch_not_supported": "Esta aplicación sólo puede instalarse en arquitecturas {required} pero la arquitectura de su servidor es {current}",
     "app_resource_failed": "Falló la asignación, desasignación o actualización de recursos para {app}: {error}",
     "app_not_enough_disk": "Esta aplicación requiere {required} espacio libre.",
     "app_not_enough_ram": "Esta aplicación requiere {required} de RAM para ser instalada/actualizada, pero solo hay {current} disponible actualmente.",
@@ -749,4 +749,4 @@
     "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Se intentará reconstruir el virtualenv para las siguientes apps (NB: ¡la operación puede llevar algún tiempo!): {rebuild_apps}",
     "migration_description_0025_global_settings_to_configpanel": "Migración de la nomenclatura de ajustes globales heredada a la nomenclatura nueva y moderna",
     "registrar_infos": "Información sobre el registrador"
-}
+}

locales/eu.json

@@ -740,7 +740,7 @@
     "domain_cannot_add_muc_upload": "Ezin duzu 'muc.'-ekin hasten den domeinurik gehitu. Mota honetako izenak YunoHosten integratuta dagoen XMPP taldeko txatek erabil ditzaten gordeta daude.",
     "confirm_app_insufficient_ram": "KONTUZ! Aplikazio honek {required} RAM behar ditu instalatu edo bertsio-berritzeko baina unean {current} bakarrik daude erabilgarri. Aplikazioa ibiliko balitz ere, instalazioak edo bertsio-berritzeak RAM kopuru handia eskatzen du eta zure zerbitzaria izoztu eta huts egin lezake. Hala ere arriskatu nahi baduzu idatzi '{answers}'",
     "confirm_notifications_read": "ADI: ikuskatu aplikazioaren jakinarazpenak jarraitu baino lehen, baliteke jakin beharreko zerbait esatea. [{answers}]",
-    "app_arch_not_supported": "Aplikazio hau {', '.join(required)} arkitekturan instala daiteke bakarrik, baina zure zerbitzariaren arkitektura {current} da",
+    "app_arch_not_supported": "Aplikazio hau {required} arkitekturan instala daiteke bakarrik, baina zure zerbitzariaren arkitektura {current} da",
     "app_resource_failed": "Huts egin du {app} aplikaziorako baliabideen eguneraketak / prestaketak / askapenak: {error}",
     "app_not_enough_disk": "Aplikazio honek {required} espazio libre behar ditu.",
     "app_yunohost_version_not_supported": "Aplikazio honek YunoHost >= {required} behar du baina unean instalatutako bertsioa {current} da",

locales/fr.json

@@ -741,7 +741,7 @@
     "global_settings_setting_portal_theme": "Thème du portail",
     "global_settings_setting_portal_theme_help": "Pour plus d'informations sur la création de thèmes de portail personnalisés, voir https://yunohost.org/theming",
     "global_settings_setting_passwordless_sudo": "Permettre aux administrateurs d'utiliser 'sudo' sans retaper leur mot de passe",
-    "app_arch_not_supported": "Cette application ne peut être installée que sur les architectures {', '.join(required)}. L'architecture de votre serveur est {current}",
+    "app_arch_not_supported": "Cette application ne peut être installée que sur les architectures {required}. L'architecture de votre serveur est {current}",
     "app_resource_failed": "L'allocation automatique des ressources (provisioning), la suppression d'accès à ces ressources (déprovisioning) ou la mise à jour des ressources pour {app} a échoué : {error}",
     "confirm_app_insufficient_ram": "ATTENTION ! Cette application requiert {required} de RAM pour l'installation/mise à niveau mais il n'y a que {current} de disponible actuellement. Même si cette application pouvait fonctionner, son processus d'installation/mise à niveau nécessite une grande quantité de RAM. Votre serveur pourrait donc geler et planter lamentablement. Si vous êtes prêt à prendre ce risque, tapez '{answers}'",
     "app_not_enough_disk": "Cette application nécessite {required} d'espace libre.",

locales/gl.json

@@ -740,7 +740,7 @@
     "global_settings_setting_passwordless_sudo": "Permitir a Admins usar 'sudo' sen ter que volver a escribir o contrasinal",
     "global_settings_setting_portal_theme": "Decorado do Portal",
     "global_settings_setting_portal_theme_help": "Tes máis info acerca da creación de decorados para o portal de acceso en https://yunohost.org/theming",
-    "app_arch_not_supported": "Esta app só pode ser instalada e arquitecturas {', '.join(required)} pero a arquitectura do teu servidor é {current}",
+    "app_arch_not_supported": "Esta app só pode ser instalada e arquitecturas {required} pero a arquitectura do teu servidor é {current}",
     "app_not_enough_disk": "Esta app precisa {required} de espazo libre.",
     "app_yunohost_version_not_supported": "Esta app require YunoHost >= {required} pero a versión actual instalada é {current}",
     "confirm_app_insufficient_ram": "PERIGO! Esta app precisa {required} de RAM para instalar/actualizar pero só hai {current} dispoñibles. Incluso se a app funcionase, o seu proceso de instalación/actualización require gran cantidade de RAM e o teu servidor podería colgarse e fallar. Se queres asumir o risco, escribe '{answers}'",

locales/pl.json

@@ -73,7 +73,7 @@
     "app_action_broke_system": "Wydaje się, że ta akcja przerwała te ważne usługi: {services}",
     "additional_urls_already_removed": "Dodatkowy URL '{url}' już usunięty w dodatkowym URL dla uprawnienia '{permission}'",
     "additional_urls_already_added": "Dodatkowy URL '{url}' już dodany w dodatkowym URL dla uprawnienia '{permission}'",
-    "app_arch_not_supported": "Ta aplikacja może być zainstalowana tylko na architekturach {', '.join(required)}, a twoja architektura serwera to {current}",
+    "app_arch_not_supported": "Ta aplikacja może być zainstalowana tylko na architekturach {required}, a twoja architektura serwera to {current}",
     "app_argument_invalid": "Wybierz poprawną wartość dla argumentu '{name}': {błąd}",
     "all_users": "Wszyscy użytkownicy YunoHost",
     "app_action_failed": "Nie udało się uruchomić akcji {action} dla aplikacji {app}",

locales/tr.json

@@ -15,5 +15,5 @@
     "additional_urls_already_added": "Ek URL '{url}' zaten '{permission}' izni için ek URL'ye eklendi",
     "additional_urls_already_removed": "Ek URL '{url}', '{permission}' izni için ek URL'de zaten kaldırıldı",
     "app_action_cannot_be_ran_because_required_services_down": "Bu eylemi gerçekleştirmek için şu servisler çalışıyor olmalıdır: {services}. Devam etmek için onları yeniden başlatın (ve muhtemelen neden çalışmadığını araştırın).",
-    "app_arch_not_supported": "Bu uygulama yalnızca {', '.join(required)} işlemci mimarisi üzerine kurulabilir ancak sunucunuzun işlemci mimarisi {current}."
+    "app_arch_not_supported": "Bu uygulama yalnızca {required} işlemci mimarisi üzerine kurulabilir ancak sunucunuzun işlemci mimarisi {current}."
-}
+}

src/app.py

@@ -743,7 +743,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
                     "Upgrade failed ... attempting to restore the satefy backup (Yunohost first need to remove the app for this) ..."
                 )
 
-                app_remove(app_instance_name)
+                app_remove(app_instance_name, force_workdir=extracted_app_folder)
                 backup_restore(
                     name=safety_backup_name, apps=[app_instance_name], force=True
                 )
@@ -1270,14 +1270,14 @@ def app_install(
 
 
 @is_unit_operation()
-def app_remove(operation_logger, app, purge=False):
+def app_remove(operation_logger, app, purge=False, force_workdir=None):
     """
     Remove app
 
     Keyword arguments:
         app -- App(s) to delete
         purge -- Remove with all app data
-
+        force_workdir -- Special var to force the working directoy to use, in context such as remove-after-failed-upgrade or remove-after-failed-restore
     """
     from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers
     from yunohost.hook import hook_exec, hook_remove, hook_callback
@@ -1296,7 +1296,6 @@ def app_remove(operation_logger, app, purge=False):
     operation_logger.start()
 
     logger.info(m18n.n("app_start_remove", app=app))
-
     app_setting_path = os.path.join(APPS_SETTING_PATH, app)
 
     # Attempt to patch legacy helpers ...
@@ -1306,8 +1305,20 @@ def app_remove(operation_logger, app, purge=False):
     # script might date back from jessie install)
     _patch_legacy_php_versions(app_setting_path)
 
-    manifest = _get_manifest_of_app(app_setting_path)
+    if force_workdir:
-    tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
+        # This is when e.g. calling app_remove() from the upgrade-failed case
+        # where we want to remove using the *new* remove script and not the old one
+        # and also get the new manifest
+        # It's especially important during v1->v2 app format transition where the
+        # setting names change (e.g. install_dir instead of final_path) and
+        # running the old remove script doesnt make sense anymore ...
+        tmp_workdir_for_app = tempfile.mkdtemp(prefix="app_", dir=APP_TMP_WORKDIRS)
+        os.system(f"cp -a {force_workdir}/* {tmp_workdir_for_app}/")
+    else:
+        tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
+
+    manifest = _get_manifest_of_app(tmp_workdir_for_app)
+
     remove_script = f"{tmp_workdir_for_app}/scripts/remove"
 
     env_dict = {}
@@ -2582,7 +2593,7 @@ def _check_manifest_requirements(
     yield (
         "arch",
         arch_requirement in ["all", "?"] or arch in arch_requirement,
-        {"current": arch, "required": arch_requirement},
+        {"current": arch, "required": ', '.join(arch_requirement)},
         "app_arch_not_supported",  # i18n: app_arch_not_supported
     )
 

src/backup.py

@@ -52,6 +52,7 @@ from yunohost.app import (
     _make_environment_for_app_script,
     _make_tmp_workdir_for_app,
     _get_manifest_of_app,
+    app_remove,
 )
 from yunohost.hook import (
     hook_list,
@@ -1368,8 +1369,6 @@ class RestoreManager:
         from yunohost.user import user_group_list
         from yunohost.permission import (
             permission_create,
-            permission_delete,
-            user_permission_list,
             permission_sync_to_user,
         )
 
@@ -1550,36 +1549,7 @@ class RestoreManager:
             else:
                 self.targets.set_result("apps", app_instance_name, "Error")
 
-                remove_script = os.path.join(app_scripts_in_archive, "remove")
+                app_remove(app_instance_name, force_workdir=app_workdir)
-
-                # Setup environment for remove script
-                env_dict_remove = _make_environment_for_app_script(
-                    app_instance_name, workdir=app_workdir
-                )
-                remove_operation_logger = OperationLogger(
-                    "remove_on_failed_restore",
-                    [("app", app_instance_name)],
-                    env=env_dict_remove,
-                )
-                remove_operation_logger.start()
-
-                # Execute remove script
-                if hook_exec(remove_script, env=env_dict_remove)[0] != 0:
-                    msg = m18n.n("app_not_properly_removed", app=app_instance_name)
-                    logger.warning(msg)
-                    remove_operation_logger.error(msg)
-                else:
-                    remove_operation_logger.success()
-
-                # Cleaning app directory
-                shutil.rmtree(app_settings_new_path, ignore_errors=True)
-
-                # Remove all permission in LDAP for this app
-                for permission_name in user_permission_list()["permissions"].keys():
-                    if permission_name.startswith(app_instance_name + "."):
-                        permission_delete(permission_name, force=True)
-
-                # TODO Cleaning app hooks
 
                 logger.error(failure_message_with_debug_instructions)
 

src/utils/resources.py

@@ -152,6 +152,9 @@ class AppResource:
         for key, value in properties.items():
             if isinstance(value, str):
                 value = value.replace("__APP__", self.app)
+                # This one is needed for custom permission urls where the domain might be used
+                if "__DOMAIN__" in value:
+                    value.replace("__DOMAIN__", self.get_setting("domain"))
             setattr(self, key, value)
 
     def get_setting(self, key):
@@ -298,11 +301,12 @@ class PermissionsResource(AppResource):
                 properties[perm]["show_tile"] = bool(properties[perm]["url"])
 
         if (
-            isinstance(properties["main"]["url"], str)
+            not isinstance(properties["main"].get("url"), str)
-            and properties["main"]["url"] != "/"
+            or properties["main"]["url"] != "/"
         ):
             raise YunohostError(
-                "URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app"
+                "URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app, i.e $domain.tld/$path/",
+                raw_msg=True
             )
 
         super().__init__({"permissions": properties}, *args, **kwargs)
@@ -470,12 +474,12 @@ class SystemuserAppResource(AppResource):
         if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
             os.system(f"deluser {self.app} >/dev/null")
         if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
-            raise YunohostError(f"Failed to delete system user for {self.app}")
+            raise YunohostError(f"Failed to delete system user for {self.app}", raw_msg=True)
 
         if check_output(f"getent group {self.app} &>/dev/null || true").strip():
             os.system(f"delgroup {self.app} >/dev/null")
         if check_output(f"getent group {self.app} &>/dev/null || true").strip():
-            raise YunohostError(f"Failed to delete system user for {self.app}")
+            raise YunohostError(f"Failed to delete system user for {self.app}", raw_msg=True)
 
         # FIXME : better logging and error handling, add stdout/stderr from the deluser/delgroup commands...
 
@@ -743,7 +747,8 @@ class AptDependenciesAppResource(AppResource):
                 isinstance(values.get(k), str) for k in ["repo", "key", "packages"]
             ):
                 raise YunohostError(
-                    "In apt resource in the manifest: 'extras' repo should have the keys 'repo', 'key' and 'packages' defined and be strings"
+                    "In apt resource in the manifest: 'extras' repo should have the keys 'repo', 'key' and 'packages' defined and be strings",
+                    raw_msg=True
                 )
 
         super().__init__(properties, *args, **kwargs)
@@ -860,7 +865,8 @@ class PortsResource(AppResource):
                 if infos["fixed"]:
                     if self._port_is_used(port_value):
                         raise YunohostValidationError(
-                            f"Port {port_value} is already used by another process or app."
+                            f"Port {port_value} is already used by another process or app.",
+                            raw_msg=True
                         )
                 else:
                     while self._port_is_used(port_value):
@@ -950,7 +956,7 @@ class DatabaseAppResource(AppResource):
 
     def db_exists(self, db_name):
         if self.dbtype == "mysql":
-            return os.system(f"mysqlshow '{db_name}' >/dev/null 2>/dev/null") == 0
+            return os.system(f"mysqlshow | grep -q -w '{db_name}' 2>/dev/null") == 0
         elif self.dbtype == "postgresql":
             return (
                 os.system(