Merge branch 'dev' into weblate-yunohost-core

2024-09-03 20:06:10 +02:00 · 2023-02-15 20:02:05 +01:00 · 2023-02-15 20:02:05 +01:00 · 39141fbb6d
commit 39141fbb6d
parent 135dbec8b6 aa585963c0
13 changed files with 71 additions and 61 deletions
--- a/.gitlab/ci/lint.gitlab-ci.yml
+++ b/.gitlab/ci/lint.gitlab-ci.yml
@ -42,6 +42,6 @@ black:
    - '[ $(git diff | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
    - git commit -am "[CI] Format code with Black" || true
    - git push -f origin "ci-format-${CI_COMMIT_REF_NAME}":"ci-format-${CI_COMMIT_REF_NAME}"
-    - hub pull-request -m "[CI] Format code with Black" -b Yunohost:$CI_COMMIT_REF_NAME -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
+    - hub pull-request -m "[CI] Format code with Black" -b Yunohost:dev -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
  only:
    - tags
--- a/helpers/apt
+++ b/helpers/apt
@ -277,7 +277,10 @@ ynh_install_app_dependencies() {
        ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version

        # Set the default php version back as the default version for php-cli.
-        update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        then
+            update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        fi
    elif grep --quiet 'php' <<< "$dependencies"; then
        ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION
    fi
--- a/helpers/php
+++ b/helpers/php
@ -57,6 +57,7 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}
 #
 # Requires YunoHost version 4.1.0 or higher.
 ynh_add_fpm_config() {
+    local _globalphpversion=${phpversion-:}
    # Declare an array to define the options of this helper.
    local legacy_args=vtufpd
    local -A args_array=([v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service)
@ -81,11 +82,16 @@ ynh_add_fpm_config() {
    dedicated_service=${dedicated_service:-0}

    # Set the default PHP-FPM version by default
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi

    local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)

    # If the PHP version changed, remove the old fpm conf
+    # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper)
    if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then
        local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
        local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
@ -100,6 +106,7 @@ ynh_add_fpm_config() {
    # Legacy args (packager should just list their php dependency as regular apt dependencies...
    if [ -n "$package" ]; then
        # Install the additionnal packages from the default repository
+        ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future"
        ynh_install_app_dependencies "$package"
    fi

@ -481,6 +488,7 @@ YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION}
 #
 # Requires YunoHost version 4.2 or higher.
 ynh_composer_exec() {
+    local _globalphpversion=${phpversion-:}
    # Declare an array to define the options of this helper.
    local legacy_args=vwc
    declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=)
@ -490,7 +498,12 @@ ynh_composer_exec() {
    # Manage arguments with getopts
    ynh_handle_getopts_args "$@"
    workdir="${workdir:-${install_dir:-$final_path}}"
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi

    COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \
        php${phpversion} "$workdir/composer.phar" $commands \
@ -507,6 +520,7 @@ ynh_composer_exec() {
 #
 # Requires YunoHost version 4.2 or higher.
 ynh_install_composer() {
+    local _globalphpversion=${phpversion-:}
    # Declare an array to define the options of this helper.
    local legacy_args=vwac
    declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=)
@ -521,7 +535,13 @@ ynh_install_composer() {
    else
        workdir="${workdir:-$install_dir}"
    fi
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi
+
    install_args="${install_args:-}"
    composerversion="${composerversion:-$YNH_COMPOSER_VERSION}"

--- a/locales/en.json
+++ b/locales/en.json
@ -13,7 +13,7 @@
    "app_already_installed": "{app} is already installed",
    "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.",
    "app_already_up_to_date": "{app} is already up-to-date",
-    "app_arch_not_supported": "This app can only be installed on architectures {', '.join(required)} but your server architecture is {current}",
+    "app_arch_not_supported": "This app can only be installed on architectures {required} but your server architecture is {current}",
    "app_argument_choice_invalid": "Pick a valid value for argument '{name}': '{value}' is not among the available choices ({choices})",
    "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}",
    "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reasons",
--- a/locales/es.json
+++ b/locales/es.json
@ -704,7 +704,7 @@
    "global_settings_setting_security_experimental_enabled": "Funciones de seguridad experimentales",
    "migration_0024_rebuild_python_venv_disclaimer_ignored": "Virtualenvs no puede reconstruirse automáticamente para esas aplicaciones. Necesitas forzar una actualización para ellas, lo que puede hacerse desde la línea de comandos con: `yunohost app upgrade --force APP`: {ignored_apps}",
    "migration_0024_rebuild_python_venv_failed": "Error al reconstruir el virtualenv de Python para {app}. La aplicación puede no funcionar mientras esto no se resuelva. Deberías arreglar la situación forzando la actualización de esta app usando `yunohost app upgrade --force {app}`.",
-    "app_arch_not_supported": "Esta aplicación sólo puede instalarse en arquitecturas {', '.join(required)} pero la arquitectura de su servidor es {current}",
+    "app_arch_not_supported": "Esta aplicación sólo puede instalarse en arquitecturas {required} pero la arquitectura de su servidor es {current}",
    "app_resource_failed": "Falló la asignación, desasignación o actualización de recursos para {app}: {error}",
    "app_not_enough_disk": "Esta aplicación requiere {required} espacio libre.",
    "app_not_enough_ram": "Esta aplicación requiere {required} de RAM para ser instalada/actualizada, pero solo hay {current} disponible actualmente.",
@ -749,4 +749,4 @@
    "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Se intentará reconstruir el virtualenv para las siguientes apps (NB: ¡la operación puede llevar algún tiempo!): {rebuild_apps}",
    "migration_description_0025_global_settings_to_configpanel": "Migración de la nomenclatura de ajustes globales heredada a la nomenclatura nueva y moderna",
    "registrar_infos": "Información sobre el registrador"
-}
+}
--- a/locales/eu.json
+++ b/locales/eu.json
@ -740,7 +740,7 @@
    "domain_cannot_add_muc_upload": "Ezin duzu 'muc.'-ekin hasten den domeinurik gehitu. Mota honetako izenak YunoHosten integratuta dagoen XMPP taldeko txatek erabil ditzaten gordeta daude.",
    "confirm_app_insufficient_ram": "KONTUZ! Aplikazio honek {required} RAM behar ditu instalatu edo bertsio-berritzeko baina unean {current} bakarrik daude erabilgarri. Aplikazioa ibiliko balitz ere, instalazioak edo bertsio-berritzeak RAM kopuru handia eskatzen du eta zure zerbitzaria izoztu eta huts egin lezake. Hala ere arriskatu nahi baduzu idatzi '{answers}'",
    "confirm_notifications_read": "ADI: ikuskatu aplikazioaren jakinarazpenak jarraitu baino lehen, baliteke jakin beharreko zerbait esatea. [{answers}]",
-    "app_arch_not_supported": "Aplikazio hau {', '.join(required)} arkitekturan instala daiteke bakarrik, baina zure zerbitzariaren arkitektura {current} da",
+    "app_arch_not_supported": "Aplikazio hau {required} arkitekturan instala daiteke bakarrik, baina zure zerbitzariaren arkitektura {current} da",
    "app_resource_failed": "Huts egin du {app} aplikaziorako baliabideen eguneraketak / prestaketak / askapenak: {error}",
    "app_not_enough_disk": "Aplikazio honek {required} espazio libre behar ditu.",
    "app_yunohost_version_not_supported": "Aplikazio honek YunoHost >= {required} behar du baina unean instalatutako bertsioa {current} da",
--- a/locales/fr.json
+++ b/locales/fr.json
@ -741,7 +741,7 @@
    "global_settings_setting_portal_theme": "Thème du portail",
    "global_settings_setting_portal_theme_help": "Pour plus d'informations sur la création de thèmes de portail personnalisés, voir https://yunohost.org/theming",
    "global_settings_setting_passwordless_sudo": "Permettre aux administrateurs d'utiliser 'sudo' sans retaper leur mot de passe",
-    "app_arch_not_supported": "Cette application ne peut être installée que sur les architectures {', '.join(required)}. L'architecture de votre serveur est {current}",
+    "app_arch_not_supported": "Cette application ne peut être installée que sur les architectures {required}. L'architecture de votre serveur est {current}",
    "app_resource_failed": "L'allocation automatique des ressources (provisioning), la suppression d'accès à ces ressources (déprovisioning) ou la mise à jour des ressources pour {app} a échoué : {error}",
    "confirm_app_insufficient_ram": "ATTENTION ! Cette application requiert {required} de RAM pour l'installation/mise à niveau mais il n'y a que {current} de disponible actuellement. Même si cette application pouvait fonctionner, son processus d'installation/mise à niveau nécessite une grande quantité de RAM. Votre serveur pourrait donc geler et planter lamentablement. Si vous êtes prêt à prendre ce risque, tapez '{answers}'",
    "app_not_enough_disk": "Cette application nécessite {required} d'espace libre.",
--- a/locales/gl.json
+++ b/locales/gl.json
@ -740,7 +740,7 @@
    "global_settings_setting_passwordless_sudo": "Permitir a Admins usar 'sudo' sen ter que volver a escribir o contrasinal",
    "global_settings_setting_portal_theme": "Decorado do Portal",
    "global_settings_setting_portal_theme_help": "Tes máis info acerca da creación de decorados para o portal de acceso en https://yunohost.org/theming",
-    "app_arch_not_supported": "Esta app só pode ser instalada e arquitecturas {', '.join(required)} pero a arquitectura do teu servidor é {current}",
+    "app_arch_not_supported": "Esta app só pode ser instalada e arquitecturas {required} pero a arquitectura do teu servidor é {current}",
    "app_not_enough_disk": "Esta app precisa {required} de espazo libre.",
    "app_yunohost_version_not_supported": "Esta app require YunoHost >= {required} pero a versión actual instalada é {current}",
    "confirm_app_insufficient_ram": "PERIGO! Esta app precisa {required} de RAM para instalar/actualizar pero só hai {current} dispoñibles. Incluso se a app funcionase, o seu proceso de instalación/actualización require gran cantidade de RAM e o teu servidor podería colgarse e fallar. Se queres asumir o risco, escribe '{answers}'",
--- a/locales/pl.json
+++ b/locales/pl.json
@ -73,7 +73,7 @@
    "app_action_broke_system": "Wydaje się, że ta akcja przerwała te ważne usługi: {services}",
    "additional_urls_already_removed": "Dodatkowy URL '{url}' już usunięty w dodatkowym URL dla uprawnienia '{permission}'",
    "additional_urls_already_added": "Dodatkowy URL '{url}' już dodany w dodatkowym URL dla uprawnienia '{permission}'",
-    "app_arch_not_supported": "Ta aplikacja może być zainstalowana tylko na architekturach {', '.join(required)}, a twoja architektura serwera to {current}",
+    "app_arch_not_supported": "Ta aplikacja może być zainstalowana tylko na architekturach {required}, a twoja architektura serwera to {current}",
    "app_argument_invalid": "Wybierz poprawną wartość dla argumentu '{name}': {błąd}",
    "all_users": "Wszyscy użytkownicy YunoHost",
    "app_action_failed": "Nie udało się uruchomić akcji {action} dla aplikacji {app}",
--- a/locales/tr.json
+++ b/locales/tr.json
@ -15,5 +15,5 @@
    "additional_urls_already_added": "Ek URL '{url}' zaten '{permission}' izni için ek URL'ye eklendi",
    "additional_urls_already_removed": "Ek URL '{url}', '{permission}' izni için ek URL'de zaten kaldırıldı",
    "app_action_cannot_be_ran_because_required_services_down": "Bu eylemi gerçekleştirmek için şu servisler çalışıyor olmalıdır: {services}. Devam etmek için onları yeniden başlatın (ve muhtemelen neden çalışmadığını araştırın).",
-    "app_arch_not_supported": "Bu uygulama yalnızca {', '.join(required)} işlemci mimarisi üzerine kurulabilir ancak sunucunuzun işlemci mimarisi {current}."
-}
+    "app_arch_not_supported": "Bu uygulama yalnızca {required} işlemci mimarisi üzerine kurulabilir ancak sunucunuzun işlemci mimarisi {current}."
+}
--- a/src/app.py
+++ b/src/app.py
@ -743,7 +743,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
                    "Upgrade failed ... attempting to restore the satefy backup (Yunohost first need to remove the app for this) ..."
                )

-                app_remove(app_instance_name)
+                app_remove(app_instance_name, force_workdir=extracted_app_folder)
                backup_restore(
                    name=safety_backup_name, apps=[app_instance_name], force=True
                )
@ -1270,14 +1270,14 @@ def app_install(


@is_unit_operation()
-def app_remove(operation_logger, app, purge=False):
+def app_remove(operation_logger, app, purge=False, force_workdir=None):
    """
    Remove app

    Keyword arguments:
        app -- App(s) to delete
        purge -- Remove with all app data
-
+        force_workdir -- Special var to force the working directoy to use, in context such as remove-after-failed-upgrade or remove-after-failed-restore
    """
    from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers
    from yunohost.hook import hook_exec, hook_remove, hook_callback
@ -1296,7 +1296,6 @@ def app_remove(operation_logger, app, purge=False):
    operation_logger.start()

    logger.info(m18n.n("app_start_remove", app=app))
-
    app_setting_path = os.path.join(APPS_SETTING_PATH, app)

    # Attempt to patch legacy helpers ...
@ -1306,8 +1305,20 @@ def app_remove(operation_logger, app, purge=False):
    # script might date back from jessie install)
    _patch_legacy_php_versions(app_setting_path)

-    manifest = _get_manifest_of_app(app_setting_path)
-    tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
+    if force_workdir:
+        # This is when e.g. calling app_remove() from the upgrade-failed case
+        # where we want to remove using the *new* remove script and not the old one
+        # and also get the new manifest
+        # It's especially important during v1->v2 app format transition where the
+        # setting names change (e.g. install_dir instead of final_path) and
+        # running the old remove script doesnt make sense anymore ...
+        tmp_workdir_for_app = tempfile.mkdtemp(prefix="app_", dir=APP_TMP_WORKDIRS)
+        os.system(f"cp -a {force_workdir}/* {tmp_workdir_for_app}/")
+    else:
+        tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
+
+    manifest = _get_manifest_of_app(tmp_workdir_for_app)
+
    remove_script = f"{tmp_workdir_for_app}/scripts/remove"

    env_dict = {}
@ -2582,7 +2593,7 @@ def _check_manifest_requirements(
    yield (
        "arch",
        arch_requirement in ["all", "?"] or arch in arch_requirement,
-        {"current": arch, "required": arch_requirement},
+        {"current": arch, "required": ', '.join(arch_requirement)},
        "app_arch_not_supported",  # i18n: app_arch_not_supported
    )

--- a/src/backup.py
+++ b/src/backup.py
@ -52,6 +52,7 @@ from yunohost.app import (
    _make_environment_for_app_script,
    _make_tmp_workdir_for_app,
    _get_manifest_of_app,
+    app_remove,
 )
 from yunohost.hook import (
    hook_list,
@ -1368,8 +1369,6 @@ class RestoreManager:
        from yunohost.user import user_group_list
        from yunohost.permission import (
            permission_create,
-            permission_delete,
-            user_permission_list,
            permission_sync_to_user,
        )

@ -1550,36 +1549,7 @@ class RestoreManager:
            else:
                self.targets.set_result("apps", app_instance_name, "Error")

-                remove_script = os.path.join(app_scripts_in_archive, "remove")
-
-                # Setup environment for remove script
-                env_dict_remove = _make_environment_for_app_script(
-                    app_instance_name, workdir=app_workdir
-                )
-                remove_operation_logger = OperationLogger(
-                    "remove_on_failed_restore",
-                    [("app", app_instance_name)],
-                    env=env_dict_remove,
-                )
-                remove_operation_logger.start()
-
-                # Execute remove script
-                if hook_exec(remove_script, env=env_dict_remove)[0] != 0:
-                    msg = m18n.n("app_not_properly_removed", app=app_instance_name)
-                    logger.warning(msg)
-                    remove_operation_logger.error(msg)
-                else:
-                    remove_operation_logger.success()
-
-                # Cleaning app directory
-                shutil.rmtree(app_settings_new_path, ignore_errors=True)
-
-                # Remove all permission in LDAP for this app
-                for permission_name in user_permission_list()["permissions"].keys():
-                    if permission_name.startswith(app_instance_name + "."):
-                        permission_delete(permission_name, force=True)
-
-                # TODO Cleaning app hooks
+                app_remove(app_instance_name, force_workdir=app_workdir)

                logger.error(failure_message_with_debug_instructions)

--- a/src/utils/resources.py
+++ b/src/utils/resources.py
@ -152,6 +152,9 @@ class AppResource:
        for key, value in properties.items():
            if isinstance(value, str):
                value = value.replace("__APP__", self.app)
+                # This one is needed for custom permission urls where the domain might be used
+                if "__DOMAIN__" in value:
+                    value.replace("__DOMAIN__", self.get_setting("domain"))
            setattr(self, key, value)

    def get_setting(self, key):
@ -298,11 +301,12 @@ class PermissionsResource(AppResource):
                properties[perm]["show_tile"] = bool(properties[perm]["url"])

        if (
-            isinstance(properties["main"]["url"], str)
-            and properties["main"]["url"] != "/"
+            not isinstance(properties["main"].get("url"), str)
+            or properties["main"]["url"] != "/"
        ):
            raise YunohostError(
-                "URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app"
+                "URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app, i.e $domain.tld/$path/",
+                raw_msg=True
            )

        super().__init__({"permissions": properties}, *args, **kwargs)
@ -470,12 +474,12 @@ class SystemuserAppResource(AppResource):
        if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
            os.system(f"deluser {self.app} >/dev/null")
        if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
-            raise YunohostError(f"Failed to delete system user for {self.app}")
+            raise YunohostError(f"Failed to delete system user for {self.app}", raw_msg=True)

        if check_output(f"getent group {self.app} &>/dev/null || true").strip():
            os.system(f"delgroup {self.app} >/dev/null")
        if check_output(f"getent group {self.app} &>/dev/null || true").strip():
-            raise YunohostError(f"Failed to delete system user for {self.app}")
+            raise YunohostError(f"Failed to delete system user for {self.app}", raw_msg=True)

        # FIXME : better logging and error handling, add stdout/stderr from the deluser/delgroup commands...

@ -743,7 +747,8 @@ class AptDependenciesAppResource(AppResource):
                isinstance(values.get(k), str) for k in ["repo", "key", "packages"]
            ):
                raise YunohostError(
-                    "In apt resource in the manifest: 'extras' repo should have the keys 'repo', 'key' and 'packages' defined and be strings"
+                    "In apt resource in the manifest: 'extras' repo should have the keys 'repo', 'key' and 'packages' defined and be strings",
+                    raw_msg=True
                )

        super().__init__(properties, *args, **kwargs)
@ -860,7 +865,8 @@ class PortsResource(AppResource):
                if infos["fixed"]:
                    if self._port_is_used(port_value):
                        raise YunohostValidationError(
-                            f"Port {port_value} is already used by another process or app."
+                            f"Port {port_value} is already used by another process or app.",
+                            raw_msg=True
                        )
                else:
                    while self._port_is_used(port_value):
@ -950,7 +956,7 @@ class DatabaseAppResource(AppResource):

    def db_exists(self, db_name):
        if self.dbtype == "mysql":
-            return os.system(f"mysqlshow '{db_name}' >/dev/null 2>/dev/null") == 0
+            return os.system(f"mysqlshow | grep -q -w '{db_name}' 2>/dev/null") == 0
        elif self.dbtype == "postgresql":
            return (
                os.system(