Merge pull request #1346 from YunoHost/refactor-app-py-zbleh

Refactor app.py / prevent change_url from being used to move a fulldomain app to a subpath

.gitlab/ci/test.gitlab-ci.yml

@@ -113,10 +113,10 @@ test-apps:
 test-appscatalog:
   extends: .test-stage
   script:
-    - python3 -m pytest src/yunohost/tests/test_appscatalog.py
+    - python3 -m pytest src/yunohost/tests/test_app_catalog.py
   only:
     changes:
-      - src/yunohost/app.py
+      - src/yunohost/app_calalog.py
 
 test-appurl:
   extends: .test-stage

locales/en.json

@@ -36,7 +36,6 @@
     "app_manifest_install_ask_is_public": "Should this app be exposed to anonymous visitors?",
     "app_manifest_install_ask_password": "Choose an administration password for this app",
     "app_manifest_install_ask_path": "Choose the URL path (after the domain) where this app should be installed",
-    "app_manifest_invalid": "Something is wrong with the app manifest: {error}",
     "app_not_correctly_installed": "{app} seems to be incorrectly installed",
     "app_not_installed": "Could not find {app} in the list of installed apps: {all_apps}",
     "app_not_properly_removed": "{app} has not been properly removed",

src/yunohost/app_catalog.py

@@ -0,0 +1,255 @@
+import os
+import re
+
+from moulinette import m18n
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.network import download_json
+from moulinette.utils.filesystem import (
+    read_json,
+    read_yaml,
+    write_to_json,
+    write_to_yaml,
+    mkdir,
+)
+
+from yunohost.utils.i18n import _value_for_locale
+from yunohost.utils.error import YunohostError
+
+logger = getActionLogger("yunohost.app_catalog")
+
+APPS_CATALOG_CACHE = "/var/cache/yunohost/repo"
+APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml"
+APPS_CATALOG_API_VERSION = 2
+APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
+
+
+# Old legacy function...
+def app_fetchlist():
+    logger.warning(
+        "'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead"
+    )
+    from yunohost.tools import tools_update
+
+    tools_update(target="apps")
+
+
+def app_catalog(full=False, with_categories=False):
+    """
+    Return a dict of apps available to installation from Yunohost's app catalog
+    """
+
+    from yunohost.app import _installed_apps, _set_default_ask_questions
+
+    # Get app list from catalog cache
+    catalog = _load_apps_catalog()
+    installed_apps = set(_installed_apps())
+
+    # Trim info for apps if not using --full
+    for app, infos in catalog["apps"].items():
+        infos["installed"] = app in installed_apps
+
+        infos["manifest"]["description"] = _value_for_locale(
+            infos["manifest"]["description"]
+        )
+
+        if not full:
+            catalog["apps"][app] = {
+                "description": infos["manifest"]["description"],
+                "level": infos["level"],
+            }
+        else:
+            infos["manifest"]["arguments"] = _set_default_ask_questions(
+                infos["manifest"].get("arguments", {})
+            )
+
+    # Trim info for categories if not using --full
+    for category in catalog["categories"]:
+        category["title"] = _value_for_locale(category["title"])
+        category["description"] = _value_for_locale(category["description"])
+        for subtags in category.get("subtags", []):
+            subtags["title"] = _value_for_locale(subtags["title"])
+
+    if not full:
+        catalog["categories"] = [
+            {"id": c["id"], "description": c["description"]}
+            for c in catalog["categories"]
+        ]
+
+    if not with_categories:
+        return {"apps": catalog["apps"]}
+    else:
+        return {"apps": catalog["apps"], "categories": catalog["categories"]}
+
+
+def app_search(string):
+    """
+    Return a dict of apps whose description or name match the search string
+    """
+
+    # Retrieve a simple dict listing all apps
+    catalog_of_apps = app_catalog()
+
+    # Selecting apps according to a match in app name or description
+    matching_apps = {"apps": {}}
+    for app in catalog_of_apps["apps"].items():
+        if re.search(string, app[0], flags=re.IGNORECASE) or re.search(
+            string, app[1]["description"], flags=re.IGNORECASE
+        ):
+            matching_apps["apps"][app[0]] = app[1]
+
+    return matching_apps
+
+
+def _initialize_apps_catalog_system():
+    """
+    This function is meant to intialize the apps_catalog system with YunoHost's default app catalog.
+    """
+
+    default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}]
+
+    try:
+        logger.debug(
+            "Initializing apps catalog system with YunoHost's default app list"
+        )
+        write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list)
+    except Exception as e:
+        raise YunohostError(
+            "Could not initialize the apps catalog system... : %s" % str(e)
+        )
+
+    logger.success(m18n.n("apps_catalog_init_success"))
+
+
+def _read_apps_catalog_list():
+    """
+    Read the json corresponding to the list of apps catalogs
+    """
+
+    try:
+        list_ = read_yaml(APPS_CATALOG_CONF)
+        # Support the case where file exists but is empty
+        # by returning [] if list_ is None
+        return list_ if list_ else []
+    except Exception as e:
+        raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e))
+
+
+def _actual_apps_catalog_api_url(base_url):
+
+    return "{base_url}/v{version}/apps.json".format(
+        base_url=base_url, version=APPS_CATALOG_API_VERSION
+    )
+
+
+def _update_apps_catalog():
+    """
+    Fetches the json for each apps_catalog and update the cache
+
+    apps_catalog_list is for example :
+     [   {"id": "default", "url": "https://app.yunohost.org/default/"}  ]
+
+    Then for each apps_catalog, the actual json URL to be fetched is like :
+       https://app.yunohost.org/default/vX/apps.json
+
+    And store it in :
+        /var/cache/yunohost/repo/default.json
+    """
+
+    apps_catalog_list = _read_apps_catalog_list()
+
+    logger.info(m18n.n("apps_catalog_updating"))
+
+    # Create cache folder if needed
+    if not os.path.exists(APPS_CATALOG_CACHE):
+        logger.debug("Initialize folder for apps catalog cache")
+        mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root")
+
+    for apps_catalog in apps_catalog_list:
+        apps_catalog_id = apps_catalog["id"]
+        actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"])
+
+        # Fetch the json
+        try:
+            apps_catalog_content = download_json(actual_api_url)
+        except Exception as e:
+            raise YunohostError(
+                "apps_catalog_failed_to_download",
+                apps_catalog=apps_catalog_id,
+                error=str(e),
+            )
+
+        # Remember the apps_catalog api version for later
+        apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION
+
+        # Save the apps_catalog data in the cache
+        cache_file = "{cache_folder}/{list}.json".format(
+            cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
+        )
+        try:
+            write_to_json(cache_file, apps_catalog_content)
+        except Exception as e:
+            raise YunohostError(
+                "Unable to write cache data for %s apps_catalog : %s"
+                % (apps_catalog_id, str(e))
+            )
+
+    logger.success(m18n.n("apps_catalog_update_success"))
+
+
+def _load_apps_catalog():
+    """
+    Read all the apps catalog cache files and build a single dict (merged_catalog)
+    corresponding to all known apps and categories
+    """
+
+    merged_catalog = {"apps": {}, "categories": []}
+
+    for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]:
+
+        # Let's load the json from cache for this catalog
+        cache_file = "{cache_folder}/{list}.json".format(
+            cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
+        )
+
+        try:
+            apps_catalog_content = (
+                read_json(cache_file) if os.path.exists(cache_file) else None
+            )
+        except Exception as e:
+            raise YunohostError(
+                "Unable to read cache for apps_catalog %s : %s" % (cache_file, e),
+                raw_msg=True,
+            )
+
+        # Check that the version of the data matches version ....
+        # ... otherwise it means we updated yunohost in the meantime
+        # and need to update the cache for everything to be consistent
+        if (
+            not apps_catalog_content
+            or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION
+        ):
+            logger.info(m18n.n("apps_catalog_obsolete_cache"))
+            _update_apps_catalog()
+            apps_catalog_content = read_json(cache_file)
+
+        del apps_catalog_content["from_api_version"]
+
+        # Add apps from this catalog to the output
+        for app, info in apps_catalog_content["apps"].items():
+
+            # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ...
+            #         in which case we keep only the first one found)
+            if app in merged_catalog["apps"]:
+                logger.warning(
+                    "Duplicate app %s found between apps catalog %s and %s"
+                    % (app, apps_catalog_id, merged_catalog["apps"][app]["repository"])
+                )
+                continue
+
+            info["repository"] = apps_catalog_id
+            merged_catalog["apps"][app] = info
+
+        # Annnnd categories
+        merged_catalog["categories"] += apps_catalog_content["categories"]
+
+    return merged_catalog

src/yunohost/backup.py

@@ -49,10 +49,6 @@ from yunohost.app import (
     app_info,
     _is_installed,
     _make_environment_for_app_script,
-    _patch_legacy_helpers,
-    _patch_legacy_php_versions,
-    _patch_legacy_php_versions_in_settings,
-    LEGACY_PHP_VERSION_REPLACEMENTS,
     _make_tmp_workdir_for_app,
 )
 from yunohost.hook import (
@@ -1190,6 +1186,7 @@ class RestoreManager:
         """
         Apply dirty patch to redirect php5 and php7.0 files to php7.3
         """
+        from yunohost.utils.legacy import LEGACY_PHP_VERSION_REPLACEMENTS
 
         backup_csv = os.path.join(self.work_dir, "backup.csv")
 
@@ -1351,6 +1348,7 @@ class RestoreManager:
         app_instance_name -- (string) The app name to restore (no app with this
                              name should be already install)
         """
+        from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_php_versions_in_settings, _patch_legacy_helpers
         from yunohost.user import user_group_list
         from yunohost.permission import (
             permission_create,
@@ -1485,7 +1483,9 @@ class RestoreManager:
         logger.debug(m18n.n("restore_running_app_script", app=app_instance_name))
 
         # Prepare env. var. to pass to script
-        env_dict = _make_environment_for_app_script(app_instance_name)
+        # FIXME : workdir should be a tmp workdir
+        app_workdir = os.path.join(self.work_dir, "apps", app_instance_name, "settings")
+        env_dict = _make_environment_for_app_script(app_instance_name, workdir=app_workdir)
         env_dict.update(
             {
                 "YNH_BACKUP_DIR": self.work_dir,
@@ -1493,9 +1493,6 @@ class RestoreManager:
                 "YNH_APP_BACKUP_DIR": os.path.join(
                     self.work_dir, "apps", app_instance_name, "backup"
                 ),
-                "YNH_APP_BASEDIR": os.path.join(
-                    self.work_dir, "apps", app_instance_name, "settings"
-                ),
             }
         )
 
@@ -1532,11 +1529,7 @@ class RestoreManager:
                 remove_script = os.path.join(app_scripts_in_archive, "remove")
 
                 # Setup environment for remove script
-                env_dict_remove = _make_environment_for_app_script(app_instance_name)
+                env_dict_remove = _make_environment_for_app_script(app_instance_name, workdir=app_workdir)
-                env_dict_remove["YNH_APP_BASEDIR"] = os.path.join(
-                    self.work_dir, "apps", app_instance_name, "settings"
-                )
-
                 remove_operation_logger = OperationLogger(
                     "remove_on_failed_restore",
                     [("app", app_instance_name)],

src/yunohost/data_migrations/0016_php70_to_php73_pools.py

@@ -4,7 +4,8 @@ from shutil import copy2
 
 from moulinette.utils.log import getActionLogger
 
-from yunohost.app import _is_installed, _patch_legacy_php_versions_in_settings
+from yunohost.app import _is_installed
+from yunohost.utils.legacy import _patch_legacy_php_versions_in_settings
 from yunohost.tools import Migration
 from yunohost.service import _run_service_command
 

src/yunohost/dns.py

@@ -32,7 +32,7 @@ from collections import OrderedDict
 
 from moulinette import m18n, Moulinette
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, write_to_file, read_toml
+from moulinette.utils.filesystem import read_file, write_to_file, read_toml, mkdir
 
 from yunohost.domain import (
     domain_list,
@@ -471,7 +471,7 @@ def _get_dns_zone_for_domain(domain):
         # Check if there's a NS record for that domain
         answer = dig(parent, rdtype="NS", full_answers=True, resolvers="force_external")
         if answer[0] == "ok":
-            os.system(f"mkdir -p {cache_folder}")
+            mkdir(cache_folder, parents=True, force=True)
             write_to_file(cache_file, parent)
             return parent
 

src/yunohost/domain.py

@@ -29,7 +29,7 @@ from typing import Dict, Any
 from moulinette import m18n, Moulinette
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml
+from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml, rm
 
 from yunohost.app import (
     app_ssowatconf,
@@ -328,7 +328,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False):
     ]
 
     for stuff in stuff_to_delete:
-        os.system("rm -rf {stuff}")
+        rm(stuff, force=True, recursive=True)
 
     # Sometime we have weird issues with the regenconf where some files
     # appears as manually modified even though they weren't touched ...

src/yunohost/dyndns.py

@@ -33,7 +33,7 @@ import subprocess
 from moulinette import m18n
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import write_to_file, read_file
+from moulinette.utils.filesystem import write_to_file, read_file, rm, chown, chmod
 from moulinette.utils.network import download_json
 
 from yunohost.utils.error import YunohostError, YunohostValidationError
@@ -152,13 +152,12 @@ def dyndns_subscribe(
 
             os.system(
                 "cd /etc/yunohost/dyndns && "
-                "dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s"
+                f"dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER {domain}"
-                % domain
-            )
-            os.system(
-                "chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private"
             )
 
+            chmod("/etc/yunohost/dyndns", 0o600, recursive=True)
+            chown("/etc/yunohost/dyndns", "root", recursive=True)
+
         private_file = glob.glob("/etc/yunohost/dyndns/*%s*.private" % domain)[0]
         key_file = glob.glob("/etc/yunohost/dyndns/*%s*.key" % domain)[0]
         with open(key_file) as f:
@@ -175,12 +174,12 @@ def dyndns_subscribe(
             timeout=30,
         )
     except Exception as e:
-        os.system("rm -f %s" % private_file)
+        rm(private_file, force=True)
-        os.system("rm -f %s" % key_file)
+        rm(key_file, force=True)
         raise YunohostError("dyndns_registration_failed", error=str(e))
     if r.status_code != 201:
-        os.system("rm -f %s" % private_file)
+        rm(private_file, force=True)
-        os.system("rm -f %s" % key_file)
+        rm(key_file, force=True)
         try:
             error = json.loads(r.text)["error"]
         except Exception:

src/yunohost/hook.py

@@ -34,7 +34,7 @@ from importlib import import_module
 from moulinette import m18n, Moulinette
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from moulinette.utils import log
-from moulinette.utils.filesystem import read_yaml
+from moulinette.utils.filesystem import read_yaml, cp
 
 HOOK_FOLDER = "/usr/share/yunohost/hooks/"
 CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/"
@@ -60,8 +60,7 @@ def hook_add(app, file):
         os.makedirs(CUSTOM_HOOK_FOLDER + action)
 
     finalpath = CUSTOM_HOOK_FOLDER + action + "/" + priority + "-" + app
-    os.system("cp %s %s" % (file, finalpath))
+    cp(file, finalpath)
-    os.system("chown -hR admin: %s" % HOOK_FOLDER)
 
     return {"hook": finalpath}
 

src/yunohost/tests/test_app_catalog.py

@@ -9,7 +9,7 @@ from moulinette import m18n
 from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml
 
 from yunohost.utils.error import YunohostError
-from yunohost.app import (
+from yunohost.app_catalog import (
     _initialize_apps_catalog_system,
     _read_apps_catalog_list,
     _update_apps_catalog,

src/yunohost/tests/test_appurl.py

@@ -4,7 +4,7 @@ import os
 from .conftest import get_test_apps_dir
 
 from yunohost.utils.error import YunohostError
-from yunohost.app import app_install, app_remove
+from yunohost.app import app_install, app_remove, _is_app_repo_url
 from yunohost.domain import _get_maindomain, domain_url_available
 from yunohost.permission import _validate_and_sanitize_permission_url
 
@@ -28,6 +28,34 @@ def teardown_function(function):
         pass
 
 
+def test_repo_url_definition():
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh")
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh/")
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh.git")
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh/tree/testing")
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/foobar123_ynh/tree/testing/")
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/foo-bar-123_ynh")
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/foo_bar_123_ynh")
+    assert _is_app_repo_url("https://github.com/YunoHost-Apps/FooBar123_ynh")
+    assert _is_app_repo_url("https://github.com/labriqueinternet/vpnclient_ynh")
+    assert _is_app_repo_url("https://framagit.org/YunoHost/apps/nodebb_ynh")
+    assert _is_app_repo_url("https://framagit.org/YunoHost/apps/nodebb_ynh/-/tree/testing")
+    assert _is_app_repo_url("https://gitlab.com/yunohost-apps/foobar_ynh")
+    assert _is_app_repo_url("https://code.antopie.org/miraty/qr_ynh")
+    assert _is_app_repo_url("https://gitlab.domainepublic.net/Neutrinet/neutrinet_ynh/-/tree/unstable")
+    assert _is_app_repo_url("git@github.com:YunoHost-Apps/foobar_ynh.git")
+
+    assert not _is_app_repo_url("github.com/YunoHost-Apps/foobar_ynh")
+    assert not _is_app_repo_url("http://github.com/YunoHost-Apps/foobar_ynh")
+    assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar_wat")
+    assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar_ynh_wat")
+    assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar/tree/testing")
+    assert not _is_app_repo_url("https://github.com/YunoHost-Apps/foobar_ynh_wat/tree/testing")
+    assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/")
+    assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/pwet")
+    assert not _is_app_repo_url("https://framagit.org/YunoHost/apps/pwet_foo")
+
+
 def test_urlavailable():
 
     # Except the maindomain/macnuggets to be available

src/yunohost/tests/test_permission.py

@@ -1049,7 +1049,7 @@ def test_permission_app_remove():
 def test_permission_app_change_url():
     app_install(
         os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
-        args="domain=%s&domain_2=%s&path=%s&admin=%s"
+        args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s"
         % (maindomain, other_domains[0], "/urlpermissionapp", "alice"),
         force=True,
     )
@@ -1072,7 +1072,7 @@ def test_permission_app_change_url():
 def test_permission_protection_management_by_helper():
     app_install(
         os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
-        args="domain=%s&domain_2=%s&path=%s&admin=%s"
+        args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s"
         % (maindomain, other_domains[0], "/urlpermissionapp", "alice"),
         force=True,
     )
@@ -1135,7 +1135,7 @@ def test_permission_legacy_app_propagation_on_ssowat():
 
     app_install(
         os.path.join(get_test_apps_dir(), "legacy_app_ynh"),
-        args="domain=%s&domain_2=%s&path=%s"
+        args="domain=%s&domain_2=%s&path=%s&is_public=1"
         % (maindomain, other_domains[0], "/legacy"),
         force=True,
     )

src/yunohost/tools.py

@@ -34,13 +34,15 @@ from typing import List
 from moulinette import Moulinette, m18n
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import check_output, call_async_output
-from moulinette.utils.filesystem import read_yaml, write_to_yaml
+from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm
 
 from yunohost.app import (
-    _update_apps_catalog,
     app_info,
     app_upgrade,
+)
+from yunohost.app_catalog import (
     _initialize_apps_catalog_system,
+    _update_apps_catalog,
 )
 from yunohost.domain import domain_add
 from yunohost.dyndns import _dyndns_available, _dyndns_provides
@@ -1145,13 +1147,11 @@ class Migration(object):
                 backup_folder = "/home/yunohost.backup/premigration/" + time.strftime(
                     "%Y%m%d-%H%M%S", time.gmtime()
                 )
-                os.makedirs(backup_folder, 0o750)
+                mkdir(backup_folder, 0o750, parents=True)
                 os.system("systemctl stop slapd")
-                os.system(f"cp -r --preserve /etc/ldap {backup_folder}/ldap_config")
+                cp("/etc/ldap", f"{backup_folder}/ldap_config", recursive=True)
-                os.system(f"cp -r --preserve /var/lib/ldap {backup_folder}/ldap_db")
+                cp("/var/lib/ldap", f"{backup_folder}/ldap_db", recursive=True)
-                os.system(
+                cp("/etc/yunohost/apps", f"{backup_folder}/apps_settings", recursive=True)
-                    f"cp -r --preserve /etc/yunohost/apps {backup_folder}/apps_settings"
-                )
             except Exception as e:
                 raise YunohostError(
                     "migration_ldap_can_not_backup_before_migration", error=str(e)
@@ -1167,17 +1167,15 @@ class Migration(object):
                 )
                 os.system("systemctl stop slapd")
                 # To be sure that we don't keep some part of the old config
-                os.system("rm -r /etc/ldap/slapd.d")
+                rm("/etc/ldap/slapd.d", force=True, recursive=True)
-                os.system(f"cp -r --preserve {backup_folder}/ldap_config/. /etc/ldap/")
+                cp(f"{backup_folder}/ldap_config", "/etc/ldap", recursive=True)
-                os.system(f"cp -r --preserve {backup_folder}/ldap_db/. /var/lib/ldap/")
+                cp(f"{backup_folder}/ldap_db", "/var/lib/ldap", recursive=True)
-                os.system(
+                cp(f"{backup_folder}/apps_settings", "/etc/yunohost/apps", recursive=True)
-                    f"cp -r --preserve {backup_folder}/apps_settings/. /etc/yunohost/apps/"
-                )
                 os.system("systemctl start slapd")
-                os.system(f"rm -r {backup_folder}")
+                rm(backup_folder, force=True, recursive=True)
                 logger.info(m18n.n("migration_ldap_rollback_success"))
                 raise
             else:
-                os.system(f"rm -r {backup_folder}")
+                rm(backup_folder, force=True, recursive=True)
 
         return func

src/yunohost/utils/legacy.py

@@ -1,7 +1,10 @@
 import os
+import re
+import glob
 from moulinette import m18n
+from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import write_to_json, read_yaml
+from moulinette.utils.filesystem import read_file, write_to_file, write_to_json, write_to_yaml, read_yaml
 
 from yunohost.user import user_list
 from yunohost.app import (
@@ -14,6 +17,8 @@ from yunohost.permission import (
     user_permission_update,
     permission_sync_to_user,
 )
+from yunohost.utils.error import YunohostValidationError
+
 
 logger = getActionLogger("yunohost.legacy")
 
@@ -237,3 +242,213 @@ def translate_legacy_rules_in_ssowant_conf_json_persistent():
     logger.warning(
         "YunoHost automatically translated some legacy rules in /etc/ssowat/conf.json.persistent to match the new permission system"
     )
+
+
+LEGACY_PHP_VERSION_REPLACEMENTS = [
+    ("/etc/php5", "/etc/php/7.3"),
+    ("/etc/php/7.0", "/etc/php/7.3"),
+    ("/var/run/php5-fpm", "/var/run/php/php7.3-fpm"),
+    ("/var/run/php/php7.0-fpm", "/var/run/php/php7.3-fpm"),
+    ("php5", "php7.3"),
+    ("php7.0", "php7.3"),
+    (
+        'phpversion="${phpversion:-7.0}"',
+        'phpversion="${phpversion:-7.3}"',
+    ),  # Many helpers like the composer ones use 7.0 by default ...
+    (
+        '"$phpversion" == "7.0"',
+        '$(bc <<< "$phpversion >= 7.3") -eq 1',
+    ),  # patch ynh_install_php to refuse installing/removing php <= 7.3
+]
+
+
+def _patch_legacy_php_versions(app_folder):
+
+    files_to_patch = []
+    files_to_patch.extend(glob.glob("%s/conf/*" % app_folder))
+    files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder))
+    files_to_patch.extend(glob.glob("%s/scripts/*/*" % app_folder))
+    files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder))
+    files_to_patch.append("%s/manifest.json" % app_folder)
+    files_to_patch.append("%s/manifest.toml" % app_folder)
+
+    for filename in files_to_patch:
+
+        # Ignore non-regular files
+        if not os.path.isfile(filename):
+            continue
+
+        c = (
+            "sed -i "
+            + "".join(
+                "-e 's@{pattern}@{replace}@g' ".format(pattern=p, replace=r)
+                for p, r in LEGACY_PHP_VERSION_REPLACEMENTS
+            )
+            + "%s" % filename
+        )
+        os.system(c)
+
+
+def _patch_legacy_php_versions_in_settings(app_folder):
+
+    settings = read_yaml(os.path.join(app_folder, "settings.yml"))
+
+    if settings.get("fpm_config_dir") == "/etc/php/7.0/fpm":
+        settings["fpm_config_dir"] = "/etc/php/7.3/fpm"
+    if settings.get("fpm_service") == "php7.0-fpm":
+        settings["fpm_service"] = "php7.3-fpm"
+    if settings.get("phpversion") == "7.0":
+        settings["phpversion"] = "7.3"
+
+    # We delete these checksums otherwise the file will appear as manually modified
+    list_to_remove = ["checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d"]
+    settings = {
+        k: v
+        for k, v in settings.items()
+        if not any(k.startswith(to_remove) for to_remove in list_to_remove)
+    }
+
+    write_to_yaml(app_folder + "/settings.yml", settings)
+
+
+def _patch_legacy_helpers(app_folder):
+
+    files_to_patch = []
+    files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder))
+    files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder))
+
+    stuff_to_replace = {
+        # Replace
+        #    sudo yunohost app initdb $db_user -p $db_pwd
+        # by
+        #    ynh_mysql_setup_db --db_user=$db_user --db_name=$db_user --db_pwd=$db_pwd
+        "yunohost app initdb": {
+            "pattern": r"(sudo )?yunohost app initdb \"?(\$\{?\w+\}?)\"?\s+-p\s\"?(\$\{?\w+\}?)\"?",
+            "replace": r"ynh_mysql_setup_db --db_user=\2 --db_name=\2 --db_pwd=\3",
+            "important": True,
+        },
+        # Replace
+        #    sudo yunohost app checkport whaterver
+        # by
+        #    ynh_port_available whatever
+        "yunohost app checkport": {
+            "pattern": r"(sudo )?yunohost app checkport",
+            "replace": r"ynh_port_available",
+            "important": True,
+        },
+        # We can't migrate easily port-available
+        # .. but at the time of writing this code, only two non-working apps are using it.
+        "yunohost tools port-available": {"important": True},
+        # Replace
+        #    yunohost app checkurl "${domain}${path_url}" -a "${app}"
+        # by
+        #    ynh_webpath_register --app=${app} --domain=${domain} --path_url=${path_url}
+        "yunohost app checkurl": {
+            "pattern": r"(sudo )?yunohost app checkurl \"?(\$\{?\w+\}?)\/?(\$\{?\w+\}?)\"?\s+-a\s\"?(\$\{?\w+\}?)\"?",
+            "replace": r"ynh_webpath_register --app=\4 --domain=\2 --path_url=\3",
+            "important": True,
+        },
+        # Remove
+        #    Automatic diagnosis data from YunoHost
+        #    __PRE_TAG1__$(yunohost tools diagnosis | ...)__PRE_TAG2__"
+        #
+        "yunohost tools diagnosis": {
+            "pattern": r"(Automatic diagnosis data from YunoHost( *\n)*)? *(__\w+__)? *\$\(yunohost tools diagnosis.*\)(__\w+__)?",
+            "replace": r"",
+            "important": False,
+        },
+        # Old $1, $2 in backup/restore scripts...
+        "app=$2": {
+            "only_for": ["scripts/backup", "scripts/restore"],
+            "pattern": r"app=\$2",
+            "replace": r"app=$YNH_APP_INSTANCE_NAME",
+            "important": True,
+        },
+        # Old $1, $2 in backup/restore scripts...
+        "backup_dir=$1": {
+            "only_for": ["scripts/backup", "scripts/restore"],
+            "pattern": r"backup_dir=\$1",
+            "replace": r"backup_dir=.",
+            "important": True,
+        },
+        # Old $1, $2 in backup/restore scripts...
+        "restore_dir=$1": {
+            "only_for": ["scripts/restore"],
+            "pattern": r"restore_dir=\$1",
+            "replace": r"restore_dir=.",
+            "important": True,
+        },
+        # Old $1, $2 in install scripts...
+        # We ain't patching that shit because it ain't trivial to patch all args...
+        "domain=$1": {"only_for": ["scripts/install"], "important": True},
+    }
+
+    for helper, infos in stuff_to_replace.items():
+        infos["pattern"] = (
+            re.compile(infos["pattern"]) if infos.get("pattern") else None
+        )
+        infos["replace"] = infos.get("replace")
+
+    for filename in files_to_patch:
+
+        # Ignore non-regular files
+        if not os.path.isfile(filename):
+            continue
+
+        try:
+            content = read_file(filename)
+        except MoulinetteError:
+            continue
+
+        replaced_stuff = False
+        show_warning = False
+
+        for helper, infos in stuff_to_replace.items():
+
+            # Ignore if not relevant for this file
+            if infos.get("only_for") and not any(
+                filename.endswith(f) for f in infos["only_for"]
+            ):
+                continue
+
+            # If helper is used, attempt to patch the file
+            if helper in content and infos["pattern"]:
+                content = infos["pattern"].sub(infos["replace"], content)
+                replaced_stuff = True
+                if infos["important"]:
+                    show_warning = True
+
+            # If the helper is *still* in the content, it means that we
+            # couldn't patch the deprecated helper in the previous lines.  In
+            # that case, abort the install or whichever step is performed
+            if helper in content and infos["important"]:
+                raise YunohostValidationError(
+                    "This app is likely pretty old and uses deprecated / outdated helpers that can't be migrated easily. It can't be installed anymore.",
+                    raw_msg=True,
+                )
+
+        if replaced_stuff:
+
+            # Check the app do load the helper
+            # If it doesn't, add the instruction ourselve (making sure it's after the #!/bin/bash if it's there...
+            if filename.split("/")[-1] in [
+                "install",
+                "remove",
+                "upgrade",
+                "backup",
+                "restore",
+            ]:
+                source_helpers = "source /usr/share/yunohost/helpers"
+                if source_helpers not in content:
+                    content.replace("#!/bin/bash", "#!/bin/bash\n" + source_helpers)
+                if source_helpers not in content:
+                    content = source_helpers + "\n" + content
+
+            # Actually write the new content in the file
+            write_to_file(filename, content)
+
+        if show_warning:
+            # And complain about those damn deprecated helpers
+            logger.error(
+                r"/!\ Packagers ! This app uses a very old deprecated helpers ... Yunohost automatically patched the helpers to use the new recommended practice, but please do consider fixing the upstream code right now ..."
+            )