Merge pull request #1289 from YunoHost/manifestv2

[enh] App manifest v2

.gitlab/ci/test.gitlab-ci.yml

@@ -125,6 +125,15 @@ test-app-config:
       - src/app.py
       - src/utils/config.py
 
+test-app-resources:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/tests/test_app_resources.py
+  only:
+    changes:
+      - src/app.py
+      - src/utils/resources.py
+
 test-changeurl:
   extends: .test-stage
   script:

doc/generate_resource_doc.py

@@ -0,0 +1,12 @@
+from yunohost.utils.resources import AppResourceClassesByType
+
+resources = sorted(AppResourceClassesByType.values(), key=lambda r: r.priority)
+
+for klass in resources:
+
+    doc = klass.__doc__.replace("\n    ", "\n")
+
+    print("")
+    print(f"## {klass.type.replace('_', ' ').title()}")
+    print("")
+    print(doc)

helpers/apt

@@ -227,9 +227,8 @@ ynh_install_app_dependencies() {
     # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See  below)
     dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')"
     local dependencies=${dependencies//|/ | }
-    local manifest_path="$YNH_APP_BASEDIR/manifest.json"
 
-    local version=$(jq -r '.version' "$manifest_path")
+    local version=$(ynh_read_manifest --manifest_key="version")
     if [ -z "${version}" ] || [ "$version" == "null" ]; then
         version="1.0"
     fi

helpers/utils

@@ -61,6 +61,12 @@ ynh_abort_if_errors() {
     trap ynh_exit_properly EXIT # Capturing exit signals on shell script
 }
 
+# When running an app script with packaging format >= 2, auto-enable ynh_abort_if_errors except for remove script
+if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 && [[ ${YNH_APP_ACTION} != "remove" ]]
+then
+    ynh_abort_if_errors
+fi
+
 # Download, check integrity, uncompress and patch the source from app.src
 #
 # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"]
@@ -758,12 +764,25 @@ ynh_read_manifest() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if [ ! -e "$manifest" ]; then
+    if [ ! -e "${manifest:-}" ]; then
         # If the manifest isn't found, try the common place for backup and restore script.
-        manifest="$YNH_APP_BASEDIR/manifest.json"
+        if [ -e "$YNH_APP_BASEDIR/manifest.json" ]
+        then
+            manifest="$YNH_APP_BASEDIR/manifest.json"
+        elif [ -e "$YNH_APP_BASEDIR/manifest.toml" ]
+        then
+            manifest="$YNH_APP_BASEDIR/manifest.toml"
+        else
+            ynh_die --message "No manifest found !?"
+        fi
     fi
 
-    jq ".$manifest_key" "$manifest" --raw-output
+    if echo "$manifest" | grep -q '\.json$'
+    then
+        jq ".$manifest_key" "$manifest" --raw-output
+    else
+        cat "$manifest" | python3 -c 'import json, toml, sys; print(json.dumps(toml.load(sys.stdin)))' | jq ".$manifest_key" --raw-output
+    fi
 }
 
 # Read the upstream version from the manifest or `$YNH_APP_MANIFEST_VERSION`
@@ -907,9 +926,9 @@ ynh_compare_current_package_version() {
 _ynh_apply_default_permissions() {
     local target=$1
 
-    local ynh_requirement=$(jq -r '.requirements.yunohost' $YNH_APP_BASEDIR/manifest.json | tr -d '>= ')
+    local ynh_requirement=$(ynh_read_manifest --manifest_key="requirements.yunohost")
 
-    if [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then
         chmod o-rwx $target
         chmod g-w $target
         chown -R root:root $target

locales/en.json

@@ -43,8 +43,7 @@
     "app_packaging_format_not_supported": "This app cannot be installed because its packaging format is not supported by your YunoHost version. You should probably consider upgrading your system.",
     "app_remove_after_failed_install": "Removing the app following the installation failure...",
     "app_removed": "{app} uninstalled",
-    "app_requirements_checking": "Checking required packages for {app}...",
-    "app_requirements_unmeet": "Requirements are not met for {app}, the package {pkgname} ({version}) must be {spec}",
+    "app_requirements_checking": "Checking requirements for {app}...",
     "app_restore_failed": "Could not restore {app}: {error}",
     "app_restore_script_failed": "An error occured inside the app restore script",
     "app_sources_fetch_failed": "Could not fetch sources files, is the URL correct?",
@@ -469,6 +468,7 @@
     "log_regen_conf": "Regenerate system configurations '{}'",
     "log_remove_on_failed_install": "Remove '{}' after a failed installation",
     "log_remove_on_failed_restore": "Remove '{}' after a failed restore from a backup archive",
+    "log_resource_snippet": "Provisioning/deprovisioning/updating a resource",
     "log_selfsigned_cert_install": "Install self-signed certificate on '{}' domain",
     "log_tools_migrations_migrate_forward": "Run migrations",
     "log_tools_postinstall": "Postinstall your YunoHost server",

share/actionsmap.yml

@@ -43,7 +43,7 @@ _global:
             help: Display YunoHost packages versions
             action: callback
             callback:
-                method: yunohost.utils.packages.ynh_packages_version
+                method: yunohost.utils.system.ynh_packages_version
                 return: true
 
 #############################

src/app.py

@@ -23,6 +23,7 @@
 
     Manage apps
 """
+import glob
 import os
 import toml
 import json
@@ -32,8 +33,10 @@ import time
 import re
 import subprocess
 import tempfile
+import copy
 from collections import OrderedDict
 from typing import List, Tuple, Dict, Any
+from packaging import version
 
 from moulinette import Moulinette, m18n
 from moulinette.utils.log import getActionLogger
@@ -51,7 +54,6 @@ from moulinette.utils.filesystem import (
     chmod,
 )
 
-from yunohost.utils import packages
 from yunohost.utils.config import (
     ConfigPanel,
     ask_questions_and_parse_answers,
@@ -61,7 +63,15 @@ from yunohost.utils.config import (
 )
 from yunohost.utils.i18n import _value_for_locale
 from yunohost.utils.error import YunohostError, YunohostValidationError
-from yunohost.utils.filesystem import free_space_in_directory
+from yunohost.utils.system import (
+    free_space_in_directory,
+    dpkg_is_broken,
+    get_ynh_package_version,
+    system_arch,
+    human_to_binary,
+    binary_to_human,
+    ram_available,
+)
 from yunohost.log import is_unit_operation, OperationLogger
 from yunohost.app_catalog import (  # noqa
     app_catalog,
@@ -165,13 +175,25 @@ def app_info(app, full=False, upgradable=False):
 
     ret["setting_path"] = setting_path
     ret["manifest"] = local_manifest
-    ret["manifest"]["arguments"] = _set_default_ask_questions(
-        ret["manifest"].get("arguments", {})
+
+    # FIXME: maybe this is not needed ? default ask questions are
+    # already set during the _get_manifest_of_app earlier ?
+    ret["manifest"]["install"] = _set_default_ask_questions(
+        ret["manifest"].get("install", {})
     )
     ret["settings"] = settings
 
     ret["from_catalog"] = from_catalog
 
+    # Hydrate app notifications and doc
+    for pagename, content_per_lang in ret["manifest"]["doc"].items():
+        for lang, content in content_per_lang.items():
+            ret["manifest"]["doc"][pagename][lang] = _hydrate_app_template(content, settings)
+    for step, notifications in ret["manifest"]["notifications"].items():
+        for name, content_per_lang in notifications.items():
+            for lang, content in content_per_lang.items():
+                notifications[name][lang] = _hydrate_app_template(content, settings)
+
     ret["is_webapp"] = "domain" in settings and "path" in settings
 
     if ret["is_webapp"]:
@@ -185,9 +207,7 @@ def app_info(app, full=False, upgradable=False):
     ret["supports_backup_restore"] = os.path.exists(
         os.path.join(setting_path, "scripts", "backup")
     ) and os.path.exists(os.path.join(setting_path, "scripts", "restore"))
-    ret["supports_multi_instance"] = is_true(
-        local_manifest.get("multi_instance", False)
-    )
+    ret["supports_multi_instance"] = local_manifest.get("integration", {}).get("multi_instance", False)
     ret["supports_config_panel"] = os.path.exists(
         os.path.join(setting_path, "config_panel.toml")
     )
@@ -202,7 +222,6 @@ def app_info(app, full=False, upgradable=False):
 
 
 def _app_upgradable(app_infos):
-    from packaging import version
 
     # Determine upgradability
 
@@ -416,7 +435,7 @@ def app_change_url(operation_logger, app, domain, path):
     tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
 
     # Prepare env. var. to pass to script
-    env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app)
+    env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app, action="change_url")
     env_dict["YNH_APP_OLD_DOMAIN"] = old_domain
     env_dict["YNH_APP_OLD_PATH"] = old_path
     env_dict["YNH_APP_NEW_DOMAIN"] = domain
@@ -467,7 +486,6 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
         no_safety_backup -- Disable the safety backup during upgrade
 
     """
-    from packaging import version
     from yunohost.hook import (
         hook_add,
         hook_remove,
@@ -477,6 +495,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
     from yunohost.permission import permission_sync_to_user
     from yunohost.regenconf import manually_modified_files
     from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers
+    from yunohost.backup import backup_list, backup_create, backup_delete, backup_restore
 
     apps = app
     # Check if disk space available
@@ -563,34 +582,69 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
                     upgrade_type = "UPGRADE_FULL"
 
         # Check requirements
-        _check_manifest_requirements(manifest)
+        _check_manifest_requirements(manifest, action="upgrade")
+
+        if manifest["packaging_format"] >= 2:
+            if no_safety_backup:
+                # FIXME: i18n
+                logger.warning("Skipping the creation of a backup prior to the upgrade.")
+            else:
+                # FIXME: i18n
+                logger.info("Creating a safety backup prior to the upgrade")
+
+                # Switch between pre-upgrade1 or pre-upgrade2
+                safety_backup_name = f"{app_instance_name}-pre-upgrade1"
+                other_safety_backup_name = f"{app_instance_name}-pre-upgrade2"
+                if safety_backup_name in backup_list()["archives"]:
+                    safety_backup_name = f"{app_instance_name}-pre-upgrade2"
+                    other_safety_backup_name = f"{app_instance_name}-pre-upgrade1"
+
+                backup_create(name=safety_backup_name, apps=[app_instance_name])
+
+                if safety_backup_name in backup_list()["archives"]:
+                    # if the backup suceeded, delete old safety backup to save space
+                    if other_safety_backup_name in backup_list()["archives"]:
+                        backup_delete(other_safety_backup_name)
+                else:
+                    # Is this needed ? Shouldn't backup_create report an expcetion if backup failed ?
+                    raise YunohostError("Uhoh the safety backup failed ?! Aborting the upgrade process.", raw_msg=True)
+
         _assert_system_is_sane_for_app(manifest, "pre")
 
-        app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
-
-        # Prepare env. var. to pass to script
-        env_dict = _make_environment_for_app_script(
-            app_instance_name, workdir=extracted_app_folder
-        )
-        env_dict["YNH_APP_UPGRADE_TYPE"] = upgrade_type
-        env_dict["YNH_APP_MANIFEST_VERSION"] = str(app_new_version)
-        env_dict["YNH_APP_CURRENT_VERSION"] = str(app_current_version)
-        env_dict["NO_BACKUP_UPGRADE"] = "1" if no_safety_backup else "0"
-
         # We'll check that the app didn't brutally edit some system configuration
         manually_modified_files_before_install = manually_modified_files()
 
+        app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
+
         # Attempt to patch legacy helpers ...
         _patch_legacy_helpers(extracted_app_folder)
 
         # Apply dirty patch to make php5 apps compatible with php7
         _patch_legacy_php_versions(extracted_app_folder)
 
+        # Prepare env. var. to pass to script
+        env_dict = _make_environment_for_app_script(
+            app_instance_name, workdir=extracted_app_folder, action="upgrade"
+        )
+        env_dict["YNH_APP_UPGRADE_TYPE"] = upgrade_type
+        env_dict["YNH_APP_MANIFEST_VERSION"] = str(app_new_version)
+        env_dict["YNH_APP_CURRENT_VERSION"] = str(app_current_version)
+        if manifest["packaging_format"] < 2:
+            env_dict["NO_BACKUP_UPGRADE"] = "1" if no_safety_backup else "0"
+
         # Start register change on system
         related_to = [("app", app_instance_name)]
         operation_logger = OperationLogger("app_upgrade", related_to, env=env_dict)
         operation_logger.start()
 
+        if manifest["packaging_format"] >= 2:
+            from yunohost.utils.resources import AppResourceManager
+            try:
+                AppResourceManager(app_instance_name, wanted=manifest, current=app_dict["manifest"]).apply(rollback_if_failure=True)
+            except Exception:
+                # FIXME : improve error handling ....
+                raise
+
         # Execute the app upgrade script
         upgrade_failed = True
         try:
@@ -607,6 +661,16 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
                 ),
             )
         finally:
+
+            # If upgrade failed, try to restore the safety backup
+            if upgrade_failed and manifest["packaging_format"] >= 2 and not no_safety_backup:
+                logger.warning("Upgrade failed ... attempting to restore the satefy backup (Yunohost first need to remove the app for this) ...")
+
+                app_remove(app_instance_name)
+                backup_restore(name=safety_backup_name, apps=[app_instance_name], force=True)
+                if not _is_installed(app_instance_name):
+                    logger.error("Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|")
+
             # Whatever happened (install success or failure) we check if it broke the system
             # and warn the user about it
             try:
@@ -698,12 +762,43 @@ def app_manifest(app):
 
     shutil.rmtree(extracted_app_folder)
 
-    raw_questions = manifest.get("arguments", {}).get("install", [])
-    manifest["arguments"]["install"] = hydrate_questions_with_choices(raw_questions)
+    raw_questions = manifest.get("install", {}).values()
+    manifest["install"] = hydrate_questions_with_choices(raw_questions)
 
     return manifest
 
 
+def _confirm_app_install(app, force=False):
+
+    # Ignore if there's nothing for confirm (good quality app), if --force is used
+    # or if request on the API (confirm already implemented on the API side)
+    if force or Moulinette.interface.type == "api":
+        return
+
+    quality = _app_quality(app)
+    if quality == "success":
+        return
+
+    # i18n: confirm_app_install_warning
+    # i18n: confirm_app_install_danger
+    # i18n: confirm_app_install_thirdparty
+
+    if quality in ["danger", "thirdparty"]:
+        answer = Moulinette.prompt(
+            m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"),
+            color="red",
+        )
+        if answer != "Yes, I understand":
+            raise YunohostError("aborting")
+
+    else:
+        answer = Moulinette.prompt(
+            m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow"
+        )
+        if answer.upper() != "Y":
+            raise YunohostError("aborting")
+
+
 @is_unit_operation()
 def app_install(
     operation_logger,
@@ -745,63 +840,41 @@ def app_install(
     if free_space_in_directory("/") <= 512 * 1000 * 1000:
         raise YunohostValidationError("disk_space_not_sufficient_install")
 
-    def confirm_install(app):
-
-        # Ignore if there's nothing for confirm (good quality app), if --force is used
-        # or if request on the API (confirm already implemented on the API side)
-        if force or Moulinette.interface.type == "api":
-            return
-
-        quality = _app_quality(app)
-        if quality == "success":
-            return
-
-        # i18n: confirm_app_install_warning
-        # i18n: confirm_app_install_danger
-        # i18n: confirm_app_install_thirdparty
-
-        if quality in ["danger", "thirdparty"]:
-            answer = Moulinette.prompt(
-                m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"),
-                color="red",
-            )
-            if answer != "Yes, I understand":
-                raise YunohostError("aborting")
-
-        else:
-            answer = Moulinette.prompt(
-                m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow"
-            )
-            if answer.upper() != "Y":
-                raise YunohostError("aborting")
-
-    confirm_install(app)
+    _confirm_app_install(app, force)
     manifest, extracted_app_folder = _extract_app(app)
 
+    # Display pre_install notices in cli mode
+    if manifest["notifications"]["pre_install"] and Moulinette.interface.type == "cli":
+        for notice in manifest["notifications"]["pre_install"].values():
+            # Should we render the markdown maybe? idk
+            print("==========")
+            print(_value_for_locale(notice))
+            print("==========")
+
+    packaging_format = manifest["packaging_format"]
+
     # Check ID
     if "id" not in manifest or "__" in manifest["id"] or "." in manifest["id"]:
         raise YunohostValidationError("app_id_invalid")
 
     app_id = manifest["id"]
-    label = label if label else manifest["name"]
 
     # Check requirements
-    _check_manifest_requirements(manifest)
+    _check_manifest_requirements(manifest, action="install")
     _assert_system_is_sane_for_app(manifest, "pre")
 
     # Check if app can be forked
     instance_number = _next_instance_number_for_app(app_id)
     if instance_number > 1:
-        if "multi_instance" not in manifest or not is_true(manifest["multi_instance"]):
-            raise YunohostValidationError("app_already_installed", app=app_id)
-
         # Change app_id to the forked app id
         app_instance_name = app_id + "__" + str(instance_number)
     else:
         app_instance_name = app_id
 
+    app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
+
     # Retrieve arguments list for install script
-    raw_questions = manifest.get("arguments", {}).get("install", {})
+    raw_questions = manifest["install"]
     questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args)
     args = {
         question.name: question.value
@@ -810,11 +883,13 @@ def app_install(
     }
 
     # Validate domain / path availability for webapps
+    # (ideally this should be handled by the resource system for manifest v >= 2
     path_requirement = _guess_webapp_path_requirement(extracted_app_folder)
     _validate_webpath_requirement(args, path_requirement)
 
-    # Attempt to patch legacy helpers ...
-    _patch_legacy_helpers(extracted_app_folder)
+    if packaging_format < 2:
+        # Attempt to patch legacy helpers ...
+        _patch_legacy_helpers(extracted_app_folder)
 
     # Apply dirty patch to make php5 apps compatible with php7
     _patch_legacy_php_versions(extracted_app_folder)
@@ -831,7 +906,6 @@ def app_install(
     logger.info(m18n.n("app_start_install", app=app_id))
 
     # Create app directory
-    app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
     if os.path.exists(app_setting_path):
         shutil.rmtree(app_setting_path)
     os.makedirs(app_setting_path)
@@ -842,6 +916,17 @@ def app_install(
         "install_time": int(time.time()),
         "current_revision": manifest.get("remote", {}).get("revision", "?"),
     }
+
+    # If packaging_format v2+, save all install questions as settings
+    if packaging_format >= 2:
+        for question in questions:
+
+            # Except user-provider passwords
+            if question.type == "password":
+                continue
+
+            app_settings[question.name] = question.value
+
     _set_app_settings(app_instance_name, app_settings)
 
     # Move scripts and manifest to the right place
@@ -853,21 +938,30 @@ def app_install(
                 recursive=True,
             )
 
-    # Initialize the main permission for the app
-    # The permission is initialized with no url associated, and with tile disabled
-    # For web app, the root path of the app will be added as url and the tile
-    # will be enabled during the app install. C.f. 'app_register_url()' below.
-    permission_create(
-        app_instance_name + ".main",
-        allowed=["all_users"],
-        label=label,
-        show_tile=False,
-        protected=False,
-    )
+    if packaging_format >= 2:
+        from yunohost.utils.resources import AppResourceManager
+        try:
+            AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(rollback_if_failure=True)
+        except Exception:
+            # FIXME : improve error handling ....
+            raise
+    else:
+        # Initialize the main permission for the app
+        # The permission is initialized with no url associated, and with tile disabled
+        # For web app, the root path of the app will be added as url and the tile
+        # will be enabled during the app install. C.f. 'app_register_url()' below
+        # or the webpath resource
+        permission_create(
+            app_instance_name + ".main",
+            allowed=["all_users"],
+            label=label if label else manifest["name"],
+            show_tile=False,
+            protected=False,
+        )
 
     # Prepare env. var. to pass to script
     env_dict = _make_environment_for_app_script(
-        app_instance_name, args=args, workdir=extracted_app_folder
+        app_instance_name, args=args, workdir=extracted_app_folder, action="install"
     )
 
     env_dict_for_logging = env_dict.copy()
@@ -929,7 +1023,7 @@ def app_install(
 
             # Setup environment for remove script
             env_dict_remove = _make_environment_for_app_script(
-                app_instance_name, workdir=extracted_app_folder
+                app_instance_name, workdir=extracted_app_folder, action="remove"
             )
 
             # Execute remove script
@@ -960,10 +1054,18 @@ def app_install(
                     m18n.n("unexpected_error", error="\n" + traceback.format_exc())
                 )
 
-            # Remove all permission in LDAP
-            for permission_name in user_permission_list()["permissions"].keys():
-                if permission_name.startswith(app_instance_name + "."):
-                    permission_delete(permission_name, force=True, sync_perm=False)
+            if packaging_format >= 2:
+                from yunohost.utils.resources import AppResourceManager
+                try:
+                    AppResourceManager(app_instance_name, wanted={}, current=manifest).apply(rollback_if_failure=False)
+                except Exception:
+                    # FIXME : improve error handling ....
+                    raise
+            else:
+                # Remove all permission in LDAP
+                for permission_name in user_permission_list()["permissions"].keys():
+                    if permission_name.startswith(app_instance_name + "."):
+                        permission_delete(permission_name, force=True, sync_perm=False)
 
             if remove_retcode != 0:
                 msg = m18n.n("app_not_properly_removed", app=app_instance_name)
@@ -999,6 +1101,17 @@ def app_install(
 
     logger.success(m18n.n("installation_complete"))
 
+    # Display post_install notices in cli mode
+    if manifest["notifications"]["post_install"] and Moulinette.interface.type == "cli":
+        # (Call app_info to get the version hydrated with settings)
+        infos = app_info(app_instance_name, full=True)
+        for notice in infos["manifest"]["notifications"]["post_install"].values():
+            # Should we render the markdown maybe? idk
+            print("==========")
+            print(_value_for_locale(notice))
+            print("==========")
+
+    # Call postinstall hook
     hook_callback("post_app_install", env=env_dict)
 
 
@@ -1044,7 +1157,7 @@ def app_remove(operation_logger, app, purge=False):
     remove_script = f"{tmp_workdir_for_app}/scripts/remove"
 
     env_dict = {}
-    env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app)
+    env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app, action="remove")
     env_dict["YNH_APP_PURGE"] = str(1 if purge else 0)
 
     operation_logger.extra.update({"env": env_dict})
@@ -1064,15 +1177,18 @@ def app_remove(operation_logger, app, purge=False):
     finally:
         shutil.rmtree(tmp_workdir_for_app)
 
-    if ret == 0:
-        logger.success(m18n.n("app_removed", app=app))
-        hook_callback("post_app_remove", env=env_dict)
+    packaging_format = manifest["packaging_format"]
+    if packaging_format >= 2:
+        try:
+            from yunohost.utils.resources import AppResourceManager
+            AppResourceManager(app, wanted={}, current=manifest).apply(rollback_if_failure=False)
+        except Exception:
+            # FIXME : improve error handling ....
+            raise
     else:
-        logger.warning(m18n.n("app_not_properly_removed", app=app))
-
-    # Remove all permission in LDAP
-    for permission_name in user_permission_list(apps=[app])["permissions"].keys():
-        permission_delete(permission_name, force=True, sync_perm=False)
+        # Remove all permission in LDAP
+        for permission_name in user_permission_list(apps=[app])["permissions"].keys():
+            permission_delete(permission_name, force=True, sync_perm=False)
 
     if os.path.exists(app_setting_path):
         shutil.rmtree(app_setting_path)
@@ -1083,6 +1199,12 @@ def app_remove(operation_logger, app, purge=False):
         if domain_config_get(domain, "feature.app.default_app") == app:
             domain_config_set(domain, "feature.app.default_app", "_none")
 
+    if ret == 0:
+        logger.success(m18n.n("app_removed", app=app))
+        hook_callback("post_app_remove", env=env_dict)
+    else:
+        logger.warning(m18n.n("app_not_properly_removed", app=app))
+
     permission_sync_to_user()
     _assert_system_is_sane_for_app(manifest, "post")
 
@@ -1291,7 +1413,7 @@ def app_register_url(app, domain, path):
             raise YunohostValidationError("app_already_installed_cant_change_url")
 
     # Check the url is available
-    _assert_no_conflicting_apps(domain, path)
+    _assert_no_conflicting_apps(domain, path, ignore_app=app)
 
     app_setting(app, "domain", value=domain)
     app_setting(app, "path", value=path)
@@ -1474,9 +1596,8 @@ def app_action_run(operation_logger, app, action, args=None):
     tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
 
     env_dict = _make_environment_for_app_script(
-        app, args=args, args_prefix="ACTION_", workdir=tmp_workdir_for_app
+        app, args=args, args_prefix="ACTION_", workdir=tmp_workdir_for_app, action=action
     )
-    env_dict["YNH_ACTION"] = action
 
     _, action_script = tempfile.mkstemp(dir=tmp_workdir_for_app)
 
@@ -1680,15 +1801,7 @@ def _get_app_actions(app_id):
         for key, value in toml_actions.items():
             action = dict(**value)
             action["id"] = key
-
-            arguments = []
-            for argument_name, argument in value.get("arguments", {}).items():
-                argument = dict(**argument)
-                argument["name"] = argument_name
-
-                arguments.append(argument)
-
-            action["arguments"] = arguments
+            action["arguments"] = value.get("arguments", {})
             actions.append(action)
 
         return actions
@@ -1868,21 +1981,7 @@ def _get_manifest_of_app(path):
     #     ¦   ¦   },
 
     if os.path.exists(os.path.join(path, "manifest.toml")):
-        manifest_toml = read_toml(os.path.join(path, "manifest.toml"))
-
-        manifest = manifest_toml.copy()
-
-        install_arguments = []
-        for name, values in (
-            manifest_toml.get("arguments", {}).get("install", {}).items()
-        ):
-            args = values.copy()
-            args["name"] = name
-
-            install_arguments.append(args)
-
-        manifest["arguments"]["install"] = install_arguments
-
+        manifest = read_toml(os.path.join(path, "manifest.toml"))
     elif os.path.exists(os.path.join(path, "manifest.json")):
         manifest = read_json(os.path.join(path, "manifest.json"))
     else:
@@ -1891,25 +1990,150 @@ def _get_manifest_of_app(path):
             raw_msg=True,
         )
 
-    manifest["arguments"] = _set_default_ask_questions(manifest.get("arguments", {}))
+    manifest["packaging_format"] = float(str(manifest.get("packaging_format", "")).strip() or "0")
+
+    if manifest["packaging_format"] < 2:
+        manifest = _convert_v1_manifest_to_v2(manifest)
+
+    manifest["install"] = _set_default_ask_questions(manifest.get("install", {}))
+    manifest["doc"], manifest["notifications"] = _parse_app_doc_and_notifications(path)
+
     return manifest
 
 
-def _set_default_ask_questions(arguments):
+def _parse_app_doc_and_notifications(path):
+
+    doc = {}
+
+    for filepath in glob.glob(os.path.join(path, "doc") + "/*.md"):
+
+        # to be improved : [a-z]{2,3} is a clumsy way of parsing the
+        # lang code ... some lang code are more complex that this é_è
+        m = re.match("([A-Z]*)(_[a-z]{2,3})?.md", filepath.split("/")[-1])
+
+        if not m:
+            # FIXME: shall we display a warning ? idk
+            continue
+        pagename, lang = m.groups()
+        lang = lang.strip("_") if lang else "en"
+
+        if pagename not in doc:
+            doc[pagename] = {}
+        doc[pagename][lang] = read_file(filepath).strip()
+
+    notifications = {}
+
+    for step in ["pre_install", "post_install", "pre_upgrade", "post_upgrade"]:
+        notifications[step] = {}
+        for filepath in glob.glob(os.path.join(path, "doc", "notifications", f"{step}*.md")):
+            m = re.match(step + "(_[a-z]{2,3})?.md", filepath.split("/")[-1])
+            if not m:
+                continue
+            pagename = "main"
+            lang = m.groups()[0].strip("_") if m.groups()[0] else "en"
+            if pagename not in notifications[step]:
+                notifications[step][pagename] = {}
+            notifications[step][pagename][lang] = read_file(filepath).strip()
+
+        for filepath in glob.glob(os.path.join(path, "doc", "notifications", f"{step}.d") + "/*.md"):
+            m = re.match(r"([A-Za-z0-9\.\~]*)(_[a-z]{2,3})?.md", filepath.split("/")[-1])
+            if not m:
+                continue
+            pagename, lang = m.groups()
+            lang = lang.strip("_") if lang else "en"
+            if pagename not in notifications[step]:
+                notifications[step][pagename] = {}
+            notifications[step][pagename][lang] = read_file(filepath).strip()
+
+    return doc, notifications
+
+
+def _hydrate_app_template(template, data):
+
+    stuff_to_replace = set(re.findall(r'__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__', template))
+
+    for stuff in stuff_to_replace:
+
+        varname = stuff.strip("_").lower()
+
+        if varname in data:
+            template = template.replace(stuff, data[varname])
+
+    return template
+
+
+def _convert_v1_manifest_to_v2(manifest):
+
+    manifest = copy.deepcopy(manifest)
+
+    if "upstream" not in manifest:
+        manifest["upstream"] = {}
+
+    if "license" in manifest and "license" not in manifest["upstream"]:
+        manifest["upstream"]["license"] = manifest["license"]
+
+    if "url" in manifest and "website" not in manifest["upstream"]:
+        manifest["upstream"]["website"] = manifest["url"]
+
+    manifest["integration"] = {
+        "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""),
+        "architectures": "all",
+        "multi_instance": manifest.get("multi_instance", False),
+        "ldap": "?",
+        "sso": "?",
+        "disk": "50M",
+        "ram": {"build": "50M", "runtime": "10M"}
+    }
+
+    maintainers = manifest.get("maintainer", {})
+    if isinstance(maintainers, list):
+        maintainers = [m['name'] for m in maintainers]
+    else:
+        maintainers = [maintainers["name"]] if maintainers.get("name") else []
+
+    manifest["maintainers"] = maintainers
+
+    install_questions = manifest["arguments"]["install"]
+
+    manifest["install"] = {}
+    for question in install_questions:
+        name = question.pop("name")
+        if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]:
+            question.pop("ask")
+        if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]:
+            question.pop("example")
+
+        manifest["install"][name] = question
+
+    manifest["resources"] = {
+        "system_user": {},
+        "install_dir": {
+            "alias": "final_path"
+        }
+    }
+
+    keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"]
+
+    keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep]
+    for key in keys_to_del:
+        del manifest[key]
+
+    return manifest
+
+
+def _set_default_ask_questions(questions, script_name="install"):
 
     # arguments is something like
-    # { "install": [
-    #       { "name": "domain",
+    # { "domain":
+    #       {
     #         "type": "domain",
     #         ....
     #       },
-    #       { "name": "path",
-    #         "type": "path"
+    #    "path": {
+    #         "type": "path",
     #         ...
     #       },
     #       ...
-    #   ],
-    #  "upgrade": [ ... ]
     # }
 
     # We set a default for any question with these matching (type, name)
@@ -1921,39 +2145,28 @@ def _set_default_ask_questions(arguments):
         ("path", "path"),  # i18n: app_manifest_install_ask_path
         ("password", "password"),  # i18n: app_manifest_install_ask_password
         ("user", "admin"),  # i18n: app_manifest_install_ask_admin
-        ("boolean", "is_public"),
-    ]  # i18n: app_manifest_install_ask_is_public
+        ("boolean", "is_public"),  # i18n: app_manifest_install_ask_is_public
+    ]
 
-    for script_name, arg_list in arguments.items():
+    for question_name, question in questions.items():
+        question["name"] = question_name
 
-        # We only support questions for install so far, and for other
-        if script_name != "install":
-            continue
-
-        for arg in arg_list:
-
-            # Do not override 'ask' field if provided by app ?... Or shall we ?
-            # if "ask" in arg:
-            #    continue
-
-            # If this arg corresponds to a question with default ask message...
-            if any(
-                (arg.get("type"), arg["name"]) == question
-                for question in questions_with_default
-            ):
-                # The key is for example "app_manifest_install_ask_domain"
-                arg_name = arg["name"]
-                key = f"app_manifest_{script_name}_ask_{arg_name}"
-                arg["ask"] = m18n.n(key)
+        # If this question corresponds to a question with default ask message...
+        if any(
+            (question.get("type"), question["name"]) == question_with_default
+            for question_with_default in questions_with_default
+        ):
+            # The key is for example "app_manifest_install_ask_domain"
+            question["ask"] = m18n.n(f"app_manifest_{script_name}_ask_{question['name']}")
 
             # Also it in fact doesn't make sense for any of those questions to have an example value nor a default value...
-            if arg.get("type") in ["domain", "user", "password"]:
-                if "example" in arg:
-                    del arg["example"]
-                if "default" in arg:
-                    del arg["default"]
+            if question.get("type") in ["domain", "user", "password"]:
+                if "example" in question:
+                    del question["example"]
+                if "default" in question:
+                    del question["default"]
 
-    return arguments
+    return questions
 
 
 def _is_app_repo_url(string: str) -> bool:
@@ -2183,33 +2396,61 @@ def _get_all_installed_apps_id():
     return all_apps_ids_formatted
 
 
-def _check_manifest_requirements(manifest: Dict):
+def _check_manifest_requirements(manifest: Dict, action: str):
     """Check if required packages are met from the manifest"""
 
-    packaging_format = int(manifest.get("packaging_format", 0))
-    if packaging_format not in [0, 1]:
+    if manifest["packaging_format"] not in [1, 2]:
         raise YunohostValidationError("app_packaging_format_not_supported")
 
-    requirements = manifest.get("requirements", dict())
+    app_id = manifest["id"]
 
-    if not requirements:
-        return
+    logger.debug(m18n.n("app_requirements_checking", app=app_id))
 
-    app = manifest.get("id", "?")
+    # Yunohost version requirement
 
-    logger.debug(m18n.n("app_requirements_checking", app=app))
+    yunohost_requirement = version.parse(manifest["integration"]["yunohost"] or "4.3")
+    yunohost_installed_version = version.parse(get_ynh_package_version("yunohost")["version"])
+    if yunohost_requirement > yunohost_installed_version:
+        # FIXME : i18n
+        raise YunohostValidationError(f"This app requires Yunohost >= {yunohost_requirement} but current installed version is {yunohost_installed_version}")
 
-    # Iterate over requirements
-    for pkgname, spec in requirements.items():
-        if not packages.meets_version_specifier(pkgname, spec):
-            version = packages.ynh_packages_version()[pkgname]["version"]
-            raise YunohostValidationError(
-                "app_requirements_unmeet",
-                pkgname=pkgname,
-                version=version,
-                spec=spec,
-                app=app,
-            )
+    # Architectures
+    arch_requirement = manifest["integration"]["architectures"]
+    if arch_requirement != "all":
+        arch = system_arch()
+        if arch not in arch_requirement:
+            # FIXME: i18n
+            raise YunohostValidationError(f"This app can only be installed on architectures {', '.join(arch_requirement)} but your server architecture is {arch}")
+
+    # Multi-instance
+    if action == "install" and manifest["integration"]["multi_instance"] == False:
+        apps = _installed_apps()
+        sibling_apps = [a for a in apps if a == app_id or a.startswith(f"{app_id}__")]
+        if len(sibling_apps) > 0:
+            raise YunohostValidationError("app_already_installed", app=app_id)
+
+    # Disk
+    if action == "install":
+        disk_requirement = manifest["integration"]["disk"]
+
+        if free_space_in_directory("/") <= human_to_binary(disk_requirement) \
+        or free_space_in_directory("/var") <= human_to_binary(disk_requirement):
+            # FIXME : i18m
+            raise YunohostValidationError(f"This app requires {disk_requirement} free space.")
+
+    # Ram for build
+    ram_build_requirement = manifest["integration"]["ram"]["build"]
+    # Is "include_swap" really useful ? We should probably decide wether to always include it or not instead
+    ram_include_swap = manifest["integration"]["ram"].get("include_swap", False)
+
+    ram, swap = ram_available()
+    if ram_include_swap:
+        ram += swap
+
+    if ram < human_to_binary(ram_build_requirement):
+        # FIXME : i18n
+        ram_human = binary_to_human(ram)
+        raise YunohostValidationError(f"This app requires {ram_build_requirement} RAM to install/upgrade but only {ram_human} is available right now.")
 
 
 def _guess_webapp_path_requirement(app_folder: str) -> str:
@@ -2218,13 +2459,13 @@ def _guess_webapp_path_requirement(app_folder: str) -> str:
     # is an available url and normalize the path.
 
     manifest = _get_manifest_of_app(app_folder)
-    raw_questions = manifest.get("arguments", {}).get("install", {})
+    raw_questions = manifest["install"]
 
     domain_questions = [
-        question for question in raw_questions if question.get("type") == "domain"
+        question for question in raw_questions.values() if question.get("type") == "domain"
     ]
     path_questions = [
-        question for question in raw_questions if question.get("type") == "path"
+        question for question in raw_questions.values() if question.get("type") == "path"
     ]
 
     if len(domain_questions) == 0 and len(path_questions) == 0:
@@ -2324,7 +2565,11 @@ def _assert_no_conflicting_apps(domain, path, ignore_app=None, full_domain=False
 
 
 def _make_environment_for_app_script(
-    app, args={}, args_prefix="APP_ARG_", workdir=None
+    app,
+    args={},
+    args_prefix="APP_ARG_",
+    workdir=None,
+    action=None
 ):
 
     app_setting_path = os.path.join(APPS_SETTING_PATH, app)
@@ -2337,16 +2582,37 @@ def _make_environment_for_app_script(
         "YNH_APP_INSTANCE_NAME": app,
         "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
         "YNH_APP_MANIFEST_VERSION": manifest.get("version", "?"),
-        "YNH_ARCH": check_output("dpkg --print-architecture"),
+        "YNH_APP_PACKAGING_FORMAT": str(manifest["packaging_format"]),
+        "YNH_ARCH": system_arch(),
     }
 
     if workdir:
         env_dict["YNH_APP_BASEDIR"] = workdir
 
+    if action:
+        env_dict["YNH_APP_ACTION"] = action
+
     for arg_name, arg_value in args.items():
         arg_name_upper = arg_name.upper()
         env_dict[f"YNH_{args_prefix}{arg_name_upper}"] = str(arg_value)
 
+    # If packaging format v2, load all settings
+    if manifest["packaging_format"] >= 2:
+        env_dict["app"] = app
+        for setting_name, setting_value in _get_app_settings(app).items():
+
+            # Ignore special internal settings like checksum__
+            # (not a huge deal to load them but idk...)
+            if setting_name.startswith("checksum__"):
+                continue
+
+            env_dict[setting_name] = str(setting_value)
+
+        # Special weird case for backward compatibility...
+        # 'path' was loaded into 'path_url' .....
+        if 'path' in env_dict:
+            env_dict["path_url"] = env_dict["path"]
+
     return env_dict
 
 
@@ -2426,26 +2692,6 @@ def _make_tmp_workdir_for_app(app=None):
     return tmpdir
 
 
-def is_true(arg):
-    """
-    Convert a string into a boolean
-
-    Keyword arguments:
-        arg -- The string to convert
-
-    Returns:
-        Boolean
-
-    """
-    if isinstance(arg, bool):
-        return arg
-    elif isinstance(arg, str):
-        return arg.lower() in ["yes", "true", "on"]
-    else:
-        logger.debug(f"arg should be a boolean or a string, got {arg}")
-        return True if arg else False
-
-
 def unstable_apps():
 
     output = []
@@ -2523,8 +2769,9 @@ def _assert_system_is_sane_for_app(manifest, when):
                 "app_action_broke_system", services=", ".join(faulty_services)
             )
 
-    if packages.dpkg_is_broken():
+    if dpkg_is_broken():
         if when == "pre":
             raise YunohostValidationError("dpkg_is_broken")
         elif when == "post":
             raise YunohostError("this_action_broke_dpkg")
+

src/app_catalog.py

@@ -19,7 +19,7 @@ logger = getActionLogger("yunohost.app_catalog")
 
 APPS_CATALOG_CACHE = "/var/cache/yunohost/repo"
 APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml"
-APPS_CATALOG_API_VERSION = 2
+APPS_CATALOG_API_VERSION = 3
 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
 
 
@@ -48,8 +48,8 @@ def app_catalog(full=False, with_categories=False):
                 "level": infos["level"],
             }
         else:
-            infos["manifest"]["arguments"] = _set_default_ask_questions(
-                infos["manifest"].get("arguments", {})
+            infos["manifest"]["install"] = _set_default_ask_questions(
+                infos["manifest"].get("install", {})
             )
 
     # Trim info for categories if not using --full
@@ -232,6 +232,8 @@ def _load_apps_catalog():
                 )
                 continue
 
+            # FIXME: we may want to autoconvert all v0/v1 manifest to v2 here
+            # so that everything is consistent in terms of APIs, datastructure format etc
             info["repository"] = apps_catalog_id
             merged_catalog["apps"][app] = info
 

src/backup.py

@@ -39,9 +39,8 @@ from functools import reduce
 from packaging import version
 
 from moulinette import Moulinette, m18n
-from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml
+from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml, rm, chown, chmod
 from moulinette.utils.process import check_output
 
 import yunohost.domain
@@ -50,6 +49,7 @@ from yunohost.app import (
     _is_installed,
     _make_environment_for_app_script,
     _make_tmp_workdir_for_app,
+    _get_manifest_of_app,
 )
 from yunohost.hook import (
     hook_list,
@@ -67,8 +67,12 @@ from yunohost.tools import (
 from yunohost.regenconf import regen_conf
 from yunohost.log import OperationLogger, is_unit_operation
 from yunohost.utils.error import YunohostError, YunohostValidationError
-from yunohost.utils.packages import ynh_packages_version
-from yunohost.utils.filesystem import free_space_in_directory
+from yunohost.utils.system import (
+    free_space_in_directory,
+    get_ynh_package_version,
+    binary_to_human,
+    space_used_by_directory,
+)
 from yunohost.settings import settings_get
 
 BACKUP_PATH = "/home/yunohost.backup"
@@ -312,7 +316,7 @@ class BackupManager:
             "size_details": self.size_details,
             "apps": self.apps_return,
             "system": self.system_return,
-            "from_yunohost_version": ynh_packages_version()["yunohost"]["version"],
+            "from_yunohost_version": get_ynh_package_version("yunohost")["version"],
         }
 
     @property
@@ -342,7 +346,7 @@ class BackupManager:
         # FIXME replace isdir by exists ? manage better the case where the path
         # exists
         if not os.path.isdir(self.work_dir):
-            filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin")
+            mkdir(self.work_dir, 0o750, parents=True, uid="admin")
         elif self.is_tmp_work_dir:
 
             logger.debug(
@@ -357,8 +361,8 @@ class BackupManager:
                 # If umount succeeded, remove the directory (we checked that
                 # we're in /home/yunohost.backup/tmp so that should be okay...
                 # c.f. method clean() which also does this)
-                filesystem.rm(self.work_dir, recursive=True, force=True)
-                filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin")
+                rm(self.work_dir, recursive=True, force=True)
+                mkdir(self.work_dir, 0o750, parents=True, uid="admin")
 
     #
     # Backup target management                                              #
@@ -535,7 +539,7 @@ class BackupManager:
         successfull_system = self.targets.list("system", include=["Success", "Warning"])
 
         if not successfull_apps and not successfull_system:
-            filesystem.rm(self.work_dir, True, True)
+            rm(self.work_dir, True, True)
             raise YunohostError("backup_nothings_done")
 
         # Add unlisted files from backup tmp dir
@@ -577,7 +581,7 @@ class BackupManager:
         env_var["YNH_BACKUP_CSV"] = tmp_csv
 
         if app is not None:
-            env_var.update(_make_environment_for_app_script(app))
+            env_var.update(_make_environment_for_app_script(app, action="backup"))
             env_var["YNH_APP_BACKUP_DIR"] = os.path.join(
                 self.work_dir, "apps", app, "backup"
             )
@@ -647,7 +651,7 @@ class BackupManager:
 
         restore_hooks_dir = os.path.join(self.work_dir, "hooks", "restore")
         if not os.path.exists(restore_hooks_dir):
-            filesystem.mkdir(restore_hooks_dir, mode=0o700, parents=True, uid="root")
+            mkdir(restore_hooks_dir, mode=0o700, parents=True, uid="root")
 
         restore_hooks = hook_list("restore")["hooks"]
 
@@ -714,7 +718,7 @@ class BackupManager:
         tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
         try:
             # Prepare backup directory for the app
-            filesystem.mkdir(tmp_app_bkp_dir, 0o700, True, uid="root")
+            mkdir(tmp_app_bkp_dir, 0o700, True, uid="root")
 
             # Copy the app settings to be able to call _common.sh
             shutil.copytree(app_setting_path, settings_dir)
@@ -753,7 +757,7 @@ class BackupManager:
         # Remove tmp files in all situations
         finally:
             shutil.rmtree(tmp_workdir_for_app)
-            filesystem.rm(env_dict["YNH_BACKUP_CSV"], force=True)
+            rm(env_dict["YNH_BACKUP_CSV"], force=True)
 
     #
     # Actual backup archive creation / method management                    #
@@ -796,7 +800,7 @@ class BackupManager:
             if row["dest"] == "info.json":
                 continue
 
-            size = disk_usage(row["source"])
+            size = space_used_by_directory(row["source"], follow_symlinks=False)
 
             # Add size to apps details
             splitted_dest = row["dest"].split("/")
@@ -949,7 +953,7 @@ class RestoreManager:
             ret = subprocess.call(["umount", self.work_dir])
             if ret != 0:
                 logger.warning(m18n.n("restore_cleaning_failed"))
-        filesystem.rm(self.work_dir, recursive=True, force=True)
+        rm(self.work_dir, recursive=True, force=True)
 
     #
     # Restore target manangement                                            #
@@ -979,7 +983,7 @@ class RestoreManager:
         available_restore_system_hooks = hook_list("restore")["hooks"]
 
         custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, "restore")
-        filesystem.mkdir(custom_restore_hook_folder, 755, parents=True, force=True)
+        mkdir(custom_restore_hook_folder, 755, parents=True, force=True)
 
         for system_part in target_list:
             # By default, we'll use the restore hooks on the current install
@@ -1084,7 +1088,7 @@ class RestoreManager:
             else:
                 raise YunohostError("restore_removing_tmp_dir_failed")
 
-        filesystem.mkdir(self.work_dir, parents=True)
+        mkdir(self.work_dir, parents=True)
 
         self.method.mount()
 
@@ -1402,7 +1406,7 @@ class RestoreManager:
 
         # Delete _common.sh file in backup
         common_file = os.path.join(app_backup_in_archive, "_common.sh")
-        filesystem.rm(common_file, force=True)
+        rm(common_file, force=True)
 
         # Check if the app has a restore script
         app_restore_script_in_archive = os.path.join(app_scripts_in_archive, "restore")
@@ -1418,14 +1422,14 @@ class RestoreManager:
             )
             app_scripts_new_path = os.path.join(app_settings_new_path, "scripts")
             shutil.copytree(app_settings_in_archive, app_settings_new_path)
-            filesystem.chmod(app_settings_new_path, 0o400, 0o400, True)
-            filesystem.chown(app_scripts_new_path, "root", None, True)
+            chmod(app_settings_new_path, 0o400, 0o400, True)
+            chown(app_scripts_new_path, "root", None, True)
 
             # Copy the app scripts to a writable temporary folder
             tmp_workdir_for_app = _make_tmp_workdir_for_app()
             copytree(app_scripts_in_archive, tmp_workdir_for_app)
-            filesystem.chmod(tmp_workdir_for_app, 0o700, 0o700, True)
-            filesystem.chown(tmp_workdir_for_app, "root", None, True)
+            chmod(tmp_workdir_for_app, 0o700, 0o700, True)
+            chown(tmp_workdir_for_app, "root", None, True)
             restore_script = os.path.join(tmp_workdir_for_app, "restore")
 
             # Restore permissions
@@ -1494,7 +1498,7 @@ class RestoreManager:
         # FIXME : workdir should be a tmp workdir
         app_workdir = os.path.join(self.work_dir, "apps", app_instance_name, "settings")
         env_dict = _make_environment_for_app_script(
-            app_instance_name, workdir=app_workdir
+            app_instance_name, workdir=app_workdir, action="restore"
         )
         env_dict.update(
             {
@@ -1509,6 +1513,15 @@ class RestoreManager:
         operation_logger.extra["env"] = env_dict
         operation_logger.flush()
 
+        manifest = _get_manifest_of_app(app_settings_in_archive)
+        if manifest["packaging_format"] >= 2:
+            from yunohost.utils.resources import AppResourceManager
+            try:
+                AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(rollback_if_failure=True)
+            except Exception:
+                # FIXME : improve error handling ....
+                raise
+
         # Execute the app install script
         restore_failed = True
         try:
@@ -1727,7 +1740,7 @@ class BackupMethod:
                 raise YunohostError("backup_cleaning_failed")
 
         if self.manager.is_tmp_work_dir:
-            filesystem.rm(self.work_dir, True, True)
+            rm(self.work_dir, True, True)
 
     def _check_is_enough_free_space(self):
         """
@@ -1775,11 +1788,11 @@ class BackupMethod:
 
             # Be sure the parent dir of destination exists
             if not os.path.isdir(dest_dir):
-                filesystem.mkdir(dest_dir, parents=True)
+                mkdir(dest_dir, parents=True)
 
             # For directory, attempt to mount bind
             if os.path.isdir(src):
-                filesystem.mkdir(dest, parents=True, force=True)
+                mkdir(dest, parents=True, force=True)
 
                 try:
                     subprocess.check_call(["mount", "--rbind", src, dest])
@@ -1832,7 +1845,7 @@ class BackupMethod:
         # to mounting error
 
         # Compute size to copy
-        size = sum(disk_usage(path["source"]) for path in paths_needed_to_be_copied)
+        size = sum(space_used_by_directory(path["source"], follow_symlinks=False) for path in paths_needed_to_be_copied)
         size /= 1024 * 1024  # Convert bytes to megabytes
 
         # Ask confirmation for copying
@@ -1884,7 +1897,7 @@ class CopyBackupMethod(BackupMethod):
 
             dest_parent = os.path.dirname(dest)
             if not os.path.exists(dest_parent):
-                filesystem.mkdir(dest_parent, 0o700, True, uid="admin")
+                mkdir(dest_parent, 0o700, True, uid="admin")
 
             if os.path.isdir(source):
                 shutil.copytree(source, dest)
@@ -1902,7 +1915,7 @@ class CopyBackupMethod(BackupMethod):
         if not os.path.isdir(self.repo):
             raise YunohostError("backup_no_uncompress_archive_dir")
 
-        filesystem.mkdir(self.work_dir, parent=True)
+        mkdir(self.work_dir, parent=True)
         ret = subprocess.call(["mount", "-r", "--rbind", self.repo, self.work_dir])
         if ret == 0:
             return
@@ -1946,7 +1959,7 @@ class TarBackupMethod(BackupMethod):
         """
 
         if not os.path.exists(self.repo):
-            filesystem.mkdir(self.repo, 0o750, parents=True, uid="admin")
+            mkdir(self.repo, 0o750, parents=True, uid="admin")
 
         # Check free space in output
         self._check_is_enough_free_space()
@@ -2667,31 +2680,3 @@ def _recursive_umount(directory):
             continue
 
     return everything_went_fine
-
-
-def disk_usage(path):
-    # We don't do this in python with os.stat because we don't want
-    # to follow symlinks
-
-    du_output = check_output(["du", "-sb", path], shell=False)
-    return int(du_output.split()[0])
-
-
-def binary_to_human(n, customary=False):
-    """
-    Convert bytes or bits into human readable format with binary prefix
-    Keyword argument:
-        n -- Number to convert
-        customary -- Use customary symbol instead of IEC standard
-    """
-    symbols = ("Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
-    if customary:
-        symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
-    prefix = {}
-    for i, s in enumerate(symbols):
-        prefix[s] = 1 << (i + 1) * 10
-    for s in reversed(symbols):
-        if n >= prefix[s]:
-            value = float(n) / prefix[s]
-            return "{:.1f}{}".format(value, s)
-    return "%s" % n

src/diagnosers/00-basesystem.py

@@ -9,7 +9,11 @@ from moulinette.utils import log
 from moulinette.utils.process import check_output
 from moulinette.utils.filesystem import read_file, read_json, write_to_json
 from yunohost.diagnosis import Diagnoser
-from yunohost.utils.packages import ynh_packages_version
+from yunohost.utils.system import (
+    ynh_packages_version,
+    system_virt,
+    system_arch,
+)
 
 logger = log.getActionLogger("yunohost.diagnosis")
 
@@ -22,15 +26,12 @@ class MyDiagnoser(Diagnoser):
 
     def run(self):
 
-        # Detect virt technology (if not bare metal) and arch
-        # Gotta have this "|| true" because it systemd-detect-virt return 'none'
-        # with an error code on bare metal ~.~
-        virt = check_output("systemd-detect-virt || true", shell=True)
+        virt = system_virt()
         if virt.lower() == "none":
             virt = "bare-metal"
 
         # Detect arch
-        arch = check_output("dpkg --print-architecture")
+        arch = system_arch()
         hardware = dict(
             meta={"test": "hardware"},
             status="INFO",

src/log.py

@@ -38,7 +38,7 @@ from io import IOBase
 from moulinette import m18n, Moulinette
 from moulinette.core import MoulinetteError
 from yunohost.utils.error import YunohostError, YunohostValidationError
-from yunohost.utils.packages import get_ynh_package_version
+from yunohost.utils.system import get_ynh_package_version
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file, read_yaml
 

src/migrations/0021_migrate_to_bullseye.py

@@ -15,8 +15,8 @@ from yunohost.tools import (
 )
 from yunohost.app import unstable_apps
 from yunohost.regenconf import manually_modified_files, _force_clear_hashes
-from yunohost.utils.filesystem import free_space_in_directory
-from yunohost.utils.packages import (
+from yunohost.utils.system import (
+    free_space_in_directory,
     get_ynh_package_version,
     _list_upgradable_apt_packages,
 )

src/migrations/0023_postgresql_11_to_13.py

@@ -7,7 +7,7 @@ from yunohost.utils.error import YunohostError, YunohostValidationError
 from moulinette.utils.log import getActionLogger
 
 from yunohost.tools import Migration
-from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory
+from yunohost.utils.system import free_space_in_directory, space_used_by_directory
 
 logger = getActionLogger("yunohost.migration")
 

src/tests/test_app_resources.py

@@ -0,0 +1,392 @@
+import os
+import pytest
+
+from moulinette.utils.process import check_output
+
+from yunohost.app import app_setting
+from yunohost.domain import _get_maindomain
+from yunohost.utils.resources import AppResource, AppResourceManager, AppResourceClassesByType
+from yunohost.permission import user_permission_list, permission_delete
+
+dummyfile = "/tmp/dummyappresource-testapp"
+
+
+class DummyAppResource(AppResource):
+
+    type = "dummy"
+
+    default_properties = {
+        "file": "/tmp/dummyappresource-__APP__",
+        "content": "foo",
+    }
+
+    def provision_or_update(self, context):
+
+        open(self.file, "w").write(self.content)
+
+        if self.content == "forbiddenvalue":
+            raise Exception("Emeged you used the forbidden value!1!£&")
+
+    def deprovision(self, context):
+
+        os.system(f"rm -f {self.file}")
+
+
+AppResourceClassesByType["dummy"] = DummyAppResource
+
+
+def setup_function(function):
+
+    clean()
+
+    os.system("mkdir /etc/yunohost/apps/testapp")
+    os.system("echo 'id: testapp' > /etc/yunohost/apps/testapp/settings.yml")
+    os.system("echo 'packaging_format = 2' > /etc/yunohost/apps/testapp/manifest.toml")
+    os.system("echo 'id = \"testapp\"' >> /etc/yunohost/apps/testapp/manifest.toml")
+
+
+def teardown_function(function):
+
+    clean()
+
+
+def clean():
+
+    os.system(f"rm -f {dummyfile}")
+    os.system("rm -rf /etc/yunohost/apps/testapp")
+    os.system("rm -rf /var/www/testapp")
+    os.system("rm -rf /home/yunohost.app/testapp")
+    os.system("apt remove lolcat sl nyancat yarn >/dev/null 2>/dev/null")
+    os.system("userdel testapp 2>/dev/null")
+
+    for p in user_permission_list()["permissions"]:
+        if p.startswith("testapp."):
+            permission_delete(p, force=True, sync_perm=False)
+
+
+def test_provision_dummy():
+
+    current = {"resources": {}}
+    wanted = {"resources": {"dummy": {}}}
+
+    assert not os.path.exists(dummyfile)
+    AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False)
+    assert open(dummyfile).read().strip() == "foo"
+
+
+def test_deprovision_dummy():
+
+    current = {"resources": {"dummy": {}}}
+    wanted = {"resources": {}}
+
+    open(dummyfile, "w").write("foo")
+
+    assert open(dummyfile).read().strip() == "foo"
+    AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False)
+    assert not os.path.exists(dummyfile)
+
+
+def test_provision_dummy_nondefaultvalue():
+
+    current = {"resources": {}}
+    wanted = {"resources": {"dummy": {"content": "bar"}}}
+
+    assert not os.path.exists(dummyfile)
+    AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False)
+    assert open(dummyfile).read().strip() == "bar"
+
+
+def test_update_dummy():
+
+    current = {"resources": {"dummy": {}}}
+    wanted = {"resources": {"dummy": {"content": "bar"}}}
+
+    open(dummyfile, "w").write("foo")
+
+    assert open(dummyfile).read().strip() == "foo"
+    AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False)
+    assert open(dummyfile).read().strip() == "bar"
+
+
+def test_update_dummy_fail():
+
+    current = {"resources": {"dummy": {}}}
+    wanted = {"resources": {"dummy": {"content": "forbiddenvalue"}}}
+
+    open(dummyfile, "w").write("foo")
+
+    assert open(dummyfile).read().strip() == "foo"
+    with pytest.raises(Exception):
+        AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False)
+    assert open(dummyfile).read().strip() == "forbiddenvalue"
+
+
+def test_update_dummy_failwithrollback():
+
+    current = {"resources": {"dummy": {}}}
+    wanted = {"resources": {"dummy": {"content": "forbiddenvalue"}}}
+
+    open(dummyfile, "w").write("foo")
+
+    assert open(dummyfile).read().strip() == "foo"
+    with pytest.raises(Exception):
+        AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=True)
+    assert open(dummyfile).read().strip() == "foo"
+
+
+def test_resource_system_user():
+
+    r = AppResourceClassesByType["system_user"]
+
+    conf = {}
+
+    assert os.system("getent passwd testapp 2>/dev/null") != 0
+
+    r(conf, "testapp").provision_or_update()
+
+    assert os.system("getent passwd testapp >/dev/null") == 0
+    assert os.system("groups testapp | grep -q 'sftp.app'") != 0
+
+    conf["allow_sftp"] = True
+    r(conf, "testapp").provision_or_update()
+
+    assert os.system("getent passwd testapp >/dev/null") == 0
+    assert os.system("groups testapp | grep -q 'sftp.app'") == 0
+
+    r(conf, "testapp").deprovision()
+
+    assert os.system("getent passwd testapp 2>/dev/null") != 0
+
+
+def test_resource_install_dir():
+
+    r = AppResourceClassesByType["install_dir"]
+    conf = {"owner": "nobody:rx", "group": "nogroup:rx"}
+
+    # FIXME: should also check settings ?
+    # FIXME: should also check automigrate from final_path
+    # FIXME: should also test changing the install folder location ?
+
+    assert not os.path.exists("/var/www/testapp")
+
+    r(conf, "testapp").provision_or_update()
+
+    assert os.path.exists("/var/www/testapp")
+    unixperms = check_output("ls -ld /var/www/testapp").split()
+    assert unixperms[0] == "dr-xr-x---"
+    assert unixperms[2] == "nobody"
+    assert unixperms[3] == "nogroup"
+
+    conf["owner"] = "nobody:rwx"
+    conf["group"] = "www-data:x"
+
+    r(conf, "testapp").provision_or_update()
+
+    assert os.path.exists("/var/www/testapp")
+    unixperms = check_output("ls -ld /var/www/testapp").split()
+    assert unixperms[0] == "drwx--x---"
+    assert unixperms[2] == "nobody"
+    assert unixperms[3] == "www-data"
+
+    r(conf, "testapp").deprovision()
+
+    assert not os.path.exists("/var/www/testapp")
+
+
+def test_resource_data_dir():
+
+    r = AppResourceClassesByType["data_dir"]
+    conf = {"owner": "nobody:rx", "group": "nogroup:rx"}
+
+    assert not os.path.exists("/home/yunohost.app/testapp")
+
+    r(conf, "testapp").provision_or_update()
+
+    assert os.path.exists("/home/yunohost.app/testapp")
+    unixperms = check_output("ls -ld /home/yunohost.app/testapp").split()
+    assert unixperms[0] == "dr-xr-x---"
+    assert unixperms[2] == "nobody"
+    assert unixperms[3] == "nogroup"
+
+    conf["owner"] = "nobody:rwx"
+    conf["group"] = "www-data:x"
+
+    r(conf, "testapp").provision_or_update()
+
+    assert os.path.exists("/home/yunohost.app/testapp")
+    unixperms = check_output("ls -ld /home/yunohost.app/testapp").split()
+    assert unixperms[0] == "drwx--x---"
+    assert unixperms[2] == "nobody"
+    assert unixperms[3] == "www-data"
+
+    r(conf, "testapp").deprovision()
+
+    # FIXME : implement and check purge option
+    #assert not os.path.exists("/home/yunohost.app/testapp")
+
+
+def test_resource_ports():
+
+    r = AppResourceClassesByType["ports"]
+    conf = {}
+
+    assert not app_setting("testapp", "port")
+
+    r(conf, "testapp").provision_or_update()
+
+    assert app_setting("testapp", "port")
+
+    r(conf, "testapp").deprovision()
+
+    assert not app_setting("testapp", "port")
+
+
+def test_resource_ports_several():
+
+    r = AppResourceClassesByType["ports"]
+    conf = {"main": {"default": 12345}, "foobar": {"default": 23456}}
+
+    assert not app_setting("testapp", "port")
+    assert not app_setting("testapp", "port_foobar")
+
+    r(conf, "testapp").provision_or_update()
+
+    assert app_setting("testapp", "port")
+    assert app_setting("testapp", "port_foobar")
+
+    r(conf, "testapp").deprovision()
+
+    assert not app_setting("testapp", "port")
+    assert not app_setting("testapp", "port_foobar")
+
+
+def test_resource_database():
+
+    r = AppResourceClassesByType["database"]
+    conf = {"type": "mysql"}
+
+    assert os.system("mysqlshow 'testapp' >/dev/null 2>/dev/null") != 0
+    assert not app_setting("testapp", "db_name")
+    assert not app_setting("testapp", "db_user")
+    assert not app_setting("testapp", "db_pwd")
+
+    r(conf, "testapp").provision_or_update()
+
+    assert os.system("mysqlshow 'testapp' >/dev/null 2>/dev/null") == 0
+    assert app_setting("testapp", "db_name")
+    assert app_setting("testapp", "db_user")
+    assert app_setting("testapp", "db_pwd")
+
+    r(conf, "testapp").deprovision()
+
+    assert os.system("mysqlshow 'testapp' >/dev/null 2>/dev/null") != 0
+    assert not app_setting("testapp", "db_name")
+    assert not app_setting("testapp", "db_user")
+    assert not app_setting("testapp", "db_pwd")
+
+
+def test_resource_apt():
+
+    r = AppResourceClassesByType["apt"]
+    conf = {
+        "packages": "nyancat, sl",
+        "extras": {
+            "yarn": {
+                "repo": "deb https://dl.yarnpkg.com/debian/ stable main",
+                "key": "https://dl.yarnpkg.com/debian/pubkey.gpg",
+                "packages": "yarn",
+            }
+        }
+    }
+
+    assert os.system("dpkg --list | grep -q 'ii *nyancat '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *sl '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *yarn '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") != 0
+
+    r(conf, "testapp").provision_or_update()
+
+    assert os.system("dpkg --list | grep -q 'ii *nyancat '") == 0
+    assert os.system("dpkg --list | grep -q 'ii *sl '") == 0
+    assert os.system("dpkg --list | grep -q 'ii *yarn '") == 0
+    assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0  # Lolcat shouldnt be installed yet
+    assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") == 0
+
+    conf["packages"] += ", lolcat"
+    r(conf, "testapp").provision_or_update()
+
+    assert os.system("dpkg --list | grep -q 'ii *nyancat '") == 0
+    assert os.system("dpkg --list | grep -q 'ii *sl '") == 0
+    assert os.system("dpkg --list | grep -q 'ii *yarn '") == 0
+    assert os.system("dpkg --list | grep -q 'ii *lolcat '") == 0
+    assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") == 0
+
+    r(conf, "testapp").deprovision()
+
+    assert os.system("dpkg --list | grep -q 'ii *nyancat '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *sl '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *yarn '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0
+    assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") != 0
+
+
+def test_resource_permissions():
+
+    maindomain = _get_maindomain()
+    os.system(f"echo 'domain: {maindomain}' >> /etc/yunohost/apps/testapp/settings.yml")
+    os.system("echo 'path: /testapp' >> /etc/yunohost/apps/testapp/settings.yml")
+
+    # A manager object is required to set the label of the app...
+    manager = AppResourceManager("testapp", current={}, wanted={"name": "Test App"})
+    r = AppResourceClassesByType["permissions"]
+    conf = {
+        "main": {
+            "url": "/",
+            "allowed": "visitors"
+            # TODO: test protected?
+        },
+    }
+
+    res = user_permission_list(full=True)["permissions"]
+    assert not any(key.startswith("testapp.") for key in res)
+
+    r(conf, "testapp", manager).provision_or_update()
+
+    res = user_permission_list(full=True)["permissions"]
+    assert "testapp.main" in res
+    assert "visitors" in res["testapp.main"]["allowed"]
+    assert res["testapp.main"]["url"] == "/"
+    assert "testapp.admin" not in res
+
+    conf["admin"] = {
+        "url": "/admin",
+        "allowed": ""
+    }
+
+    r(conf, "testapp", manager).provision_or_update()
+
+    res = user_permission_list(full=True)["permissions"]
+
+    assert "testapp.main" in list(res.keys())
+    assert "visitors" in res["testapp.main"]["allowed"]
+    assert res["testapp.main"]["url"] == "/"
+
+    assert "testapp.admin" in res
+    assert not res["testapp.admin"]["allowed"]
+    assert res["testapp.admin"]["url"] == "/admin"
+
+    conf["admin"]["url"] = "/adminpanel"
+
+    r(conf, "testapp", manager).provision_or_update()
+
+    res = user_permission_list(full=True)["permissions"]
+
+    # FIXME FIXME FIXME : this is the current behavior but
+    # it is NOT okay. c.f. comment in the code
+    assert res["testapp.admin"]["url"] == "/admin"  # should be '/adminpanel'
+
+    r(conf, "testapp").deprovision()
+
+    res = user_permission_list(full=True)["permissions"]
+    assert "testapp.main" not in res

src/tests/test_apps.py

@@ -15,6 +15,8 @@ from yunohost.app import (
     _is_installed,
     app_upgrade,
     app_map,
+    app_manifest,
+    app_info,
 )
 from yunohost.domain import _get_maindomain, domain_add, domain_remove, domain_list
 from yunohost.utils.error import YunohostError
@@ -45,6 +47,7 @@ def clean():
         "break_yo_system",
         "legacy_app",
         "legacy_app__2",
+        "manifestv2_app",
         "full_domain_app",
         "my_webapp",
     ]
@@ -115,7 +118,10 @@ def app_expected_files(domain, app):
     if app.startswith("legacy_app"):
         yield "/var/www/%s/index.html" % app
     yield "/etc/yunohost/apps/%s/settings.yml" % app
-    yield "/etc/yunohost/apps/%s/manifest.json" % app
+    if "manifestv2" in app:
+        yield "/etc/yunohost/apps/%s/manifest.toml" % app
+    else:
+        yield "/etc/yunohost/apps/%s/manifest.json" % app
     yield "/etc/yunohost/apps/%s/scripts/install" % app
     yield "/etc/yunohost/apps/%s/scripts/remove" % app
 
@@ -157,6 +163,15 @@ def install_legacy_app(domain, path, public=True):
     )
 
 
+def install_manifestv2_app(domain, path, public=True):
+
+    app_install(
+        os.path.join(get_test_apps_dir(), "manifestv2_app_ynh"),
+        args="domain={}&path={}&init_main_permission={}".format(domain, path, "visitors" if public else "all_users"),
+        force=True,
+    )
+
+
 def install_full_domain_app(domain):
 
     app_install(
@@ -195,6 +210,105 @@ def test_legacy_app_install_main_domain():
     assert app_is_not_installed(main_domain, "legacy_app")
 
 
+def test_legacy_app_manifest_preinstall():
+
+    m = app_manifest(os.path.join(get_test_apps_dir(), "legacy_app_ynh"))
+    # v1 manifesto are expected to have been autoconverted to v2
+
+    assert "id" in m
+    assert "description" in m
+    assert "integration" in m
+    assert "install" in m
+    assert m["doc"] == {}
+    assert m["notifications"] == {"pre_install": {}, "pre_upgrade": {}, "post_install": {}, "post_upgrade": {}}
+
+
+def test_manifestv2_app_manifest_preinstall():
+
+    m = app_manifest(os.path.join(get_test_apps_dir(), "manifestv2_app_ynh"))
+
+    assert "id" in m
+    assert "install" in m
+    assert "description" in m
+    assert "doc" in m
+    assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"]["en"]
+    assert "Ceci est une fausse description des fonctionalités de l'app" in m["doc"]["DESCRIPTION"]["fr"]
+    assert "notifications" in m
+    assert "This is a dummy disclaimer to display prior to the install" in m["notifications"]["pre_install"]["main"]["en"]
+    assert "Ceci est un faux disclaimer à présenter avant l'installation" in m["notifications"]["pre_install"]["main"]["fr"]
+
+
+def test_manifestv2_app_install_main_domain():
+
+    main_domain = _get_maindomain()
+
+    install_manifestv2_app(main_domain, "/manifestv2")
+
+    app_map_ = app_map(raw=True)
+    assert main_domain in app_map_
+    assert "/manifestv2" in app_map_[main_domain]
+    assert "id" in app_map_[main_domain]["/manifestv2"]
+    assert app_map_[main_domain]["/manifestv2"]["id"] == "manifestv2_app"
+
+    assert app_is_installed(main_domain, "manifestv2_app")
+    assert app_is_exposed_on_http(main_domain, "/manifestv2", "Hextris")
+
+    app_remove("manifestv2_app")
+
+    assert app_is_not_installed(main_domain, "manifestv2_app")
+
+
+def test_manifestv2_app_info_postinstall():
+
+    main_domain = _get_maindomain()
+    install_manifestv2_app(main_domain, "/manifestv2")
+    m = app_info("manifestv2_app", full=True)["manifest"]
+
+    assert "id" in m
+    assert "install" in m
+    assert "description" in m
+    assert "doc" in m
+    assert "The app install dir is /var/www/manifestv2_app" in m["doc"]["ADMIN"]["en"]
+    assert "Le dossier d'install de l'app est /var/www/manifestv2_app" in m["doc"]["ADMIN"]["fr"]
+    assert "notifications" in m
+    assert "The app install dir is /var/www/manifestv2_app" in m["notifications"]["post_install"]["main"]["en"]
+    assert "The app id is manifestv2_app" in m["notifications"]["post_install"]["main"]["en"]
+    assert f"The app url is {main_domain}/manifestv2" in m["notifications"]["post_install"]["main"]["en"]
+
+
+def test_manifestv2_app_info_preupgrade(monkeypatch):
+
+    manifest = app_manifest(os.path.join(get_test_apps_dir(), "manifestv2_app_ynh"))
+
+    from yunohost.app_catalog import _load_apps_catalog as original_load_apps_catalog
+    def custom_load_apps_catalog(*args, **kwargs):
+
+        res = original_load_apps_catalog(*args, **kwargs)
+        res["apps"]["manifestv2_app"] = {
+            "id": "manifestv2_app",
+            "level": 10,
+            "lastUpdate": 999999999,
+            "maintained": True,
+            "manifest": manifest,
+            "state": "working",
+        }
+        res["apps"]["manifestv2_app"]["manifest"]["version"] = "99999~ynh1"
+
+        return res
+    monkeypatch.setattr("yunohost.app._load_apps_catalog", custom_load_apps_catalog)
+
+    main_domain = _get_maindomain()
+    install_manifestv2_app(main_domain, "/manifestv2")
+    i = app_info("manifestv2_app", full=True)
+
+    assert i["upgradable"] == "yes"
+    assert i["new_version"] == "99999~ynh1"
+    # FIXME : as I write this test, I realize that this implies the catalog API
+    # does provide the notifications, which means the list builder script
+    # should parse the files in the original app repo, possibly with proper i18n etc
+    assert "This is a dummy disclaimer to display prior to any upgrade" \
+            in i["from_catalog"]["manifest"]["notifications"]["pre_upgrade"]["main"]["en"]
+
 def test_app_from_catalog():
     main_domain = _get_maindomain()
 

src/tests/test_backuprestore.py

@@ -355,13 +355,13 @@ def test_backup_script_failure_handling(monkeypatch, mocker):
 
 @pytest.mark.with_backup_recommended_app_installed
 def test_backup_not_enough_free_space(monkeypatch, mocker):
-    def custom_disk_usage(path):
+    def custom_space_used_by_directory(path, *args, **kwargs):
         return 99999999999999999
 
     def custom_free_space_in_directory(dirpath):
         return 0
 
-    monkeypatch.setattr("yunohost.backup.disk_usage", custom_disk_usage)
+    monkeypatch.setattr("yunohost.backup.space_used_by_directory", custom_space_used_by_directory)
     monkeypatch.setattr(
         "yunohost.backup.free_space_in_directory", custom_free_space_in_directory
     )

src/tests/test_permission.py

@@ -78,6 +78,7 @@ def _permission_create_with_dummy_app(
                     "name": app,
                     "id": app,
                     "description": {"en": "Dummy app to test permissions"},
+                    "arguments": {"install": []}
                 },
                 f,
             )

src/tools.py

@@ -45,10 +45,12 @@ from yunohost.domain import domain_add
 from yunohost.firewall import firewall_upnp
 from yunohost.service import service_start, service_enable
 from yunohost.regenconf import regen_conf
-from yunohost.utils.packages import (
+from yunohost.utils.system import (
     _dump_sources_list,
     _list_upgradable_apt_packages,
     ynh_packages_version,
+    dpkg_is_broken,
+    dpkg_lock_available,
 )
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.log import is_unit_operation, OperationLogger
@@ -166,20 +168,6 @@ def _set_hostname(hostname, pretty_hostname=None):
             logger.debug(out)
 
 
-def _detect_virt():
-    """
-    Returns the output of systemd-detect-virt (so e.g. 'none' or 'lxc' or ...)
-    You can check the man of the command to have a list of possible outputs...
-    """
-
-    p = subprocess.Popen(
-        "systemd-detect-virt".split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-    )
-
-    out, _ = p.communicate()
-    return out.split()[0]
-
-
 @is_unit_operation()
 def tools_postinstall(
     operation_logger,
@@ -410,13 +398,12 @@ def tools_upgrade(operation_logger, target=None):
        apps -- List of apps to upgrade (or [] to update all apps)
        system -- True to upgrade system
     """
-    from yunohost.utils import packages
 
-    if packages.dpkg_is_broken():
+    if dpkg_is_broken():
         raise YunohostValidationError("dpkg_is_broken")
 
     # Check for obvious conflict with other dpkg/apt commands already running in parallel
-    if not packages.dpkg_lock_available():
+    if not dpkg_lock_available():
         raise YunohostValidationError("dpkg_lock_not_available")
 
     if target not in ["apps", "system"]:

src/user.py

@@ -40,6 +40,7 @@ from moulinette.utils.process import check_output
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.service import service_status
 from yunohost.log import is_unit_operation
+from yunohost.utils.system import binary_to_human
 
 logger = getActionLogger("yunohost.user")
 
@@ -596,7 +597,7 @@ def user_info(username):
 
             if has_value:
                 storage_use = int(has_value.group(1))
-                storage_use = _convertSize(storage_use)
+                storage_use = binary_to_human(storage_use)
 
                 if is_limited:
                     has_percent = re.search(r"%=(\d+)", cmd_result)
@@ -1323,15 +1324,6 @@ def user_ssh_remove_key(username, key):
 # End SSH subcategory
 #
 
-
-def _convertSize(num, suffix=""):
-    for unit in ["K", "M", "G", "T", "P", "E", "Z"]:
-        if abs(num) < 1024.0:
-            return "{:3.1f}{}{}".format(num, unit, suffix)
-        num /= 1024.0
-    return "{:.1f}{}{}".format(num, "Yi", suffix)
-
-
 def _hash_user_password(password):
     """
     This function computes and return a salted hash for the password in input.

src/utils/config.py

@@ -581,7 +581,7 @@ class ConfigPanel:
             prefilled_answers.update(self.new_values)
 
             questions = ask_questions_and_parse_answers(
-                section["options"],
+                {question["name"]: question for question in section["options"]},
                 prefilled_answers=prefilled_answers,
                 current_values=self.values,
                 hooks=self.hooks,
@@ -1191,6 +1191,21 @@ class UserQuestion(Question):
                     break
 
 
+class GroupQuestion(Question):
+    argument_type = "group"
+
+    def __init__(self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {}):
+
+        from yunohost.user import user_group_list
+
+        super().__init__(question, context)
+
+        self.choices = list(user_group_list(short=True)["groups"])
+
+        if self.default is None:
+            self.default = "all_users"
+
+
 class NumberQuestion(Question):
     argument_type = "number"
     default_value = None
@@ -1349,6 +1364,7 @@ ARGUMENTS_TYPE_PARSERS = {
     "boolean": BooleanQuestion,
     "domain": DomainQuestion,
     "user": UserQuestion,
+    "group": GroupQuestion,
     "number": NumberQuestion,
     "range": NumberQuestion,
     "display_text": DisplayTextQuestion,
@@ -1393,9 +1409,10 @@ def ask_questions_and_parse_answers(
     context = {**current_values, **answers}
     out = []
 
-    for raw_question in raw_questions:
+    for name, raw_question in raw_questions.items():
+        raw_question['name'] = name
         question_class = ARGUMENTS_TYPE_PARSERS[raw_question.get("type", "string")]
-        raw_question["value"] = answers.get(raw_question["name"])
+        raw_question["value"] = answers.get(name)
         question = question_class(raw_question, context=context, hooks=hooks)
         new_values = question.ask_if_needed()
         answers.update(new_values)

src/utils/filesystem.py

@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-
-""" License
-
-    Copyright (C) 2018 YUNOHOST.ORG
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published
-    by the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program; if not, see http://www.gnu.org/licenses
-
-"""
-import os
-
-
-def free_space_in_directory(dirpath):
-    stat = os.statvfs(dirpath)
-    return stat.f_frsize * stat.f_bavail
-
-
-def space_used_by_directory(dirpath):
-    stat = os.statvfs(dirpath)
-    return stat.f_frsize * stat.f_blocks

src/utils/resources.py

@@ -0,0 +1,905 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2021 YUNOHOST.ORG
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+import os
+import copy
+import shutil
+import random
+from typing import Dict, Any, List
+
+from moulinette.utils.process import check_output
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file
+from moulinette.utils.filesystem import (
+    rm,
+)
+
+from yunohost.utils.error import YunohostError
+
+logger = getActionLogger("yunohost.app_resources")
+
+
+class AppResourceManager:
+
+    # FIXME : add some sort of documentation mechanism
+    # to create a have a detailed description of each resource behavior
+
+    def __init__(self, app: str, current: Dict, wanted: Dict):
+
+        self.app = app
+        self.current = current
+        self.wanted = wanted
+
+        if "resources" not in self.current:
+            self.current["resources"] = {}
+        if "resources" not in self.wanted:
+            self.wanted["resources"] = {}
+
+    def apply(self, rollback_if_failure, **context):
+
+        todos = list(self.compute_todos())
+        completed = []
+        rollback = False
+        exception = None
+
+        for todo, name, old, new in todos:
+            try:
+                if todo == "deprovision":
+                    # FIXME : i18n, better info strings
+                    logger.info(f"Deprovisionning {name} ...")
+                    old.deprovision(context=context)
+                elif todo == "provision":
+                    logger.info(f"Provisionning {name} ...")
+                    new.provision_or_update(context=context)
+                elif todo == "update":
+                    logger.info(f"Updating {name} ...")
+                    new.provision_or_update(context=context)
+            # FIXME FIXME FIXME : this exception doesnt catch Ctrl+C ?!?!
+            except Exception as e:
+                exception = e
+                # FIXME: better error handling ? display stacktrace ?
+                logger.warning(f"Failed to {todo} for {name} : {e}")
+                if rollback_if_failure:
+                    rollback = True
+                    completed.append((todo, name, old, new))
+                    break
+                else:
+                    pass
+            else:
+                completed.append((todo, name, old, new))
+
+        if rollback:
+            for todo, name, old, new in completed:
+                try:
+                    # (NB. here we want to undo the todo)
+                    if todo == "deprovision":
+                        # FIXME : i18n, better info strings
+                        logger.info(f"Reprovisionning {name} ...")
+                        old.provision_or_update(context=context)
+                    elif todo == "provision":
+                        logger.info(f"Deprovisionning {name} ...")
+                        new.deprovision(context=context)
+                    elif todo == "update":
+                        logger.info(f"Reverting {name} ...")
+                        old.provision_or_update(context=context)
+                except Exception as e:
+                    # FIXME: better error handling ? display stacktrace ?
+                    logger.error(f"Failed to rollback {name} : {e}")
+
+        if exception:
+            raise exception
+
+    def compute_todos(self):
+
+        for name, infos in reversed(self.current["resources"].items()):
+            if name not in self.wanted["resources"].keys():
+                resource = AppResourceClassesByType[name](infos, self.app, self)
+                yield ("deprovision", name, resource, None)
+
+        for name, infos in self.wanted["resources"].items():
+            wanted_resource = AppResourceClassesByType[name](infos, self.app, self)
+            if name not in self.current["resources"].keys():
+                yield ("provision", name, None, wanted_resource)
+            else:
+                infos_ = self.current["resources"][name]
+                current_resource = AppResourceClassesByType[name](infos_, self.app, self)
+                yield ("update", name, current_resource, wanted_resource)
+
+
+class AppResource:
+
+    type: str = ""
+    default_properties: Dict[str, Any] = {}
+
+    def __init__(self, properties: Dict[str, Any], app: str, manager=None):
+
+        self.app = app
+        self.manager = manager
+
+        for key, value in self.default_properties.items():
+            if isinstance(value, str):
+                value = value.replace("__APP__", self.app)
+            setattr(self, key, value)
+
+        for key, value in properties.items():
+            if isinstance(value, str):
+                value = value.replace("__APP__", self.app)
+            setattr(self, key, value)
+
+    def get_setting(self, key):
+        from yunohost.app import app_setting
+        return app_setting(self.app, key)
+
+    def set_setting(self, key, value):
+        from yunohost.app import app_setting
+        app_setting(self.app, key, value=value)
+
+    def delete_setting(self, key):
+        from yunohost.app import app_setting
+        app_setting(self.app, key, delete=True)
+
+    def _run_script(self, action, script, env={}, user="root"):
+
+        from yunohost.app import _make_tmp_workdir_for_app, _make_environment_for_app_script
+        from yunohost.hook import hook_exec_with_script_debug_if_failure
+
+        tmpdir = _make_tmp_workdir_for_app(app=self.app)
+
+        env_ = _make_environment_for_app_script(self.app, workdir=tmpdir, action=f"{action}_{self.type}")
+        env_.update(env)
+
+        script_path = f"{tmpdir}/{action}_{self.type}"
+        script = f"""
+source /usr/share/yunohost/helpers
+ynh_abort_if_errors
+
+{script}
+"""
+
+        write_to_file(script_path, script)
+
+        from yunohost.log import OperationLogger
+
+        if OperationLogger._instances:
+            # FIXME ? : this is an ugly hack :(
+            operation_logger = OperationLogger._instances[-1]
+        else:
+            operation_logger = OperationLogger("resource_snippet", [("app", self.app)], env=env_)
+            operation_logger.start()
+
+        try:
+            (
+                call_failed,
+                failure_message_with_debug_instructions,
+            ) = hook_exec_with_script_debug_if_failure(
+                script_path,
+                env=env_,
+                operation_logger=operation_logger,
+                error_message_if_script_failed="An error occured inside the script snippet",
+                error_message_if_failed=lambda e: f"{action} failed for {self.type} : {e}"
+            )
+        finally:
+            if call_failed:
+                raise YunohostError(
+                    failure_message_with_debug_instructions, raw_msg=True
+                )
+            else:
+                # FIXME: currently in app install code, we have
+                # more sophisticated code checking if this broke something on the system etc ...
+                # dunno if we want to do this here or manage it elsewhere
+                pass
+
+        #print(ret)
+
+
+class PermissionsResource(AppResource):
+    """
+    Configure the SSO permissions/tiles. Typically, webapps are expected to have a 'main' permission mapped to '/', meaning that a tile pointing to the `$domain/$path` will be available in the SSO for users allowed to access that app.
+
+    Additional permissions can be created, typically to have a specific tile and/or access rules for the admin part of a webapp.
+
+    The list of allowed user/groups may be initialized using the content of the `init_{perm}_permission` question from the manifest, hence `init_main_permission` replaces the `is_public` question and shall contain a group name (typically, `all_users` or `visitors`).
+
+    ##### Example:
+    ```toml
+    [resources.permissions]
+    main.url = "/"
+    # (these two previous lines should be enough in the majority of cases)
+
+    admin.url = "/admin"
+    admin.show_tile = false
+    admin.allowed = "admins"   # Assuming the "admins" group exists (cf future developments ;))
+    ```
+
+    ##### Properties (for each perm name):
+    - `url`: The relative URI corresponding to this permission. Typically `/` or `/something`. This property may be omitted for non-web permissions.
+    - `show_tile`: (default: `true` if `url` is defined) Wether or not a tile should be displayed for that permission in the user portal
+    - `allowed`: (default: nobody) The group initially allowed to access this perm, if `init_{perm}_permission` is not defined in the manifest questions. Note that the admin may tweak who is allowed/unallowed on that permission later on, this is only meant to **initialize** the permission.
+    - `auth_header`: (default: `true`) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true
+    - `protected`: (default: `false`) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'.
+    - `additional_urls`: (default: none) List of additional URL for which access will be allowed/forbidden
+
+    ##### Provision/Update:
+    - Delete any permissions that may exist and be related to this app yet is not declared anymore
+    - Loop over the declared permissions and create them if needed or update them with the new values (FIXME : update ain't implemented yet >_>)
+
+    ##### Deprovision:
+    - Delete all permission related to this app
+
+    ##### Legacy management:
+    - Legacy `is_public` setting will be deleted if it exists
+    """
+
+    # Notes for future ?
+    # deep_clean  -> delete permissions for any __APP__.foobar where app not in app list...
+    # backup -> handled elsewhere by the core, should be integrated in there (dump .ldif/yml?)
+    # restore -> handled by the core, should be integrated in there (restore .ldif/yml?)
+
+    type = "permissions"
+    priority = 80
+
+    default_properties: Dict[str, Any] = {
+    }
+
+    default_perm_properties: Dict[str, Any] = {
+        "url": None,
+        "additional_urls": [],
+        "auth_header": True,
+        "allowed": None,
+        "show_tile": None,  # To be automagically set to True by default if an url is defined and show_tile not provided
+        "protected": False,
+    }
+
+    permissions: Dict[str, Dict[str, Any]] = {}
+
+    def __init__(self, properties: Dict[str, Any], *args, **kwargs):
+
+        # FIXME : if url != None, we should check that there's indeed a domain/path defined ? ie that app is a webapp
+
+        for perm, infos in properties.items():
+            properties[perm] = copy.copy(self.default_perm_properties)
+            properties[perm].update(infos)
+            if properties[perm]["show_tile"] is None:
+                properties[perm]["show_tile"] = bool(properties[perm]["url"])
+
+        if isinstance(properties["main"]["url"], str) and properties["main"]["url"] != "/":
+            raise YunohostError("URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app")
+
+        super().__init__({"permissions": properties}, *args, **kwargs)
+
+    def provision_or_update(self, context: Dict={}):
+
+        from yunohost.permission import (
+            permission_create,
+            #permission_url,
+            permission_delete,
+            user_permission_list,
+            user_permission_update,
+            permission_sync_to_user,
+        )
+
+        # Delete legacy is_public setting if not already done
+        self.delete_setting("is_public")
+
+        existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"]
+        for perm in existing_perms:
+            if perm.split(".")[1] not in self.permissions.keys():
+                permission_delete(perm, force=True, sync_perm=False)
+
+        for perm, infos in self.permissions.items():
+            if f"{self.app}.{perm}" not in existing_perms:
+                # Use the 'allowed' key from the manifest,
+                # or use the 'init_{perm}_permission' from the install questions
+                # which is temporarily saved as a setting as an ugly hack to pass the info to this piece of code...
+                init_allowed = infos["allowed"] or self.get_setting(f"init_{perm}_permission") or []
+                permission_create(
+                    f"{self.app}.{perm}",
+                    allowed=init_allowed,
+                    # This is why the ugly hack with self.manager exists >_>
+                    label=self.manager.wanted["name"] if perm == "main" else perm,
+                    url=infos["url"],
+                    additional_urls=infos["additional_urls"],
+                    auth_header=infos["auth_header"],
+                    sync_perm=False,
+                )
+                self.delete_setting(f"init_{perm}_permission")
+
+                user_permission_update(
+                    f"{self.app}.{perm}",
+                    show_tile=infos["show_tile"],
+                    protected=infos["protected"],
+                    sync_perm=False
+                )
+            else:
+                pass
+                # FIXME : current implementation of permission_url is hell for
+                # easy declarativeness of additional_urls >_> ...
+                #permission_url(f"{self.app}.{perm}", url=infos["url"], auth_header=infos["auth_header"], sync_perm=False)
+
+        permission_sync_to_user()
+
+    def deprovision(self, context: Dict={}):
+
+        from yunohost.permission import (
+            permission_delete,
+            user_permission_list,
+            permission_sync_to_user,
+        )
+
+        existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"]
+        for perm in existing_perms:
+            permission_delete(perm, force=True, sync_perm=False)
+
+        permission_sync_to_user()
+
+
+class SystemuserAppResource(AppResource):
+    """
+    Provision a system user to be used by the app. The username is exactly equal to the app id
+
+    ##### Example:
+    ```toml
+    [resources.system_user]
+    # (empty - defaults are usually okay)
+    ```
+
+    ##### Properties:
+    - `allow_ssh`: (default: False) Adds the user to the ssh.app group, allowing SSH connection via this user
+    - `allow_sftp`: (defalt: False) Adds the user to the sftp.app group, allowing SFTP connection via this user
+
+    ##### Provision/Update:
+    - will create the system user if it doesn't exists yet
+    - will add/remove the ssh/sftp.app groups
+
+    ##### Deprovision:
+    - deletes the user and group
+    """
+
+    # Notes for future?
+    #
+    # deep_clean  -> uuuuh ? delete any user that could correspond to an app x_x ?
+    #
+    # backup -> nothing
+    # restore -> provision
+
+    type = "system_user"
+    priority = 20
+
+    default_properties: Dict[str, Any] = {
+        "allow_ssh": False,
+        "allow_sftp": False
+    }
+
+    # FIXME : wat do regarding ssl-cert, multimedia
+    # FIXME : wat do about home dir
+
+    allow_ssh: bool = False
+    allow_sftp: bool = False
+
+    def provision_or_update(self, context: Dict={}):
+
+        # FIXME : validate that no yunohost user exists with that name?
+        # and/or that no system user exists during install ?
+
+        if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
+            # FIXME: improve logging ? os.system wont log stdout / stderr
+            cmd = f"useradd --system --user-group {self.app}"
+            ret = os.system(cmd)
+            assert ret == 0, f"useradd command failed with exit code {ret}"
+
+        if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
+            raise YunohostError(f"Failed to create system user for {self.app}", raw_msg=True)
+
+        groups = set(check_output(f"groups {self.app}").strip().split()[2:])
+
+        if self.allow_ssh:
+            groups.add("ssh.app")
+        elif "ssh.app" in groups:
+            groups.remove("ssh.app")
+
+        if self.allow_sftp:
+            groups.add("sftp.app")
+        elif "sftp.app" in groups:
+            groups.remove("sftp.app")
+
+        os.system(f"usermod -G {','.join(groups)} {self.app}")
+
+    def deprovision(self, context: Dict={}):
+
+        if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
+            os.system(f"deluser {self.app} >/dev/null")
+        if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
+            raise YunohostError(f"Failed to delete system user for {self.app}")
+
+        if check_output(f"getent group {self.app} &>/dev/null || true").strip():
+            os.system(f"delgroup {self.app} >/dev/null")
+        if check_output(f"getent group {self.app} &>/dev/null || true").strip():
+            raise YunohostError(f"Failed to delete system user for {self.app}")
+
+        # FIXME : better logging and error handling, add stdout/stderr from the deluser/delgroup commands...
+
+
+class InstalldirAppResource(AppResource):
+    """
+    Creates a directory to be used by the app as the installation directory, typically where the app sources and assets are located. The corresponding path is stored in the settings as `install_dir`
+
+    ##### Example:
+    ```toml
+    [resources.install_dir]
+    # (empty - defaults are usually okay)
+    ```
+
+    ##### Properties:
+    - `dir`: (default: `/var/www/__APP__`) The full path of the install dir
+    - `owner`: (default: `__APP__:rx`) The owner (and owner permissions) for the install dir
+    - `group`: (default: `__APP__:rx`) The group (and group permissions) for the install dir
+
+    ##### Provision/Update:
+    - during install, the folder will be deleted if it already exists (FIXME: is this what we want?)
+    - if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location
+    - otherwise, creates the directory if it doesn't exists yet
+    - (re-)apply permissions (only on the folder itself, not recursively)
+    - save the value of `dir` as `install_dir` in the app's settings, which can be then used by the app scripts (`$install_dir`) and conf templates (`__INSTALL_DIR__`)
+
+    ##### Deprovision:
+    - recursively deletes the directory if it exists
+
+    ##### Legacy management:
+    - In the past, the setting was called `final_path`. The code will automatically rename it as `install_dir`.
+    - As explained in the 'Provision/Update' section, the folder will also be moved if the location changed
+
+    """
+
+    # Notes for future?
+    # deep_clean  -> uuuuh ? delete any dir in /var/www/ that would not correspond to an app x_x ?
+    # backup -> cp install dir
+    # restore -> cp install dir
+
+    type = "install_dir"
+    priority = 30
+
+    default_properties: Dict[str, Any] = {
+        "dir": "/var/www/__APP__",
+        "owner": "__APP__:rx",
+        "group": "__APP__:rx",
+    }
+
+    dir: str = ""
+    owner: str = ""
+    group: str = ""
+
+    # FIXME: change default dir to /opt/stuff if app ain't a webapp ...
+
+    def provision_or_update(self, context: Dict={}):
+
+        assert self.dir.strip() # Be paranoid about self.dir being empty...
+        assert self.owner.strip()
+        assert self.group.strip()
+
+        current_install_dir = self.get_setting("install_dir") or self.get_setting("final_path")
+
+        # If during install, /var/www/$app already exists, assume that it's okay to remove and recreate it
+        # FIXME : is this the right thing to do ?
+        if not current_install_dir and os.path.isdir(self.dir):
+            rm(self.dir, recursive=True)
+
+        if not os.path.isdir(self.dir):
+            # Handle case where install location changed, in which case we shall move the existing install dir
+            # FIXME: confirm that's what we wanna do
+            # Maybe a middle ground could be to compute the size, check that it's not too crazy (eg > 1G idk),
+            # and check for available space on the destination
+            if current_install_dir and os.path.isdir(current_install_dir):
+                logger.warning(f"Moving {current_install_dir} to {self.dir} ... (this may take a while)")
+                shutil.move(current_install_dir, self.dir)
+            else:
+                mkdir(self.dir)
+
+        owner, owner_perm = self.owner.split(":")
+        group, group_perm = self.group.split(":")
+        owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0)
+        group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0)
+
+        perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal
+
+        chmod(self.dir, perm_octal)
+        chown(self.dir, owner, group)
+        # FIXME: shall we apply permissions recursively ?
+
+        self.set_setting("install_dir", self.dir)
+        self.delete_setting("final_path")  # Legacy
+
+    def deprovision(self, context: Dict={}):
+
+        assert self.dir.strip() # Be paranoid about self.dir being empty...
+        assert self.owner.strip()
+        assert self.group.strip()
+
+        # FIXME : check that self.dir has a sensible value to prevent catastrophes
+        if os.path.isdir(self.dir):
+            rm(self.dir, recursive=True)
+        # FIXME : in fact we should delete settings to be consistent
+
+
+class DatadirAppResource(AppResource):
+    """
+    Creates a directory to be used by the app as the data store directory, typically where the app multimedia or large assets added by users are located. The corresponding path is stored in the settings as `data_dir`. This resource behaves very similarly to install_dir.
+
+    ##### Example:
+    ```toml
+    [resources.data_dir]
+    # (empty - defaults are usually okay)
+    ```
+
+    ##### Properties:
+    - `dir`: (default: `/home/yunohost.app/__APP__`) The full path of the data dir
+    - `owner`: (default: `__APP__:rx`) The owner (and owner permissions) for the data dir
+    - `group`: (default: `__APP__:rx`) The group (and group permissions) for the data dir
+
+    ##### Provision/Update:
+    - if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location
+    - otherwise, creates the directory if it doesn't exists yet
+    - (re-)apply permissions (only on the folder itself, not recursively)
+    - save the value of `dir` as `data_dir` in the app's settings, which can be then used by the app scripts (`$data_dir`) and conf templates (`__DATA_DIR__`)
+
+    ##### Deprovision:
+    - recursively deletes the directory if it exists
+    - FIXME: this should only be done if the PURGE option is set
+    - FIXME: this should also delete the corresponding setting
+
+    ##### Legacy management:
+    - In the past, the setting may have been called `datadir`. The code will automatically rename it as `data_dir`.
+    - As explained in the 'Provision/Update' section, the folder will also be moved if the location changed
+
+    """
+
+    # notes for future ?
+    # deep_clean  -> zblerg idk nothing
+    # backup -> cp data dir ? (if not backup_core_only)
+    # restore -> cp data dir ? (if in backup)
+
+    type = "data_dir"
+    priority = 40
+
+    default_properties: Dict[str, Any] = {
+        "dir": "/home/yunohost.app/__APP__",
+        "owner": "__APP__:rx",
+        "group": "__APP__:rx",
+    }
+
+    dir: str = ""
+    owner: str = ""
+    group: str = ""
+
+    def provision_or_update(self, context: Dict={}):
+
+        assert self.dir.strip() # Be paranoid about self.dir being empty...
+        assert self.owner.strip()
+        assert self.group.strip()
+
+        current_data_dir = self.get_setting("data_dir") or self.get_setting("datadir")
+
+        if not os.path.isdir(self.dir):
+            # Handle case where install location changed, in which case we shall move the existing install dir
+            # FIXME: same as install_dir, is this what we want ?
+            # FIXME: What if people manually mved the data dir and changed the setting value and dont want the folder to be moved ? x_x
+            if current_data_dir and os.path.isdir(current_data_dir):
+                shutil.move(current_data_dir, self.dir)
+            else:
+                mkdir(self.dir)
+
+        owner, owner_perm = self.owner.split(":")
+        group, group_perm = self.group.split(":")
+        owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0)
+        group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0)
+        perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal
+
+        chmod(self.dir, perm_octal)
+        chown(self.dir, owner, group)
+
+        self.set_setting("data_dir", self.dir)
+        self.delete_setting("datadir")  # Legacy
+
+    def deprovision(self, context: Dict={}):
+
+        assert self.dir.strip() # Be paranoid about self.dir being empty...
+        assert self.owner.strip()
+        assert self.group.strip()
+
+        # FIXME: This should rm the datadir only if purge is enabled
+        pass
+        #if os.path.isdir(self.dir):
+        #    rm(self.dir, recursive=True)
+        # FIXME : in fact we should delete settings to be consistent
+
+
+class AptDependenciesAppResource(AppResource):
+    """
+    Create a virtual package in apt, depending on the list of specified packages that the app needs. The virtual packages is called `$app-ynh-deps` (with `_` being replaced by `-` in the app name, see `ynh_install_app_dependencies`)
+
+    ##### Example:
+    ```toml
+    [resources.apt]
+    packages = "nyancat, lolcat, sl"
+
+    # (this part is optional and corresponds to the legacy ynh_install_extra_app_dependencies helper)
+    extras.yarn.repo = "deb https://dl.yarnpkg.com/debian/ stable main"
+    extras.yarn.key = "https://dl.yarnpkg.com/debian/pubkey.gpg"
+    extras.yarn.packages = "yarn"
+    ```
+
+    ##### Properties:
+    - `packages`: Comma-separated list of packages to be installed via `apt`
+    - `extras`: A dict of (repo, key, packages) corresponding to "extra" repositories to fetch dependencies from
+
+    ##### Provision/Update:
+    - The code literally calls the bash helpers `ynh_install_app_dependencies` and `ynh_install_extra_app_dependencies`, similar to what happens in v1.
+
+    ##### Deprovision:
+    - The code literally calls the bash helper `ynh_remove_app_dependencies`
+    """
+
+    # Notes for future?
+    # deep_clean  -> remove any __APP__-ynh-deps for app not in app list
+    # backup -> nothing
+    # restore = provision
+
+    type = "apt"
+    priority = 50
+
+    default_properties: Dict[str, Any] = {
+        "packages": [],
+        "extras": {}
+    }
+
+    packages: List = []
+    extras: Dict[str, Dict[str, str]] = {}
+
+    def __init__(self, properties: Dict[str, Any], *args, **kwargs):
+
+        for key, values in properties.get("extras", {}).items():
+            if not all(isinstance(values.get(k), str) for k in ["repo", "key", "packages"]):
+                raise YunohostError("In apt resource in the manifest: 'extras' repo should have the keys 'repo', 'key' and 'packages' defined and be strings")
+
+        super().__init__(properties, *args, **kwargs)
+
+    def provision_or_update(self, context: Dict={}):
+
+        script = [f"ynh_install_app_dependencies {self.packages}"]
+        for repo, values in self.extras.items():
+            script += [f"ynh_install_extra_app_dependencies --repo='{values['repo']}' --key='{values['key']}' --package='{values['packages']}'"]
+            # FIXME : we're feeding the raw value of values['packages'] to the helper .. if we want to be consistent, may they should be comma-separated, though in the majority of cases, only a single package is installed from an extra repo..
+
+        self._run_script("provision_or_update", '\n'.join(script))
+
+    def deprovision(self, context: Dict={}):
+
+        self._run_script("deprovision", "ynh_remove_app_dependencies")
+
+
+class PortsResource(AppResource):
+    """
+    Book port(s) to be used by the app, typically to be used to the internal reverse-proxy between nginx and the app process.
+
+    Note that because multiple ports can be booked, each properties is prefixed by the name of the port. `main` is a special name and will correspond to the setting `$port`, whereas for example `xmpp_client` will correspond to the setting `$port_xmpp_client`.
+
+    ##### Example:
+    ```toml
+    [resources.port]
+    # (empty should be fine for most apps ... though you can customize stuff if absolutely needed)
+
+    main.default = 12345    # if you really want to specify a prefered value .. but shouldnt matter in the majority of cases
+
+    xmpp_client.default = 5222  # if you need another port, pick a name for it (here, "xmpp_client")
+    xmpp_client.exposed = "TCP" # here, we're telling that the port needs to be publicly exposed on TCP on the firewall
+    ```
+
+    ##### Properties (for every port name):
+    - `default`: The prefered value for the port. If this port is already being used by another process right now, or is booked in another app's setting, the code will increment the value until it finds a free port and store that value as the setting. If no value is specified, a random value between 10000 and 60000 is used.
+    - `exposed`: (default: `false`) Wether this port should be opened on the firewall and be publicly reachable. This should be kept to `false` for the majority of apps than only need a port for internal reverse-proxying! Possible values: `false`, `true`(=`Both`), `Both`, `TCP`, `UDP`. This will result in the port being opened on the firewall, and the diagnosis checking that a program answers on that port.  (FIXME: this is not implemented yet)
+    - `fixed`: (default: `false`) Tells that the app absolutely needs the specific value provided in `default`, typically because it's needed for a specific protocol (FIXME: this is not implemented yet)
+
+    ##### Provision/Update (for every port name):
+    - If not already booked, look for a free port, starting with the `default` value (or a random value between 10000 and 60000 if no `default` set)
+    - (FIXME) If `exposed` is not `false`, open the port in the firewall accordingly - otherwise make sure it's closed.
+    - The value of the port is stored in the `$port` setting for the `main` port, or `$port_NAME` for other `NAME`s
+
+    ##### Deprovision:
+    - (FIXME) Close the ports on the firewall
+    - Deletes all the port settings
+
+    ##### Legacy management:
+    - In the past, some settings may have been named `NAME_port` instead of `port_NAME`, in which case the code will automatically rename the old setting.
+    """
+
+    # Notes for future?
+    #deep_clean  -> ?
+    #backup -> nothing (backup port setting)
+    #restore -> nothing (restore port setting)
+
+    type = "ports"
+    priority = 70
+
+    default_properties: Dict[str, Any] = {
+    }
+
+    default_port_properties = {
+        "default": None,
+        "exposed": False,   # or True(="Both"), "TCP", "UDP"   # FIXME : implement logic for exposed port (allow/disallow in firewall ?)
+        "fixed": False,     # FIXME: implement logic. Corresponding to wether or not the port is "fixed" or any random port is ok
+    }
+
+    ports: Dict[str, Dict[str, Any]]
+
+    def __init__(self, properties: Dict[str, Any], *args, **kwargs):
+
+        if "main" not in properties:
+            properties["main"] = {}
+
+        for port, infos in properties.items():
+            properties[port] = copy.copy(self.default_port_properties)
+            properties[port].update(infos)
+
+            if properties[port]["default"] is None:
+                properties[port]["default"] = random.randint(10000, 60000)
+
+        super().__init__({"ports": properties}, *args, **kwargs)
+
+    def _port_is_used(self, port):
+
+        # FIXME : this could be less brutal than two os.system ...
+        cmd1 = "ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ':%s$'" % port
+        # This second command is mean to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up)
+        cmd2 = f"grep --quiet \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml"
+        return os.system(cmd1) == 0 and os.system(cmd2) == 0
+
+    def provision_or_update(self, context: Dict={}):
+
+        for name, infos in self.ports.items():
+
+            setting_name = f"port_{name}" if name != "main" else "port"
+            port_value = self.get_setting(setting_name)
+            if not port_value and name != "main":
+                # Automigrate from legacy setting foobar_port (instead of port_foobar)
+                legacy_setting_name = "{name}_port"
+                port_value = self.get_setting(legacy_setting_name)
+                if port_value:
+                    self.set_setting(setting_name, port_value)
+                    self.delete_setting(legacy_setting_name)
+                    continue
+
+            if not port_value:
+                port_value = infos["default"]
+                while self._port_is_used(port_value):
+                    port_value += 1
+
+            self.set_setting(setting_name, port_value)
+
+    def deprovision(self, context: Dict={}):
+
+        for name, infos in self.ports.items():
+            setting_name = f"port_{name}" if name != "main" else "port"
+            self.delete_setting(setting_name)
+
+
+class DatabaseAppResource(AppResource):
+    """
+    Initialize a database, either using MySQL or Postgresql. Relevant DB infos are stored in settings `$db_name`, `$db_user` and `$db_pwd`.
+
+    NB: only one DB can be handled in such a way (is there really an app that would need two completely different DB ?...)
+
+    NB2: no automagic migration will happen in an suddenly change `type` from `mysql` to `postgresql` or viceversa in its life
+
+    ##### Example:
+    ```toml
+    [resources.database]
+    type = "mysql"   # or : "postgresql". Only these two values are supported
+    ```
+
+    ##### Properties:
+    - `type`: The database type, either `mysql` or `postgresql`
+
+    ##### Provision/Update:
+    - (Re)set the `$db_name` and `$db_user` settings with the sanitized app name (replacing `-` and `.` with `_`)
+    - If `$db_pwd` doesn't already exists, pick a random database password and store it in that setting
+    - If the database doesn't exists yet, create the SQL user and DB using `ynh_mysql_create_db` or `ynh_psql_create_db`.
+
+    ##### Deprovision:
+    - Drop the DB using `ynh_mysql_remove_db` or `ynh_psql_remove_db`
+    - Deletes the `db_name`, `db_user` and `db_pwd` settings
+
+    ##### Legacy management:
+    - In the past, the sql passwords may have been named `mysqlpwd` or `psqlpwd`, in which case it will automatically be renamed as `db_pwd`
+    """
+
+    # Notes for future?
+    # deep_clean  -> ... idk look into any db name that would not be related to any app ...
+    # backup -> dump db
+    # restore -> setup + inject db dump
+
+    type = "database"
+    priority = 90
+
+    default_properties: Dict[str, Any] = {
+        "type": None,   # FIXME: eeeeeeeh is this really a good idea considering 'type' is supposed to be the resource type x_x
+    }
+
+    def __init__(self, properties: Dict[str, Any], *args, **kwargs):
+
+        if "type" not in properties or properties["type"] not in ["mysql", "postgresql"]:
+            raise YunohostError("Specifying the type of db ('mysql' or 'postgresql') is mandatory for db resources")
+
+        super().__init__(properties, *args, **kwargs)
+
+    def db_exists(self, db_name):
+
+        if self.type == "mysql":
+            return os.system(f"mysqlshow '{db_name}' >/dev/null 2>/dev/null") == 0
+        elif self.type == "postgresql":
+            return os.system(f"sudo --login --user=postgres psql -c '' '{db_name}' >/dev/null 2>/dev/null") == 0
+        else:
+            return False
+
+    def provision_or_update(self, context: Dict={}):
+
+        # This is equivalent to ynh_sanitize_dbid
+        db_name = self.app.replace('-', '_').replace('.', '_')
+        db_user = db_name
+        self.set_setting("db_name", db_name)
+        self.set_setting("db_user", db_user)
+
+        db_pwd = None
+        if self.get_setting("db_pwd"):
+            db_pwd = self.get_setting("db_pwd")
+        else:
+            # Legacy setting migration
+            legacypasswordsetting = "psqlpwd" if self.type == "postgresql" else "mysqlpwd"
+            if self.get_setting(legacypasswordsetting):
+                db_pwd = self.get_setting(legacypasswordsetting)
+                self.delete_setting(legacypasswordsetting)
+                self.set_setting("db_pwd", db_pwd)
+
+        if not db_pwd:
+            from moulinette.utils.text import random_ascii
+            db_pwd = random_ascii(24)
+            self.set_setting("db_pwd", db_pwd)
+
+        if not self.db_exists(db_name):
+
+            if self.type == "mysql":
+                self._run_script("provision", f"ynh_mysql_create_db '{db_name}' '{db_user}' '{db_pwd}'")
+            elif self.type == "postgresql":
+                self._run_script("provision", f"ynh_psql_create_user '{db_user}' '{db_pwd}'; ynh_psql_create_db '{db_name}' '{db_user}'")
+
+    def deprovision(self, context: Dict={}):
+
+        db_name = self.app.replace('-', '_').replace('.', '_')
+        db_user = db_name
+
+        if self.type == "mysql":
+            self._run_script("deprovision", f"ynh_mysql_remove_db '{db_name}' '{db_user}'")
+        elif self.type == "postgresql":
+            self._run_script("deprovision", f"ynh_psql_remove_db '{db_name}' '{db_user}'")
+
+        self.delete_setting("db_name")
+        self.delete_setting("db_user")
+        self.delete_setting("db_pwd")
+
+
+AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()}

src/utils/system.py

@@ -23,13 +23,85 @@ import os
 import logging
 
 from moulinette.utils.process import check_output
-from packaging import version
+from yunohost.utils.error import YunohostError
 
 logger = logging.getLogger("yunohost.utils.packages")
 
 YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat"]
 
 
+def system_arch():
+    return check_output("dpkg --print-architecture")
+
+
+def system_virt():
+    """
+    Returns the output of systemd-detect-virt (so e.g. 'none' or 'lxc' or ...)
+    You can check the man of the command to have a list of possible outputs...
+    """
+    # Detect virt technology (if not bare metal) and arch
+    # Gotta have this "|| true" because it systemd-detect-virt return 'none'
+    # with an error code on bare metal ~.~
+    return check_output("systemd-detect-virt || true")
+
+
+def free_space_in_directory(dirpath):
+    stat = os.statvfs(dirpath)
+    return stat.f_frsize * stat.f_bavail
+
+
+def space_used_by_directory(dirpath, follow_symlinks=True):
+
+    if not follow_symlinks:
+        du_output = check_output(["du", "-sb", dirpath], shell=False)
+        return int(du_output.split()[0])
+
+    stat = os.statvfs(dirpath)
+    return stat.f_frsize * stat.f_blocks  # FIXME : this doesnt do what the function name suggest this does ...
+
+
+def human_to_binary(size: str) -> int:
+
+    symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
+    factor = {}
+    for i, s in enumerate(symbols):
+        factor[s] = 1 << (i + 1) * 10
+
+    suffix = size[-1]
+    size = size[:-1]
+
+    if suffix not in symbols:
+        raise YunohostError(f"Invalid size suffix '{suffix}', expected one of {symbols}")
+
+    try:
+        size_ = float(size)
+    except Exception:
+        raise YunohostError(f"Failed to convert size {size} to float")
+
+    return int(size_ * factor[suffix])
+
+
+def binary_to_human(n: int) -> str:
+    """
+    Convert bytes or bits into human readable format with binary prefix
+    """
+    symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
+    prefix = {}
+    for i, s in enumerate(symbols):
+        prefix[s] = 1 << (i + 1) * 10
+    for s in reversed(symbols):
+        if n >= prefix[s]:
+            value = float(n) / prefix[s]
+            return "%.1f%s" % (value, s)
+    return "%s" % n
+
+
+def ram_available():
+
+    import psutil
+    return (psutil.virtual_memory().available, psutil.swap_memory().free)
+
+
 def get_ynh_package_version(package):
 
     # Returns the installed version and release version ('stable' or 'testing'
@@ -48,43 +120,6 @@ def get_ynh_package_version(package):
     return {"version": out[1].strip("()"), "repo": out[2].strip(";")}
 
 
-def meets_version_specifier(pkg_name, specifier):
-    """
-    Check if a package installed version meets specifier
-
-    specifier is something like ">> 1.2.3"
-    """
-
-    # In practice, this function is only used to check the yunohost version
-    # installed.
-    # We'll trim any ~foobar in the current installed version because it's not
-    # handled correctly by version.parse, but we don't care so much in that
-    # context
-    assert pkg_name in YUNOHOST_PACKAGES
-    pkg_version = get_ynh_package_version(pkg_name)["version"]
-    pkg_version = re.split(r"\~|\+|\-", pkg_version)[0]
-    pkg_version = version.parse(pkg_version)
-
-    # Extract operator and version specifier
-    op, req_version = re.search(r"(<<|<=|=|>=|>>) *([\d\.]+)", specifier).groups()
-    req_version = version.parse(req_version)
-
-    # Python2 had a builtin that returns (-1, 0, 1) depending on comparison
-    # c.f. https://stackoverflow.com/a/22490617
-    def cmp(a, b):
-        return (a > b) - (a < b)
-
-    deb_operators = {
-        "<<": lambda v1, v2: cmp(v1, v2) in [-1],
-        "<=": lambda v1, v2: cmp(v1, v2) in [-1, 0],
-        "=": lambda v1, v2: cmp(v1, v2) in [0],
-        ">=": lambda v1, v2: cmp(v1, v2) in [0, 1],
-        ">>": lambda v1, v2: cmp(v1, v2) in [1],
-    }
-
-    return deb_operators[op](pkg_version, req_version)
-
-
 def ynh_packages_version(*args, **kwargs):
     # from cli the received arguments are:
     # (Namespace(_callbacks=deque([]), _tid='_global', _to_return={}), []) {}