appsv2/sources: change 'sources.toml' to a new 'sources' app resource instead

helpers/utils

@@ -160,17 +160,19 @@ ynh_setup_source() {
     keep="${keep:-}"
     full_replace="${full_replace:-0}"
 
-    if test -e $YNH_APP_BASEDIR/sources.toml
+    if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null
     then
         source_id="${source_id:-main}"
-        local sources_json=$(cat $YNH_APP_BASEDIR/sources.toml | toml_to_json)
+        local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq '.resources.sources')
-        if [[ "$(echo "$sources_json" | jq -r ".$source_id.autoswitch_per_arch")" == "true" ]]
+        if ! echo "$sources_json" | jq -re ".$source_id.url"
         then
-            source_id=$source_id.$YNH_ARCH
+            local arch_prefix=".$YNH_ARCH"
+        else
+            local arch_prefix=""
         fi
 
-        local src_url="$(echo "$sources_json" | jq -r ".$source_id.url" | sed 's/^null$//')"
+        local src_url="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.url" | sed 's/^null$//')"
-        local src_sum="$(echo "$sources_json" | jq -r ".$source_id.sha256" | sed 's/^null$//')"
+        local src_sum="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.sha256" | sed 's/^null$//')"
         local src_sumprg="sha256sum"
         local src_format="$(echo "$sources_json" | jq -r ".$source_id.format" | sed 's/^null$//')"
         local src_in_subdir="$(echo "$sources_json" | jq -r ".$source_id.in_subdir" | sed 's/^null$//')"
@@ -178,8 +180,8 @@ ynh_setup_source() {
         local src_platform="$(echo "$sources_json" | jq -r ".$source_id.platform" | sed 's/^null$//')"
         local src_rename="$(echo "$sources_json" | jq -r ".$source_id.rename" | sed 's/^null$//')"
 
-        [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id ?"
+        [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?"
-        [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id ?"
+        [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?"
 
         if [[ -z "$src_format" ]]
         then
@@ -222,7 +224,6 @@ ynh_setup_source() {
     src_format=${src_format:-tar.gz}
     src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]')
     src_extract=${src_extract:-true}
-    src_filename="${source_id}.${src_format}"
 
     if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]]
     then
@@ -231,10 +232,10 @@ ynh_setup_source() {
 
 
     # (Unused?) mecanism where one can have the file in a special local cache to not have to download it...
-    local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}"
+    local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}"
 
     mkdir -p /var/cache/yunohost/download/${YNH_APP_ID}/
-    src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${src_filename}"
+    src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}"
 
     if [ "$src_format" = "docker" ]; then
         src_platform="${src_platform:-"linux/$YNH_ARCH"}"
@@ -243,6 +244,15 @@ ynh_setup_source() {
     else
         [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?"
 
+        # If the file was prefetched but somehow doesn't match the sum, rm and redownload it
+        if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
+        then
+            rm -f "$src_filename"
+        fi
+
+        # Only redownload the file if it wasnt prefetched
+        if [ ! -e "$src_filename" ]
+        then
             # NB. we have to declare the var as local first,
             # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work
             # because local always return 0 ...
@@ -250,9 +260,14 @@ ynh_setup_source() {
             # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
             out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \
                 || ynh_die --message="$out"
+        fi
+
         # Check the control sum
-        echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status \
+        if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
-            || ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got $(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1) (size: $(du -hs ${src_filename} | cut --delimiter=' ' --fields=1))."
+        then
+            rm ${src_filename}
+            ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got $(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1) (size: $(du -hs ${src_filename} | cut --delimiter=' ' --fields=1))."
+        fi
     fi
 
     # Keep files to be backup/restored at the end of the helper

src/app.py

@@ -747,6 +747,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
             ).apply(
                 rollback_and_raise_exception_if_failure=True,
                 operation_logger=operation_logger,
+                action="upgrade",
             )
 
             # Boring stuff : the resource upgrade may have added/remove/updated setting
@@ -1122,6 +1123,7 @@ def app_install(
             AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(
                 rollback_and_raise_exception_if_failure=True,
                 operation_logger=operation_logger,
+                action="install",
             )
         except (KeyboardInterrupt, EOFError, Exception) as e:
             shutil.rmtree(app_setting_path)
@@ -1253,7 +1255,7 @@ def app_install(
 
                 AppResourceManager(
                     app_instance_name, wanted={}, current=manifest
-                ).apply(rollback_and_raise_exception_if_failure=False)
+                ).apply(rollback_and_raise_exception_if_failure=False, action="remove")
             else:
                 # Remove all permission in LDAP
                 for permission_name in user_permission_list()["permissions"].keys():
@@ -1392,7 +1394,7 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None):
         from yunohost.utils.resources import AppResourceManager
 
         AppResourceManager(app, wanted={}, current=manifest).apply(
-            rollback_and_raise_exception_if_failure=False, purge_data_dir=purge
+            rollback_and_raise_exception_if_failure=False, purge_data_dir=purge, action="remove"
         )
     else:
         # Remove all permission in LDAP

src/backup.py

@@ -1528,6 +1528,7 @@ class RestoreManager:
             AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(
                 rollback_and_raise_exception_if_failure=True,
                 operation_logger=operation_logger,
+                action="restore",
             )
 
         # Execute the app install script

src/utils/resources.py

@@ -21,6 +21,7 @@ import copy
 import shutil
 import random
 import tempfile
+import subprocess
 from typing import Dict, Any, List
 
 from moulinette import m18n
@@ -30,7 +31,7 @@ from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file
 from moulinette.utils.filesystem import (
     rm,
 )
-
+from yunohost.utils.system import system_arch
 from yunohost.utils.error import YunohostError, YunohostValidationError
 
 logger = getActionLogger("yunohost.app_resources")
@@ -257,6 +258,146 @@ ynh_abort_if_errors
 
         # print(ret)
 
+class SourcesResource(AppResource):
+    """
+    Declare what are the sources / assets used by this app. Typically, this corresponds to some tarball published by the upstream project, that needs to be downloaded and extracted in the install dir using the ynh_setup_source helper.
+
+    This resource is intended both to declare the assets, which will be parsed by ynh_setup_source during the app script runtime, AND to prefetch and validate the sha256sum of those asset before actually running the script, to be able to report an error early when the asset turns out to not be available for some reason.
+
+    Various options are available to accomodate the behavior according to the asset structure
+
+    ##### Example:
+
+    ```toml
+    [resources.sources]
+
+        [resources.sources.main]
+        url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.tar.gz"
+        sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
+    ```
+
+    Or more complex examples with several element, including one with asset that depends on the arch
+
+    ```toml
+    [resources.sources]
+
+        [resources.sources.main]
+        in_subdir = false
+        amd64.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz"
+        amd64.sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
+        i386.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz"
+        i386.sha256 = "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3"
+        armhf.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.armhf.tar.gz"
+        armhf.sha256 = "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865"
+
+        [resources.sources.zblerg]
+        url = "https://zblerg.com/download/zblerg"
+        sha256sum = "1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2"
+        format = "script"
+        rename = "zblerg.sh"
+
+    ```
+
+    ##### Properties (for each source):
+
+    - `prefetch` : `true` (default) or `false`, wether or not to pre-fetch this asset during the provisioning phase of the resource. If several arch-dependent url are provided, YunoHost will only prefetch the one for the current system architecture.
+    - `url` : the asset's URL
+        - If the asset's URL depend on the architecture, you may instead provide `amd64.url`, `i386.url`, `armhf.url` and `arm64.url` (depending on what architectures are supported), using the same `dpkg --print-architecture` nomenclature as for the supported architecture key in the manifest
+    - `sha256` : the asset's sha256sum. This is used both as an integrity check, and as a layer of security to protect against malicious actors which could have injected malicious code inside the asset...
+        - Same as `url` : if the asset's URL depend on the architecture, you may instead provide `amd64.sha256`, `i386.sha256`, ...
+    - `format` : The "format" of the asset. It is typically automatically guessed from the extension of the URL (or the mention of "tarball", "zipball" in the URL), but can be set explicitly:
+        - `tar.gz`, `tar.xz`, `tar.bz2` : will use `tar` to extract the archive
+        - `zip` : will use `unzip` to extract the archive
+        - `docker` : useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract`
+        - `whatever`: whatever arbitrary value, not really meaningful except to imply that the file won't be extracted (eg because it's a .deb to be manually installed with dpkg/apt, or a script, or ...)
+    - `in_subdir`: `true` (default) or `false`, depending on if there's an intermediate subdir in the archive before accessing the actual files. Can also be `N` (an integer) to handle special cases where there's `N` level of subdir to get rid of to actually access the files
+    - `extract` : `true` or `false`. Defaults to `true` for archives such as `zip`, `tar.gz`, `tar.bz2`, ... Or defaults to `false` when `format` is not something that should be extracted. When `extract = false`, the file will only be `mv`ed to the location, possibly renamed using the `rename` value
+    - `rename`: some string like `whatever_your_want`, to be used for convenience when `extract` is `false` and the default name of the file is not practical
+    - `platform`: for exampl `linux/amd64` (defaults to `linux/$YNH_ARCH`) to be used in conjonction with `format = "docker"` to specify which architecture to extract for
+
+
+    ##### Provision/Update:
+    - For elements with `prefetch = true`, will download the asset (for the appropriate architecture) and store them in `/var/cache/yunohost/download/$app/$source_id`, to be later picked up by `ynh_setup_source`. (NB: this only happens during install and upgrade, not restore)
+
+    ##### Deprovision:
+    - Nothing
+    """
+
+    type = "sources"
+    priority = 10
+
+    default_sources_properties: Dict[str, Any] = {
+        "prefetch": True,
+        "url": None,
+        "sha256": None,
+    }
+
+    sources: Dict[str, Dict[str, Any]] = {}
+
+    def __init__(self, properties: Dict[str, Any], *args, **kwargs):
+
+        for source_id, infos in properties.items():
+            properties[source_id] = copy.copy(self.default_sources_properties)
+            properties[source_id].update(infos)
+
+        super().__init__({"sources": properties}, *args, **kwargs)
+
+    def deprovision(self, context: Dict = {}):
+        if os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"):
+            rm(f"/var/cache/yunohost/download/{self.app}/", recursive=True)
+        pass
+
+    def provision_or_update(self, context: Dict = {}):
+
+        # Don't prefetch stuff during restore
+        if context.get("action") == "restore":
+            return
+
+        import pdb; pdb.set_trace()
+
+        for source_id, infos in self.sources.items():
+
+            if not infos["prefetch"]:
+                continue
+
+            if infos["url"] is None:
+                arch = system_arch()
+                if arch in infos and isinstance(infos[arch], dict) and isinstance(infos[arch].get("url"), str) and isinstance(infos[arch].get("sha256"), str):
+                    self.prefetch(source_id, infos[arch]["url"], infos[arch]["sha256"])
+                else:
+                    raise YunohostError(f"In resources.sources: it looks like you forgot to define url/sha256 or {arch}.url/{arch}.sha256", raw_msg=True)
+            else:
+                if infos["sha256"] is None:
+                    raise YunohostError(f"In resources.sources: it looks like the sha256 is missing for {source_id}", raw_msg=True)
+                self.prefetch(source_id, infos["url"], infos["sha256"])
+
+    def prefetch(self, source_id, url, expected_sha256):
+
+        logger.debug(f"Prefetching asset {source_id}: {url} ...")
+
+        if not os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"):
+            mkdir(f"/var/cache/yunohost/download/{self.app}/", parents=True)
+        filename = f"/var/cache/yunohost/download/{self.app}/{source_id}"
+
+        # NB: we use wget and not requests.get() because we want to output to a file (ie avoid ending up with the full archive in RAM)
+        # AND the nice --tries, --no-dns-cache, --timeout options ...
+        p = subprocess.Popen(["/usr/bin/wget", "--tries=3", "--no-dns-cache", "--timeout=900", "--no-verbose", "--output-document=" + filename, url], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        out, _ = p.communicate()
+        returncode = p.returncode
+        if returncode != 0:
+            if os.path.exists(filename):
+                rm(filename)
+            out = out.decode()
+            raise YunohostError(f"Failed to download asset {source_id} ({url}) for {self.app}: {out}", raw_msg=True)
+
+        assert os.path.exists(filename), f"For some reason, wget worked but {filename} doesnt exists?"
+
+        computed_sha256 = check_output(f"sha256sum {filename}").split()[0]
+        if computed_sha256 != expected_sha256:
+            size = check_output(f"du -hs {filename}").split()[0]
+            rm(filename)
+            raise YunohostError(f"Corrupt source for {url} : expected to find {expected_sha256} as sha256sum, but got {computed_sha256} instead ... (file size : {size})", raw_msg=True)
+
 
 class PermissionsResource(AppResource):
     """