From eb6d9df92f7256821bf56a523c81f5e554e65075 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 26 Feb 2023 20:08:59 +0100 Subject: [PATCH 1/7] helpers: add support for a sources.toml to modernize and replace app.src format --- helpers/utils | 166 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 130 insertions(+), 36 deletions(-) diff --git a/helpers/utils b/helpers/utils index f80c22901..d958ae02e 100644 --- a/helpers/utils +++ b/helpers/utils @@ -71,39 +71,78 @@ fi # # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] # | arg: -d, --dest_dir= - Directory where to setup sources -# | arg: -s, --source_id= - Name of the source, defaults to `app` +# | arg: -s, --source_id= - Name of the source, defaults to `main` (when sources.toml exists) or (legacy) `app` (when no sources.toml exists) # | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs/' # | arg: -r, --full_replace= - Remove previous sources before installing new sources # +# #### New format `.toml` +# +# This helper will read infos from a sources.toml at the root of the app package +# and expect a structure like: +# +# ```toml +# [main] +# url = "https://some.address.to/download/the/app/archive" +# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL +# +# +# # Optional flags: +# format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract +# "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract +# "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract +# "whatever" # an arbitrary value, not really meaningful except to imply that the file won't be extracted +# +# in_subdir = true # default, there's an intermediate subdir in the archive before accessing the actual files +# false # sources are directly in the archive root +# n # (special cases) an integer representing a number of subdirs levels to get rid of +# +# extract = true # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ... +# = false # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset. +# # in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value +# +# rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical +# platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for +# ``` +# +# You may also define sublevels for each architectures such as: +# ```toml +# [main] +# autoswitch_per_arch = true +# +# [main.amd64] +# url = "https://some.address.to/download/the/app/archive/when/amd64" +# sha256 = "0123456789abcdef" +# +# [main.armhf] +# url = "https://some.address.to/download/the/app/archive/when/armhf" +# sha256 = "fedcba9876543210" +# ``` +# +# In which case ynh_setup_source --dest_dir="$install_dir" will automatically pick the appropriate source depending on the arch +# +# +# +# #### Legacy format '.src' +# # This helper will read `conf/${source_id}.src`, download and install the sources. # # The src file need to contains: # ``` # SOURCE_URL=Address to download the app archive -# SOURCE_SUM=Control sum -# # (Optional) Program to check the integrity (sha256sum, md5sum...). Default: sha256 -# SOURCE_SUM_PRG=sha256 -# # (Optional) Archive format. Default: tar.gz +# SOURCE_SUM=Sha256 sum # SOURCE_FORMAT=tar.gz -# # (Optional) Put false if sources are directly in the archive root. Default: true -# # Instead of true, SOURCE_IN_SUBDIR could be the number of sub directories to remove. # SOURCE_IN_SUBDIR=false -# # (Optionnal) Name of the local archive (offline setup support). Default: ${src_id}.${src_format} # SOURCE_FILENAME=example.tar.gz -# # (Optional) If it set as false don't extract the source. Default: true -# # (Useful to get a debian package or a python wheel.) # SOURCE_EXTRACT=(true|false) -# # (Optionnal) Name of the plateform. Default: "linux/$YNH_ARCH" # SOURCE_PLATFORM=linux/arm64/v8 # ``` # # The helper will: -# - Check if there is a local source archive in `/opt/yunohost-apps-src/$APP_ID/$SOURCE_FILENAME` -# - Download `$SOURCE_URL` if there is no local archive -# - Check the integrity with `$SOURCE_SUM_PRG -c --status` +# - Download the specific URL if there is no local archive +# - Check the integrity with the specific sha256 sum # - Uncompress the archive to `$dest_dir`. -# - If `$SOURCE_IN_SUBDIR` is true, the first level directory of the archive will be removed. -# - If `$SOURCE_IN_SUBDIR` is a numeric value, the N first level directories will be removed. +# - If `in_subdir` is true, the first level directory of the archive will be removed. +# - If `in_subdir` is a numeric value, the N first level directories will be removed. # - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` # - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir # @@ -118,22 +157,64 @@ ynh_setup_source() { local full_replace # Manage arguments with getopts ynh_handle_getopts_args "$@" - source_id="${source_id:-app}" keep="${keep:-}" full_replace="${full_replace:-0}" - local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" + if test -e $YNH_APP_BASEDIR/sources.toml + then + source_id="${source_id:-main}" + local sources_json=$(cat $YNH_APP_BASEDIR/sources.toml | toml_to_json) + if [[ "$(echo "$sources_json" | jq -r ".$source_id.autoswitch_per_arch")" == "true" ]] + then + source_id=$source_id.$YNH_ARCH + fi - # Load value from configuration file (see above for a small doc about this file - # format) - local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_filename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_plateform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_url="$(echo "$sources_json" | jq -r ".$source_id.url" | sed 's/^null$//')" + local src_sum="$(echo "$sources_json" | jq -r ".$source_id.sha256" | sed 's/^null$//')" + local src_sumprg="sha256sum" + local src_format="$(echo "$sources_json" | jq -r ".$source_id.format" | sed 's/^null$//')" + local src_in_subdir="$(echo "$sources_json" | jq -r ".$source_id.in_subdir" | sed 's/^null$//')" + local src_extract="$(echo "$sources_json" | jq -r ".$source_id.extract" | sed 's/^null$//')" + local src_platform="$(echo "$sources_json" | jq -r ".$source_id.platform" | sed 's/^null$//')" + local src_rename="$(echo "$sources_json" | jq -r ".$source_id.rename" | sed 's/^null$//')" + + [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id ?" + [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id ?" + + if [[ -z "$src_format" ]] + then + if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] + then + src_format="zip" + elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] + then + src_format="tar.gz" + elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] + then + src_format="tar.xz" + elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] + then + src_format="tar.bz2" + elif [[ -z "$src_extract" ]] + then + src_extract="false" + fi + fi + else + source_id="${source_id:-app}" + local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" + + # Load value from configuration file (see above for a small doc about this file + # format) + local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_rename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_platform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + fi # Default value src_sumprg=${src_sumprg:-sha256sum} @@ -141,10 +222,14 @@ ynh_setup_source() { src_format=${src_format:-tar.gz} src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') src_extract=${src_extract:-true} - if [ "$src_filename" = "" ]; then - src_filename="${source_id}.${src_format}" + src_filename="${source_id}.${src_format}" + + if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] + then + ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" fi + # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}" @@ -152,7 +237,7 @@ ynh_setup_source() { src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${src_filename}" if [ "$src_format" = "docker" ]; then - src_plateform="${src_plateform:-"linux/$YNH_ARCH"}" + src_platform="${src_platform:-"linux/$YNH_ARCH"}" elif test -e "$local_src"; then cp $local_src $src_filename else @@ -199,11 +284,16 @@ ynh_setup_source() { _ynh_apply_default_permissions $dest_dir fi - if ! "$src_extract"; then - mv $src_filename $dest_dir - elif [ "$src_format" = "docker" ]; then - /usr/share/yunohost/helpers.d/vendor/docker-image-extract/docker-image-extract -p $src_plateform -o $dest_dir $src_url 2>&1 - elif [ "$src_format" = "zip" ]; then + if [[ "$src_extract" == "false" ]]; then + if [[ -z "$src_rename" ]] + then + mv $src_filename $dest_dir + else + mv $src_filename $dest_dir/$src_rename + fi + elif [[ "$src_format" == "docker" ]]; then + /usr/share/yunohost/helpers.d/vendor/docker-image-extract/docker-image-extract -p $src_platform -o $dest_dir $src_url 2>&1 + elif [[ "$src_format" == "zip" ]]; then # Zip format # Using of a temp directory, because unzip doesn't manage --strip-components if $src_in_subdir; then @@ -970,3 +1060,7 @@ _ynh_apply_default_permissions() { int_to_bool() { sed -e 's/^1$/True/g' -e 's/^0$/False/g' } + +toml_to_json() { + python3 -c 'import toml, json, sys; print(json.dumps(toml.load(sys.stdin)))' +} From 4102d626e5e381dd57887acca4000d37bb2e1be4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Mar 2023 19:57:12 +0100 Subject: [PATCH 2/7] appsv2/sources: change 'sources.toml' to a new 'sources' app resource instead --- helpers/utils | 55 ++++++++++------ src/app.py | 6 +- src/backup.py | 1 + src/utils/resources.py | 143 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 182 insertions(+), 23 deletions(-) diff --git a/helpers/utils b/helpers/utils index d958ae02e..3ef7c2246 100644 --- a/helpers/utils +++ b/helpers/utils @@ -160,17 +160,19 @@ ynh_setup_source() { keep="${keep:-}" full_replace="${full_replace:-0}" - if test -e $YNH_APP_BASEDIR/sources.toml + if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null then source_id="${source_id:-main}" - local sources_json=$(cat $YNH_APP_BASEDIR/sources.toml | toml_to_json) - if [[ "$(echo "$sources_json" | jq -r ".$source_id.autoswitch_per_arch")" == "true" ]] + local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq '.resources.sources') + if ! echo "$sources_json" | jq -re ".$source_id.url" then - source_id=$source_id.$YNH_ARCH + local arch_prefix=".$YNH_ARCH" + else + local arch_prefix="" fi - local src_url="$(echo "$sources_json" | jq -r ".$source_id.url" | sed 's/^null$//')" - local src_sum="$(echo "$sources_json" | jq -r ".$source_id.sha256" | sed 's/^null$//')" + local src_url="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.url" | sed 's/^null$//')" + local src_sum="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.sha256" | sed 's/^null$//')" local src_sumprg="sha256sum" local src_format="$(echo "$sources_json" | jq -r ".$source_id.format" | sed 's/^null$//')" local src_in_subdir="$(echo "$sources_json" | jq -r ".$source_id.in_subdir" | sed 's/^null$//')" @@ -178,8 +180,8 @@ ynh_setup_source() { local src_platform="$(echo "$sources_json" | jq -r ".$source_id.platform" | sed 's/^null$//')" local src_rename="$(echo "$sources_json" | jq -r ".$source_id.rename" | sed 's/^null$//')" - [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id ?" - [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id ?" + [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" if [[ -z "$src_format" ]] then @@ -222,7 +224,6 @@ ynh_setup_source() { src_format=${src_format:-tar.gz} src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') src_extract=${src_extract:-true} - src_filename="${source_id}.${src_format}" if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] then @@ -231,10 +232,10 @@ ynh_setup_source() { # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... - local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}" + local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" mkdir -p /var/cache/yunohost/download/${YNH_APP_ID}/ - src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${src_filename}" + src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" if [ "$src_format" = "docker" ]; then src_platform="${src_platform:-"linux/$YNH_ARCH"}" @@ -243,16 +244,30 @@ ynh_setup_source() { else [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" - # NB. we have to declare the var as local first, - # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work - # because local always return 0 ... - local out - # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) - out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ - || ynh_die --message="$out" + # If the file was prefetched but somehow doesn't match the sum, rm and redownload it + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + rm -f "$src_filename" + fi + + # Only redownload the file if it wasnt prefetched + if [ ! -e "$src_filename" ] + then + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ + || ynh_die --message="$out" + fi + # Check the control sum - echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status \ - || ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got $(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1) (size: $(du -hs ${src_filename} | cut --delimiter=' ' --fields=1))." + if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + rm ${src_filename} + ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got $(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1) (size: $(du -hs ${src_filename} | cut --delimiter=' ' --fields=1))." + fi fi # Keep files to be backup/restored at the end of the helper diff --git a/src/app.py b/src/app.py index 17ebe96ca..753f17339 100644 --- a/src/app.py +++ b/src/app.py @@ -747,6 +747,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False ).apply( rollback_and_raise_exception_if_failure=True, operation_logger=operation_logger, + action="upgrade", ) # Boring stuff : the resource upgrade may have added/remove/updated setting @@ -1122,6 +1123,7 @@ def app_install( AppResourceManager(app_instance_name, wanted=manifest, current={}).apply( rollback_and_raise_exception_if_failure=True, operation_logger=operation_logger, + action="install", ) except (KeyboardInterrupt, EOFError, Exception) as e: shutil.rmtree(app_setting_path) @@ -1253,7 +1255,7 @@ def app_install( AppResourceManager( app_instance_name, wanted={}, current=manifest - ).apply(rollback_and_raise_exception_if_failure=False) + ).apply(rollback_and_raise_exception_if_failure=False, action="remove") else: # Remove all permission in LDAP for permission_name in user_permission_list()["permissions"].keys(): @@ -1392,7 +1394,7 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): from yunohost.utils.resources import AppResourceManager AppResourceManager(app, wanted={}, current=manifest).apply( - rollback_and_raise_exception_if_failure=False, purge_data_dir=purge + rollback_and_raise_exception_if_failure=False, purge_data_dir=purge, action="remove" ) else: # Remove all permission in LDAP diff --git a/src/backup.py b/src/backup.py index 0cf73c4ae..ee218607d 100644 --- a/src/backup.py +++ b/src/backup.py @@ -1528,6 +1528,7 @@ class RestoreManager: AppResourceManager(app_instance_name, wanted=manifest, current={}).apply( rollback_and_raise_exception_if_failure=True, operation_logger=operation_logger, + action="restore", ) # Execute the app install script diff --git a/src/utils/resources.py b/src/utils/resources.py index cff6c6b19..6c5e4890d 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -21,6 +21,7 @@ import copy import shutil import random import tempfile +import subprocess from typing import Dict, Any, List from moulinette import m18n @@ -30,7 +31,7 @@ from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file from moulinette.utils.filesystem import ( rm, ) - +from yunohost.utils.system import system_arch from yunohost.utils.error import YunohostError, YunohostValidationError logger = getActionLogger("yunohost.app_resources") @@ -257,6 +258,146 @@ ynh_abort_if_errors # print(ret) +class SourcesResource(AppResource): + """ + Declare what are the sources / assets used by this app. Typically, this corresponds to some tarball published by the upstream project, that needs to be downloaded and extracted in the install dir using the ynh_setup_source helper. + + This resource is intended both to declare the assets, which will be parsed by ynh_setup_source during the app script runtime, AND to prefetch and validate the sha256sum of those asset before actually running the script, to be able to report an error early when the asset turns out to not be available for some reason. + + Various options are available to accomodate the behavior according to the asset structure + + ##### Example: + + ```toml + [resources.sources] + + [resources.sources.main] + url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.tar.gz" + sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + ``` + + Or more complex examples with several element, including one with asset that depends on the arch + + ```toml + [resources.sources] + + [resources.sources.main] + in_subdir = false + amd64.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz" + amd64.sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + i386.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz" + i386.sha256 = "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3" + armhf.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.armhf.tar.gz" + armhf.sha256 = "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + + [resources.sources.zblerg] + url = "https://zblerg.com/download/zblerg" + sha256sum = "1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2" + format = "script" + rename = "zblerg.sh" + + ``` + + ##### Properties (for each source): + + - `prefetch` : `true` (default) or `false`, wether or not to pre-fetch this asset during the provisioning phase of the resource. If several arch-dependent url are provided, YunoHost will only prefetch the one for the current system architecture. + - `url` : the asset's URL + - If the asset's URL depend on the architecture, you may instead provide `amd64.url`, `i386.url`, `armhf.url` and `arm64.url` (depending on what architectures are supported), using the same `dpkg --print-architecture` nomenclature as for the supported architecture key in the manifest + - `sha256` : the asset's sha256sum. This is used both as an integrity check, and as a layer of security to protect against malicious actors which could have injected malicious code inside the asset... + - Same as `url` : if the asset's URL depend on the architecture, you may instead provide `amd64.sha256`, `i386.sha256`, ... + - `format` : The "format" of the asset. It is typically automatically guessed from the extension of the URL (or the mention of "tarball", "zipball" in the URL), but can be set explicitly: + - `tar.gz`, `tar.xz`, `tar.bz2` : will use `tar` to extract the archive + - `zip` : will use `unzip` to extract the archive + - `docker` : useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` + - `whatever`: whatever arbitrary value, not really meaningful except to imply that the file won't be extracted (eg because it's a .deb to be manually installed with dpkg/apt, or a script, or ...) + - `in_subdir`: `true` (default) or `false`, depending on if there's an intermediate subdir in the archive before accessing the actual files. Can also be `N` (an integer) to handle special cases where there's `N` level of subdir to get rid of to actually access the files + - `extract` : `true` or `false`. Defaults to `true` for archives such as `zip`, `tar.gz`, `tar.bz2`, ... Or defaults to `false` when `format` is not something that should be extracted. When `extract = false`, the file will only be `mv`ed to the location, possibly renamed using the `rename` value + - `rename`: some string like `whatever_your_want`, to be used for convenience when `extract` is `false` and the default name of the file is not practical + - `platform`: for exampl `linux/amd64` (defaults to `linux/$YNH_ARCH`) to be used in conjonction with `format = "docker"` to specify which architecture to extract for + + + ##### Provision/Update: + - For elements with `prefetch = true`, will download the asset (for the appropriate architecture) and store them in `/var/cache/yunohost/download/$app/$source_id`, to be later picked up by `ynh_setup_source`. (NB: this only happens during install and upgrade, not restore) + + ##### Deprovision: + - Nothing + """ + + type = "sources" + priority = 10 + + default_sources_properties: Dict[str, Any] = { + "prefetch": True, + "url": None, + "sha256": None, + } + + sources: Dict[str, Dict[str, Any]] = {} + + def __init__(self, properties: Dict[str, Any], *args, **kwargs): + + for source_id, infos in properties.items(): + properties[source_id] = copy.copy(self.default_sources_properties) + properties[source_id].update(infos) + + super().__init__({"sources": properties}, *args, **kwargs) + + def deprovision(self, context: Dict = {}): + if os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"): + rm(f"/var/cache/yunohost/download/{self.app}/", recursive=True) + pass + + def provision_or_update(self, context: Dict = {}): + + # Don't prefetch stuff during restore + if context.get("action") == "restore": + return + + import pdb; pdb.set_trace() + + for source_id, infos in self.sources.items(): + + if not infos["prefetch"]: + continue + + if infos["url"] is None: + arch = system_arch() + if arch in infos and isinstance(infos[arch], dict) and isinstance(infos[arch].get("url"), str) and isinstance(infos[arch].get("sha256"), str): + self.prefetch(source_id, infos[arch]["url"], infos[arch]["sha256"]) + else: + raise YunohostError(f"In resources.sources: it looks like you forgot to define url/sha256 or {arch}.url/{arch}.sha256", raw_msg=True) + else: + if infos["sha256"] is None: + raise YunohostError(f"In resources.sources: it looks like the sha256 is missing for {source_id}", raw_msg=True) + self.prefetch(source_id, infos["url"], infos["sha256"]) + + def prefetch(self, source_id, url, expected_sha256): + + logger.debug(f"Prefetching asset {source_id}: {url} ...") + + if not os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"): + mkdir(f"/var/cache/yunohost/download/{self.app}/", parents=True) + filename = f"/var/cache/yunohost/download/{self.app}/{source_id}" + + # NB: we use wget and not requests.get() because we want to output to a file (ie avoid ending up with the full archive in RAM) + # AND the nice --tries, --no-dns-cache, --timeout options ... + p = subprocess.Popen(["/usr/bin/wget", "--tries=3", "--no-dns-cache", "--timeout=900", "--no-verbose", "--output-document=" + filename, url], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + out, _ = p.communicate() + returncode = p.returncode + if returncode != 0: + if os.path.exists(filename): + rm(filename) + out = out.decode() + raise YunohostError(f"Failed to download asset {source_id} ({url}) for {self.app}: {out}", raw_msg=True) + + assert os.path.exists(filename), f"For some reason, wget worked but {filename} doesnt exists?" + + computed_sha256 = check_output(f"sha256sum {filename}").split()[0] + if computed_sha256 != expected_sha256: + size = check_output(f"du -hs {filename}").split()[0] + rm(filename) + raise YunohostError(f"Corrupt source for {url} : expected to find {expected_sha256} as sha256sum, but got {computed_sha256} instead ... (file size : {size})", raw_msg=True) + class PermissionsResource(AppResource): """ From 0a937ab0bd9f3dda8345cfeb6434c71a866b55d5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Mar 2023 20:06:11 +0100 Subject: [PATCH 3/7] Unecessary pass statement --- src/utils/resources.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 6c5e4890d..6202f0353 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -320,7 +320,7 @@ class SourcesResource(AppResource): - For elements with `prefetch = true`, will download the asset (for the appropriate architecture) and store them in `/var/cache/yunohost/download/$app/$source_id`, to be later picked up by `ynh_setup_source`. (NB: this only happens during install and upgrade, not restore) ##### Deprovision: - - Nothing + - Nothing (just cleanup the cache) """ type = "sources" @@ -345,7 +345,6 @@ class SourcesResource(AppResource): def deprovision(self, context: Dict = {}): if os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"): rm(f"/var/cache/yunohost/download/{self.app}/", recursive=True) - pass def provision_or_update(self, context: Dict = {}): From acb359bdbfc4cc6e4e8a600011cf258eaee4eaf7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 6 Mar 2023 20:15:20 +0100 Subject: [PATCH 4/7] Forgot to remove pdb D: --- src/utils/resources.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 6202f0353..2b2ba97d8 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -352,8 +352,6 @@ class SourcesResource(AppResource): if context.get("action") == "restore": return - import pdb; pdb.set_trace() - for source_id, infos in self.sources.items(): if not infos["prefetch"]: From ebc9e645fc02eb319c68e222de3ae1a48d732e56 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 8 Mar 2023 16:23:58 +0100 Subject: [PATCH 5/7] Typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Éric Gaspar <46165813+ericgaspar@users.noreply.github.com> --- src/utils/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 2b2ba97d8..8427b4811 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -313,7 +313,7 @@ class SourcesResource(AppResource): - `in_subdir`: `true` (default) or `false`, depending on if there's an intermediate subdir in the archive before accessing the actual files. Can also be `N` (an integer) to handle special cases where there's `N` level of subdir to get rid of to actually access the files - `extract` : `true` or `false`. Defaults to `true` for archives such as `zip`, `tar.gz`, `tar.bz2`, ... Or defaults to `false` when `format` is not something that should be extracted. When `extract = false`, the file will only be `mv`ed to the location, possibly renamed using the `rename` value - `rename`: some string like `whatever_your_want`, to be used for convenience when `extract` is `false` and the default name of the file is not practical - - `platform`: for exampl `linux/amd64` (defaults to `linux/$YNH_ARCH`) to be used in conjonction with `format = "docker"` to specify which architecture to extract for + - `platform`: for example `linux/amd64` (defaults to `linux/$YNH_ARCH`) to be used in conjonction with `format = "docker"` to specify which architecture to extract for ##### Provision/Update: From 0d524220e5c679142139ce1e2836ae8664ddf64d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 8 Mar 2023 16:44:52 +0100 Subject: [PATCH 6/7] appsv2/sources: i18n --- locales/en.json | 2 ++ src/utils/resources.py | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/locales/en.json b/locales/en.json index 7cc1b96b6..083ecdc8d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -26,6 +26,8 @@ "app_change_url_success": "{app} URL is now {domain}{path}", "app_config_unable_to_apply": "Failed to apply config panel values.", "app_config_unable_to_read": "Failed to read config panel values.", + "app_failed_to_download_asset": "Failed to download asset '{source_id}' ({url}) for {app}: {out}", + "app_corrupt_source": "YunoHost was able to download the asset '{source_id}' ({url}) for {app}, but the asset doesn't match the expected checksum. This could mean that some temporary network failure happened on your server, OR the asset was somehow changed by the upstream maintainer (or a malicious actor?) and YunoHost packagers need to investigate and update the app manifest to reflect this change.\n Expected sha256 checksum: {expected_sha256}\n Downloaded sha256 checksum: {computed_sha256}\n Downloaded file size: {size}", "app_extraction_failed": "Could not extract the installation files", "app_full_domain_unavailable": "Sorry, this app must be installed on a domain of its own, but other apps are already installed on the domain '{domain}'. You could use a subdomain dedicated to this app instead.", "app_id_invalid": "Invalid app ID", diff --git a/src/utils/resources.py b/src/utils/resources.py index 8427b4811..f6ff6bb46 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -292,7 +292,7 @@ class SourcesResource(AppResource): [resources.sources.zblerg] url = "https://zblerg.com/download/zblerg" - sha256sum = "1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2" + sha256 = "1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2" format = "script" rename = "zblerg.sh" @@ -384,8 +384,7 @@ class SourcesResource(AppResource): if returncode != 0: if os.path.exists(filename): rm(filename) - out = out.decode() - raise YunohostError(f"Failed to download asset {source_id} ({url}) for {self.app}: {out}", raw_msg=True) + raise YunohostError("app_failed_to_download_asset", source_id=source_id, url=url, app=self.app, out=out.decode()) assert os.path.exists(filename), f"For some reason, wget worked but {filename} doesnt exists?" @@ -393,7 +392,7 @@ class SourcesResource(AppResource): if computed_sha256 != expected_sha256: size = check_output(f"du -hs {filename}").split()[0] rm(filename) - raise YunohostError(f"Corrupt source for {url} : expected to find {expected_sha256} as sha256sum, but got {computed_sha256} instead ... (file size : {size})", raw_msg=True) + raise YunohostError("app_corrupt_source", source_id=source_id, url=url, app=self.app, expected_sha256=expected_sha256, computed_sha256=computed_sha256, size=size) class PermissionsResource(AppResource): From cb324232366f8c4c857e85fa728efe442caaacac Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 9 Mar 2023 15:18:29 +0100 Subject: [PATCH 7/7] appsv2/sources: Reflect changes in ynh_setup_source doc --- helpers/utils | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/helpers/utils b/helpers/utils index 3ef7c2246..695b165c0 100644 --- a/helpers/utils +++ b/helpers/utils @@ -71,20 +71,23 @@ fi # # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] # | arg: -d, --dest_dir= - Directory where to setup sources -# | arg: -s, --source_id= - Name of the source, defaults to `main` (when sources.toml exists) or (legacy) `app` (when no sources.toml exists) +# | arg: -s, --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise # | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs/' # | arg: -r, --full_replace= - Remove previous sources before installing new sources # -# #### New format `.toml` +# #### New 'sources' resources # -# This helper will read infos from a sources.toml at the root of the app package +# (See also the resources documentation which may be more complete?) +# +# This helper will read infos from the 'sources' resources in the manifest.toml of the app # and expect a structure like: # # ```toml -# [main] -# url = "https://some.address.to/download/the/app/archive" -# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL -# +# [resources.sources] +# [resources.sources.main] +# url = "https://some.address.to/download/the/app/archive" +# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL +# ``` # # # Optional flags: # format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract @@ -102,20 +105,16 @@ fi # # rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical # platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for -# ``` +# # -# You may also define sublevels for each architectures such as: +# You may also define assets url and checksum per-architectures such as: # ```toml -# [main] -# autoswitch_per_arch = true -# -# [main.amd64] -# url = "https://some.address.to/download/the/app/archive/when/amd64" -# sha256 = "0123456789abcdef" -# -# [main.armhf] -# url = "https://some.address.to/download/the/app/archive/when/armhf" -# sha256 = "fedcba9876543210" +# [resources.sources] +# [resources.sources.main] +# amd64.url = "https://some.address.to/download/the/app/archive/when/amd64" +# amd64.sha256 = "0123456789abcdef" +# armhf.url = "https://some.address.to/download/the/app/archive/when/armhf" +# armhf.sha256 = "fedcba9876543210" # ``` # # In which case ynh_setup_source --dest_dir="$install_dir" will automatically pick the appropriate source depending on the arch