YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

appsv2/sources: change 'sources.toml' to a new 'sources' app resource instead

Browse source
This commit is contained in:
Alexandre Aubin 2023-03-06 19:57:12 +01:00
parent eb6d9df92f
commit 4102d626e5
4 changed files with 182 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
helpers/utils
View file

@ -160,17 +160,19 @@ ynh_setup_source() {
keep="${keep:-}"
full_replace="${full_replace:-0}"
if test -e $YNH_APP_BASEDIR/sources.toml
if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null
then
source_id="${source_id:-main}"
local sources_json=$(cat $YNH_APP_BASEDIR/sources.toml | toml_to_json)
if [[ "$(echo "$sources_json" | jq -r ".$source_id.autoswitch_per_arch")" == "true" ]]
local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq '.resources.sources')
if ! echo "$sources_json" | jq -re ".$source_id.url"
then
source_id=$source_id.$YNH_ARCH
local arch_prefix=".$YNH_ARCH"
else
local arch_prefix=""
fi
local src_url="$(echo "$sources_json" | jq -r ".$source_id.url" | sed 's/^null$//')"
local src_sum="$(echo "$sources_json" | jq -r ".$source_id.sha256" | sed 's/^null$//')"
local src_url="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.url" | sed 's/^null$//')"
local src_sum="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.sha256" | sed 's/^null$//')"
local src_sumprg="sha256sum"
local src_format="$(echo "$sources_json" | jq -r ".$source_id.format" | sed 's/^null$//')"
local src_in_subdir="$(echo "$sources_json" | jq -r ".$source_id.in_subdir" | sed 's/^null$//')"
@ -178,8 +180,8 @@ ynh_setup_source() {
local src_platform="$(echo "$sources_json" | jq -r ".$source_id.platform" | sed 's/^null$//')"
local src_rename="$(echo "$sources_json" | jq -r ".$source_id.rename" | sed 's/^null$//')"
[[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id ?"
[[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id ?"
[[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?"
[[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?"
if [[ -z "$src_format" ]]
then
@ -222,7 +224,6 @@ ynh_setup_source() {
src_format=${src_format:-tar.gz}
src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]')
src_extract=${src_extract:-true}
src_filename="${source_id}.${src_format}"
if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]]
then
@ -231,10 +232,10 @@ ynh_setup_source() {
# (Unused?) mecanism where one can have the file in a special local cache to not have to download it...
local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}"
local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}"
mkdir -p /var/cache/yunohost/download/${YNH_APP_ID}/
src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${src_filename}"
src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}"
if [ "$src_format" = "docker" ]; then
src_platform="${src_platform:-"linux/$YNH_ARCH"}"
@ -243,6 +244,15 @@ ynh_setup_source() {
else
[ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?"
# If the file was prefetched but somehow doesn't match the sum, rm and redownload it
if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
then
rm -f "$src_filename"
fi
# Only redownload the file if it wasnt prefetched
if [ ! -e "$src_filename" ]
then
# NB. we have to declare the var as local first,
# otherwise 'local foo=$(false) || echo 'pwet'" does'nt work
# because local always return 0 ...
@ -250,9 +260,14 @@ ynh_setup_source() {
# Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \
|| ynh_die --message="$out"
fi
# Check the control sum
echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status \
|| ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got $(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1) (size: $(du -hs ${src_filename} | cut --delimiter=' ' --fields=1))."
if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
then
rm ${src_filename}
ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got $(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1) (size: $(du -hs ${src_filename} | cut --delimiter=' ' --fields=1))."
fi
fi
# Keep files to be backup/restored at the end of the helper

6
src/app.py
View file

@ -747,6 +747,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
).apply(
rollback_and_raise_exception_if_failure=True,
operation_logger=operation_logger,
action="upgrade",
)
# Boring stuff : the resource upgrade may have added/remove/updated setting
@ -1122,6 +1123,7 @@ def app_install(
AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(
rollback_and_raise_exception_if_failure=True,
operation_logger=operation_logger,
action="install",
)
except (KeyboardInterrupt, EOFError, Exception) as e:
shutil.rmtree(app_setting_path)
@ -1253,7 +1255,7 @@ def app_install(
AppResourceManager(
app_instance_name, wanted={}, current=manifest
).apply(rollback_and_raise_exception_if_failure=False)
).apply(rollback_and_raise_exception_if_failure=False, action="remove")
else:
# Remove all permission in LDAP
for permission_name in user_permission_list()["permissions"].keys():
@ -1392,7 +1394,7 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None):
from yunohost.utils.resources import AppResourceManager
AppResourceManager(app, wanted={}, current=manifest).apply(
rollback_and_raise_exception_if_failure=False, purge_data_dir=purge
rollback_and_raise_exception_if_failure=False, purge_data_dir=purge, action="remove"
)
else:
# Remove all permission in LDAP

1
src/backup.py
View file

@ -1528,6 +1528,7 @@ class RestoreManager:
AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(
rollback_and_raise_exception_if_failure=True,
operation_logger=operation_logger,
action="restore",
)
# Execute the app install script

143
src/utils/resources.py
View file

@ -21,6 +21,7 @@ import copy
import shutil
import random
import tempfile
import subprocess
from typing import Dict, Any, List
from moulinette import m18n
@ -30,7 +31,7 @@ from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file
from moulinette.utils.filesystem import (
rm,
)
from yunohost.utils.system import system_arch
from yunohost.utils.error import YunohostError, YunohostValidationError
logger = getActionLogger("yunohost.app_resources")
@ -257,6 +258,146 @@ ynh_abort_if_errors
# print(ret)
class SourcesResource(AppResource):
"""
Declare what are the sources / assets used by this app. Typically, this corresponds to some tarball published by the upstream project, that needs to be downloaded and extracted in the install dir using the ynh_setup_source helper.
This resource is intended both to declare the assets, which will be parsed by ynh_setup_source during the app script runtime, AND to prefetch and validate the sha256sum of those asset before actually running the script, to be able to report an error early when the asset turns out to not be available for some reason.
Various options are available to accomodate the behavior according to the asset structure
##### Example:
```toml
[resources.sources]
[resources.sources.main]
url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.tar.gz"
sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
```
Or more complex examples with several element, including one with asset that depends on the arch
```toml
[resources.sources]
[resources.sources.main]
in_subdir = false
amd64.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz"
amd64.sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
i386.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz"
i386.sha256 = "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3"
armhf.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.armhf.tar.gz"
armhf.sha256 = "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865"
[resources.sources.zblerg]
url = "https://zblerg.com/download/zblerg"
sha256sum = "1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2"
format = "script"
rename = "zblerg.sh"
```
##### Properties (for each source):
- `prefetch` : `true` (default) or `false`, wether or not to pre-fetch this asset during the provisioning phase of the resource. If several arch-dependent url are provided, YunoHost will only prefetch the one for the current system architecture.
- `url` : the asset's URL
- If the asset's URL depend on the architecture, you may instead provide `amd64.url`, `i386.url`, `armhf.url` and `arm64.url` (depending on what architectures are supported), using the same `dpkg --print-architecture` nomenclature as for the supported architecture key in the manifest
- `sha256` : the asset's sha256sum. This is used both as an integrity check, and as a layer of security to protect against malicious actors which could have injected malicious code inside the asset...
- Same as `url` : if the asset's URL depend on the architecture, you may instead provide `amd64.sha256`, `i386.sha256`, ...
- `format` : The "format" of the asset. It is typically automatically guessed from the extension of the URL (or the mention of "tarball", "zipball" in the URL), but can be set explicitly:
- `tar.gz`, `tar.xz`, `tar.bz2` : will use `tar` to extract the archive
- `zip` : will use `unzip` to extract the archive
- `docker` : useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract`
- `whatever`: whatever arbitrary value, not really meaningful except to imply that the file won't be extracted (eg because it's a .deb to be manually installed with dpkg/apt, or a script, or ...)
- `in_subdir`: `true` (default) or `false`, depending on if there's an intermediate subdir in the archive before accessing the actual files. Can also be `N` (an integer) to handle special cases where there's `N` level of subdir to get rid of to actually access the files
- `extract` : `true` or `false`. Defaults to `true` for archives such as `zip`, `tar.gz`, `tar.bz2`, ... Or defaults to `false` when `format` is not something that should be extracted. When `extract = false`, the file will only be `mv`ed to the location, possibly renamed using the `rename` value
- `rename`: some string like `whatever_your_want`, to be used for convenience when `extract` is `false` and the default name of the file is not practical
- `platform`: for exampl `linux/amd64` (defaults to `linux/$YNH_ARCH`) to be used in conjonction with `format = "docker"` to specify which architecture to extract for
##### Provision/Update:
- For elements with `prefetch = true`, will download the asset (for the appropriate architecture) and store them in `/var/cache/yunohost/download/$app/$source_id`, to be later picked up by `ynh_setup_source`. (NB: this only happens during install and upgrade, not restore)
##### Deprovision:
- Nothing
"""
type = "sources"
priority = 10
default_sources_properties: Dict[str, Any] = {
"prefetch": True,
"url": None,
"sha256": None,
}
sources: Dict[str, Dict[str, Any]] = {}
def __init__(self, properties: Dict[str, Any], *args, **kwargs):
for source_id, infos in properties.items():
properties[source_id] = copy.copy(self.default_sources_properties)
properties[source_id].update(infos)
super().__init__({"sources": properties}, *args, **kwargs)
def deprovision(self, context: Dict = {}):
if os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"):
rm(f"/var/cache/yunohost/download/{self.app}/", recursive=True)
pass
def provision_or_update(self, context: Dict = {}):
# Don't prefetch stuff during restore
if context.get("action") == "restore":
return
import pdb; pdb.set_trace()
for source_id, infos in self.sources.items():
if not infos["prefetch"]:
continue
if infos["url"] is None:
arch = system_arch()
if arch in infos and isinstance(infos[arch], dict) and isinstance(infos[arch].get("url"), str) and isinstance(infos[arch].get("sha256"), str):
self.prefetch(source_id, infos[arch]["url"], infos[arch]["sha256"])
else:
raise YunohostError(f"In resources.sources: it looks like you forgot to define url/sha256 or {arch}.url/{arch}.sha256", raw_msg=True)
else:
if infos["sha256"] is None:
raise YunohostError(f"In resources.sources: it looks like the sha256 is missing for {source_id}", raw_msg=True)
self.prefetch(source_id, infos["url"], infos["sha256"])
def prefetch(self, source_id, url, expected_sha256):
logger.debug(f"Prefetching asset {source_id}: {url} ...")
if not os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"):
mkdir(f"/var/cache/yunohost/download/{self.app}/", parents=True)
filename = f"/var/cache/yunohost/download/{self.app}/{source_id}"
# NB: we use wget and not requests.get() because we want to output to a file (ie avoid ending up with the full archive in RAM)
# AND the nice --tries, --no-dns-cache, --timeout options ...
p = subprocess.Popen(["/usr/bin/wget", "--tries=3", "--no-dns-cache", "--timeout=900", "--no-verbose", "--output-document=" + filename, url], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, _ = p.communicate()
returncode = p.returncode
if returncode != 0:
if os.path.exists(filename):
rm(filename)
out = out.decode()
raise YunohostError(f"Failed to download asset {source_id} ({url}) for {self.app}: {out}", raw_msg=True)
assert os.path.exists(filename), f"For some reason, wget worked but {filename} doesnt exists?"
computed_sha256 = check_output(f"sha256sum {filename}").split()[0]
if computed_sha256 != expected_sha256:
size = check_output(f"du -hs {filename}").split()[0]
rm(filename)
raise YunohostError(f"Corrupt source for {url} : expected to find {expected_sha256} as sha256sum, but got {computed_sha256} instead ... (file size : {size})", raw_msg=True)
class PermissionsResource(AppResource):
"""