From 90f7446b6b218634cd3a953a304fb97aa4c4f415 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 7 Sep 2022 07:15:40 +0200 Subject: [PATCH] Implement ynh_setup_source_docker --- conf/app.src | 7 + scripts/_common.sh | 4 +- scripts/install | 2 +- scripts/upgrade | 2 +- scripts/vendor/docker-image-extract | 254 ++++++++++++++++++++++++++++ scripts/ynh_docker_image_extract | 238 +++++++++++++++++++------- 6 files changed, 443 insertions(+), 64 deletions(-) create mode 100644 conf/app.src create mode 100644 scripts/vendor/docker-image-extract diff --git a/conf/app.src b/conf/app.src new file mode 100644 index 0000000..6653cf1 --- /dev/null +++ b/conf/app.src @@ -0,0 +1,7 @@ +SOURCE_URL=vaultwarden/server:$(ynh_app_upstream_version) +SOURCE_SUM=sha256 checksum +SOURCE_SUM_PRG=sha256sum +SOURCE_FORMAT=docker +SOURCE_IN_SUBDIR=true +SOURCE_FILENAME= +SOURCE_EXTRACT=false diff --git a/scripts/_common.sh b/scripts/_common.sh index 44aed53..2ea6d07 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -4,11 +4,9 @@ # COMMON VARIABLES #================================================= -# dependencies used by the app +# dependencies used by the app (must be on a single line) pkg_dependencies="libpq5" -pkg_image="vaultwarden/server" - #================================================= # PERSONAL HELPERS #================================================= diff --git a/scripts/install b/scripts/install index 1e26415..c8e24ee 100644 --- a/scripts/install +++ b/scripts/install @@ -93,7 +93,7 @@ ynh_script_progression --message="Setting up source files..." ynh_app_setting_set --app=$app --key=final_path --value=$final_path # Download, check integrity, uncompress and patch the source from app.src -ynh_docker_image_extract --dest_dir="$final_path/build/" --image_spec="$pkg_image:$(ynh_app_upstream_version)" +ynh_setup_source_docker --dest_dir="$final_path" mkdir -p "$final_path/live/" chmod 750 "$final_path" diff --git a/scripts/upgrade b/scripts/upgrade index e719e01..10f5128 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -143,7 +143,7 @@ then ynh_script_progression --message="Upgrading source files..." # Download, check integrity, uncompress the source of vaultwarden from app.src to his build directory - ynh_docker_image_extract --dest_dir="$final_path/build/" --image_spec="$pkg_image:$(ynh_app_upstream_version)" + ynh_setup_source_docker --dest_dir="$final_path" mkdir -p "$final_path/live/" fi diff --git a/scripts/vendor/docker-image-extract b/scripts/vendor/docker-image-extract new file mode 100644 index 0000000..50a9751 --- /dev/null +++ b/scripts/vendor/docker-image-extract @@ -0,0 +1,254 @@ +#!/bin/sh +# +# This script pulls and extracts all files from an image in Docker Hub. +# +# Copyright (c) 2020-2022, Jeremy Lin +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +PLATFORM_DEFAULT="linux/amd64" +PLATFORM="${PLATFORM_DEFAULT}" +OUT_DIR="./output" + +usage() { + echo "This script pulls and extracts all files from an image in Docker Hub." + echo + echo "$0 [OPTIONS...] IMAGE[:REF]" + echo + echo "IMAGE can be a community user image (like 'some-user/some-image') or a" + echo "Docker official image (like 'hello-world', which contains no '/')." + echo + echo "REF is either a tag name or a full SHA-256 image digest (with a 'sha256:' prefix)." + echo + echo "Options:" + echo + echo " -p PLATFORM Pull image for the specified platform (default: ${PLATFORM})" + echo " For a given image on Docker Hub, the 'Tags' tab lists the" + echo " platforms supported for that image." + echo " -o OUT_DIR Extract image to the specified output dir (default: ${OUT_DIR})" + echo " -h Show help with usage examples" +} + +usage_detailed() { + usage + echo + echo "Examples:" + echo + echo "# Pull and extract all files in the 'hello-world' image tagged 'latest'." + echo "\$ $0 hello-world:latest" + echo + echo "# Same as above; ref defaults to the 'latest' tag." + echo "\$ $0 hello-world" + echo + echo "# Pull the 'hello-world' image for the 'linux/arm64/v8' platform." + echo "\$ $0 -p linux/arm64/v8 hello-world" + echo + echo "# Pull an image by digest." + echo "\$ $0 hello-world:sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042" +} + +if [ $# -eq 0 ]; then + usage_detailed + exit 0 +fi + +while getopts ':ho:p:' opt; do + case $opt in + o) + OUT_DIR="${OPTARG}" + ;; + p) + PLATFORM="${OPTARG}" + ;; + h) + usage_detailed + exit 0 + ;; + \?) + echo "ERROR: Invalid option '-$OPTARG'" + echo + usage + exit 1 + ;; + \:) echo "ERROR: Argument required for option '-$OPTARG'" + echo + usage + exit 1 + ;; + esac +done +shift $(($OPTIND - 1)) + +have_curl() { + command -v curl >/dev/null +} + +have_wget() { + command -v wget >/dev/null +} + +if ! have_curl && ! have_wget; then + echo "This script requires either curl or wget." + exit 1 +fi + +image_spec="$1" +image="${image_spec%%:*}" +if [ "${image#*/}" = "${image}" ]; then + # Docker official images are in the 'library' namespace. + image="library/${image}" +fi +ref="${image_spec#*:}" +if [ "${ref}" = "${image_spec}" ]; then + echo "Defaulting ref to tag 'latest'..." + ref=latest +fi + +# Split platform (OS/arch/variant) into separate variables. +# A platform specifier doesn't always include the `variant` component. +OLD_IFS="${IFS}" +IFS=/ read -r OS ARCH VARIANT <":"" (assumes key/val won't contain double quotes). + # The colon may have whitespace on either side. + grep -o "\"${key}\"[[:space:]]*:[[:space:]]*\"[^\"]\+\"" | + # Extract just by deleting the last '"', and then greedily deleting + # everything up to '"'. + sed -e 's/"$//' -e 's/.*"//' +} + +# Fetch a URL to stdout. Up to two header arguments may be specified: +# +# fetch [name1: value1] [name2: value2] +# +fetch() { + if have_curl; then + if [ $# -eq 2 ]; then + set -- -H "$2" "$1" + elif [ $# -eq 3 ]; then + set -- -H "$2" -H "$3" "$1" + fi + curl -sSL "$@" + else + if [ $# -eq 2 ]; then + set -- --header "$2" "$1" + elif [ $# -eq 3 ]; then + set -- --header "$2" --header "$3" "$1" + fi + wget -qO- "$@" + fi +} + +# https://docs.docker.com/docker-hub/api/latest/#tag/repositories +manifest_list_url="https://hub.docker.com/v2/repositories/${image}/tags/${ref}" + +# If we're pulling the image for the default platform, or the ref is already +# a SHA-256 image digest, then we don't need to look up anything. +if [ "${PLATFORM}" = "${PLATFORM_DEFAULT}" ] || [ -z "${ref##sha256:*}" ]; then + digest="${ref}" +else + echo "Getting multi-arch manifest list..." + digest=$(fetch "${manifest_list_url}" | + # Break up the single-line JSON output into separate lines by adding + # newlines before and after the chars '[', ']', '{', and '}'. + sed -e 's/\([][{}]\)/\n\1\n/g' | + # Extract the "images":[...] list. + sed -n '/"images":/,/]/ p' | + # Each image's details are now on a separate line, e.g. + # "architecture":"arm64","features":"","variant":"v8","digest":"sha256:054c85801c4cb41511b176eb0bf13a2c4bbd41611ddd70594ec3315e88813524","os":"linux","os_features":"","os_version":null,"size":828724,"status":"active","last_pulled":"2022-09-02T22:46:48.240632Z","last_pushed":"2022-09-02T00:42:45.69226Z" + # The image details are interspersed with lines of stray punctuation, + # so grep for an arbitrary string that must be in these lines. + grep architecture | + # Search for an image that matches the platform. + while read -r image; do + # Arch is probably most likely to be unique, so check that first. + arch="$(echo ${image} | extract 'architecture')" + if [ "${arch}" != "${ARCH}" ]; then continue; fi + + os="$(echo ${image} | extract 'os')" + if [ "${os}" != "${OS}" ]; then continue; fi + + variant="$(echo ${image} | extract 'variant')" + if [ "${variant}" = "${VARIANT}" ]; then + echo ${image} | extract 'digest' + break + fi + done) +fi + +if [ -n "${digest}" ]; then + echo "Platform ${PLATFORM} resolved to '${digest}'..." +else + echo "No image digest found. Verify that the image, ref, and platform are valid." + exit 1 +fi + +# https://docs.docker.com/registry/spec/auth/token/#how-to-authenticate +api_token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" + +# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-an-image-manifest +manifest_url="https://registry-1.docker.io/v2/${image}/manifests/${digest}" + +# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-a-layer +blobs_base_url="https://registry-1.docker.io/v2/${image}/blobs" + +echo "Getting API token..." +token=$(fetch "${api_token_url}" | extract 'token') +auth_header="Authorization: Bearer $token" +v2_header="Accept: application/vnd.docker.distribution.manifest.v2+json" + +echo "Getting image manifest for $image:$ref..." +layers=$(fetch "${manifest_url}" "${auth_header}" "${v2_header}" | + # Extract `digest` values only after the `layers` section appears. + sed -n '/"layers":/,$ p' | + extract 'digest') + +if [ -z "${layers}" ]; then + echo "No layers returned. Verify that the image and ref are valid." + exit 1 +fi + +mkdir -p "${OUT_DIR}" + +for layer in $layers; do + hash="${layer#sha256:}" + echo "Fetching and extracting layer ${hash}..." + fetch "${blobs_base_url}/${layer}" "${auth_header}" | gzip -d | tar -C "${OUT_DIR}" -xf - + # Ref: https://github.com/moby/moby/blob/master/image/spec/v1.2.md#creating-an-image-filesystem-changeset + # https://github.com/moby/moby/blob/master/pkg/archive/whiteouts.go + # Search for "whiteout" files to indicate files deleted in this layer. + OLD_IFS="${IFS}" + find "${OUT_DIR}" -name '.wh.*' | while IFS= read -r f; do + dir="${f%/*}" + wh_file="${f##*/}" + file="${wh_file#.wh.}" + # Delete both the whiteout file and the whited-out file. + rm -rf "${dir}/${wh_file}" "${dir}/${file}" + done + IFS="${OLD_IFS}" +done + +echo "Image contents extracted into ${OUT_DIR}." diff --git a/scripts/ynh_docker_image_extract b/scripts/ynh_docker_image_extract index 4c19d13..80de2e7 100644 --- a/scripts/ynh_docker_image_extract +++ b/scripts/ynh_docker_image_extract @@ -1,76 +1,196 @@ #!/bin/bash +# Download, check integrity, uncompress and patch the source from app.src # -# This script pulls and extracts all files from an image in Docker Hub. +# usage: ynh_setup_source_docker --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] +# | arg: -d, --dest_dir= - Directory where to setup sources +# | arg: -s, --source_id= - Name of the source, defaults to `app` +# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs/' # -# usage: ynh_docker_image_extract --dest_dir=dest_dir --image_spec=image_spec [--os_arch_variant=os_arch_variant] -# | arg: -d, --dest_dir= - Directory where to setup sources -# | arg: -i, --image_spec= - Image specification -# | arg: -o, --os_arch_variant= - OS, architecture and variant seen as OS/ARCH on Docker Hub default:linux/$YNH_ARCH. +# This helper will read `conf/${source_id}.src`, download and install the sources. # -# Pull and extract all files from the 'hello-world' image tagged 'latest'. -# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world:latest" -# -# Same as above; tag defaults to 'latest'. -# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world" -# -# Same as above; tag defaults from 'latest' for a specific OS/ARCH. -# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world "--os_arch_variant=="linux/arm/v6" -# -# Same as above, but specify the image by digest, don't require the specific OS/ARCH. -# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world:sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042" -# -# This helper will pulls and extracts all files from the image $image_spec in Docker Hub. +# The src file need to contains: +# ``` +# SOURCE_URL=Address to download the app archive +# SOURCE_SUM=Control sum +# # (Optional) Program to check the integrity (sha256sum, md5sum...). Default: sha256 +# SOURCE_SUM_PRG=sha256 +# # (Optional) Archive format. Default: tar.gz +# SOURCE_FORMAT=tar.gz +# # (Optional) Put false if sources are directly in the archive root. Default: true +# # Instead of true, SOURCE_IN_SUBDIR could be the number of sub directories to remove. +# SOURCE_IN_SUBDIR=false +# # (Optionnal) Name of the local archive (offline setup support). Default: ${src_id}.${src_format} +# SOURCE_FILENAME=example.tar.gz +# # (Optional) If it set as false don't extract the source. Default: true +# # (Useful to get a debian package or a python wheel.) +# SOURCE_EXTRACT=(true|false) +# # (Optionnal) Name of the plateform. Default: "linux/$YNH_ARCH" +# SOURCE_FILENAME=linux/arm64/v8 +# ``` # # The helper will: -# - Download `$image_spec` and extract it to `$dest_dir`. +# - Check if there is a local source archive in `/opt/yunohost-apps-src/$APP_ID/$SOURCE_FILENAME` +# - Download `$SOURCE_URL` if there is no local archive +# - Check the integrity with `$SOURCE_SUM_PRG -c --status` +# - Uncompress the archive to `$dest_dir`. +# - If `$SOURCE_IN_SUBDIR` is true, the first level directory of the archive will be removed. +# - If `$SOURCE_IN_SUBDIR` is a numeric value, the N first level directories will be removed. # - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` # - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir # -# Requires YunoHost version *.*.* or higher. -ynh_docker_image_extract() { - # Declare an array to define the options of this helper. - local legacy_args=dio - local -A args_array=([d]=dest_dir= [i]=image_spec= [o]=os_arch_variant=) - local dest_dir - local image_spec - local os_arch_variant - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - os_arch_variant="${os_arch_variant:-"linux/$YNH_ARCH"}" +# Requires YunoHost version 2.6.4 or higher. +ynh_setup_source_docker() { + # Declare an array to define the options of this helper. + local legacy_args=dsk + local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep=) + local dest_dir + local source_id + local keep + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + source_id="${source_id:-app}" + keep="${keep:-}" - # Extract source into the app dir - mkdir --parents "$dest_dir" + local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" - if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then - _ynh_apply_default_permissions $dest_dir + # Load value from configuration file (see above for a small doc about this file + # format) + local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_filename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_plateform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + + # Default value + src_sumprg=${src_sumprg:-sha256sum} + src_in_subdir=${src_in_subdir:-true} + src_format=${src_format:-tar.gz} + src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') + src_extract=${src_extract:-true} + if [ "$src_filename" = "" ]; then + src_filename="${source_id}.${src_format}" + fi + + # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... + local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}" + + mkdir -p /var/cache/yunohost/download/${YNH_APP_ID}/ + src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${src_filename}" + + if [ "$src_format" = "docker" ]; then + src_plateform="${src_plateform:-"linux/$YNH_ARCH"}" + else + if test -e "$local_src"; then + cp $local_src $src_filename + else + [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" + + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ + || ynh_die --message="$out" + fi + + # Check the control sum + echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status \ + || ynh_die --message="Corrupt source" fi - tempdir="$(mktemp -d)" - pushd $tempdir - git init -q - git remote add -f -t main origin https://github.com/jjlin/docker-image-extract.git > /dev/null 2>&1 - git checkout -q -b main origin/main - ./docker-image-extract -p $os_arch_variant -o $dest_dir $image_spec 2>&1 - popd - rm -rf $tempdir + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ + if [ -n "$keep" ] && [ -e "$dest_dir" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + mkdir -p $keep_dir + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$dest_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" + cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" + fi + done + fi - # Apply patches - if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then - local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) - if (($(find $patches_folder -type f -name "${image_spec}-*.patch" 2>/dev/null | wc --lines) > "0")); then - ( - cd "$dest_dir" - for p in $patches_folder/${image_spec}-*.patch; do - echo $p - patch --strip=1 <$p - done - ) || ynh_die --message="Unable to apply patches" - fi - fi + # Extract source into the app dir + mkdir --parents "$dest_dir" - # Add supplementary files - if test -e "$YNH_APP_BASEDIR/sources/extra_files/${image_spec}"; then - cp --archive $YNH_APP_BASEDIR/sources/extra_files/$image_spec/. "$dest_dir" - fi + if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then + _ynh_apply_default_permissions $dest_dir + fi + + if ! "$src_extract"; then + mv $src_filename $dest_dir + elif [ "$src_format" = "docker" ]; then + $YNH_APP_BASEDIR/script/vendor/docker-image-extract -p $src_plateform -o $src_filename $src_url 2>&1 + mv $src_filename $dest_dir + ynh_secure_remove --file="$src_filename" + elif [ "$src_format" = "zip" ]; then + # Zip format + # Using of a temp directory, because unzip doesn't manage --strip-components + if $src_in_subdir; then + local tmp_dir=$(mktemp --directory) + unzip -quo $src_filename -d "$tmp_dir" + cp --archive $tmp_dir/*/. "$dest_dir" + ynh_secure_remove --file="$tmp_dir" + else + unzip -quo $src_filename -d "$dest_dir" + fi + ynh_secure_remove --file="$src_filename" + else + local strip="" + if [ "$src_in_subdir" != "false" ]; then + if [ "$src_in_subdir" == "true" ]; then + local sub_dirs=1 + else + local sub_dirs="$src_in_subdir" + fi + strip="--strip-components $sub_dirs" + fi + if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then + tar --extract --file=$src_filename --directory="$dest_dir" $strip + else + ynh_die --message="Archive format unrecognized." + fi + ynh_secure_remove --file="$src_filename" + fi + + # Apply patches + if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then + local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) + if (($(find $patches_folder -type f -name "${source_id}-*.patch" 2>/dev/null | wc --lines) > "0")); then + ( + cd "$dest_dir" + for p in $patches_folder/${source_id}-*.patch; do + echo $p + patch --strip=1 <$p + done + ) || ynh_die --message="Unable to apply patches" + fi + fi + + # Add supplementary files + if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then + cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir" + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + if [ -n "$keep" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$keep_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" + cp --archive "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" + fi + done + fi + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ }