From fab4455a08cf09a7dfa3a4ec2a3fe7d1050cdca6 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Fri, 2 Sep 2022 22:17:22 +0200 Subject: [PATCH] Implement ynh_docker_image_extract --- conf/docker-image-extract.src | 7 - scripts/install | 11 +- scripts/upgrade | 11 +- scripts/ynh_docker_image_extract | 237 +++++++++++++++++++++++++++++++ 4 files changed, 245 insertions(+), 21 deletions(-) delete mode 100644 conf/docker-image-extract.src create mode 100644 scripts/ynh_docker_image_extract diff --git a/conf/docker-image-extract.src b/conf/docker-image-extract.src deleted file mode 100644 index 64fe1a1..0000000 --- a/conf/docker-image-extract.src +++ /dev/null @@ -1,7 +0,0 @@ -SOURCE_URL=https://codeload.github.com/jjlin/docker-image-extract/tar.gz/a9e455e44bbbfba897bf3342d9661b182cee67a9 -SOURCE_SUM=9eb0c734e83a3fd7102fc7209af4977024ec467fbc819782491af47295675f67 -SOURCE_SUM_PRG=sha256sum -SOURCE_FORMAT=tar.gz -SOURCE_IN_SUBDIR=true -SOURCE_FILENAME= -SOURCE_EXTRACT=true diff --git a/scripts/install b/scripts/install index 0a6c775..44c183a 100644 --- a/scripts/install +++ b/scripts/install @@ -7,6 +7,7 @@ #================================================= source _common.sh +source ynh_docker_image_extract source ynh_send_readme_to_admin__2 source /usr/share/yunohost/helpers @@ -92,7 +93,7 @@ ynh_script_progression --message="Setting up source files..." ynh_app_setting_set --app=$app --key=final_path --value=$final_path # Download, check integrity, uncompress and patch the source from app.src -ynh_setup_source --dest_dir="$final_path/build/" --source_id="docker-image-extract" +ynh_docker_image_extract --dest_dir="$final_path/build/" --image_spec="vaultwarden/server:$(ynh_app_upstream_version)" mkdir -p "$final_path/live/" chmod 750 "$final_path" @@ -128,12 +129,8 @@ chown -R $app:$app "$datadir" #================================================= ynh_script_progression --message="Making install..." -pushd "$final_path"/build - ./docker-image-extract vaultwarden/server:$(ynh_app_upstream_version) -popd - -mv -f "$final_path/build/output/vaultwarden" "$final_path/live/vaultwarden" -rsync -a "$final_path/build/output/web-vault/" "$final_path/live/web-vault/" +mv -f "$final_path/build/vaultwarden" "$final_path/live/vaultwarden" +rsync -a "$final_path/build/web-vault/" "$final_path/live/web-vault/" ynh_secure_remove --file="$final_path/build" chmod 750 "$final_path" diff --git a/scripts/upgrade b/scripts/upgrade index e1372e0..1a5526e 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -7,6 +7,7 @@ #================================================= source _common.sh +source ynh_docker_image_extract source ynh_handle_app_migration source /usr/share/yunohost/helpers @@ -142,7 +143,7 @@ then ynh_script_progression --message="Upgrading source files..." # Download, check integrity, uncompress the source of vaultwarden from app.src to his build directory - ynh_setup_source --dest_dir="$final_path/build/" --source_id="docker-image-extract" + ynh_docker_image_extract --dest_dir="$final_path/build/" --image_spec="vaultwarden/server:$(ynh_app_upstream_version)" mkdir -p "$final_path/live/" fi @@ -174,13 +175,9 @@ ynh_script_progression --message="Making upgrade..." if [ "$upgrade_type" == "UPGRADE_APP" ] then - pushd "$final_path"/build - ./docker-image-extract vaultwarden/server:$(ynh_app_upstream_version) - popd - - mv -f "$final_path/build/output/vaultwarden" "$final_path/live/vaultwarden" + mv -f "$final_path/build/vaultwarden" "$final_path/live/vaultwarden" ynh_secure_remove --file="$final_path/live/web-vault/" - rsync -a "$final_path/build/output/web-vault/" "$final_path/live/web-vault/" + rsync -a "$final_path/build/web-vault/" "$final_path/live/web-vault/" ynh_secure_remove --file="$final_path/build" fi diff --git a/scripts/ynh_docker_image_extract b/scripts/ynh_docker_image_extract new file mode 100644 index 0000000..4dc6061 --- /dev/null +++ b/scripts/ynh_docker_image_extract @@ -0,0 +1,237 @@ +#!/bin/bash + +# +# This script pulls and extracts all files from an image in Docker Hub. +# +# usage: ynh_docker_image_extract --dest_dir=dest_dir --image_spec=image_spec [--os_arch_variant=os_arch_variant] [--keep="file1 file2"] +# | arg: -d, --dest_dir= - Directory where to setup sources +# | arg: -i, --image_spec= - Image specification +# | arg: -o, --os_arch_variant= - OS, architecture and variant seen as OS/ARCH. on Docker Hub +# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs/' +# +# Pull and extract all files from the 'hello-world' image tagged 'latest'. +# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world:latest" +# +# Same as above; tag defaults to 'latest'. +# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world" +# +# Same as above; tag defaults from 'latest' for a specific OS/ARCH. +# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world "--os_arch_variant=="linux/arm/v6" +# +# Same as above, but specify the image by digest, don't require the specific OS/ARCH. +# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world:sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042" +# +# This helper will pulls and extracts all files from the image $image_spec in Docker Hub. +# +# The helper will: +# - Download `$image_spec` and extract it to `$dest_dir`. +# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` +# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir +# +# Requires YunoHost version *.*.* or higher. +ynh_docker_image_extract() { + # Declare an array to define the options of this helper. + local legacy_args=diok + local -A args_array=([d]=dest_dir= [i]=image_spec= [o]=os_arch_variant= [k]=keep=) + local dest_dir + local image_spec + local os_arch_variant + local keep + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + os_arch_variant="${os_arch_variant:-}" + keep="${keep:-}" + + have_curl() { + command -v curl >/dev/null + } + + have_wget() { + command -v wget >/dev/null + } + + if ! have_curl && ! have_wget; then + ynh_die --message="This script requires either curl or wget." + fi + + # Given a JSON input on stdin, extract the string value associated with the + # specified key. This avoids an extra dependency on a tool like `jq`. + extract() { + local key="$1" + # Extract "":"" (assumes key/val won't contain double quotes). + # The colon may have whitespace on either side. + grep -o "\"${key}\"[[:space:]]*:[[:space:]]*\"[^\"]\+\"" | + # Extract just by deleting the last '"', and then greedily deleting + # everything up to '"'. + sed -e 's/"$//' -e 's/.*"//' + } + + # Fetch a URL to stdout. Up to two header arguments may be specified: + # + # fetch [name1: value1] [name2: value2] + # + fetch() { + if have_curl; then + if [ $# -eq 2 ]; then + set -- -H "$2" "$1" + elif [ $# -eq 3 ]; then + set -- -H "$2" -H "$3" "$1" + fi + curl -sSL "$@" + else + if [ $# -eq 2 ]; then + set -- --header "$2" "$1" + elif [ $# -eq 3 ]; then + set -- --header "$2" --header "$3" "$1" + fi + wget -qO- "$@" + fi + } + + IFS='/' read -ra newarr <<< "$os_arch_variant" + os=${newarr[0]:-'linux'} + arch=${newarr[1]:-$YNH_ARCH} + variant=${newarr[2]:-} + + image="${image_spec%%:*}" + if [ "${image#*/}" = "${image}" ]; then + # Docker official images are in the 'library' namespace. + image="library/${image}" + fi + + tag="${image_spec#*:}" + if [ "${tag}" = "${image_spec}" ]; then + tag=latest + fi + + digest="${image_spec#*:}" + if [[ "${digest}" != sha* ]]; then + if [ -z "$variant" ] + then + digest=$(fetch https://hub.docker.com/v2/repositories/$image/tags/$tag/ | jq -r '.images[] | select(.os=="'$os'" and .architecture=="'$arch'").digest') + else + digest=$(fetch https://hub.docker.com/v2/repositories/$image/tags/$tag/ | jq -r '.images[] | select(.os=="'$os'" and .architecture=="'$arch'" and .variant=="'$variant'").digest') + fi + else + tag=unknow + fi + + # https://docs.docker.com/registry/spec/auth/token/#how-to-authenticate + api_token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" + + # https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-an-image-manifest + manifest_url="https://registry-1.docker.io/v2/${image}/manifests/$digest" + + # https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-a-layer + blobs_base_url="https://registry-1.docker.io/v2/${image}/blobs" + + # Getting API token + token=$(fetch "${api_token_url}" | extract 'token') + auth_header="Authorization: Bearer $token" + v2_header="Accept: application/vnd.docker.distribution.manifest.v2+json" + + # Getting image manifest for $image:$tag $digest + layers=$(fetch "${manifest_url}" "${auth_header}" "${v2_header}" | + # Extract `digest` values only after the `layers` section appears. + sed -n '/"layers":/,$ p' | + extract 'digest') + + if [ -z "${layers}" ]; then + ynh_die --message="No layers returned. Verify that the image and tag are valid." + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ + if [ -n "$keep" ] && [ -e "$dest_dir" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + mkdir -p $keep_dir + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$dest_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" + cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" + fi + done + fi + + # Extract source into the app dir + mkdir --parents "$dest_dir" + + if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then + _ynh_apply_default_permissions $dest_dir + fi + + for layer in $layers; do + hash="${layer#sha256:}" + # Fetching and extracting layer ${hash} + fetch "${blobs_base_url}/${layer}" "${auth_header}" | gzip -d | tar -C "${dest_dir}" -xf - + # Ref: https://github.com/moby/moby/blob/master/image/spec/v1.2.md#creating-an-image-filesystem-changeset + # https://github.com/moby/moby/blob/master/pkg/archive/whiteouts.go + # Search for "whiteout" files to indicate files deleted in this layer. + OLD_IFS="${IFS}" + find "${dest_dir}" -name '.wh.*' | while IFS= read -r f; do + dir="${f%/*}" + wh_file="${f##*/}" + file="${wh_file#.wh.}" + # Delete both the whiteout file and the whited-out file. + rm -rf "${dir}/${wh_file}" "${dir}/${file}" + done + IFS="${OLD_IFS}" + done + + # Apply patches + if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then + local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) + if (($(find $patches_folder -type f -name "${image_spec}-*.patch" 2>/dev/null | wc --lines) > "0")); then + ( + cd "$dest_dir" + for p in $patches_folder/${image_spec}-*.patch; do + echo $p + patch --strip=1 <$p + done + ) || ynh_die --message="Unable to apply patches" + fi + fi + + # Add supplementary files + if test -e "$YNH_APP_BASEDIR/sources/extra_files/${image_spec}"; then + cp --archive $YNH_APP_BASEDIR/sources/extra_files/$image_spec/. "$dest_dir" + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + if [ -n "$keep" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$keep_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" + cp --archive "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" + fi + done + fi + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ +} + +# +# Copyright (c) 2020-2021, Jeremy Lin +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +#