Merge pull request #211 from YunoHost-Apps/fix

Implement ynh_docker_image_extract

conf/docker-image-extract.src

@@ -1,7 +0,0 @@
-SOURCE_URL=https://codeload.github.com/jjlin/docker-image-extract/tar.gz/a9e455e44bbbfba897bf3342d9661b182cee67a9
-SOURCE_SUM=9eb0c734e83a3fd7102fc7209af4977024ec467fbc819782491af47295675f67
-SOURCE_SUM_PRG=sha256sum
-SOURCE_FORMAT=tar.gz
-SOURCE_IN_SUBDIR=true
-SOURCE_FILENAME=
-SOURCE_EXTRACT=true

scripts/install

@@ -7,6 +7,7 @@
 #=================================================
 
 source _common.sh
+source ynh_docker_image_extract
 source ynh_send_readme_to_admin__2
 source /usr/share/yunohost/helpers
 
@@ -92,7 +93,7 @@ ynh_script_progression --message="Setting up source files..."
 
 ynh_app_setting_set --app=$app --key=final_path --value=$final_path
 # Download, check integrity, uncompress and patch the source from app.src
-ynh_setup_source --dest_dir="$final_path/build/" --source_id="docker-image-extract"
+ynh_docker_image_extract --dest_dir="$final_path/build/" --image_spec="vaultwarden/server:$(ynh_app_upstream_version)"
 mkdir -p "$final_path/live/"
 
 chmod 750 "$final_path"
@@ -128,12 +129,8 @@ chown -R $app:$app "$datadir"
 #=================================================
 ynh_script_progression --message="Making install..."
 
-pushd "$final_path"/build
-	./docker-image-extract vaultwarden/server:$(ynh_app_upstream_version)
-popd
-
-mv -f "$final_path/build/output/vaultwarden" "$final_path/live/vaultwarden"
-rsync -a "$final_path/build/output/web-vault/" "$final_path/live/web-vault/"
+mv -f "$final_path/build/vaultwarden" "$final_path/live/vaultwarden"
+rsync -a "$final_path/build/web-vault/" "$final_path/live/web-vault/"
 ynh_secure_remove --file="$final_path/build"
 
 chmod 750 "$final_path"

scripts/upgrade

@@ -7,6 +7,7 @@
 #=================================================
 
 source _common.sh
+source ynh_docker_image_extract
 source ynh_handle_app_migration
 source /usr/share/yunohost/helpers
 
@@ -142,7 +143,7 @@ then
 	ynh_script_progression --message="Upgrading source files..."
 
 	# Download, check integrity, uncompress the source of vaultwarden from app.src to his build directory
-	ynh_setup_source --dest_dir="$final_path/build/" --source_id="docker-image-extract"
+	ynh_docker_image_extract --dest_dir="$final_path/build/" --image_spec="vaultwarden/server:$(ynh_app_upstream_version)"
 	mkdir -p "$final_path/live/"
 fi
 
@@ -174,13 +175,9 @@ ynh_script_progression --message="Making upgrade..."
 
 if [ "$upgrade_type" == "UPGRADE_APP" ]
 then
-	pushd "$final_path"/build
-		./docker-image-extract vaultwarden/server:$(ynh_app_upstream_version)
-	popd
-
-	mv -f "$final_path/build/output/vaultwarden" "$final_path/live/vaultwarden"
+	mv -f "$final_path/build/vaultwarden" "$final_path/live/vaultwarden"
 	ynh_secure_remove --file="$final_path/live/web-vault/"
-	rsync -a "$final_path/build/output/web-vault/" "$final_path/live/web-vault/"
+	rsync -a "$final_path/build/web-vault/" "$final_path/live/web-vault/"
 	ynh_secure_remove --file="$final_path/build"
 fi
 

scripts/ynh_docker_image_extract

@@ -0,0 +1,237 @@
+#!/bin/bash
+
+#
+# This script pulls and extracts all files from an image in Docker Hub.
+#
+# usage: ynh_docker_image_extract --dest_dir=dest_dir --image_spec=image_spec [--os_arch_variant=os_arch_variant] [--keep="file1 file2"]
+# | arg: -d, --dest_dir=		- Directory where to setup sources
+# | arg: -i, --image_spec=		- Image specification
+# | arg: -o, --os_arch_variant=	- OS, architecture and variant seen as OS/ARCH. on Docker Hub
+# | arg: -k, --keep=			- Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs/'
+#
+# Pull and extract all files from the 'hello-world' image tagged 'latest'.
+# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world:latest"
+#
+# Same as above; tag defaults to 'latest'.
+# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world"
+#
+# Same as above; tag defaults from 'latest' for a specific OS/ARCH.
+# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world "--os_arch_variant=="linux/arm/v6"
+#
+# Same as above, but specify the image by digest, don't require the specific OS/ARCH.
+# example: ynh_docker_image_extract --dest_dir="dest_dir" --image_spec="hello-world:sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042"
+#
+# This helper will pulls and extracts all files from the image $image_spec in Docker Hub.
+#
+# The helper will:
+# - Download `$image_spec` and extract it to `$dest_dir`.
+# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir`
+# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir
+#
+# Requires YunoHost version *.*.* or higher.
+ynh_docker_image_extract() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=diok
+	local -A args_array=([d]=dest_dir= [i]=image_spec= [o]=os_arch_variant= [k]=keep=)
+	local dest_dir
+	local image_spec
+	local os_arch_variant
+	local keep
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	os_arch_variant="${os_arch_variant:-}"
+	keep="${keep:-}"
+
+	have_curl() {
+		command -v curl >/dev/null
+	}
+
+	have_wget() {
+		command -v wget >/dev/null
+	}
+
+	if ! have_curl && ! have_wget; then
+		ynh_die --message="This script requires either curl or wget."
+	fi
+
+	# Given a JSON input on stdin, extract the string value associated with the
+	# specified key. This avoids an extra dependency on a tool like `jq`.
+	extract() {
+		local key="$1"
+		# Extract "<key>":"<val>" (assumes key/val won't contain double quotes).
+		# The colon may have whitespace on either side.
+		grep -o "\"${key}\"[[:space:]]*:[[:space:]]*\"[^\"]\+\"" |
+		# Extract just <val> by deleting the last '"', and then greedily deleting
+		# everything up to '"'.
+		sed -e 's/"$//' -e 's/.*"//'
+	}
+
+	# Fetch a URL to stdout. Up to two header arguments may be specified:
+	#
+	#   fetch <url> [name1: value1] [name2: value2]
+	#
+	fetch() {
+		if have_curl; then
+			if [ $# -eq 2 ]; then
+				set -- -H "$2" "$1"
+			elif [ $# -eq 3 ]; then
+				set -- -H "$2" -H "$3" "$1"
+			fi
+			curl -sSL "$@"
+		else
+			if [ $# -eq 2 ]; then
+				set -- --header "$2" "$1"
+			elif [ $# -eq 3 ]; then
+				set -- --header "$2" --header "$3" "$1"
+			fi
+			wget -qO- "$@"
+		fi
+	}
+
+	IFS='/' read -ra newarr <<< "$os_arch_variant"
+	os=${newarr[0]:-'linux'}
+	arch=${newarr[1]:-$YNH_ARCH}
+	variant=${newarr[2]:-}
+
+	image="${image_spec%%:*}"
+	if [ "${image#*/}" = "${image}" ]; then
+		# Docker official images are in the 'library' namespace.
+		image="library/${image}"
+	fi
+
+	tag="${image_spec#*:}"
+	if [ "${tag}" = "${image_spec}" ]; then
+		tag=latest
+	fi
+
+	digest="${image_spec#*:}"
+	if [[ "${digest}" != sha* ]]; then
+		if [ -z "$variant" ]
+		then
+			digest=$(fetch https://hub.docker.com/v2/repositories/$image/tags/$tag/ | jq -r '.images[] | select(.os=="'$os'" and .architecture=="'$arch'").digest')
+		else
+			digest=$(fetch https://hub.docker.com/v2/repositories/$image/tags/$tag/ | jq -r '.images[] | select(.os=="'$os'" and .architecture=="'$arch'" and .variant=="'$variant'").digest')
+		fi
+	else
+		tag=unknow
+	fi
+
+	# https://docs.docker.com/registry/spec/auth/token/#how-to-authenticate
+	api_token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull"
+
+	# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-an-image-manifest
+	manifest_url="https://registry-1.docker.io/v2/${image}/manifests/$digest"
+
+	# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-a-layer
+	blobs_base_url="https://registry-1.docker.io/v2/${image}/blobs"
+
+	# Getting API token
+	token=$(fetch "${api_token_url}" | extract 'token')
+	auth_header="Authorization: Bearer $token"
+	v2_header="Accept: application/vnd.docker.distribution.manifest.v2+json"
+
+	# Getting image manifest for $image:$tag $digest
+	layers=$(fetch "${manifest_url}" "${auth_header}" "${v2_header}" |
+				 # Extract `digest` values only after the `layers` section appears.
+				 sed -n '/"layers":/,$ p' |
+				 extract 'digest')
+
+	if [ -z "${layers}" ]; then
+		ynh_die --message="No layers returned. Verify that the image and tag are valid."
+	fi
+
+	# Keep files to be backup/restored at the end of the helper
+	# Assuming $dest_dir already exists
+	rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
+	if [ -n "$keep" ] && [ -e "$dest_dir" ]; then
+		local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
+		mkdir -p $keep_dir
+		local stuff_to_keep
+		for stuff_to_keep in $keep; do
+			if [ -e "$dest_dir/$stuff_to_keep" ]; then
+				mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")"
+				cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep"
+			fi
+		done
+	fi
+
+	# Extract source into the app dir
+	mkdir --parents "$dest_dir"
+
+	if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then
+		_ynh_apply_default_permissions $dest_dir
+	fi
+
+	for layer in $layers; do
+		hash="${layer#sha256:}"
+		# Fetching and extracting layer ${hash}
+		fetch "${blobs_base_url}/${layer}" "${auth_header}" | gzip -d | tar -C "${dest_dir}" -xf -
+		# Ref: https://github.com/moby/moby/blob/master/image/spec/v1.2.md#creating-an-image-filesystem-changeset
+		#	  https://github.com/moby/moby/blob/master/pkg/archive/whiteouts.go
+		# Search for "whiteout" files to indicate files deleted in this layer.
+		OLD_IFS="${IFS}"
+		find "${dest_dir}" -name '.wh.*' | while IFS= read -r f; do
+			dir="${f%/*}"
+			wh_file="${f##*/}"
+			file="${wh_file#.wh.}"
+			# Delete both the whiteout file and the whited-out file.
+			rm -rf "${dir}/${wh_file}" "${dir}/${file}"
+		done
+		IFS="${OLD_IFS}"
+	done
+
+	# Apply patches
+	if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then
+		local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/)
+		if (($(find $patches_folder -type f -name "${image_spec}-*.patch" 2>/dev/null | wc --lines) > "0")); then
+			(
+				cd "$dest_dir"
+				for p in $patches_folder/${image_spec}-*.patch; do
+					echo $p
+					patch --strip=1 <$p
+				done
+			) || ynh_die --message="Unable to apply patches"
+		fi
+	fi
+
+	# Add supplementary files
+	if test -e "$YNH_APP_BASEDIR/sources/extra_files/${image_spec}"; then
+		cp --archive $YNH_APP_BASEDIR/sources/extra_files/$image_spec/. "$dest_dir"
+	fi
+
+	# Keep files to be backup/restored at the end of the helper
+	# Assuming $dest_dir already exists
+	if [ -n "$keep" ]; then
+		local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
+		local stuff_to_keep
+		for stuff_to_keep in $keep; do
+			if [ -e "$keep_dir/$stuff_to_keep" ]; then
+				mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")"
+				cp --archive "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep"
+			fi
+		done
+	fi
+	rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
+}
+
+#
+# Copyright (c) 2020-2021, Jeremy Lin
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#