clean and add helpers

README.md

@@ -80,5 +80,5 @@ Use the field `image` to switch between `before-install`, `before-postinstall` o
 ## TODO
 
 - Support more YunoHost Core projects (for now only `yunohost` is supported, not `moulinette`...)
-- Be sure that the runner can run several jobs in parallel (The `rebuild_base_container` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?).
+- Be sure that the runner can run several jobs in parallel (The `rebuild_base_containers` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?).
 - Git pull this repo before running tests to keep these files up-to-date.

auto_upgrade_container.sh

@@ -0,0 +1,15 @@
+current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $current_dir/utils.sh # Get utils functions.
+
+for debian_version in "stretch" "buster"
+do
+    for ynh_version in "stable" "testing" "unstable"
+    do
+        for snapshot in "before-install" "before-postinstall" "after-postinstall"
+        do
+            local image="yunohost-$debian_version-$ynh_version-$snapshot"
+
+            update_image $image
+        done
+    done
+done

base.sh

@@ -30,4 +30,7 @@ then
     SNAPSHOT_NAME="after-postinstall"
 fi
 PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR"
-PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME"
+PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME"
+
+# For example yunohost-stretch-unstable
+BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"

ci_cron

@@ -0,0 +1,2 @@
+# Upgrade all continers
+30 2 * * * root "/opt/yunohost-ciyunohost/auto_upgrade_container.sh"

cleanup.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-currentDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source ${currentDir}/base.sh # Get variables from base.
+source $current_dir/base.sh # Get variables from base.
 
 echo "Deleting container $CONTAINER_ID"
 

prepare.sh

@@ -1,115 +1,14 @@
 #!/usr/bin/env bash
 
-currentDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source ${currentDir}/base.sh # Get variables from base.
+source $current_dir/base.sh # Get variables from base.
+source $current_dir/utils.sh # Get utils functions.
 
 set -eo pipefail
 
 # trap any error, and mark it as a system failure.
 trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR
 
-clean_containers()
-{
-	for image_to_delete in "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"{,"-tmp"}
-	do
-		if lxc info $image_to_delete &>/dev/null
-		then
-			lxc delete $image_to_delete --force
-		fi
-	done
-
-	for image_to_delete in "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-"{"before-install","before-postinstall","after-postinstall"}
-	do
-		if lxc image info $image_to_delete &>/dev/null
-		then
-			lxc image delete $image_to_delete
-		fi
-	done
-}
-
-wait_container()
-{
-	# Wait for container to start, we are using systemd to check this,
-	# for the sake of brevity.
-	for i in $(seq 1 10); do
-		if lxc exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then
-			break
-		fi
-
-		if [ "$i" == "10" ]; then
-			echo 'Waited for 10 seconds to start container, exiting..'
-			# Inform GitLab Runner that this is a system failure, so it
-			# should be retried.
-			exit "$SYSTEM_FAILURE_EXIT_CODE"
-		fi
-
-		sleep 1s
-	done
-}
-
-rebuild_base_container()
-{
-	clean_containers
-
-	lxc launch images:debian/$DEBIAN_VERSION/$ARCH "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-	
-	wait_container "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get update"
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get install curl -y"
-	# Install Git LFS, git comes pre installed with ubuntu image.
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get install git-lfs -y"
-	# Install gitlab-runner binary since we need for cache/artifacts.
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash"
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y"
-	lxc stop "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-
-	# Create image before install
-	lxc publish "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" --alias "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-before-install"
-	lxc start "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-	wait_container "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-
-	# Install yunohost
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "curl https://install.yunohost.org | bash -s -- -a -d $CURRENT_VERSION"
-	lxc stop "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-
-	# Create image before postinstall
-	lxc publish "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" --alias "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-before-postinstall"
-	lxc start "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-	wait_container "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-
-	# Running post Install
-	lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns"
-	lxc stop "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-
-	# Create image after postinstall
-	lxc publish "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" --alias "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-after-postinstall"
-
-	lxc delete "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp"
-}
-
-update_image() {
-	image_to_update=$1
-
-	# Start and run upgrade
-	lxc launch "$image_to_update" "$image_to_update-tmp"
-	
-	wait_container "$image_to_update-tmp"
-
-	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update"
-	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y"
-	lxc stop "$image_to_update-tmp"
-
-	# Remove old image
-	lxc image delete "$image_to_update"
-
-	# Add new image updated
-	lxc publish "$image_to_update-tmp" --alias "$image_to_update"
-
-	lxc delete "$image_to_update-tmp"
-}
-
 start_container () {
 	set -x
 
@@ -118,18 +17,21 @@ start_container () {
 		lxc delete -f "$CONTAINER_ID"
 	fi
 
-	if ! lxc image info "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-$SNAPSHOT_NAME" &>/dev/null
+	if ! lxc image info "$BASE_IMAGE-$SNAPSHOT_NAME" &>/dev/null
 	then
-		rebuild_base_container
+		echo "$BASE_IMAGE not found, please rebuild with rebuild_all.sh"
+		# Inform GitLab Runner that this is a system failure, so it
+		# should be retried.
+		exit $SYSTEM_FAILURE_EXIT_CODE
 	fi
 
-	update_image "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-$SNAPSHOT_NAME"
+	update_image "$BASE_IMAGE-$SNAPSHOT_NAME"
 
-	lxc launch "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-$SNAPSHOT_NAME" "$CONTAINER_ID" 2>/dev/null
+	lxc launch "$BASE_IMAGE-$SNAPSHOT_NAME" "$CONTAINER_ID" 2>/dev/null
 
-	mkdir -p ${currentDir}/cache
+	mkdir -p $current_dir/cache
-	chmod 777 ${currentDir}/cache
+	chmod 777 $current_dir/cache
-	lxc config device add "$CONTAINER_ID" cache-folder disk path=/cache source="${currentDir}/cache"
+	lxc config device add "$CONTAINER_ID" cache-folder disk path=/cache source="$current_dir/cache"
 
 	set +x
 

rebuild_all.sh

@@ -0,0 +1,12 @@
+current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $current_dir/utils.sh # Get utils functions.
+
+for debian_version in "stretch" "buster"
+do
+    for ynh_version in "stable" "testing" "unstable"
+    do
+        local base_image="yunohost-$debian_version-$ynh_version"
+
+        rebuild_base_containers $BASE_IMAGE
+    done
+done

run.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-currentDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source ${currentDir}/base.sh # Get variables from base.
+source $current_dir/base.sh # Get variables from base.
 
 create_sym_link() {
 	local DEST=$1

utils.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $current_dir/base.sh # Get variables from base.
+
+clean_containers()
+{
+    local base_image_to_clean=$1
+
+	for image_to_delete in "$base_image_to_clean"{,"-tmp"}
+	do
+		if lxc info $image_to_delete &>/dev/null
+		then
+			lxc delete $base_image_to_clean --force
+		fi
+	done
+
+	for image_to_delete in "$base_image_to_clean-"{"before-install","before-postinstall","after-postinstall"}
+	do
+		if lxc image info $image_to_delete &>/dev/null
+		then
+			lxc image delete $image_to_delete
+		fi
+	done
+}
+
+wait_container()
+{
+	# Wait for container to start, we are using systemd to check this,
+	# for the sake of brevity.
+	for i in $(seq 1 10); do
+		if lxc exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then
+			break
+		fi
+
+		if [ "$i" == "10" ]; then
+			echo 'Waited for 10 seconds to start container, exiting..'
+			# Inform GitLab Runner that this is a system failure, so it
+			# should be retried.
+			exit "$SYSTEM_FAILURE_EXIT_CODE"
+		fi
+
+		sleep 1s
+	done
+}
+
+rebuild_base_containers()
+{
+    local base_image_to_rebuild=$1
+	clean_containers $base_image_to_rebuild
+
+	lxc launch images:debian/$DEBIAN_VERSION/$ARCH "$base_image_to_rebuild-tmp"
+	
+	wait_container "$base_image_to_rebuild-tmp"
+
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install curl -y"
+	# Install Git LFS, git comes pre installed with ubuntu image.
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install git-lfs -y"
+	# Install gitlab-runner binary since we need for cache/artifacts.
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y"
+	lxc stop "$base_image_to_rebuild-tmp"
+
+	# Create image before install
+	lxc publish "$base_image_to_rebuild-tmp" --alias "$base_image_to_rebuild-before-install"
+	lxc start "$base_image_to_rebuild-tmp"
+
+	wait_container "$base_image_to_rebuild-tmp"
+
+	# Install yunohost
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl https://install.yunohost.org | bash -s -- -a -d $CURRENT_VERSION"
+	lxc stop "$base_image_to_rebuild-tmp"
+
+	# Create image before postinstall
+	lxc publish "$base_image_to_rebuild-tmp" --alias "$base_image_to_rebuild-before-postinstall"
+	lxc start "$base_image_to_rebuild-tmp"
+
+	wait_container "$base_image_to_rebuild-tmp"
+
+	# Running post Install
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns"
+	lxc stop "$base_image_to_rebuild-tmp"
+
+	# Create image after postinstall
+	lxc publish "$base_image_to_rebuild-tmp" --alias "$base_image_to_rebuild-after-postinstall"
+
+	lxc delete "$base_image_to_rebuild-tmp"
+}
+
+update_image() {
+	local image_to_update=$1
+
+    if ! lxc image info "$image_to_update" &>/dev/null
+    then
+        echo "Unable to upgrade image $image_to_update"
+        return
+    fi
+
+    local finger_print_to_delete=$(lxc image info "$image_to_update" | grep Fingerprint | cut -d' ' -f2)
+
+	# Start and run upgrade
+	lxc launch "$image_to_update" "$image_to_update-tmp"
+	
+	wait_container "$image_to_update-tmp"
+
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update"
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y"
+	lxc stop "$image_to_update-tmp"
+
+	# Add new image updated
+	lxc publish "$image_to_update-tmp" --alias "$image_to_update"
+
+	# Remove old image
+	lxc image delete "$finger_print_to_delete"
+
+	lxc delete "$image_to_update-tmp"
+}