From 777149191b645211b46721d7b439177d43f5db70 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 16 May 2023 15:54:24 +0200 Subject: [PATCH] use a snapshot instead of recreating a container from scratch --- cleanup.sh | 4 +- prepare.sh | 35 +++++++------- utils.sh | 126 +++++++++++++++++---------------------------------- variables.sh | 3 +- 4 files changed, 63 insertions(+), 105 deletions(-) diff --git a/cleanup.sh b/cleanup.sh index 858570a..96bceb9 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -4,6 +4,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/prints.sh source $current_dir/variables.sh # Get variables from variables. -info "Deleting container $CONTAINER_ID" +#info "Deleting container $CONTAINER_ID" -lxc delete -f "$CONTAINER_ID" \ No newline at end of file +#lxc delete -f "$CONTAINER_ID" \ No newline at end of file diff --git a/prepare.sh b/prepare.sh index 0c63406..d635a0c 100755 --- a/prepare.sh +++ b/prepare.sh @@ -10,32 +10,33 @@ set -eo pipefail trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR start_container () { - if lxc info "$CONTAINER_ID" >/dev/null 2>/dev/null ; then - warn 'Found old container, deleting' - lxc delete -f "$CONTAINER_ID" + if ! lxc info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then + warn 'Container not found, copying it from the prebuilt image' + if ! lxc image info "$BASE_IMAGE" &>/dev/null + then + error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" + # Inform GitLab Runner that this is a system failure, so it + # should be retried. + exit $SYSTEM_FAILURE_EXIT_CODE + fi + lxc copy "$BASE_IMAGE" "$CONTAINER_IMAGE" fi - if ! lxc image info "$BASE_IMAGE-$SNAPSHOT_NAME" &>/dev/null - then - error "$BASE_IMAGE-$SNAPSHOT_NAME not found, please rebuild with rebuild_all.sh" - # Inform GitLab Runner that this is a system failure, so it - # should be retried. - exit $SYSTEM_FAILURE_EXIT_CODE - fi + info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE, Snapshot: $SNAPSHOT_NAME" - info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE-$SNAPSHOT_NAME" - - lxc launch "$BASE_IMAGE-$SNAPSHOT_NAME" "$CONTAINER_ID" -c security.nesting=true 2>/dev/null + lxc restore $CONTAINER_IMAGE $SNAPSHOT_NAME mkdir -p $current_dir/cache chmod 777 $current_dir/cache - lxc config device add "$CONTAINER_ID" cache-folder disk path=/cache source="$current_dir/cache" + lxc config device add "$CONTAINER_IMAGE" cache-folder disk path=/cache source="$current_dir/cache" - wait_container $CONTAINER_ID + lxc restart $CONTAINER_IMAGE + + wait_container $CONTAINER_IMAGE } -info "Starting $CONTAINER_ID" +info "Starting $CONTAINER_IMAGE" start_container -info "$CONTAINER_ID started properly" +info "$CONTAINER_IMAGE started properly" diff --git a/utils.sh b/utils.sh index 955f4f3..d325bcd 100755 --- a/utils.sh +++ b/utils.sh @@ -4,27 +4,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/prints.sh source $current_dir/variables.sh # Get variables from variables. -clean_containers() -{ - local base_image_to_clean=$1 - - for image_to_delete in "$base_image_to_clean"{,"-tmp"} - do - if lxc info $image_to_delete &>/dev/null - then - lxc delete $image_to_delete --force - fi - done - - for image_to_delete in "$base_image_to_clean-"{"before-install","after-install"} - do - if lxc image info $image_to_delete &>/dev/null - then - lxc image delete $image_to_delete - fi - done -} - wait_container() { restart_container() @@ -108,32 +87,13 @@ wait_container() done } -rotate_image() +create_snapshot() { local instance_to_publish=$1 - local alias_image=$2 + local snapshot=$2 - # Save the finger print to delete the old image later - local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}') - local should_restart=0 - - # If the container is running, stop it - if [ "$(lxc info $instance_to_publish | grep Status | awk '{print tolower($2)}')" = "running" ] - then - should_restart=1 - lxc stop "$instance_to_publish" - fi - - # Create image before install - lxc publish "$instance_to_publish" --alias "$alias_image" - # Remove old image - lxc image delete "$finger_print_to_delete" - - if [ $should_restart = 1 ] - then - lxc start "$instance_to_publish" - wait_container "$instance_to_publish" - fi + # Create snapshot + lxc snapshot "$instance_to_publish" "$snapshot" --reuse } # These lines are used to extract the dependencies/recommendations from the debian/control file. @@ -169,67 +129,65 @@ rebuild_base_containers() local arch=$3 local base_image_to_rebuild="yunohost-$debian_version-$ynh_version" - lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild-tmp" -c security.nesting=true + lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild" -c security.nesting=true - wait_container "$base_image_to_rebuild-tmp" + wait_container "$base_image_to_rebuild" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes wget curl" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get update" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes wget curl" # Install Git LFS, git comes pre installed with ubuntu image. # Disable this line because we don't need to add a new repo to have git-lfs - #lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes git-lfs" + #lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes git-lfs" # Install gitlab-runner binary since we need for cache/artifacts. if [[ $debian_version == "bullseye" ]] then - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" else - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" fi INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/main/$debian_version" # Download the YunoHost install script - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" # Patch the YunoHost install script - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh" # Run the YunoHost install script patched - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" get_dependencies $debian_version # Pre install dependencies - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" # Disable apt-daily - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily.service" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.service" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily.service" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" - rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install" + create_snapshot "$base_image_to_rebuild" "before-install" # Install YunoHost - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" # Run postinstall - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" - rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-install" + create_snapshot "$base_image_to_rebuild" "after-install" - lxc stop "$base_image_to_rebuild-tmp" - - lxc delete "$base_image_to_rebuild-tmp" + lxc stop "$base_image_to_rebuild" } update_image() { @@ -245,21 +203,19 @@ update_image() { fi # Start and run upgrade - lxc launch "$image_to_update" "$image_to_update-tmp" -c security.nesting=true + lxc launch "$image_to_update" "$image_to_update" -c security.nesting=true - wait_container "$image_to_update-tmp" + wait_container "$image_to_update" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade --assume-yes" + lxc exec "$image_to_update" -- /bin/bash -c "apt-get update" + lxc exec "$image_to_update" -- /bin/bash -c "apt-get upgrade --assume-yes" get_dependencies $debian_version - lxc exec "$image_to_update-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" + lxc exec "$image_to_update" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$image_to_update" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" - rotate_image "$image_to_update-tmp" "$image_to_update" + create_snapshot "$image_to_update" "$snapshot" - lxc stop "$image_to_update-tmp" - - lxc delete "$image_to_update-tmp" + lxc stop "$image_to_update" } diff --git a/variables.sh b/variables.sh index a7626ab..c2a07c7 100755 --- a/variables.sh +++ b/variables.sh @@ -5,7 +5,6 @@ source $current_dir/prints.sh # All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_ -CONTAINER_ID="runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID-$CUSTOM_ENV_CI_JOB_ID" ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64 DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH" @@ -30,3 +29,5 @@ PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" # For example yunohost-buster-unstable BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION" + +CONTAINER_IMAGE="$BASE_IMAGE-runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID" \ No newline at end of file