From 777149191b645211b46721d7b439177d43f5db70 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 16 May 2023 15:54:24 +0200 Subject: [PATCH 01/25] use a snapshot instead of recreating a container from scratch --- cleanup.sh | 4 +- prepare.sh | 35 +++++++------- utils.sh | 126 +++++++++++++++++---------------------------------- variables.sh | 3 +- 4 files changed, 63 insertions(+), 105 deletions(-) diff --git a/cleanup.sh b/cleanup.sh index 858570a..96bceb9 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -4,6 +4,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/prints.sh source $current_dir/variables.sh # Get variables from variables. -info "Deleting container $CONTAINER_ID" +#info "Deleting container $CONTAINER_ID" -lxc delete -f "$CONTAINER_ID" \ No newline at end of file +#lxc delete -f "$CONTAINER_ID" \ No newline at end of file diff --git a/prepare.sh b/prepare.sh index 0c63406..d635a0c 100755 --- a/prepare.sh +++ b/prepare.sh @@ -10,32 +10,33 @@ set -eo pipefail trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR start_container () { - if lxc info "$CONTAINER_ID" >/dev/null 2>/dev/null ; then - warn 'Found old container, deleting' - lxc delete -f "$CONTAINER_ID" + if ! lxc info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then + warn 'Container not found, copying it from the prebuilt image' + if ! lxc image info "$BASE_IMAGE" &>/dev/null + then + error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" + # Inform GitLab Runner that this is a system failure, so it + # should be retried. + exit $SYSTEM_FAILURE_EXIT_CODE + fi + lxc copy "$BASE_IMAGE" "$CONTAINER_IMAGE" fi - if ! lxc image info "$BASE_IMAGE-$SNAPSHOT_NAME" &>/dev/null - then - error "$BASE_IMAGE-$SNAPSHOT_NAME not found, please rebuild with rebuild_all.sh" - # Inform GitLab Runner that this is a system failure, so it - # should be retried. - exit $SYSTEM_FAILURE_EXIT_CODE - fi + info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE, Snapshot: $SNAPSHOT_NAME" - info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE-$SNAPSHOT_NAME" - - lxc launch "$BASE_IMAGE-$SNAPSHOT_NAME" "$CONTAINER_ID" -c security.nesting=true 2>/dev/null + lxc restore $CONTAINER_IMAGE $SNAPSHOT_NAME mkdir -p $current_dir/cache chmod 777 $current_dir/cache - lxc config device add "$CONTAINER_ID" cache-folder disk path=/cache source="$current_dir/cache" + lxc config device add "$CONTAINER_IMAGE" cache-folder disk path=/cache source="$current_dir/cache" - wait_container $CONTAINER_ID + lxc restart $CONTAINER_IMAGE + + wait_container $CONTAINER_IMAGE } -info "Starting $CONTAINER_ID" +info "Starting $CONTAINER_IMAGE" start_container -info "$CONTAINER_ID started properly" +info "$CONTAINER_IMAGE started properly" diff --git a/utils.sh b/utils.sh index 955f4f3..d325bcd 100755 --- a/utils.sh +++ b/utils.sh @@ -4,27 +4,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/prints.sh source $current_dir/variables.sh # Get variables from variables. -clean_containers() -{ - local base_image_to_clean=$1 - - for image_to_delete in "$base_image_to_clean"{,"-tmp"} - do - if lxc info $image_to_delete &>/dev/null - then - lxc delete $image_to_delete --force - fi - done - - for image_to_delete in "$base_image_to_clean-"{"before-install","after-install"} - do - if lxc image info $image_to_delete &>/dev/null - then - lxc image delete $image_to_delete - fi - done -} - wait_container() { restart_container() @@ -108,32 +87,13 @@ wait_container() done } -rotate_image() +create_snapshot() { local instance_to_publish=$1 - local alias_image=$2 + local snapshot=$2 - # Save the finger print to delete the old image later - local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}') - local should_restart=0 - - # If the container is running, stop it - if [ "$(lxc info $instance_to_publish | grep Status | awk '{print tolower($2)}')" = "running" ] - then - should_restart=1 - lxc stop "$instance_to_publish" - fi - - # Create image before install - lxc publish "$instance_to_publish" --alias "$alias_image" - # Remove old image - lxc image delete "$finger_print_to_delete" - - if [ $should_restart = 1 ] - then - lxc start "$instance_to_publish" - wait_container "$instance_to_publish" - fi + # Create snapshot + lxc snapshot "$instance_to_publish" "$snapshot" --reuse } # These lines are used to extract the dependencies/recommendations from the debian/control file. @@ -169,67 +129,65 @@ rebuild_base_containers() local arch=$3 local base_image_to_rebuild="yunohost-$debian_version-$ynh_version" - lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild-tmp" -c security.nesting=true + lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild" -c security.nesting=true - wait_container "$base_image_to_rebuild-tmp" + wait_container "$base_image_to_rebuild" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes wget curl" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get update" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes wget curl" # Install Git LFS, git comes pre installed with ubuntu image. # Disable this line because we don't need to add a new repo to have git-lfs - #lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes git-lfs" + #lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes git-lfs" # Install gitlab-runner binary since we need for cache/artifacts. if [[ $debian_version == "bullseye" ]] then - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" else - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" fi INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/main/$debian_version" # Download the YunoHost install script - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" # Patch the YunoHost install script - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh" # Run the YunoHost install script patched - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" get_dependencies $debian_version # Pre install dependencies - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" # Disable apt-daily - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily.service" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.service" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily.service" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" - rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install" + create_snapshot "$base_image_to_rebuild" "before-install" # Install YunoHost - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" # Run postinstall - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" - rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-install" + create_snapshot "$base_image_to_rebuild" "after-install" - lxc stop "$base_image_to_rebuild-tmp" - - lxc delete "$base_image_to_rebuild-tmp" + lxc stop "$base_image_to_rebuild" } update_image() { @@ -245,21 +203,19 @@ update_image() { fi # Start and run upgrade - lxc launch "$image_to_update" "$image_to_update-tmp" -c security.nesting=true + lxc launch "$image_to_update" "$image_to_update" -c security.nesting=true - wait_container "$image_to_update-tmp" + wait_container "$image_to_update" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade --assume-yes" + lxc exec "$image_to_update" -- /bin/bash -c "apt-get update" + lxc exec "$image_to_update" -- /bin/bash -c "apt-get upgrade --assume-yes" get_dependencies $debian_version - lxc exec "$image_to_update-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" + lxc exec "$image_to_update" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$image_to_update" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" - rotate_image "$image_to_update-tmp" "$image_to_update" + create_snapshot "$image_to_update" "$snapshot" - lxc stop "$image_to_update-tmp" - - lxc delete "$image_to_update-tmp" + lxc stop "$image_to_update" } diff --git a/variables.sh b/variables.sh index a7626ab..c2a07c7 100755 --- a/variables.sh +++ b/variables.sh @@ -5,7 +5,6 @@ source $current_dir/prints.sh # All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_ -CONTAINER_ID="runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID-$CUSTOM_ENV_CI_JOB_ID" ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64 DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH" @@ -30,3 +29,5 @@ PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" # For example yunohost-buster-unstable BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION" + +CONTAINER_IMAGE="$BASE_IMAGE-runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID" \ No newline at end of file From 9c23d090fb41f92b8180ff466681d3541fdcce3a Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 16 May 2023 17:31:19 +0200 Subject: [PATCH 02/25] fix update container --- auto_upgrade_container.sh | 4 ++-- prepare.sh | 2 +- rebuild_all.sh | 2 +- utils.sh | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index 9ee92ee..da76b9c 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -9,7 +9,7 @@ do do for snapshot in "before-install" "after-install" do - update_image $debian_version $ynh_version $snapshot + update_container $debian_version $ynh_version $snapshot done done done @@ -20,7 +20,7 @@ do do for snapshot in "before-install" "after-install" do - update_image $debian_version $ynh_version $snapshot + update_container $debian_version $ynh_version $snapshot done done done diff --git a/prepare.sh b/prepare.sh index d635a0c..1cb5017 100755 --- a/prepare.sh +++ b/prepare.sh @@ -12,7 +12,7 @@ trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR start_container () { if ! lxc info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then warn 'Container not found, copying it from the prebuilt image' - if ! lxc image info "$BASE_IMAGE" &>/dev/null + if ! lxc info "$BASE_IMAGE" &>/dev/null || ! lxc info "$BASE_IMAGE" | grep -q "$SNAPSHOT_NAME" then error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" # Inform GitLab Runner that this is a system failure, so it diff --git a/rebuild_all.sh b/rebuild_all.sh index b222bf7..93c399f 100755 --- a/rebuild_all.sh +++ b/rebuild_all.sh @@ -30,7 +30,7 @@ do lxc delete -f "yunohost-$debian_version-$ynh_version-$snapshot-tmp" - update_image $debian_version $ynh_version $snapshot + update_container $debian_version $ynh_version $snapshot done done done diff --git a/utils.sh b/utils.sh index d325bcd..490d250 100755 --- a/utils.sh +++ b/utils.sh @@ -190,20 +190,20 @@ rebuild_base_containers() lxc stop "$base_image_to_rebuild" } -update_image() { +update_container() { local debian_version=$1 local ynh_version=$2 local snapshot=$3 local image_to_update="yunohost-$debian_version-$ynh_version-$snapshot" - if ! lxc image info "$image_to_update" &>/dev/null + if ! lxc info "$image_to_update" &>/dev/null then error "Unable to upgrade image $image_to_update" return fi # Start and run upgrade - lxc launch "$image_to_update" "$image_to_update" -c security.nesting=true + lxc restore "$image_to_update" "$snapshot" wait_container "$image_to_update" From e96a3cb52802b57fde3b63b1d0f3c6e332974004 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 16 May 2023 18:11:52 +0200 Subject: [PATCH 03/25] fix prepare script --- prepare.sh | 3 ++- run.sh | 2 +- utils.sh | 5 +++++ variables.sh | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/prepare.sh b/prepare.sh index 1cb5017..cd2a5cd 100755 --- a/prepare.sh +++ b/prepare.sh @@ -30,7 +30,8 @@ start_container () { chmod 777 $current_dir/cache lxc config device add "$CONTAINER_IMAGE" cache-folder disk path=/cache source="$current_dir/cache" - lxc restart $CONTAINER_IMAGE + lxc stop $CONTAINER_IMAGE || true + lxc start $CONTAINER_IMAGE wait_container $CONTAINER_IMAGE } diff --git a/run.sh b/run.sh index d1769ff..2cace7b 100755 --- a/run.sh +++ b/run.sh @@ -36,7 +36,7 @@ case ${2} in esac -lxc exec "$CONTAINER_ID" /bin/bash < "${1}" +lxc exec "$CONTAINER_IMAGE" /bin/bash < "${1}" if [ $? -ne 0 ]; then # Exit using the variable, to make the build as failure in GitLab # CI. diff --git a/utils.sh b/utils.sh index 490d250..956dab2 100755 --- a/utils.sh +++ b/utils.sh @@ -46,6 +46,11 @@ wait_container() if [ "$j" == "10" ]; then error 'Failed to access the internet' failstart=1 + lxc exec "$1" -- /bin/bash -c "echo 'resolv-file=/etc/resolv.dnsmasq.conf' > /etc/dnsmasq.d/resolvconf" + lxc exec "$1" -- /bin/bash -c "echo 'nameserver 8.8.8.8' > /etc/resolv.dnsmasq.conf" + lxc exec "$1" -- /bin/bash -c "sed -i 's/#IGNORE/IGNORE/g' /etc/default/dnsmasq" + lxc exec "$1" -- /bin/bash -c "systemctl restart dnsmasq" + lxc exec "$1" -- /bin/bash -c "journalctl -u dnsmasq -n 100 --no-pager" restart_container "$1" fi diff --git a/variables.sh b/variables.sh index c2a07c7..872c7e3 100755 --- a/variables.sh +++ b/variables.sh @@ -30,4 +30,4 @@ PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" # For example yunohost-buster-unstable BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION" -CONTAINER_IMAGE="$BASE_IMAGE-runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID" \ No newline at end of file +CONTAINER_IMAGE="$BASE_IMAGE-r-$CUSTOM_ENV_CI_RUNNER_ID-p-$CUSTOM_ENV_CI_PROJECT_ID-c-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID" \ No newline at end of file From 7e367e3fc7837354dc407734fd966cfb0d4de79f Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 12:12:21 +0200 Subject: [PATCH 04/25] remove runner's containers after auto upgrade --- auto_upgrade_container.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index da76b9c..d16edef 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -22,5 +22,6 @@ do do update_container $debian_version $ynh_version $snapshot done + lxc delete -f $(lxc list yunohost-$debian_version-$ynh_version-r -c n -f csv) done done From 9a552cdab3eadfe8beb9bb633a434e9b90b70d2f Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 12:13:08 +0200 Subject: [PATCH 05/25] fix IP address of the copy containers --- utils.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils.sh b/utils.sh index 956dab2..720f7b8 100755 --- a/utils.sh +++ b/utils.sh @@ -97,6 +97,9 @@ create_snapshot() local instance_to_publish=$1 local snapshot=$2 + # Unset the mac address to ensure the copy will get a new one and will be able to get new IP + lxc config unset "$instance_to_publish" volatile.eth0.hwaddr + # Create snapshot lxc snapshot "$instance_to_publish" "$snapshot" --reuse } From 35ba94d08a3ed35ad663a9927e0e6bc1358b5ec3 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 12:13:27 +0200 Subject: [PATCH 06/25] preinstall php8.2-cli mariadb-client --- utils.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils.sh b/utils.sh index 720f7b8..d7e9f24 100755 --- a/utils.sh +++ b/utils.sh @@ -118,6 +118,10 @@ get_dependencies() # To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'. YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ") + + # We add php8.2-cli and mariadb-client to the dependencies for test_app_resources + YUNOHOST_DEPENDENCIES="$YUNOHOST_DEPENDENCIES php8.2-cli mariadb-client" + YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") # Same as above, except that all dependencies are in the same line From 13682de9dea06765fe8b7c010f50a8f19224a15e Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 12:19:44 +0200 Subject: [PATCH 07/25] fix update_container --- utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.sh b/utils.sh index d7e9f24..c950ece 100755 --- a/utils.sh +++ b/utils.sh @@ -206,7 +206,7 @@ update_container() { local debian_version=$1 local ynh_version=$2 local snapshot=$3 - local image_to_update="yunohost-$debian_version-$ynh_version-$snapshot" + local image_to_update="yunohost-$debian_version-$ynh_version" if ! lxc info "$image_to_update" &>/dev/null then From 71f07c134bd970d95abd3fe89b87f9b9698217e6 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 12:48:04 +0200 Subject: [PATCH 08/25] fix tabs --- utils.sh | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/utils.sh b/utils.sh index c950ece..d470dda 100755 --- a/utils.sh +++ b/utils.sh @@ -108,30 +108,30 @@ create_snapshot() # /!\ There's a high risk of lamentable failure if we change the format of this file get_dependencies() { - local debian_version=$1 - if [[ "$debian_version" == "bullseye" ]] - then - local branch="dev" - else - local branch="$debian_version" - fi + local debian_version=$1 + if [[ "$debian_version" == "bullseye" ]] + then + local branch="dev" + else + local branch="$debian_version" + fi - # To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'. - YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ") + # To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'. + YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ") # We add php8.2-cli and mariadb-client to the dependencies for test_app_resources YUNOHOST_DEPENDENCIES="$YUNOHOST_DEPENDENCIES php8.2-cli mariadb-client" - YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") - MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") - # Same as above, except that all dependencies are in the same line - SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$branch/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") - BUILD_DEPENDENCIES="git-buildpackage postfix python3-setuptools python3-pip devscripts" - PIP3_PKG='mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 types-ipaddress types-enum34 types-cryptography types-toml types-requests types-PyYAML types-pyOpenSSL types-mock "packaging<22"' - if [[ "$debian_version" == "bookworm" ]] - then - PIP3_PKG="$PIP3_PKG --break-system-packages" - fi + YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") + MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") + # Same as above, except that all dependencies are in the same line + SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$branch/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") + BUILD_DEPENDENCIES="git-buildpackage postfix python3-setuptools python3-pip devscripts" + PIP3_PKG='mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 types-ipaddress types-enum34 types-cryptography types-toml types-requests types-PyYAML types-pyOpenSSL types-mock "packaging<22"' + if [[ "$debian_version" == "bookworm" ]] + then + PIP3_PKG="$PIP3_PKG --break-system-packages" + fi } rebuild_base_containers() From 6a6db406ef820accc4343a12a8787c9707d35ab6 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 13:03:46 +0200 Subject: [PATCH 09/25] pre add the cache folder --- prepare.sh | 5 ----- rebuild_all.sh | 3 +++ utils.sh | 2 ++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/prepare.sh b/prepare.sh index cd2a5cd..6f45fa8 100755 --- a/prepare.sh +++ b/prepare.sh @@ -26,11 +26,6 @@ start_container () { lxc restore $CONTAINER_IMAGE $SNAPSHOT_NAME - mkdir -p $current_dir/cache - chmod 777 $current_dir/cache - lxc config device add "$CONTAINER_IMAGE" cache-folder disk path=/cache source="$current_dir/cache" - - lxc stop $CONTAINER_IMAGE || true lxc start $CONTAINER_IMAGE wait_container $CONTAINER_IMAGE diff --git a/rebuild_all.sh b/rebuild_all.sh index 93c399f..4e3a029 100755 --- a/rebuild_all.sh +++ b/rebuild_all.sh @@ -3,6 +3,9 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/utils.sh # Get utils functions. +mkdir -p $current_dir/cache +chmod 777 $current_dir/cache + for debian_version in "bullseye" do rebuild_base_containers $debian_version "stable" "amd64" diff --git a/utils.sh b/utils.sh index d470dda..e70e5c3 100755 --- a/utils.sh +++ b/utils.sh @@ -189,6 +189,8 @@ rebuild_base_containers() lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service" lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" + lxc config device add "$base_image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache" + create_snapshot "$base_image_to_rebuild" "before-install" # Install YunoHost From c05450ad8e45d328b77651b2fcc30b60d175be97 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 15:10:29 +0200 Subject: [PATCH 10/25] rework the snapshot name --- auto_upgrade_container.sh | 6 ++--- cleanup.sh | 4 +-- prepare.sh | 2 +- rebuild_all.sh | 13 +++------ utils.sh | 57 ++++++++++++++++++++++++++------------- variables.sh | 4 +-- 6 files changed, 49 insertions(+), 37 deletions(-) diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index d16edef..1fa4b2b 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -9,7 +9,7 @@ do do for snapshot in "before-install" "after-install" do - update_container $debian_version $ynh_version $snapshot + update_container "$debian_version" "$ynh_version" "$snapshot" done done done @@ -20,8 +20,8 @@ do do for snapshot in "before-install" "after-install" do - update_container $debian_version $ynh_version $snapshot + update_container "$debian_version" "$ynh_version" "$snapshot" done - lxc delete -f $(lxc list yunohost-$debian_version-$ynh_version-r -c n -f csv) + lxc delete -f $(lxc list yunohost-$debian_version-r -c n -f csv) done done diff --git a/cleanup.sh b/cleanup.sh index 96bceb9..1dbfc16 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -4,6 +4,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/prints.sh source $current_dir/variables.sh # Get variables from variables. -#info "Deleting container $CONTAINER_ID" +info "Stopping container $CONTAINER_ID" -#lxc delete -f "$CONTAINER_ID" \ No newline at end of file +lxc stop "$CONTAINER_ID" \ No newline at end of file diff --git a/prepare.sh b/prepare.sh index 6f45fa8..549d67d 100755 --- a/prepare.sh +++ b/prepare.sh @@ -24,7 +24,7 @@ start_container () { info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE, Snapshot: $SNAPSHOT_NAME" - lxc restore $CONTAINER_IMAGE $SNAPSHOT_NAME + restore_snapshot "$CONTAINER_IMAGE" "$CURRENT_VERSION" "$SNAPSHOT_NAME" lxc start $CONTAINER_IMAGE diff --git a/rebuild_all.sh b/rebuild_all.sh index 4e3a029..33df0ad 100755 --- a/rebuild_all.sh +++ b/rebuild_all.sh @@ -3,9 +3,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/utils.sh # Get utils functions. -mkdir -p $current_dir/cache -chmod 777 $current_dir/cache - for debian_version in "bullseye" do rebuild_base_containers $debian_version "stable" "amd64" @@ -14,7 +11,7 @@ do do for snapshot in "before-install" "after-install" do - lxc launch "yunohost-$debian_version-stable-$snapshot" "yunohost-$debian_version-$ynh_version-$snapshot-tmp" + restore_snapshot "yunohost-$debian_version" "stable" "$snapshot" if [[ "$ynh_version" == "testing" ]] then @@ -24,16 +21,12 @@ do repo_version="testing unstable" fi - lxc exec "yunohost-$debian_version-$ynh_version-$snapshot-tmp" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`; + lxc exec "yunohost-$debian_version" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`; do sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE done" - rotate_image "yunohost-$debian_version-$ynh_version-$snapshot-tmp" "yunohost-$debian_version-$ynh_version-$snapshot" - - lxc delete -f "yunohost-$debian_version-$ynh_version-$snapshot-tmp" - - update_container $debian_version $ynh_version $snapshot + update_container "$debian_version" "$ynh_version" "$snapshot" done done done diff --git a/utils.sh b/utils.sh index e70e5c3..7813494 100755 --- a/utils.sh +++ b/utils.sh @@ -95,13 +95,20 @@ wait_container() create_snapshot() { local instance_to_publish=$1 - local snapshot=$2 - - # Unset the mac address to ensure the copy will get a new one and will be able to get new IP - lxc config unset "$instance_to_publish" volatile.eth0.hwaddr + local ynh_version=$2 + local snapshot=$3 # Create snapshot - lxc snapshot "$instance_to_publish" "$snapshot" --reuse + lxc snapshot "$instance_to_publish" "$ynh_version-$snapshot" --reuse +} + +restore_snapshot() +{ + local instance_to_publish=$1 + local ynh_version=$2 + local snapshot=$3 + + lxc restore "$instance_to_publish" "$ynh_version-$snapshot" } # These lines are used to extract the dependencies/recommendations from the debian/control file. @@ -118,18 +125,17 @@ get_dependencies() # To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'. YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ") - - # We add php8.2-cli and mariadb-client to the dependencies for test_app_resources - YUNOHOST_DEPENDENCIES="$YUNOHOST_DEPENDENCIES php8.2-cli mariadb-client" - YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") # Same as above, except that all dependencies are in the same line SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$branch/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") BUILD_DEPENDENCIES="git-buildpackage postfix python3-setuptools python3-pip devscripts" PIP3_PKG='mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 types-ipaddress types-enum34 types-cryptography types-toml types-requests types-PyYAML types-pyOpenSSL types-mock "packaging<22"' + if [[ "$debian_version" == "bookworm" ]] then + # We add php8.2-cli and mariadb-client to the dependencies for test_app_resources + YUNOHOST_DEPENDENCIES="$YUNOHOST_DEPENDENCIES php8.2-cli mariadb-client" PIP3_PKG="$PIP3_PKG --break-system-packages" fi } @@ -139,7 +145,12 @@ rebuild_base_containers() local debian_version=$1 local ynh_version=$2 local arch=$3 - local base_image_to_rebuild="yunohost-$debian_version-$ynh_version" + local base_image_to_rebuild="yunohost-$debian_version" + + if lxc info "$base_image_to_rebuild" &>/dev/null + then + lxc delete -f "$base_image_to_rebuild" + fi lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild" -c security.nesting=true @@ -154,11 +165,11 @@ rebuild_base_containers() # Install gitlab-runner binary since we need for cache/artifacts. if [[ $debian_version == "bullseye" ]] then - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" else - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" + lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" fi INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/main/$debian_version" @@ -189,9 +200,15 @@ rebuild_base_containers() lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service" lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" + + mkdir -p $current_dir/cache + chmod 777 $current_dir/cache lxc config device add "$base_image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache" - create_snapshot "$base_image_to_rebuild" "before-install" + # Unset the mac address to ensure the copy will get a new one and will be able to get new IP + lxc config unset "$instance_to_publish" volatile.eth0.hwaddr + + create_snapshot "$base_image_to_rebuild" "$ynh_version" "before-install" # Install YunoHost lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" @@ -199,7 +216,7 @@ rebuild_base_containers() # Run postinstall lxc exec "$base_image_to_rebuild" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" - create_snapshot "$base_image_to_rebuild" "after-install" + create_snapshot "$base_image_to_rebuild" "$ynh_version" "after-install" lxc stop "$base_image_to_rebuild" } @@ -208,7 +225,7 @@ update_container() { local debian_version=$1 local ynh_version=$2 local snapshot=$3 - local image_to_update="yunohost-$debian_version-$ynh_version" + local image_to_update="yunohost-$debian_version" if ! lxc info "$image_to_update" &>/dev/null then @@ -217,8 +234,10 @@ update_container() { fi # Start and run upgrade - lxc restore "$image_to_update" "$snapshot" + restore_snapshot "$image_to_update" "$ynh_version" "$snapshot" + lxc start "$image_to_update" 2>&1 || true + wait_container "$image_to_update" lxc exec "$image_to_update" -- /bin/bash -c "apt-get update" @@ -229,7 +248,7 @@ update_container() { lxc exec "$image_to_update" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" lxc exec "$image_to_update" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" - create_snapshot "$image_to_update" "$snapshot" + create_snapshot "$image_to_update" "$ynh_version" "$snapshot" lxc stop "$image_to_update" } diff --git a/variables.sh b/variables.sh index 872c7e3..1a1efd9 100755 --- a/variables.sh +++ b/variables.sh @@ -27,7 +27,7 @@ fi PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR" PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" -# For example yunohost-buster-unstable -BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION" +# For example yunohost-buster +BASE_IMAGE="yunohost-$DEBIAN_VERSION" CONTAINER_IMAGE="$BASE_IMAGE-r-$CUSTOM_ENV_CI_RUNNER_ID-p-$CUSTOM_ENV_CI_PROJECT_ID-c-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID" \ No newline at end of file From ec05c80f27bea55f3bc553510975cd20e2d6ddb9 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 16:28:42 +0200 Subject: [PATCH 11/25] fix container name for unset mac addr --- utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.sh b/utils.sh index 7813494..56fcb8b 100755 --- a/utils.sh +++ b/utils.sh @@ -206,7 +206,7 @@ rebuild_base_containers() lxc config device add "$base_image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache" # Unset the mac address to ensure the copy will get a new one and will be able to get new IP - lxc config unset "$instance_to_publish" volatile.eth0.hwaddr + lxc config unset "$base_image_to_rebuild" volatile.eth0.hwaddr create_snapshot "$base_image_to_rebuild" "$ynh_version" "before-install" From 2cc13f93fb45e32fe9093f95938a0cbcb1b28422 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 16:34:44 +0200 Subject: [PATCH 12/25] fix rebuild testing and unstable --- rebuild_all.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rebuild_all.sh b/rebuild_all.sh index 33df0ad..5c22544 100755 --- a/rebuild_all.sh +++ b/rebuild_all.sh @@ -26,6 +26,8 @@ do sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE done" + create_snapshot "yunohost-$debian_version" "$ynh_version" "$snapshot" + update_container "$debian_version" "$ynh_version" "$snapshot" done done From cc0bcaf6002e0b88ddabd4937374424ef25d5327 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 16:48:45 +0200 Subject: [PATCH 13/25] standardization of container names --- auto_upgrade_container.sh | 6 +-- rebuild_all.sh | 12 +++--- utils.sh | 82 +++++++++++++++++++-------------------- variables.sh | 5 ++- 4 files changed, 53 insertions(+), 52 deletions(-) diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index 1fa4b2b..461a047 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -9,7 +9,7 @@ do do for snapshot in "before-install" "after-install" do - update_container "$debian_version" "$ynh_version" "$snapshot" + update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done done done @@ -20,8 +20,8 @@ do do for snapshot in "before-install" "after-install" do - update_container "$debian_version" "$ynh_version" "$snapshot" + update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done - lxc delete -f $(lxc list yunohost-$debian_version-r -c n -f csv) + lxc delete -f $(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) done done diff --git a/rebuild_all.sh b/rebuild_all.sh index 5c22544..c0f4338 100755 --- a/rebuild_all.sh +++ b/rebuild_all.sh @@ -5,13 +5,13 @@ source $current_dir/utils.sh # Get utils functions. for debian_version in "bullseye" do - rebuild_base_containers $debian_version "stable" "amd64" + rebuild_base_containers "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "stable" "amd64" for ynh_version in "testing" "unstable" do for snapshot in "before-install" "after-install" do - restore_snapshot "yunohost-$debian_version" "stable" "$snapshot" + restore_snapshot "$PREFIX_IMAGE_NAME-$debian_version" "stable" "$snapshot" if [[ "$ynh_version" == "testing" ]] then @@ -21,14 +21,14 @@ do repo_version="testing unstable" fi - lxc exec "yunohost-$debian_version" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`; + lxc exec "$PREFIX_IMAGE_NAME-$debian_version" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`; do sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE done" - create_snapshot "yunohost-$debian_version" "$ynh_version" "$snapshot" + create_snapshot "$PREFIX_IMAGE_NAME-$debian_version" "$ynh_version" "$snapshot" - update_container "$debian_version" "$ynh_version" "$snapshot" + update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done done done @@ -36,6 +36,6 @@ done for debian_version in "bookworm" do - rebuild_base_containers $debian_version "unstable" "amd64" + rebuild_base_containers "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "unstable" "amd64" done diff --git a/utils.sh b/utils.sh index 56fcb8b..1ebcf0d 100755 --- a/utils.sh +++ b/utils.sh @@ -142,90 +142,90 @@ get_dependencies() rebuild_base_containers() { - local debian_version=$1 - local ynh_version=$2 - local arch=$3 - local base_image_to_rebuild="yunohost-$debian_version" + local image_to_rebuild=$1 + local debian_version=$2 + local ynh_version=$3 + local arch=$4 - if lxc info "$base_image_to_rebuild" &>/dev/null + if lxc info "$image_to_rebuild" &>/dev/null then - lxc delete -f "$base_image_to_rebuild" + lxc delete -f "$image_to_rebuild" fi - lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild" -c security.nesting=true + lxc launch images:debian/$debian_version/$arch "$image_to_rebuild" -c security.nesting=true - wait_container "$base_image_to_rebuild" + wait_container "$image_to_rebuild" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get update" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes wget curl" + lxc exec "$image_to_rebuild" -- /bin/bash -c "apt-get update" + lxc exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes wget curl" # Install Git LFS, git comes pre installed with ubuntu image. # Disable this line because we don't need to add a new repo to have git-lfs - #lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes git-lfs" + #lxc exec "$image_to_rebuild" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" + lxc exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes git-lfs" # Install gitlab-runner binary since we need for cache/artifacts. if [[ $debian_version == "bullseye" ]] then - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" + lxc exec "$image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" + lxc exec "$image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" else - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" + lxc exec "$image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" + lxc exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" fi INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/main/$debian_version" # Download the YunoHost install script - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" + lxc exec "$image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" # Patch the YunoHost install script - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh" + lxc exec "$image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" + lxc exec "$image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh" # Run the YunoHost install script patched - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" + lxc exec "$image_to_rebuild" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" get_dependencies $debian_version # Pre install dependencies - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" + lxc exec "$image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$image_to_rebuild" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" # Disable apt-daily - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.timer" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.timer" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.service" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.service" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.timer" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service" - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.timer" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.timer" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily.service" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q stop apt-daily-upgrade.service" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.timer" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service" + lxc exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service" mkdir -p $current_dir/cache chmod 777 $current_dir/cache - lxc config device add "$base_image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache" + lxc config device add "$image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache" # Unset the mac address to ensure the copy will get a new one and will be able to get new IP - lxc config unset "$base_image_to_rebuild" volatile.eth0.hwaddr + lxc config unset "$image_to_rebuild" volatile.eth0.hwaddr - create_snapshot "$base_image_to_rebuild" "$ynh_version" "before-install" + create_snapshot "$image_to_rebuild" "$ynh_version" "before-install" # Install YunoHost - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" + lxc exec "$image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" # Run postinstall - lxc exec "$base_image_to_rebuild" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" + lxc exec "$image_to_rebuild" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" - create_snapshot "$base_image_to_rebuild" "$ynh_version" "after-install" + create_snapshot "$image_to_rebuild" "$ynh_version" "after-install" - lxc stop "$base_image_to_rebuild" + lxc stop "$image_to_rebuild" } update_container() { - local debian_version=$1 - local ynh_version=$2 - local snapshot=$3 - local image_to_update="yunohost-$debian_version" + local image_to_update=$1 + local debian_version=$2 + local ynh_version=$3 + local snapshot=$4 if ! lxc info "$image_to_update" &>/dev/null then diff --git a/variables.sh b/variables.sh index 1a1efd9..7187d7b 100755 --- a/variables.sh +++ b/variables.sh @@ -27,7 +27,8 @@ fi PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR" PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" -# For example yunohost-buster -BASE_IMAGE="yunohost-$DEBIAN_VERSION" +PREFIX_IMAGE_NAME="ynh" +# For example ynh-buster +BASE_IMAGE="$PREFIX_IMAGE_NAME-$DEBIAN_VERSION" CONTAINER_IMAGE="$BASE_IMAGE-r-$CUSTOM_ENV_CI_RUNNER_ID-p-$CUSTOM_ENV_CI_PROJECT_ID-c-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID" \ No newline at end of file From b5e48b2e3402dddaf8700baa6da4c4a587bfa08b Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 17 May 2023 20:18:34 +0200 Subject: [PATCH 14/25] fix prepare and cleanup scripts --- cleanup.sh | 4 ++-- prepare.sh | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cleanup.sh b/cleanup.sh index 1dbfc16..27b98d6 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -4,6 +4,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/prints.sh source $current_dir/variables.sh # Get variables from variables. -info "Stopping container $CONTAINER_ID" +info "Stopping container $CONTAINER_IMAGE" -lxc stop "$CONTAINER_ID" \ No newline at end of file +lxc stop "$CONTAINER_IMAGE" \ No newline at end of file diff --git a/prepare.sh b/prepare.sh index 549d67d..42e2525 100755 --- a/prepare.sh +++ b/prepare.sh @@ -12,7 +12,7 @@ trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR start_container () { if ! lxc info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then warn 'Container not found, copying it from the prebuilt image' - if ! lxc info "$BASE_IMAGE" &>/dev/null || ! lxc info "$BASE_IMAGE" | grep -q "$SNAPSHOT_NAME" + if ! lxc info "$BASE_IMAGE" &>/dev/null || ! lxc info "$BASE_IMAGE" | grep -q "$CURRENT_VERSION-$SNAPSHOT_NAME" then error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" # Inform GitLab Runner that this is a system failure, so it @@ -21,6 +21,10 @@ start_container () { fi lxc copy "$BASE_IMAGE" "$CONTAINER_IMAGE" fi + # Stop the container if it's running + if [ "$(lxc info $CONTAINER_IMAGE | grep Status | awk '{print tolower($2)}')" == "running" ]; then + lxc stop $CONTAINER_IMAGE + fi info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE, Snapshot: $SNAPSHOT_NAME" From 79a410b8f557f6d0a8185580a8396e31cb6fdb35 Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 18 May 2023 02:06:35 +0200 Subject: [PATCH 15/25] add mariadb-server as a dependency --- utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.sh b/utils.sh index 1ebcf0d..b38549f 100755 --- a/utils.sh +++ b/utils.sh @@ -135,7 +135,7 @@ get_dependencies() if [[ "$debian_version" == "bookworm" ]] then # We add php8.2-cli and mariadb-client to the dependencies for test_app_resources - YUNOHOST_DEPENDENCIES="$YUNOHOST_DEPENDENCIES php8.2-cli mariadb-client" + YUNOHOST_DEPENDENCIES="$YUNOHOST_DEPENDENCIES php8.2-cli mariadb-client mariadb-server" PIP3_PKG="$PIP3_PKG --break-system-packages" fi } From b602dad93c0a332728fe2f89ce0b2834df2d233d Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 11:43:12 +0200 Subject: [PATCH 16/25] add a default value to $current_branch --- variables.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/variables.sh b/variables.sh index 7187d7b..2f04475 100755 --- a/variables.sh +++ b/variables.sh @@ -9,6 +9,11 @@ ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH" CURRENT_BRANCH="$CUSTOM_ENV_CI_COMMIT_REF_NAME" # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR +if [ -z "$CURRENT_BRANCH" ] +then + CURRENT_BRANCH="dev" +fi + LAST_CHANGELOG_ENTRY=$(curl https://gitlab.com/yunohost/yunohost/-/raw/$CURRENT_BRANCH/debian/changelog --silent | head -n 1) # yunohost (4.2) unstable; urgency=low CURRENT_VERSION=$(echo $LAST_CHANGELOG_ENTRY | cut -d' ' -f3 | tr -d ';') # stable, testing, unstable From a323ed0643a5e33450d52399f8935a59c4fc83cf Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 11:44:19 +0200 Subject: [PATCH 17/25] add git and hub to the preinstall deps --- utils.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils.sh b/utils.sh index b38549f..90e602e 100755 --- a/utils.sh +++ b/utils.sh @@ -130,12 +130,13 @@ get_dependencies() # Same as above, except that all dependencies are in the same line SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$branch/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") BUILD_DEPENDENCIES="git-buildpackage postfix python3-setuptools python3-pip devscripts" + TESTS_DEPENDENCIES="git hub" PIP3_PKG='mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 types-ipaddress types-enum34 types-cryptography types-toml types-requests types-PyYAML types-pyOpenSSL types-mock "packaging<22"' if [[ "$debian_version" == "bookworm" ]] then - # We add php8.2-cli and mariadb-client to the dependencies for test_app_resources - YUNOHOST_DEPENDENCIES="$YUNOHOST_DEPENDENCIES php8.2-cli mariadb-client mariadb-server" + # We add php8.2-cli, mariadb-client and mariadb-server to the dependencies for test_app_resources + TESTS_DEPENDENCIES="$TESTS_DEPENDENCIES php8.2-cli mariadb-client mariadb-server" PIP3_PKG="$PIP3_PKG --break-system-packages" fi } @@ -187,7 +188,7 @@ rebuild_base_containers() get_dependencies $debian_version # Pre install dependencies - lxc exec "$image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES $TESTS_DEPENDENCIES" lxc exec "$image_to_rebuild" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" # Disable apt-daily @@ -245,7 +246,7 @@ update_container() { get_dependencies $debian_version - lxc exec "$image_to_update" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$image_to_update" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES $TESTS_DEPENDENCIES" lxc exec "$image_to_update" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" create_snapshot "$image_to_update" "$ynh_version" "$snapshot" From a89298382ea6416f6321abf1c643039309742a39 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 11:44:32 +0200 Subject: [PATCH 18/25] remove old containers after an auto upgrade --- auto_upgrade_container.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index 461a047..f62f070 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -12,6 +12,8 @@ do update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done done + # Remove old runner containers + lxc delete -f $(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) done for debian_version in "bookworm" @@ -22,6 +24,7 @@ do do update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done - lxc delete -f $(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) done + # Remove old runner containers + lxc delete -f $(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) done From eb74089b8991edf93253468c573810b0a691373b Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 11:46:28 +0200 Subject: [PATCH 19/25] add snap/bin to the $PATH for cron task --- ci_cron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_cron b/ci_cron index 91edf88..6d2b5d6 100644 --- a/ci_cron +++ b/ci_cron @@ -1,2 +1,2 @@ # Upgrade all continers -30 2 * * * root /bin/bash -c "/opt/yunohost-ci/auto_upgrade_container.sh >> /opt/yunohost-ci/upgrade.log 2>>/opt/yunohost-ci/upgrade-error.log" +30 2 * * * root PATH=/snap/bin/:$PATH /bin/bash -c "/opt/yunohost-ci/auto_upgrade_container.sh >> /opt/yunohost-ci/upgrade.log 2>>/opt/yunohost-ci/upgrade-error.log" \ No newline at end of file From 43bc8390f706ffc128791fecda10e3c198cff26a Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 11:51:30 +0200 Subject: [PATCH 20/25] add some logs --- auto_upgrade_container.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index f62f070..3418c11 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -9,6 +9,7 @@ do do for snapshot in "before-install" "after-install" do + info "Updating container $PREFIX_IMAGE_NAME-$debian_version $ynh_version $snapshot" update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done done @@ -22,6 +23,7 @@ do do for snapshot in "before-install" "after-install" do + info "Updating container $PREFIX_IMAGE_NAME-$debian_version $ynh_version $snapshot" update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done done From e2bf59539b1bd9b42ef738830b76d5d516faad36 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 12:11:41 +0200 Subject: [PATCH 21/25] auto_upgrade_container remove old containers --- auto_upgrade_container.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index 3418c11..ce4fd87 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -13,8 +13,12 @@ do update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done done - # Remove old runner containers - lxc delete -f $(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) + containers_to_remove=$(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) + if [ -n "$containers_to_remove" ] + then + # Remove old runner containers + lxc delete -f $(echo $containers_to_remove) + fi done for debian_version in "bookworm" @@ -27,6 +31,10 @@ do update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" done done - # Remove old runner containers - lxc delete -f $(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) + containers_to_remove=$(lxc list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) + if [ -n "$containers_to_remove" ] + then + # Remove old runner containers + lxc delete -f $(echo $containers_to_remove) + fi done From 33a8934c0dc04bc7bf29feff2278ee3f751d3595 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 12:42:37 +0200 Subject: [PATCH 22/25] remove mac address each time we create a new snapshot --- utils.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils.sh b/utils.sh index 90e602e..298e26e 100755 --- a/utils.sh +++ b/utils.sh @@ -98,6 +98,9 @@ create_snapshot() local ynh_version=$2 local snapshot=$3 + # Unset the mac address to ensure the copy will get a new one and will be able to get new IP + lxc config unset "$image_to_rebuild" volatile.eth0.hwaddr 2> /dev/null + # Create snapshot lxc snapshot "$instance_to_publish" "$ynh_version-$snapshot" --reuse } @@ -206,9 +209,6 @@ rebuild_base_containers() chmod 777 $current_dir/cache lxc config device add "$image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache" - # Unset the mac address to ensure the copy will get a new one and will be able to get new IP - lxc config unset "$image_to_rebuild" volatile.eth0.hwaddr - create_snapshot "$image_to_rebuild" "$ynh_version" "before-install" # Install YunoHost From fb03d3edea984eda49f0ea1715b8fb5ef6ffc980 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 12:57:17 +0200 Subject: [PATCH 23/25] force remove mac address after restoring snapshot --- prepare.sh | 3 +++ utils.sh | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/prepare.sh b/prepare.sh index 42e2525..8611366 100755 --- a/prepare.sh +++ b/prepare.sh @@ -30,6 +30,9 @@ start_container () { restore_snapshot "$CONTAINER_IMAGE" "$CURRENT_VERSION" "$SNAPSHOT_NAME" + # Unset the mac address to ensure the copy will get a new one and will be able to get new IP + lxc config unset "$CONTAINER_IMAGE" volatile.eth0.hwaddr 2> /dev/null + lxc start $CONTAINER_IMAGE wait_container $CONTAINER_IMAGE diff --git a/utils.sh b/utils.sh index 298e26e..0d7f55a 100755 --- a/utils.sh +++ b/utils.sh @@ -98,9 +98,6 @@ create_snapshot() local ynh_version=$2 local snapshot=$3 - # Unset the mac address to ensure the copy will get a new one and will be able to get new IP - lxc config unset "$image_to_rebuild" volatile.eth0.hwaddr 2> /dev/null - # Create snapshot lxc snapshot "$instance_to_publish" "$ynh_version-$snapshot" --reuse } From 1abb4d602d3aed01197c83d14be75992d81dce69 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 12:59:19 +0200 Subject: [PATCH 24/25] simplify the detection of the presence of the BASE IMAGE --- prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare.sh b/prepare.sh index 8611366..39761df 100755 --- a/prepare.sh +++ b/prepare.sh @@ -12,7 +12,7 @@ trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR start_container () { if ! lxc info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then warn 'Container not found, copying it from the prebuilt image' - if ! lxc info "$BASE_IMAGE" &>/dev/null || ! lxc info "$BASE_IMAGE" | grep -q "$CURRENT_VERSION-$SNAPSHOT_NAME" + if ! lxc info "$BASE_IMAGE" | grep -q "$CURRENT_VERSION-$SNAPSHOT_NAME" then error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" # Inform GitLab Runner that this is a system failure, so it From 84b27c25ed139c95f955f7817168bd63f847d148 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 13:54:54 +0200 Subject: [PATCH 25/25] match the whole word --- prepare.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prepare.sh b/prepare.sh index 39761df..b2f3ee3 100755 --- a/prepare.sh +++ b/prepare.sh @@ -12,7 +12,7 @@ trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR start_container () { if ! lxc info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then warn 'Container not found, copying it from the prebuilt image' - if ! lxc info "$BASE_IMAGE" | grep -q "$CURRENT_VERSION-$SNAPSHOT_NAME" + if ! lxc info "$BASE_IMAGE" | grep -w "$CURRENT_VERSION-$SNAPSHOT_NAME" >/dev/null then error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" # Inform GitLab Runner that this is a system failure, so it @@ -23,8 +23,8 @@ start_container () { fi # Stop the container if it's running if [ "$(lxc info $CONTAINER_IMAGE | grep Status | awk '{print tolower($2)}')" == "running" ]; then - lxc stop $CONTAINER_IMAGE - fi + lxc stop $CONTAINER_IMAGE + fi info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE, Snapshot: $SNAPSHOT_NAME"