From a7eb63ff4ae5f0b432d77264a5391037b849452d Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 26 Mar 2020 17:50:12 +0100 Subject: [PATCH] clean and add helpers --- README.md | 2 +- auto_upgrade_container.sh | 15 +++++ base.sh | 5 +- ci_cron | 2 + cleanup.sh | 4 +- prepare.sh | 124 ++++---------------------------------- rebuild_all.sh | 12 ++++ run.sh | 4 +- utils.sh | 119 ++++++++++++++++++++++++++++++++++++ 9 files changed, 170 insertions(+), 117 deletions(-) create mode 100755 auto_upgrade_container.sh create mode 100644 ci_cron create mode 100755 rebuild_all.sh create mode 100755 utils.sh diff --git a/README.md b/README.md index 5fd704d..a9fadc4 100644 --- a/README.md +++ b/README.md @@ -80,5 +80,5 @@ Use the field `image` to switch between `before-install`, `before-postinstall` o ## TODO - Support more YunoHost Core projects (for now only `yunohost` is supported, not `moulinette`...) -- Be sure that the runner can run several jobs in parallel (The `rebuild_base_container` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?). +- Be sure that the runner can run several jobs in parallel (The `rebuild_base_containers` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?). - Git pull this repo before running tests to keep these files up-to-date. diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh new file mode 100755 index 0000000..2505aa6 --- /dev/null +++ b/auto_upgrade_container.sh @@ -0,0 +1,15 @@ +current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $current_dir/utils.sh # Get utils functions. + +for debian_version in "stretch" "buster" +do + for ynh_version in "stable" "testing" "unstable" + do + for snapshot in "before-install" "before-postinstall" "after-postinstall" + do + local image="yunohost-$debian_version-$ynh_version-$snapshot" + + update_image $image + done + done +done \ No newline at end of file diff --git a/base.sh b/base.sh index 4039b7c..98ff9b0 100755 --- a/base.sh +++ b/base.sh @@ -30,4 +30,7 @@ then SNAPSHOT_NAME="after-postinstall" fi PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR" -PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" \ No newline at end of file +PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" + +# For example yunohost-stretch-unstable +BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION" \ No newline at end of file diff --git a/ci_cron b/ci_cron new file mode 100644 index 0000000..462f324 --- /dev/null +++ b/ci_cron @@ -0,0 +1,2 @@ +# Upgrade all continers +30 2 * * * root "/opt/yunohost-ciyunohost/auto_upgrade_container.sh" \ No newline at end of file diff --git a/cleanup.sh b/cleanup.sh index 4dd70c5..0eaf410 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -currentDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source ${currentDir}/base.sh # Get variables from base. +current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $current_dir/base.sh # Get variables from base. echo "Deleting container $CONTAINER_ID" diff --git a/prepare.sh b/prepare.sh index 3986726..45219b9 100755 --- a/prepare.sh +++ b/prepare.sh @@ -1,115 +1,14 @@ #!/usr/bin/env bash -currentDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source ${currentDir}/base.sh # Get variables from base. +current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $current_dir/base.sh # Get variables from base. +source $current_dir/utils.sh # Get utils functions. set -eo pipefail # trap any error, and mark it as a system failure. trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR -clean_containers() -{ - for image_to_delete in "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"{,"-tmp"} - do - if lxc info $image_to_delete &>/dev/null - then - lxc delete $image_to_delete --force - fi - done - - for image_to_delete in "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-"{"before-install","before-postinstall","after-postinstall"} - do - if lxc image info $image_to_delete &>/dev/null - then - lxc image delete $image_to_delete - fi - done -} - -wait_container() -{ - # Wait for container to start, we are using systemd to check this, - # for the sake of brevity. - for i in $(seq 1 10); do - if lxc exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then - break - fi - - if [ "$i" == "10" ]; then - echo 'Waited for 10 seconds to start container, exiting..' - # Inform GitLab Runner that this is a system failure, so it - # should be retried. - exit "$SYSTEM_FAILURE_EXIT_CODE" - fi - - sleep 1s - done -} - -rebuild_base_container() -{ - clean_containers - - lxc launch images:debian/$DEBIAN_VERSION/$ARCH "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - - wait_container "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get update" - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get install curl -y" - # Install Git LFS, git comes pre installed with ubuntu image. - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get install git-lfs -y" - # Install gitlab-runner binary since we need for cache/artifacts. - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash" - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y" - lxc stop "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - - # Create image before install - lxc publish "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" --alias "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-before-install" - lxc start "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - wait_container "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - - # Install yunohost - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "curl https://install.yunohost.org | bash -s -- -a -d $CURRENT_VERSION" - lxc stop "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - - # Create image before postinstall - lxc publish "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" --alias "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-before-postinstall" - lxc start "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - wait_container "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - - # Running post Install - lxc exec "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns" - lxc stop "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" - - # Create image after postinstall - lxc publish "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" --alias "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-after-postinstall" - - lxc delete "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-tmp" -} - -update_image() { - image_to_update=$1 - - # Start and run upgrade - lxc launch "$image_to_update" "$image_to_update-tmp" - - wait_container "$image_to_update-tmp" - - lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y" - lxc stop "$image_to_update-tmp" - - # Remove old image - lxc image delete "$image_to_update" - - # Add new image updated - lxc publish "$image_to_update-tmp" --alias "$image_to_update" - - lxc delete "$image_to_update-tmp" -} - start_container () { set -x @@ -118,18 +17,21 @@ start_container () { lxc delete -f "$CONTAINER_ID" fi - if ! lxc image info "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-$SNAPSHOT_NAME" &>/dev/null + if ! lxc image info "$BASE_IMAGE-$SNAPSHOT_NAME" &>/dev/null then - rebuild_base_container + echo "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" + # Inform GitLab Runner that this is a system failure, so it + # should be retried. + exit $SYSTEM_FAILURE_EXIT_CODE fi - update_image "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-$SNAPSHOT_NAME" + update_image "$BASE_IMAGE-$SNAPSHOT_NAME" - lxc launch "yunohost-$DEBIAN_VERSION-$CURRENT_VERSION-$SNAPSHOT_NAME" "$CONTAINER_ID" 2>/dev/null + lxc launch "$BASE_IMAGE-$SNAPSHOT_NAME" "$CONTAINER_ID" 2>/dev/null - mkdir -p ${currentDir}/cache - chmod 777 ${currentDir}/cache - lxc config device add "$CONTAINER_ID" cache-folder disk path=/cache source="${currentDir}/cache" + mkdir -p $current_dir/cache + chmod 777 $current_dir/cache + lxc config device add "$CONTAINER_ID" cache-folder disk path=/cache source="$current_dir/cache" set +x diff --git a/rebuild_all.sh b/rebuild_all.sh new file mode 100755 index 0000000..8d75d63 --- /dev/null +++ b/rebuild_all.sh @@ -0,0 +1,12 @@ +current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $current_dir/utils.sh # Get utils functions. + +for debian_version in "stretch" "buster" +do + for ynh_version in "stable" "testing" "unstable" + do + local base_image="yunohost-$debian_version-$ynh_version" + + rebuild_base_containers $BASE_IMAGE + done +done \ No newline at end of file diff --git a/run.sh b/run.sh index aa3cc66..3284d5a 100755 --- a/run.sh +++ b/run.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -currentDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source ${currentDir}/base.sh # Get variables from base. +current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $current_dir/base.sh # Get variables from base. create_sym_link() { local DEST=$1 diff --git a/utils.sh b/utils.sh new file mode 100755 index 0000000..8c3c9b3 --- /dev/null +++ b/utils.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $current_dir/base.sh # Get variables from base. + +clean_containers() +{ + local base_image_to_clean=$1 + + for image_to_delete in "$base_image_to_clean"{,"-tmp"} + do + if lxc info $image_to_delete &>/dev/null + then + lxc delete $base_image_to_clean --force + fi + done + + for image_to_delete in "$base_image_to_clean-"{"before-install","before-postinstall","after-postinstall"} + do + if lxc image info $image_to_delete &>/dev/null + then + lxc image delete $image_to_delete + fi + done +} + +wait_container() +{ + # Wait for container to start, we are using systemd to check this, + # for the sake of brevity. + for i in $(seq 1 10); do + if lxc exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then + break + fi + + if [ "$i" == "10" ]; then + echo 'Waited for 10 seconds to start container, exiting..' + # Inform GitLab Runner that this is a system failure, so it + # should be retried. + exit "$SYSTEM_FAILURE_EXIT_CODE" + fi + + sleep 1s + done +} + +rebuild_base_containers() +{ + local base_image_to_rebuild=$1 + clean_containers $base_image_to_rebuild + + lxc launch images:debian/$DEBIAN_VERSION/$ARCH "$base_image_to_rebuild-tmp" + + wait_container "$base_image_to_rebuild-tmp" + + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install curl -y" + # Install Git LFS, git comes pre installed with ubuntu image. + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install git-lfs -y" + # Install gitlab-runner binary since we need for cache/artifacts. + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y" + lxc stop "$base_image_to_rebuild-tmp" + + # Create image before install + lxc publish "$base_image_to_rebuild-tmp" --alias "$base_image_to_rebuild-before-install" + lxc start "$base_image_to_rebuild-tmp" + + wait_container "$base_image_to_rebuild-tmp" + + # Install yunohost + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl https://install.yunohost.org | bash -s -- -a -d $CURRENT_VERSION" + lxc stop "$base_image_to_rebuild-tmp" + + # Create image before postinstall + lxc publish "$base_image_to_rebuild-tmp" --alias "$base_image_to_rebuild-before-postinstall" + lxc start "$base_image_to_rebuild-tmp" + + wait_container "$base_image_to_rebuild-tmp" + + # Running post Install + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns" + lxc stop "$base_image_to_rebuild-tmp" + + # Create image after postinstall + lxc publish "$base_image_to_rebuild-tmp" --alias "$base_image_to_rebuild-after-postinstall" + + lxc delete "$base_image_to_rebuild-tmp" +} + +update_image() { + local image_to_update=$1 + + if ! lxc image info "$image_to_update" &>/dev/null + then + echo "Unable to upgrade image $image_to_update" + return + fi + + local finger_print_to_delete=$(lxc image info "$image_to_update" | grep Fingerprint | cut -d' ' -f2) + + # Start and run upgrade + lxc launch "$image_to_update" "$image_to_update-tmp" + + wait_container "$image_to_update-tmp" + + lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update" + lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y" + lxc stop "$image_to_update-tmp" + + # Add new image updated + lxc publish "$image_to_update-tmp" --alias "$image_to_update" + + # Remove old image + lxc image delete "$finger_print_to_delete" + + lxc delete "$image_to_update-tmp" +} \ No newline at end of file