From bc9eec167d4e70100fc7c36ecdc15d7cd0ecf8cf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 7 Aug 2024 23:31:32 +0200 Subject: [PATCH] Part1 of epic refactoring to share image between app CI, core CI, ynh-dev --- auto_upgrade_container.sh | 40 ------ ci_cron | 2 - cleanup.sh | 8 +- common.sh | 39 ++++++ prepare.sh | 140 +++++++++++++++---- prints.sh | 37 ----- rebuild_all.sh | 41 ------ run.sh | 46 ++---- utils.sh | 284 -------------------------------------- variables.sh | 39 ------ 10 files changed, 168 insertions(+), 508 deletions(-) delete mode 100755 auto_upgrade_container.sh delete mode 100644 ci_cron create mode 100755 common.sh delete mode 100755 prints.sh delete mode 100755 rebuild_all.sh delete mode 100755 utils.sh delete mode 100755 variables.sh diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh deleted file mode 100755 index b046154..0000000 --- a/auto_upgrade_container.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $current_dir/utils.sh # Get utils functions. - -for debian_version in "bullseye" -do - for ynh_version in "stable" "testing" "unstable" - do - for snapshot in "before-install" "after-install" - do - info "Updating container $PREFIX_IMAGE_NAME-$debian_version $ynh_version $snapshot" - update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" - done - done - containers_to_remove=$(incus list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) - if [ -n "$containers_to_remove" ] - then - # Remove old runner containers - incus delete -f $(echo $containers_to_remove) - fi -done - -for debian_version in "bookworm" -do - for ynh_version in "unstable" - do - for snapshot in "before-install" "after-install" - do - info "Updating container $PREFIX_IMAGE_NAME-$debian_version $ynh_version $snapshot" - update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" - done - done - containers_to_remove=$(incus list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv) - if [ -n "$containers_to_remove" ] - then - # Remove old runner containers - incus delete -f $(echo $containers_to_remove) - fi -done diff --git a/ci_cron b/ci_cron deleted file mode 100644 index b799cb0..0000000 --- a/ci_cron +++ /dev/null @@ -1,2 +0,0 @@ -# Upgrade all continers -30 2 * * * root /bin/bash -c "/opt/yunohost-ci/auto_upgrade_container.sh >> /opt/yunohost-ci/upgrade.log 2>>/opt/yunohost-ci/upgrade-error.log" \ No newline at end of file diff --git a/cleanup.sh b/cleanup.sh index b39a8b4..2e229e9 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $current_dir/prints.sh -source $current_dir/variables.sh # Get variables from variables. +source $current_dir/common.sh -info "Stopping container $CONTAINER_IMAGE" +info "Stopping container $CONTAINER_NAME" -incus stop "$CONTAINER_IMAGE" \ No newline at end of file +incus stop "$CONTAINER_NAME" +incus delete "$CONTAINER_NAME" diff --git a/common.sh b/common.sh new file mode 100755 index 0000000..c42dd5e --- /dev/null +++ b/common.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +NORMAL=$(printf '\033[0m') +BOLD=$(printf '\033[1m') +RED=$(printf '\033[31m') +GREEN=$(printf '\033[32m') +ORANGE=$(printf '\033[33m') +BLUE=$(printf '\033[34m') + +function success() { echo "[${BOLD}${GREEN} OK ${NORMAL}] ${1}"; } +function info() { echo "[${BOLD}${BLUE}INFO${NORMAL}] ${1}"; } +function warn() { echo "[${BOLD}${ORANGE}WARN${NORMAL}] ${1}" 2>&1; } +function error() { echo "[${BOLD}${RED}FAIL${NORMAL}] ${1}" 2>&1; } + +# All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_ + +DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH" +CURRENT_BRANCH="${CUSTOM_ENV_CI_COMMIT_REF_NAME:-$DEFAULT_BRANCH}" # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR + +IMAGE="${CUSTOM_ENV_CI_JOB_IMAGE:-after-install}" + +LAST_CHANGELOG_ENTRY=$(curl $CUSTOM_ENV_CI_PROJECT_URL/-/raw/$CURRENT_BRANCH/debian/changelog --silent | head -n 1) # yunohost (4.2) unstable; urgency=low +DEBIAN_VERSION_NUMBER=$(echo $LAST_CHANGELOG_ENTRY | cut -d' ' -f2 | tr -d '(' | tr -d ')' | cut -d'.' -f1) # 11, 12 +if [[ "$DEBIAN_VERSION_NUMBER" == "11" ]] +then + BASE_IMAGE="ynh-$IMAGE-bullseye-amd64-stable-base" +elif [[ "$DEBIAN_VERSION_NUMBER" == "12" ]] +then + BASE_IMAGE="ynh-$IMAGE-bookworm-amd64-testing-base" +elif [[ "$DEBIAN_VERSION_NUMBER" == "13" ]] +then + # Upcoming somedayâ„¢ + BASE_IMAGE="ynh-$IMAGE-trixie-amd64-unstable-base" +else + echo "Uhoh, unknown debian version $DEBIAN_VERSION_NUMBER" + exit 1 +fi + +CONTAINER_NAME="job-$CUSTOM_ENV_CI_JOB_ID-$CUSTOM_ENV_CI_JOB_NAME_SLUG" diff --git a/prepare.sh b/prepare.sh index 8f88c15..11426b4 100755 --- a/prepare.sh +++ b/prepare.sh @@ -1,45 +1,129 @@ #!/usr/bin/env bash current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $current_dir/prints.sh -source $current_dir/utils.sh # Get utils functions. +source $current_dir/common.sh set -eo pipefail +wait_container() +{ + restart_container() + { + incus stop "$1" --timeout 30 || incus stop "$1" --force + incus start "$1" + } + + # Try to start the container 3 times. + local max_try=3 + local i=0 + while [ $i -lt $max_try ] + do + i=$(( i +1 )) + local failstart=0 + + # Wait for container to start, we are using systemd to check this, + # for the sake of brevity. + for j in $(seq 1 10); do + if incus exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then + break + fi + + if [ "$j" == "10" ]; then + error 'Failed to start the container' + failstart=1 + + restart_container "$1" + fi + + sleep 1s + done + + # Wait for container to access the internet + for j in $(seq 1 10); do + if incus exec "$1" -- /bin/bash -c "! which wget > /dev/null 2>&1 || wget -q --spider http://github.com"; then + break + fi + + if [ "$j" == "10" ]; then + error 'Failed to access the internet' + failstart=1 + incus exec "$1" -- /bin/bash -c "echo 'resolv-file=/etc/resolv.dnsmasq.conf' > /etc/dnsmasq.d/resolvconf" + incus exec "$1" -- /bin/bash -c "echo 'nameserver 8.8.8.8' > /etc/resolv.dnsmasq.conf" + incus exec "$1" -- /bin/bash -c "sed -i 's/#IGNORE/IGNORE/g' /etc/default/dnsmasq" + incus exec "$1" -- /bin/bash -c "systemctl restart dnsmasq" + incus exec "$1" -- /bin/bash -c "journalctl -u dnsmasq -n 100 --no-pager" + + restart_container "$1" + fi + + sleep 1s + done + + # Wait dpkg + for j in $(seq 1 10); do + if ! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock > /dev/null 2>&1" && + ! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock-frontend > /dev/null 2>&1" && + ! incus exec "$1" -- /bin/bash -c "fuser /var/cache/apt/archives/lock > /dev/null 2>&1"; then + break + fi + + if [ "$j" == "10" ]; then + error 'Waiting too long for lock release' + failstart=1 + + restart_container "$1" + fi + + sleep 1s + done + + # Has started and has access to the internet + if [ $failstart -eq 0 ] + then + break + fi + + # Fail if the container failed to start + if [ $i -eq $max_try ] && [ $failstart -eq 1 ] + then + # Inform GitLab Runner that this is a system failure, so it + # should be retried. + exit "$SYSTEM_FAILURE_EXIT_CODE" + fi + done +} + # trap any error, and mark it as a system failure. trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR -start_container () { - if ! incus info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then - warn 'Container not found, copying it from the prebuilt image' - if ! incus info "$BASE_IMAGE" | grep -w "$CURRENT_VERSION-$SNAPSHOT_NAME" >/dev/null - then - error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh" - # Inform GitLab Runner that this is a system failure, so it - # should be retried. - exit $SYSTEM_FAILURE_EXIT_CODE - fi - incus copy "$BASE_IMAGE" "$CONTAINER_IMAGE" - fi - # Stop the container if it's running - if [ "$(incus info $CONTAINER_IMAGE | grep Status | awk '{print tolower($2)}')" == "running" ]; then - incus stop $CONTAINER_IMAGE - fi +############################################################################### - info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE, Snapshot: $SNAPSHOT_NAME" +info "Starting $CONTAINER_NAME from $BASE_IMAGE ..." - restore_snapshot "$CONTAINER_IMAGE" "$CURRENT_VERSION" "$SNAPSHOT_NAME" +if ! incus info "$CONTAINER_NAME" >/dev/null 2>/dev/null ; then - # Unset the mac address to ensure the copy will get a new one and will be able to get new IP - incus config unset "$CONTAINER_IMAGE" volatile.eth0.hwaddr 2> /dev/null + # Force the usage of the fingerprint because otherwise for some reason lxd won't use the newer version + # available even though it's aware it exists -_- + BASE_HASH="$(incus image list yunohost:$BASE_IMAGE --format json | jq -r '.[].fingerprint')" - incus start $CONTAINER_IMAGE + # NB: this image comes from the 'common' image repository shared with the appci, ynh-dev etc + incus launch yunohost:$BASE_HASH $CONTAINER_NAME +fi - wait_container $CONTAINER_IMAGE -} +# Stop the container if it's running +if [ "$(incus info $CONTAINER_NAME | grep Status | awk '{print tolower($2)}')" == "running" ]; then + incus stop "$CONTAINER_NAME" --timeout 30 || incus stop "$CONTAINER_NAME" --force +fi -info "Starting $CONTAINER_IMAGE" +# Unset the mac address to ensure the copy will get a new one and will be able to get new IP +incus start $CONTAINER_NAME -start_container +sleep 2 -info "$CONTAINER_IMAGE started properly" +incus exec $CONTAINER_NAME dhclient eth0 + +info "Waiting for $CONTAINER_NAME to finish booting and making sure it's connected to the internetz..." + +wait_container $CONTAINER_NAME + +success "$CONTAINER_NAME started properly" diff --git a/prints.sh b/prints.sh deleted file mode 100755 index 240ebef..0000000 --- a/prints.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -NORMAL=$(printf '\033[0m') -BOLD=$(printf '\033[1m') -faint=$(printf '\033[2m') -UNDERLINE=$(printf '\033[4m') -NEGATIVE=$(printf '\033[7m') -RED=$(printf '\033[31m') -GREEN=$(printf '\033[32m') -ORANGE=$(printf '\033[33m') -BLUE=$(printf '\033[34m') -YELLOW=$(printf '\033[93m') -WHITE=$(printf '\033[39m') - -function success() -{ - local msg=${1} - echo "[${BOLD}${GREEN} OK ${NORMAL}] ${msg}" -} - -function info() -{ - local msg=${1} - echo "[${BOLD}${BLUE}INFO${NORMAL}] ${msg}" -} - -function warn() -{ - local msg=${1} - echo "[${BOLD}${ORANGE}WARN${NORMAL}] ${msg}" 2>&1 -} - -function error() -{ - local msg=${1} - echo "[${BOLD}${RED}FAIL${NORMAL}] ${msg}" 2>&1 -} \ No newline at end of file diff --git a/rebuild_all.sh b/rebuild_all.sh deleted file mode 100755 index b72b72b..0000000 --- a/rebuild_all.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $current_dir/utils.sh # Get utils functions. - -for debian_version in "bullseye" -do - rebuild_base_containers "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "stable" "amd64" - - for ynh_version in "testing" "unstable" - do - for snapshot in "before-install" "after-install" - do - restore_snapshot "$PREFIX_IMAGE_NAME-$debian_version" "stable" "$snapshot" - - if [[ "$ynh_version" == "testing" ]] - then - repo_version="testing" - elif [[ "$DISTRIB" == "unstable" ]] - then - repo_version="testing unstable" - fi - - incus exec "$PREFIX_IMAGE_NAME-$debian_version" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`; - do - sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE - done" - - create_snapshot "$PREFIX_IMAGE_NAME-$debian_version" "$ynh_version" "$snapshot" - - update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot" - done - done -done - - -for debian_version in "bookworm" -do - rebuild_base_containers "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "unstable" "amd64" -done - diff --git a/run.sh b/run.sh index ab5c280..f2be910 100755 --- a/run.sh +++ b/run.sh @@ -1,42 +1,22 @@ #!/usr/bin/env bash current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $current_dir/variables.sh # Get variables from variables. +source $current_dir/common.sh -case ${2} in - prepare_script) - echo "CI OPEN: $CUSTOM_ENV_CI_OPEN_MERGE_REQUESTS / CI PIPELINE SOURCE: $CUSTOM_ENV_CI_PIPELINE_SOURCE / REF NAME: $CUSTOM_ENV_CI_COMMIT_REF_NAME / COMMIT: $CUSTOM_ENV_CI_COMMIT_BRANCH, TAG: $CUSTOM_ENV_CI_COMMIT_TAG" - ;; - get_sources) - ;; - restore_cache) - ;; - download_artifacts) - ;; - # build_script is now step_script - # but we can also have "step_release" or "step_accessibility" - # More info here: - # https://docs.gitlab.com/runner/executors/custom.html#run - # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26426 - step_*) - case $PROJECT_NAME in - yunohost) - # Nothing to do? - ;; - esac - ;; - after_script) - ;; - archive_cache) - ;; - upload_artifact_on_success) - ;; - upload_artifact_on_failure) - ;; -esac +# This script is called multiple times with different stage ($2) value +# This is documented in https://docs.gitlab.com/runner/executors/custom.html#run +JOB_STAGE=$2 +if [[ "$JOB_STAGE" == "prepare_script" ]] +then + echo "CI OPEN: $CUSTOM_ENV_CI_OPEN_MERGE_REQUESTS / CI PIPELINE SOURCE: $CUSTOM_ENV_CI_PIPELINE_SOURCE / REF NAME: $CUSTOM_ENV_CI_COMMIT_REF_NAME / COMMIT: $CUSTOM_ENV_CI_COMMIT_BRANCH, TAG: $CUSTOM_ENV_CI_COMMIT_TAG" +fi +############################################################################### -incus exec "$CONTAINER_IMAGE" /bin/bash < "${1}" +# This is the real 'important' logic bit +incus exec "$CONTAINER_NAME" /bin/bash < "${1}" + +############################################################################### err=$? if [ $err -ne 0 ]; then echo "Exit with error code $err" diff --git a/utils.sh b/utils.sh deleted file mode 100755 index 5732e5c..0000000 --- a/utils.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash - -current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $current_dir/prints.sh -source $current_dir/variables.sh # Get variables from variables. - -wait_container() -{ - restart_container() - { - incus stop "$1" - incus start "$1" - } - - # Try to start the container 3 times. - local max_try=3 - local i=0 - while [ $i -lt $max_try ] - do - i=$(( i +1 )) - local failstart=0 - - # Wait for container to start, we are using systemd to check this, - # for the sake of brevity. - for j in $(seq 1 10); do - if incus exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then - break - fi - - if [ "$j" == "10" ]; then - error 'Failed to start the container' - failstart=1 - - restart_container "$1" - fi - - sleep 1s - done - - # Wait for container to access the internet - for j in $(seq 1 10); do - if incus exec "$1" -- /bin/bash -c "! which wget > /dev/null 2>&1 || wget -q --spider http://github.com"; then - break - fi - - if [ "$j" == "10" ]; then - error 'Failed to access the internet' - failstart=1 - incus exec "$1" -- /bin/bash -c "echo 'resolv-file=/etc/resolv.dnsmasq.conf' > /etc/dnsmasq.d/resolvconf" - incus exec "$1" -- /bin/bash -c "echo 'nameserver 8.8.8.8' > /etc/resolv.dnsmasq.conf" - incus exec "$1" -- /bin/bash -c "sed -i 's/#IGNORE/IGNORE/g' /etc/default/dnsmasq" - incus exec "$1" -- /bin/bash -c "systemctl restart dnsmasq" - incus exec "$1" -- /bin/bash -c "journalctl -u dnsmasq -n 100 --no-pager" - - restart_container "$1" - fi - - sleep 1s - done - - # Wait dpkg - for j in $(seq 1 10); do - if ! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock > /dev/null 2>&1" && - ! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock-frontend > /dev/null 2>&1" && - ! incus exec "$1" -- /bin/bash -c "fuser /var/cache/apt/archives/lock > /dev/null 2>&1"; then - break - fi - - if [ "$j" == "10" ]; then - error 'Waiting too long for lock release' - failstart=1 - - restart_container "$1" - fi - - sleep 1s - done - - # Has started and has access to the internet - if [ $failstart -eq 0 ] - then - break - fi - - # Fail if the container failed to start - if [ $i -eq $max_try ] && [ $failstart -eq 1 ] - then - # Inform GitLab Runner that this is a system failure, so it - # should be retried. - exit "$SYSTEM_FAILURE_EXIT_CODE" - fi - done -} - -create_snapshot() -{ - local instance_to_publish=$1 - local ynh_version=$2 - local snapshot=$3 - - # Create snapshot - incus snapshot create "$instance_to_publish" "$ynh_version-$snapshot" --reuse -} - -restore_snapshot() -{ - local incus_name=$1 - local ynh_version=$2 - local snapshot=$3 - local retry_incus=0 - - while [[ ${retry_incus} -lt 10 ]] - do - incus snapshot restore "$incus_name" "$ynh_version-$snapshot" && break || retry_incus=$(($retry_incus+1)) - info "Failed to restore snapshot? Retrying in 10 sec ..." - if [[ ${retry_incus} -ge 3 ]] - then - warn "If this keeps happening, restarting the LXD daemon might help :| ..." - fi - sleep 10 - done - - if [[ ${retry_incus} -ge 10 ]] - then - error "Failed to properly restore the snapshot zrgmblg" - return 1 - fi -} - -# These lines are used to extract the dependencies/recommendations from the debian/control file. -# /!\ There's a high risk of lamentable failure if we change the format of this file -get_dependencies() -{ - local debian_version=$1 - if [[ "$debian_version" == "bullseye" ]] - then - local branch="dev" - else - local branch="$debian_version" - fi - - # To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'. - YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ") - YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") - MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") - # Same as above, except that all dependencies are in the same line - SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$branch/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") - BUILD_DEPENDENCIES="git-buildpackage postfix python3-setuptools python3-pip devscripts" - TESTS_DEPENDENCIES="git hub" - PIP3_PKG='mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 types-ipaddress types-enum34 types-cryptography types-toml types-requests types-PyYAML types-pyOpenSSL types-mock "packaging<22"' - - if [[ "$debian_version" == "bookworm" ]] - then - # We add php8.2-cli, mariadb-client and mariadb-server to the dependencies for test_app_resources - TESTS_DEPENDENCIES="$TESTS_DEPENDENCIES php8.2-cli mariadb-client mariadb-server" - PIP3_PKG="$PIP3_PKG --break-system-packages" - fi -} - -rebuild_base_containers() -{ - local image_to_rebuild=$1 - local debian_version=$2 - local ynh_version=$3 - local arch=$4 - - if incus info "$image_to_rebuild" &>/dev/null - then - incus delete -f "$image_to_rebuild" - fi - - incus launch images:debian/$debian_version/$arch "$image_to_rebuild" -c security.nesting=true - - wait_container "$image_to_rebuild" - - incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get update" - incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes wget curl" - # Install Git LFS, git comes pre installed with ubuntu image. - # Disable this line because we don't need to add a new repo to have git-lfs - #incus exec "$image_to_rebuild" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes git-lfs" - # Install gitlab-runner binary since we need for cache/artifacts. - if [[ $debian_version == "bullseye" ]] - then - incus exec "$image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" - incus exec "$image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb" - else - incus exec "$image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash" - incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" - fi - - INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/main/$debian_version" - - # Download the YunoHost install script - incus exec "$image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" - - # Patch the YunoHost install script - incus exec "$image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" - incus exec "$image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/echo skip restart service #\1/' install.sh" - - # Run the YunoHost install script patched - incus exec "$image_to_rebuild" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" - - get_dependencies $debian_version - - # Pre install dependencies - incus exec "$image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES $TESTS_DEPENDENCIES" - incus exec "$image_to_rebuild" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" - - # Disable apt-daily - incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.timer --now" - incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer --now" - incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service --now" - incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service --now" - - - mkdir -p $current_dir/cache - chmod 777 $current_dir/cache - incus config device add "$image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache" - - create_snapshot "$image_to_rebuild" "$ynh_version" "before-install" - echo "Created snapshot base 'before-install' for $image_to_rebuild" - - # Install YunoHost - incus exec "$image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" - - # Run postinstall - incus exec "$image_to_rebuild" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace" - - # Disable services which are not really mandatory, to reduce the incus memory footprint and hopefully speed things up a bit - - if [[ "$debian_version" == "bookworm" ]] - then - incus exec "$image_to_rebuild" -- systemctl -q disable metronome --now - incus exec "$image_to_rebuild" -- systemctl -q disable rspamd --now - incus exec "$image_to_rebuild" -- systemctl -q disable postsrsd --now - incus exec "$image_to_rebuild" -- systemctl -q disable yunohost-api --now - incus exec "$image_to_rebuild" -- systemctl -q disable fake-hwclock.service --now - incus exec "$image_to_rebuild" -- systemctl -q disable yunoprompt --now - incus exec "$image_to_rebuild" -- systemctl -q disable haveged.service --now - incus exec "$image_to_rebuild" -- systemctl -q disable unattended-upgrades.service --now - incus exec "$image_to_rebuild" -- systemctl -q disable e2scrub_all.timer --now - incus exec "$image_to_rebuild" -- systemctl -q disable logrotate.timer --now - incus exec "$image_to_rebuild" -- systemctl -q disable phpsessionclean.timer --now - incus exec "$image_to_rebuild" -- systemctl -q disable systemd-tmpfiles-clean.timer --now - fi - - create_snapshot "$image_to_rebuild" "$ynh_version" "after-install" - echo "Created snapshot base 'after-install' for $image_to_rebuild" - - incus stop "$image_to_rebuild" -} - -update_container() { - local image_to_update=$1 - local debian_version=$2 - local ynh_version=$3 - local snapshot=$4 - - if ! incus info "$image_to_update" &>/dev/null - then - error "Unable to upgrade image $image_to_update" - return - fi - - # Start and run upgrade - restore_snapshot "$image_to_update" "$ynh_version" "$snapshot" - - incus start "$image_to_update" 2>&1 || true - - wait_container "$image_to_update" - - incus exec "$image_to_update" -- /bin/bash -c "apt-get update" - incus exec "$image_to_update" -- /bin/bash -c "apt-get upgrade --assume-yes" - - get_dependencies $debian_version - - incus exec "$image_to_update" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES $TESTS_DEPENDENCIES" - incus exec "$image_to_update" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG" - - create_snapshot "$image_to_update" "$ynh_version" "$snapshot" - - incus stop "$image_to_update" -} diff --git a/variables.sh b/variables.sh deleted file mode 100755 index 1befe66..0000000 --- a/variables.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $current_dir/prints.sh - -# All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_ - -ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64 -DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH" - -CURRENT_BRANCH="$CUSTOM_ENV_CI_COMMIT_REF_NAME" # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR -if [ -z "$CURRENT_BRANCH" ] -then - CURRENT_BRANCH="$DEFAULT_BRANCH" -fi - -LAST_CHANGELOG_ENTRY=$(curl $CUSTOM_ENV_CI_PROJECT_URL/-/raw/$CURRENT_BRANCH/debian/changelog --silent | head -n 1) # yunohost (4.2) unstable; urgency=low -CURRENT_VERSION=$(echo $LAST_CHANGELOG_ENTRY | cut -d' ' -f3 | tr -d ';') # stable, testing, unstable - -DEBIAN_VERSION_NUMBER=$(echo $LAST_CHANGELOG_ENTRY | cut -d' ' -f2 | tr -d '(' | tr -d ')' | cut -d'.' -f1) # 11, 12 - -declare -A DEBIAN_VERSION_TABLE -DEBIAN_VERSION_TABLE=(["11"]="bullseye" ["12"]="bookworm") - -DEBIAN_VERSION="${DEBIAN_VERSION_TABLE[$DEBIAN_VERSION_NUMBER]}" - -SNAPSHOT_NAME="$CUSTOM_ENV_CI_JOB_IMAGE" -if [ -z "$SNAPSHOT_NAME" ] -then - SNAPSHOT_NAME="after-install" -fi -PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR" -PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" - -PREFIX_IMAGE_NAME="ynh" -# For example ynh-buster -BASE_IMAGE="$PREFIX_IMAGE_NAME-$DEBIAN_VERSION" - -CONTAINER_IMAGE="$BASE_IMAGE-r-$CUSTOM_ENV_CI_RUNNER_ID-p-$CUSTOM_ENV_CI_PROJECT_ID-c-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID" \ No newline at end of file