mirror of
https://github.com/YunoHost/yunohost-ci.git
synced 2024-09-03 20:05:53 +02:00
Part1 of epic refactoring to share image between app CI, core CI, ynh-dev
This commit is contained in:
Alexandre Aubin
parent
f61b74a135
commit
bc9eec167d
10 changed files with 168 additions and 508 deletions
|
|
@ -1,40 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source $current_dir/utils.sh # Get utils functions.
|
||||
|
||||
for debian_version in "bullseye"
|
||||
do
|
||||
for ynh_version in "stable" "testing" "unstable"
|
||||
do
|
||||
for snapshot in "before-install" "after-install"
|
||||
do
|
||||
info "Updating container $PREFIX_IMAGE_NAME-$debian_version $ynh_version $snapshot"
|
||||
update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot"
|
||||
done
|
||||
done
|
||||
containers_to_remove=$(incus list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv)
|
||||
if [ -n "$containers_to_remove" ]
|
||||
then
|
||||
# Remove old runner containers
|
||||
incus delete -f $(echo $containers_to_remove)
|
||||
fi
|
||||
done
|
||||
|
||||
for debian_version in "bookworm"
|
||||
do
|
||||
for ynh_version in "unstable"
|
||||
do
|
||||
for snapshot in "before-install" "after-install"
|
||||
do
|
||||
info "Updating container $PREFIX_IMAGE_NAME-$debian_version $ynh_version $snapshot"
|
||||
update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot"
|
||||
done
|
||||
done
|
||||
containers_to_remove=$(incus list $PREFIX_IMAGE_NAME-$debian_version-r -c n -f csv)
|
||||
if [ -n "$containers_to_remove" ]
|
||||
then
|
||||
# Remove old runner containers
|
||||
incus delete -f $(echo $containers_to_remove)
|
||||
fi
|
||||
done
|
||||
2
ci_cron
2
ci_cron
|
|
@ -1,2 +0,0 @@
|
|||
# Upgrade all continers
|
||||
30 2 * * * root /bin/bash -c "/opt/yunohost-ci/auto_upgrade_container.sh >> /opt/yunohost-ci/upgrade.log 2>>/opt/yunohost-ci/upgrade-error.log"
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source $current_dir/prints.sh
|
||||
source $current_dir/variables.sh # Get variables from variables.
|
||||
source $current_dir/common.sh
|
||||
|
||||
info "Stopping container $CONTAINER_IMAGE"
|
||||
info "Stopping container $CONTAINER_NAME"
|
||||
|
||||
incus stop "$CONTAINER_IMAGE"
|
||||
incus stop "$CONTAINER_NAME"
|
||||
incus delete "$CONTAINER_NAME"
|
||||
|
|
|
|||
39
common.sh
Executable file
39
common.sh
Executable file
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
NORMAL=$(printf '\033[0m')
|
||||
BOLD=$(printf '\033[1m')
|
||||
RED=$(printf '\033[31m')
|
||||
GREEN=$(printf '\033[32m')
|
||||
ORANGE=$(printf '\033[33m')
|
||||
BLUE=$(printf '\033[34m')
|
||||
|
||||
function success() { echo "[${BOLD}${GREEN} OK ${NORMAL}] ${1}"; }
|
||||
function info() { echo "[${BOLD}${BLUE}INFO${NORMAL}] ${1}"; }
|
||||
function warn() { echo "[${BOLD}${ORANGE}WARN${NORMAL}] ${1}" 2>&1; }
|
||||
function error() { echo "[${BOLD}${RED}FAIL${NORMAL}] ${1}" 2>&1; }
|
||||
|
||||
# All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_
|
||||
|
||||
DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH"
|
||||
CURRENT_BRANCH="${CUSTOM_ENV_CI_COMMIT_REF_NAME:-$DEFAULT_BRANCH}" # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR
|
||||
|
||||
IMAGE="${CUSTOM_ENV_CI_JOB_IMAGE:-after-install}"
|
||||
|
||||
LAST_CHANGELOG_ENTRY=$(curl $CUSTOM_ENV_CI_PROJECT_URL/-/raw/$CURRENT_BRANCH/debian/changelog --silent | head -n 1) # yunohost (4.2) unstable; urgency=low
|
||||
DEBIAN_VERSION_NUMBER=$(echo $LAST_CHANGELOG_ENTRY | cut -d' ' -f2 | tr -d '(' | tr -d ')' | cut -d'.' -f1) # 11, 12
|
||||
if [[ "$DEBIAN_VERSION_NUMBER" == "11" ]]
|
||||
then
|
||||
BASE_IMAGE="ynh-$IMAGE-bullseye-amd64-stable-base"
|
||||
elif [[ "$DEBIAN_VERSION_NUMBER" == "12" ]]
|
||||
then
|
||||
BASE_IMAGE="ynh-$IMAGE-bookworm-amd64-testing-base"
|
||||
elif [[ "$DEBIAN_VERSION_NUMBER" == "13" ]]
|
||||
then
|
||||
# Upcoming someday™
|
||||
BASE_IMAGE="ynh-$IMAGE-trixie-amd64-unstable-base"
|
||||
else
|
||||
echo "Uhoh, unknown debian version $DEBIAN_VERSION_NUMBER"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CONTAINER_NAME="job-$CUSTOM_ENV_CI_JOB_ID-$CUSTOM_ENV_CI_JOB_NAME_SLUG"
|
||||
140
prepare.sh
140
prepare.sh
|
|
@ -1,45 +1,129 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source $current_dir/prints.sh
|
||||
source $current_dir/utils.sh # Get utils functions.
|
||||
source $current_dir/common.sh
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
wait_container()
|
||||
{
|
||||
restart_container()
|
||||
{
|
||||
incus stop "$1" --timeout 30 || incus stop "$1" --force
|
||||
incus start "$1"
|
||||
}
|
||||
|
||||
# Try to start the container 3 times.
|
||||
local max_try=3
|
||||
local i=0
|
||||
while [ $i -lt $max_try ]
|
||||
do
|
||||
i=$(( i +1 ))
|
||||
local failstart=0
|
||||
|
||||
# Wait for container to start, we are using systemd to check this,
|
||||
# for the sake of brevity.
|
||||
for j in $(seq 1 10); do
|
||||
if incus exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$j" == "10" ]; then
|
||||
error 'Failed to start the container'
|
||||
failstart=1
|
||||
|
||||
restart_container "$1"
|
||||
fi
|
||||
|
||||
sleep 1s
|
||||
done
|
||||
|
||||
# Wait for container to access the internet
|
||||
for j in $(seq 1 10); do
|
||||
if incus exec "$1" -- /bin/bash -c "! which wget > /dev/null 2>&1 || wget -q --spider http://github.com"; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$j" == "10" ]; then
|
||||
error 'Failed to access the internet'
|
||||
failstart=1
|
||||
incus exec "$1" -- /bin/bash -c "echo 'resolv-file=/etc/resolv.dnsmasq.conf' > /etc/dnsmasq.d/resolvconf"
|
||||
incus exec "$1" -- /bin/bash -c "echo 'nameserver 8.8.8.8' > /etc/resolv.dnsmasq.conf"
|
||||
incus exec "$1" -- /bin/bash -c "sed -i 's/#IGNORE/IGNORE/g' /etc/default/dnsmasq"
|
||||
incus exec "$1" -- /bin/bash -c "systemctl restart dnsmasq"
|
||||
incus exec "$1" -- /bin/bash -c "journalctl -u dnsmasq -n 100 --no-pager"
|
||||
|
||||
restart_container "$1"
|
||||
fi
|
||||
|
||||
sleep 1s
|
||||
done
|
||||
|
||||
# Wait dpkg
|
||||
for j in $(seq 1 10); do
|
||||
if ! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock > /dev/null 2>&1" &&
|
||||
! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock-frontend > /dev/null 2>&1" &&
|
||||
! incus exec "$1" -- /bin/bash -c "fuser /var/cache/apt/archives/lock > /dev/null 2>&1"; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$j" == "10" ]; then
|
||||
error 'Waiting too long for lock release'
|
||||
failstart=1
|
||||
|
||||
restart_container "$1"
|
||||
fi
|
||||
|
||||
sleep 1s
|
||||
done
|
||||
|
||||
# Has started and has access to the internet
|
||||
if [ $failstart -eq 0 ]
|
||||
then
|
||||
break
|
||||
fi
|
||||
|
||||
# Fail if the container failed to start
|
||||
if [ $i -eq $max_try ] && [ $failstart -eq 1 ]
|
||||
then
|
||||
# Inform GitLab Runner that this is a system failure, so it
|
||||
# should be retried.
|
||||
exit "$SYSTEM_FAILURE_EXIT_CODE"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# trap any error, and mark it as a system failure.
|
||||
trap "exit $SYSTEM_FAILURE_EXIT_CODE" ERR
|
||||
|
||||
start_container () {
|
||||
if ! incus info "$CONTAINER_IMAGE" >/dev/null 2>/dev/null ; then
|
||||
warn 'Container not found, copying it from the prebuilt image'
|
||||
if ! incus info "$BASE_IMAGE" | grep -w "$CURRENT_VERSION-$SNAPSHOT_NAME" >/dev/null
|
||||
then
|
||||
error "$BASE_IMAGE not found, please rebuild with rebuild_all.sh"
|
||||
# Inform GitLab Runner that this is a system failure, so it
|
||||
# should be retried.
|
||||
exit $SYSTEM_FAILURE_EXIT_CODE
|
||||
fi
|
||||
incus copy "$BASE_IMAGE" "$CONTAINER_IMAGE"
|
||||
fi
|
||||
# Stop the container if it's running
|
||||
if [ "$(incus info $CONTAINER_IMAGE | grep Status | awk '{print tolower($2)}')" == "running" ]; then
|
||||
incus stop $CONTAINER_IMAGE
|
||||
fi
|
||||
###############################################################################
|
||||
|
||||
info "Debian version: $DEBIAN_VERSION, YunoHost version: $CURRENT_VERSION, Image used: $BASE_IMAGE, Snapshot: $SNAPSHOT_NAME"
|
||||
info "Starting $CONTAINER_NAME from $BASE_IMAGE ..."
|
||||
|
||||
restore_snapshot "$CONTAINER_IMAGE" "$CURRENT_VERSION" "$SNAPSHOT_NAME"
|
||||
if ! incus info "$CONTAINER_NAME" >/dev/null 2>/dev/null ; then
|
||||
|
||||
# Unset the mac address to ensure the copy will get a new one and will be able to get new IP
|
||||
incus config unset "$CONTAINER_IMAGE" volatile.eth0.hwaddr 2> /dev/null
|
||||
# Force the usage of the fingerprint because otherwise for some reason lxd won't use the newer version
|
||||
# available even though it's aware it exists -_-
|
||||
BASE_HASH="$(incus image list yunohost:$BASE_IMAGE --format json | jq -r '.[].fingerprint')"
|
||||
|
||||
incus start $CONTAINER_IMAGE
|
||||
# NB: this image comes from the 'common' image repository shared with the appci, ynh-dev etc
|
||||
incus launch yunohost:$BASE_HASH $CONTAINER_NAME
|
||||
fi
|
||||
|
||||
wait_container $CONTAINER_IMAGE
|
||||
}
|
||||
# Stop the container if it's running
|
||||
if [ "$(incus info $CONTAINER_NAME | grep Status | awk '{print tolower($2)}')" == "running" ]; then
|
||||
incus stop "$CONTAINER_NAME" --timeout 30 || incus stop "$CONTAINER_NAME" --force
|
||||
fi
|
||||
|
||||
info "Starting $CONTAINER_IMAGE"
|
||||
# Unset the mac address to ensure the copy will get a new one and will be able to get new IP
|
||||
incus start $CONTAINER_NAME
|
||||
|
||||
start_container
|
||||
sleep 2
|
||||
|
||||
info "$CONTAINER_IMAGE started properly"
|
||||
incus exec $CONTAINER_NAME dhclient eth0
|
||||
|
||||
info "Waiting for $CONTAINER_NAME to finish booting and making sure it's connected to the internetz..."
|
||||
|
||||
wait_container $CONTAINER_NAME
|
||||
|
||||
success "$CONTAINER_NAME started properly"
|
||||
|
|
|
|||
37
prints.sh
37
prints.sh
|
|
@ -1,37 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
NORMAL=$(printf '\033[0m')
|
||||
BOLD=$(printf '\033[1m')
|
||||
faint=$(printf '\033[2m')
|
||||
UNDERLINE=$(printf '\033[4m')
|
||||
NEGATIVE=$(printf '\033[7m')
|
||||
RED=$(printf '\033[31m')
|
||||
GREEN=$(printf '\033[32m')
|
||||
ORANGE=$(printf '\033[33m')
|
||||
BLUE=$(printf '\033[34m')
|
||||
YELLOW=$(printf '\033[93m')
|
||||
WHITE=$(printf '\033[39m')
|
||||
|
||||
function success()
|
||||
{
|
||||
local msg=${1}
|
||||
echo "[${BOLD}${GREEN} OK ${NORMAL}] ${msg}"
|
||||
}
|
||||
|
||||
function info()
|
||||
{
|
||||
local msg=${1}
|
||||
echo "[${BOLD}${BLUE}INFO${NORMAL}] ${msg}"
|
||||
}
|
||||
|
||||
function warn()
|
||||
{
|
||||
local msg=${1}
|
||||
echo "[${BOLD}${ORANGE}WARN${NORMAL}] ${msg}" 2>&1
|
||||
}
|
||||
|
||||
function error()
|
||||
{
|
||||
local msg=${1}
|
||||
echo "[${BOLD}${RED}FAIL${NORMAL}] ${msg}" 2>&1
|
||||
}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source $current_dir/utils.sh # Get utils functions.
|
||||
|
||||
for debian_version in "bullseye"
|
||||
do
|
||||
rebuild_base_containers "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "stable" "amd64"
|
||||
|
||||
for ynh_version in "testing" "unstable"
|
||||
do
|
||||
for snapshot in "before-install" "after-install"
|
||||
do
|
||||
restore_snapshot "$PREFIX_IMAGE_NAME-$debian_version" "stable" "$snapshot"
|
||||
|
||||
if [[ "$ynh_version" == "testing" ]]
|
||||
then
|
||||
repo_version="testing"
|
||||
elif [[ "$DISTRIB" == "unstable" ]]
|
||||
then
|
||||
repo_version="testing unstable"
|
||||
fi
|
||||
|
||||
incus exec "$PREFIX_IMAGE_NAME-$debian_version" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`;
|
||||
do
|
||||
sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE
|
||||
done"
|
||||
|
||||
create_snapshot "$PREFIX_IMAGE_NAME-$debian_version" "$ynh_version" "$snapshot"
|
||||
|
||||
update_container "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "$ynh_version" "$snapshot"
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
|
||||
for debian_version in "bookworm"
|
||||
do
|
||||
rebuild_base_containers "$PREFIX_IMAGE_NAME-$debian_version" "$debian_version" "unstable" "amd64"
|
||||
done
|
||||
|
||||
46
run.sh
46
run.sh
|
|
@ -1,42 +1,22 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source $current_dir/variables.sh # Get variables from variables.
|
||||
source $current_dir/common.sh
|
||||
|
||||
case ${2} in
|
||||
prepare_script)
|
||||
echo "CI OPEN: $CUSTOM_ENV_CI_OPEN_MERGE_REQUESTS / CI PIPELINE SOURCE: $CUSTOM_ENV_CI_PIPELINE_SOURCE / REF NAME: $CUSTOM_ENV_CI_COMMIT_REF_NAME / COMMIT: $CUSTOM_ENV_CI_COMMIT_BRANCH, TAG: $CUSTOM_ENV_CI_COMMIT_TAG"
|
||||
;;
|
||||
get_sources)
|
||||
;;
|
||||
restore_cache)
|
||||
;;
|
||||
download_artifacts)
|
||||
;;
|
||||
# build_script is now step_script
|
||||
# but we can also have "step_release" or "step_accessibility"
|
||||
# More info here:
|
||||
# https://docs.gitlab.com/runner/executors/custom.html#run
|
||||
# https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26426
|
||||
step_*)
|
||||
case $PROJECT_NAME in
|
||||
yunohost)
|
||||
# Nothing to do?
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
after_script)
|
||||
;;
|
||||
archive_cache)
|
||||
;;
|
||||
upload_artifact_on_success)
|
||||
;;
|
||||
upload_artifact_on_failure)
|
||||
;;
|
||||
esac
|
||||
# This script is called multiple times with different stage ($2) value
|
||||
# This is documented in https://docs.gitlab.com/runner/executors/custom.html#run
|
||||
JOB_STAGE=$2
|
||||
if [[ "$JOB_STAGE" == "prepare_script" ]]
|
||||
then
|
||||
echo "CI OPEN: $CUSTOM_ENV_CI_OPEN_MERGE_REQUESTS / CI PIPELINE SOURCE: $CUSTOM_ENV_CI_PIPELINE_SOURCE / REF NAME: $CUSTOM_ENV_CI_COMMIT_REF_NAME / COMMIT: $CUSTOM_ENV_CI_COMMIT_BRANCH, TAG: $CUSTOM_ENV_CI_COMMIT_TAG"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
|
||||
incus exec "$CONTAINER_IMAGE" /bin/bash < "${1}"
|
||||
# This is the real 'important' logic bit
|
||||
incus exec "$CONTAINER_NAME" /bin/bash < "${1}"
|
||||
|
||||
###############################################################################
|
||||
err=$?
|
||||
if [ $err -ne 0 ]; then
|
||||
echo "Exit with error code $err"
|
||||
|
|
|
|||
284
utils.sh
284
utils.sh
|
|
@ -1,284 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source $current_dir/prints.sh
|
||||
source $current_dir/variables.sh # Get variables from variables.
|
||||
|
||||
wait_container()
|
||||
{
|
||||
restart_container()
|
||||
{
|
||||
incus stop "$1"
|
||||
incus start "$1"
|
||||
}
|
||||
|
||||
# Try to start the container 3 times.
|
||||
local max_try=3
|
||||
local i=0
|
||||
while [ $i -lt $max_try ]
|
||||
do
|
||||
i=$(( i +1 ))
|
||||
local failstart=0
|
||||
|
||||
# Wait for container to start, we are using systemd to check this,
|
||||
# for the sake of brevity.
|
||||
for j in $(seq 1 10); do
|
||||
if incus exec "$1" -- /bin/bash -c "systemctl isolate multi-user.target" >/dev/null 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$j" == "10" ]; then
|
||||
error 'Failed to start the container'
|
||||
failstart=1
|
||||
|
||||
restart_container "$1"
|
||||
fi
|
||||
|
||||
sleep 1s
|
||||
done
|
||||
|
||||
# Wait for container to access the internet
|
||||
for j in $(seq 1 10); do
|
||||
if incus exec "$1" -- /bin/bash -c "! which wget > /dev/null 2>&1 || wget -q --spider http://github.com"; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$j" == "10" ]; then
|
||||
error 'Failed to access the internet'
|
||||
failstart=1
|
||||
incus exec "$1" -- /bin/bash -c "echo 'resolv-file=/etc/resolv.dnsmasq.conf' > /etc/dnsmasq.d/resolvconf"
|
||||
incus exec "$1" -- /bin/bash -c "echo 'nameserver 8.8.8.8' > /etc/resolv.dnsmasq.conf"
|
||||
incus exec "$1" -- /bin/bash -c "sed -i 's/#IGNORE/IGNORE/g' /etc/default/dnsmasq"
|
||||
incus exec "$1" -- /bin/bash -c "systemctl restart dnsmasq"
|
||||
incus exec "$1" -- /bin/bash -c "journalctl -u dnsmasq -n 100 --no-pager"
|
||||
|
||||
restart_container "$1"
|
||||
fi
|
||||
|
||||
sleep 1s
|
||||
done
|
||||
|
||||
# Wait dpkg
|
||||
for j in $(seq 1 10); do
|
||||
if ! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock > /dev/null 2>&1" &&
|
||||
! incus exec "$1" -- /bin/bash -c "fuser /var/lib/dpkg/lock-frontend > /dev/null 2>&1" &&
|
||||
! incus exec "$1" -- /bin/bash -c "fuser /var/cache/apt/archives/lock > /dev/null 2>&1"; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$j" == "10" ]; then
|
||||
error 'Waiting too long for lock release'
|
||||
failstart=1
|
||||
|
||||
restart_container "$1"
|
||||
fi
|
||||
|
||||
sleep 1s
|
||||
done
|
||||
|
||||
# Has started and has access to the internet
|
||||
if [ $failstart -eq 0 ]
|
||||
then
|
||||
break
|
||||
fi
|
||||
|
||||
# Fail if the container failed to start
|
||||
if [ $i -eq $max_try ] && [ $failstart -eq 1 ]
|
||||
then
|
||||
# Inform GitLab Runner that this is a system failure, so it
|
||||
# should be retried.
|
||||
exit "$SYSTEM_FAILURE_EXIT_CODE"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
create_snapshot()
|
||||
{
|
||||
local instance_to_publish=$1
|
||||
local ynh_version=$2
|
||||
local snapshot=$3
|
||||
|
||||
# Create snapshot
|
||||
incus snapshot create "$instance_to_publish" "$ynh_version-$snapshot" --reuse
|
||||
}
|
||||
|
||||
restore_snapshot()
|
||||
{
|
||||
local incus_name=$1
|
||||
local ynh_version=$2
|
||||
local snapshot=$3
|
||||
local retry_incus=0
|
||||
|
||||
while [[ ${retry_incus} -lt 10 ]]
|
||||
do
|
||||
incus snapshot restore "$incus_name" "$ynh_version-$snapshot" && break || retry_incus=$(($retry_incus+1))
|
||||
info "Failed to restore snapshot? Retrying in 10 sec ..."
|
||||
if [[ ${retry_incus} -ge 3 ]]
|
||||
then
|
||||
warn "If this keeps happening, restarting the LXD daemon might help :| ..."
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
|
||||
if [[ ${retry_incus} -ge 10 ]]
|
||||
then
|
||||
error "Failed to properly restore the snapshot zrgmblg"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# These lines are used to extract the dependencies/recommendations from the debian/control file.
|
||||
# /!\ There's a high risk of lamentable failure if we change the format of this file
|
||||
get_dependencies()
|
||||
{
|
||||
local debian_version=$1
|
||||
if [[ "$debian_version" == "bullseye" ]]
|
||||
then
|
||||
local branch="dev"
|
||||
else
|
||||
local branch="$debian_version"
|
||||
fi
|
||||
|
||||
# To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'.
|
||||
YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ")
|
||||
YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$branch/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
|
||||
MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$branch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
|
||||
# Same as above, except that all dependencies are in the same line
|
||||
SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$branch/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
|
||||
BUILD_DEPENDENCIES="git-buildpackage postfix python3-setuptools python3-pip devscripts"
|
||||
TESTS_DEPENDENCIES="git hub"
|
||||
PIP3_PKG='mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 types-ipaddress types-enum34 types-cryptography types-toml types-requests types-PyYAML types-pyOpenSSL types-mock "packaging<22"'
|
||||
|
||||
if [[ "$debian_version" == "bookworm" ]]
|
||||
then
|
||||
# We add php8.2-cli, mariadb-client and mariadb-server to the dependencies for test_app_resources
|
||||
TESTS_DEPENDENCIES="$TESTS_DEPENDENCIES php8.2-cli mariadb-client mariadb-server"
|
||||
PIP3_PKG="$PIP3_PKG --break-system-packages"
|
||||
fi
|
||||
}
|
||||
|
||||
rebuild_base_containers()
|
||||
{
|
||||
local image_to_rebuild=$1
|
||||
local debian_version=$2
|
||||
local ynh_version=$3
|
||||
local arch=$4
|
||||
|
||||
if incus info "$image_to_rebuild" &>/dev/null
|
||||
then
|
||||
incus delete -f "$image_to_rebuild"
|
||||
fi
|
||||
|
||||
incus launch images:debian/$debian_version/$arch "$image_to_rebuild" -c security.nesting=true
|
||||
|
||||
wait_container "$image_to_rebuild"
|
||||
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get update"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes wget curl"
|
||||
# Install Git LFS, git comes pre installed with ubuntu image.
|
||||
# Disable this line because we don't need to add a new repo to have git-lfs
|
||||
#incus exec "$image_to_rebuild" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes git-lfs"
|
||||
# Install gitlab-runner binary since we need for cache/artifacts.
|
||||
if [[ $debian_version == "bullseye" ]]
|
||||
then
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "wget https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "dpkg -i gitlab-runner_amd64.deb"
|
||||
else
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | os=debian dist=$debian_version bash"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner"
|
||||
fi
|
||||
|
||||
INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/main/$debian_version"
|
||||
|
||||
# Download the YunoHost install script
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh"
|
||||
|
||||
# Patch the YunoHost install script
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/echo skip restart service #\1/' install.sh"
|
||||
|
||||
# Run the YunoHost install script patched
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version"
|
||||
|
||||
get_dependencies $debian_version
|
||||
|
||||
# Pre install dependencies
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES $TESTS_DEPENDENCIES"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG"
|
||||
|
||||
# Disable apt-daily
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.timer --now"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.timer --now"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily.service --now"
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "systemctl -q disable apt-daily-upgrade.service --now"
|
||||
|
||||
|
||||
mkdir -p $current_dir/cache
|
||||
chmod 777 $current_dir/cache
|
||||
incus config device add "$image_to_rebuild" cache-folder disk path=/cache source="$current_dir/cache"
|
||||
|
||||
create_snapshot "$image_to_rebuild" "$ynh_version" "before-install"
|
||||
echo "Created snapshot base 'before-install' for $image_to_rebuild"
|
||||
|
||||
# Install YunoHost
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version"
|
||||
|
||||
# Run postinstall
|
||||
incus exec "$image_to_rebuild" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace"
|
||||
|
||||
# Disable services which are not really mandatory, to reduce the incus memory footprint and hopefully speed things up a bit
|
||||
|
||||
if [[ "$debian_version" == "bookworm" ]]
|
||||
then
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable metronome --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable rspamd --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable postsrsd --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable yunohost-api --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable fake-hwclock.service --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable yunoprompt --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable haveged.service --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable unattended-upgrades.service --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable e2scrub_all.timer --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable logrotate.timer --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable phpsessionclean.timer --now
|
||||
incus exec "$image_to_rebuild" -- systemctl -q disable systemd-tmpfiles-clean.timer --now
|
||||
fi
|
||||
|
||||
create_snapshot "$image_to_rebuild" "$ynh_version" "after-install"
|
||||
echo "Created snapshot base 'after-install' for $image_to_rebuild"
|
||||
|
||||
incus stop "$image_to_rebuild"
|
||||
}
|
||||
|
||||
update_container() {
|
||||
local image_to_update=$1
|
||||
local debian_version=$2
|
||||
local ynh_version=$3
|
||||
local snapshot=$4
|
||||
|
||||
if ! incus info "$image_to_update" &>/dev/null
|
||||
then
|
||||
error "Unable to upgrade image $image_to_update"
|
||||
return
|
||||
fi
|
||||
|
||||
# Start and run upgrade
|
||||
restore_snapshot "$image_to_update" "$ynh_version" "$snapshot"
|
||||
|
||||
incus start "$image_to_update" 2>&1 || true
|
||||
|
||||
wait_container "$image_to_update"
|
||||
|
||||
incus exec "$image_to_update" -- /bin/bash -c "apt-get update"
|
||||
incus exec "$image_to_update" -- /bin/bash -c "apt-get upgrade --assume-yes"
|
||||
|
||||
get_dependencies $debian_version
|
||||
|
||||
incus exec "$image_to_update" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES $TESTS_DEPENDENCIES"
|
||||
incus exec "$image_to_update" -- /bin/bash -c "python3 -m pip install -U $PIP3_PKG"
|
||||
|
||||
create_snapshot "$image_to_update" "$ynh_version" "$snapshot"
|
||||
|
||||
incus stop "$image_to_update"
|
||||
}
|
||||
39
variables.sh
39
variables.sh
|
|
@ -1,39 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source $current_dir/prints.sh
|
||||
|
||||
# All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_
|
||||
|
||||
ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64
|
||||
DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH"
|
||||
|
||||
CURRENT_BRANCH="$CUSTOM_ENV_CI_COMMIT_REF_NAME" # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR
|
||||
if [ -z "$CURRENT_BRANCH" ]
|
||||
then
|
||||
CURRENT_BRANCH="$DEFAULT_BRANCH"
|
||||
fi
|
||||
|
||||
LAST_CHANGELOG_ENTRY=$(curl $CUSTOM_ENV_CI_PROJECT_URL/-/raw/$CURRENT_BRANCH/debian/changelog --silent | head -n 1) # yunohost (4.2) unstable; urgency=low
|
||||
CURRENT_VERSION=$(echo $LAST_CHANGELOG_ENTRY | cut -d' ' -f3 | tr -d ';') # stable, testing, unstable
|
||||
|
||||
DEBIAN_VERSION_NUMBER=$(echo $LAST_CHANGELOG_ENTRY | cut -d' ' -f2 | tr -d '(' | tr -d ')' | cut -d'.' -f1) # 11, 12
|
||||
|
||||
declare -A DEBIAN_VERSION_TABLE
|
||||
DEBIAN_VERSION_TABLE=(["11"]="bullseye" ["12"]="bookworm")
|
||||
|
||||
DEBIAN_VERSION="${DEBIAN_VERSION_TABLE[$DEBIAN_VERSION_NUMBER]}"
|
||||
|
||||
SNAPSHOT_NAME="$CUSTOM_ENV_CI_JOB_IMAGE"
|
||||
if [ -z "$SNAPSHOT_NAME" ]
|
||||
then
|
||||
SNAPSHOT_NAME="after-install"
|
||||
fi
|
||||
PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR"
|
||||
PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME"
|
||||
|
||||
PREFIX_IMAGE_NAME="ynh"
|
||||
# For example ynh-buster
|
||||
BASE_IMAGE="$PREFIX_IMAGE_NAME-$DEBIAN_VERSION"
|
||||
|
||||
CONTAINER_IMAGE="$BASE_IMAGE-r-$CUSTOM_ENV_CI_RUNNER_ID-p-$CUSTOM_ENV_CI_PROJECT_ID-c-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID"
|
||||
Loading…
Add table
Reference in a new issue