Allow the use of incus instead of LXD. In the future, we could remove LXD support, making the README way shorter (no snap, etc)

This commit is contained in:
Salamandar 2024-03-06 22:55:17 +01:00
parent 42961ea755
commit 0998271a98
6 changed files with 122 additions and 68 deletions

View file

@ -2,48 +2,48 @@
function launch_new_lxc()
{
lxc info $LXC_BASE >/dev/null && lxc delete $LXC_BASE --force
$lxc info $LXC_BASE >/dev/null && $lxc delete $LXC_BASE --force
if [ $(get_arch) = $ARCH ];
then
lxc launch images:debian/$DIST/$ARCH $LXC_BASE -c security.privileged=true -c security.nesting=true
$lxc launch images:debian/$DIST/$ARCH $LXC_BASE -c security.privileged=true -c security.nesting=true
else
lxc image info $LXC_BASE >/dev/null && lxc image delete $LXC_BASE
$lxc image info $LXC_BASE >/dev/null && $lxc image delete $LXC_BASE
tmp_dir=$(mktemp -d)
pushd $tmp_dir
lxc image export images:debian/$DIST/$ARCH
$lxc image export images:debian/$DIST/$ARCH
tar xJf lxd.tar.xz
local current_arch=$(get_arch)
sed -i "0,/architecture: $ARCH/s//architecture: $current_arch/" metadata.yaml
tar cJf lxd.tar.xz metadata.yaml templates
lxc image import lxd.tar.xz rootfs.squashfs --alias $LXC_BASE
$lxc image import lxd.tar.xz rootfs.squashfs --alias $LXC_BASE
popd
rm -rf "$tmp_dir"
lxc launch $LXC_BASE $LXC_BASE -c security.privileged=true -c security.nesting=true
$lxc launch $LXC_BASE $LXC_BASE -c security.privileged=true -c security.nesting=true
fi
}
function rebuild_base_lxc()
{
check_lxd_setup
check_lxc_setup
launch_new_lxc
sleep 5
IN_LXC="lxc exec $LXC_BASE --"
IN_LXC="$lxc exec $LXC_BASE --"
INSTALL_SCRIPT="https://install.yunohost.org/$DIST"
$IN_LXC apt install curl -y
$IN_LXC /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $YNH_BRANCH"
$IN_LXC systemctl -q stop apt-daily.timer
$IN_LXC systemctl -q stop apt-daily-upgrade.timer
$IN_LXC systemctl -q stop apt-daily.service
$IN_LXC systemctl -q stop apt-daily-upgrade.service
$IN_LXC systemctl -q stop apt-daily-upgrade.service
$IN_LXC systemctl -q disable apt-daily.timer
$IN_LXC systemctl -q disable apt-daily-upgrade.timer
$IN_LXC systemctl -q disable apt-daily.service
@ -63,8 +63,8 @@ function rebuild_base_lxc()
$IN_LXC yunohost --version
lxc stop $LXC_BASE
lxc image delete $LXC_BASE
lxc publish $LXC_BASE --alias $LXC_BASE --public
lxc delete $LXC_BASE
$lxc stop $LXC_BASE
$lxc image delete $LXC_BASE
$lxc publish $LXC_BASE --alias $LXC_BASE --public
$lxc delete $LXC_BASE
}

View file

@ -32,8 +32,14 @@ assert_we_are_connected_to_the_internets() {
}
assert_we_have_all_dependencies() {
for dep in "lxc" "lxd" "lynx" "jq" "python3" "pip3"
do
deps=("lynx" "jq" "python3" "pip3")
if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then
deps+=(incus)
else
deps+=(lxc lxd)
fi
for dep in "${deps[@]}"; do
which $dep 2>&1 > /dev/null || log_critical "Please install $dep"
done
}
@ -55,6 +61,39 @@ function check_lxd_setup()
|| log_critical "There is no 'lxdbr0' interface... Did you ran 'lxd init' ?"
}
function check_incus_setup()
{
# Check incus is installed somehow
if ! which incus &>/dev/null; then
critical "You need to have Incus installed for ynh-dev to be usable from the host machine. Refer to the README to know how to install it."
fi
if ! id -nG "$(whoami)" | grep -qw "incus-admin"; then
critical "You need to be in the incus-admin group!"
fi
ip a | grep -q incusbr0 \
|| warn "There is no 'incusbr0' interface... Did you ran 'incus admin init' ?"
set_incus_remote
}
function set_incus_remote()
{
configured=$(incus remote list -f json | jq 'has("yunohost")')
if [[ "$configured" != "true" ]]; then
incus remote add yunohost https://devbaseimgs.yunohost.org --public
fi
}
function check_lxc_setup()
{
if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then
check_incus_setup
else
check_lxd_setup
fi
}
#=================================================
# Logging helpers
#=================================================

View file

@ -1,5 +1,17 @@
#!/bin/bash
# Check for LXC or Incus
function switch_lxc_incus()
{
if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then
lxc=incus
else
lxc=lxc
fi
}
switch_lxc_incus
#=================================================
# RUNNING SNAPSHOT
#=================================================
@ -7,19 +19,19 @@
LXC_CREATE () {
log_info "Launching new LXC $LXC_NAME ..."
# Check if we can launch container from YunoHost remote image
if lxc remote list | grep -q "yunohost" && lxc image list yunohost:$LXC_BASE | grep -q -w $LXC_BASE; then
if $lxc remote list | grep -q "yunohost" && $lxc image list yunohost:$LXC_BASE | grep -q -w $LXC_BASE; then
# Force the usage of the fingerprint because otherwise for some reason lxd won't use the newer version
# available even though it's aware it exists -_-
LXC_BASE_HASH="$(lxc image list yunohost:$LXC_BASE --format json | jq -r '.[].fingerprint')"
lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME \
LXC_BASE_HASH="$($lxc image list yunohost:$LXC_BASE --format json | jq -r '.[].fingerprint')"
$lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME \
-c security.nesting=true \
-c security.privileged=true \
-c limits.memory=80% \
-c limits.cpu.allowance=80% \
>>/proc/self/fd/3
# Check if we can launch container from a local image
elif lxc image list $LXC_BASE | grep -q -w $LXC_BASE; then
lxc launch $LXC_BASE $LXC_NAME \
elif $lxc image list $LXC_BASE | grep -q -w $LXC_BASE; then
$lxc launch $LXC_BASE $LXC_NAME \
-c security.nesting=true \
-c security.privileged=true \
-c limits.memory=80% \
@ -30,12 +42,12 @@ LXC_CREATE () {
fi
pipestatus="${PIPESTATUS[0]}"
location=$(lxc list --format json | jq -e --arg LXC_NAME $LXC_NAME '.[] | select(.name==$LXC_NAME) | .location' | tr -d '"')
location=$($lxc list --format json | jq -e --arg LXC_NAME $LXC_NAME '.[] | select(.name==$LXC_NAME) | .location' | tr -d '"')
[[ "$location" != "none" ]] && log_info "... on $location"
[[ "$pipestatus" -eq 0 ]] || exit 1
if [[ "$(lxc list $LXC_NAME --format json)" == "[]" ]]
if [[ "$($lxc list $LXC_NAME --format json)" == "[]" ]]
then
log_critical "Failed to create the new LXC :/"
fi
@ -43,7 +55,7 @@ LXC_CREATE () {
_LXC_START_AND_WAIT $LXC_NAME
sleep 3
if ! lxc exec $LXC_NAME -- test -e /etc/yunohost
if ! $lxc exec $LXC_NAME -- test -e /etc/yunohost
then
log_critical "Failed to run 'test -e /etc/yunohost' on the container ... either the container did not start, or YunoHost doesn't exists yet in the container :/"
fi
@ -52,9 +64,9 @@ LXC_CREATE () {
sleep 3
log_info "Creating initial snapshot $LXC_NAME ..."
lxc snapshot $LXC_NAME snap0
$lxc snapshot create $LXC_NAME snap0
if [[ -z "$(lxc list $LXC_NAME --format json | jq '.[].snapshots[] | select(.name=="snap0")')" ]]
if [[ -z "$($lxc list $LXC_NAME --format json | jq '.[].snapshots[] | select(.name=="snap0")')" ]]
then
log_critical "Failed to create the initial snapshot :/"
fi
@ -62,7 +74,7 @@ LXC_CREATE () {
LXC_SNAPSHOT_EXISTS() {
local snapname=$1
lxc list --format json \
$lxc list --format json \
| jq -e --arg LXC_NAME $LXC_NAME --arg snapname $snapname \
'.[] | select(.name==$LXC_NAME) | .snapshots[] | select(.name==$snapname)' \
>/dev/null
@ -87,7 +99,7 @@ CREATE_LXC_SNAPSHOT () {
if ! LXC_SNAPSHOT_EXISTS "$snapname"
then
log_info "(Creating snapshot $snapname ...)"
lxc snapshot $LXC_NAME $snapname
$lxc snapshot create $LXC_NAME $snapname
fi
_LXC_START_AND_WAIT $LXC_NAME
@ -106,7 +118,7 @@ LOAD_LXC_SNAPSHOT () {
while [[ ${retry_lxc} -lt 10 ]]
do
LXC_STOP $LXC_NAME || true
lxc restore $LXC_NAME $snapname && break || retry_lxc=$(($retry_lxc+1))
$lxc snapshot restore $LXC_NAME $snapname && break || retry_lxc=$(($retry_lxc+1))
log_warning "Failed to restore snapshot? Retrying in 20 sec ..."
if [[ ${retry_lxc} -ge 3 ]]
then
@ -120,7 +132,7 @@ LOAD_LXC_SNAPSHOT () {
log_error "Failed to restore snapshot ? The next step may miserably crash because of this ... if this happens to often, maybe restarting the LXD daemon can help ..."
fi
lxc start $LXC_NAME
$lxc start $LXC_NAME
_LXC_START_AND_WAIT $LXC_NAME
}
@ -135,7 +147,7 @@ LXC_EXEC () {
start_timer
# Execute the command given in argument in the container and log its results.
lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- /bin/bash -c "$cmd" | tee -a "$full_log" $current_test_log
$lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- /bin/bash -c "$cmd" | tee -a "$full_log" $current_test_log
# Store the return code of the command
local returncode=${PIPESTATUS[0]}
@ -151,42 +163,42 @@ LXC_STOP () {
local container_to_stop=$1
# (We also use timeout 30 in front of the command because sometime lxc
# commands can hang forever despite the --timeout >_>...)
timeout 30 lxc stop --timeout 15 $container_to_stop 2>/dev/null
timeout 30 $lxc stop --timeout 15 $container_to_stop 2>/dev/null
local retry_stop_lxc=0
while [[ ${retry_stop_lxc} -lt 5 ]]
do
local status="$(lxc list $container_to_stop --format json | jq -r '.[].state.status')"
local status="$($lxc list $container_to_stop --format json | jq -r '.[].state.status')"
if [[ -z "$status" ]] || [[ "$status" == "Stopped" ]] || [[ "$status" == "null" ]]
then
break
fi
retry_stop_lxc="$(($retry_stop_lxc+1))"
sleep 10
timeout 30 lxc stop --timeout 15 $container_to_stop 2>/dev/null
timeout 30 $lxc stop --timeout 15 $container_to_stop 2>/dev/null
done
if [[ ${retry_stop_lxc} -ge 5 ]]
then
timeout 30 lxc stop --timeout 15 $container_to_stop --force 2>/dev/null
timeout 30 $lxc stop --timeout 15 $container_to_stop --force 2>/dev/null
fi
}
LXC_RESET () {
# If the container exists
if lxc info $LXC_NAME >/dev/null 2>/dev/null; then
if $lxc info $LXC_NAME >/dev/null 2>/dev/null; then
# Remove swap files before deletting the continer
CLEAN_SWAPFILES
fi
LXC_STOP $LXC_NAME
if lxc info $LXC_NAME >/dev/null 2>/dev/null; then
local current_storage=$(lxc list $LXC_NAME --format json --columns b | jq -r '.[].expanded_devices.root.pool')
swapoff "$(lxc storage get $current_storage source)/containers/$LXC_NAME/rootfs/swap" 2>/dev/null
if $lxc info $LXC_NAME >/dev/null 2>/dev/null; then
local current_storage=$($lxc list $LXC_NAME --format json --columns b | jq -r '.[].expanded_devices.root.pool')
swapoff "$($lxc storage get $current_storage source)/containers/$LXC_NAME/rootfs/swap" 2>/dev/null
fi
lxc delete $LXC_NAME --force 2>/dev/null
$lxc delete $LXC_NAME --force 2>/dev/null
}
@ -195,7 +207,7 @@ _LXC_START_AND_WAIT() {
restart_container()
{
LXC_STOP $1
lxc start "$1"
$lxc start "$1"
}
# Try to start the container 3 times.
@ -209,7 +221,7 @@ _LXC_START_AND_WAIT() {
# Wait for container to start, we are using systemd to check this,
# for the sake of brevity.
for j in $(seq 1 10); do
if lxc exec "$1" -- timeout 30 systemctl isolate multi-user.target >/dev/null 2>/dev/null; then
if $lxc exec "$1" -- timeout 30 systemctl isolate multi-user.target >/dev/null 2>/dev/null; then
break
fi
@ -225,7 +237,7 @@ _LXC_START_AND_WAIT() {
# Wait for container to access the internet
for j in $(seq 1 10); do
if lxc exec "$1" -- timeout 10 curl -s http://wikipedia.org > /dev/null 2>/dev/null; then
if $lxc exec "$1" -- timeout 10 curl -s http://wikipedia.org > /dev/null 2>/dev/null; then
break
fi
@ -249,26 +261,26 @@ _LXC_START_AND_WAIT() {
if [ $i -eq $max_try ] && [ $failstart -eq 1 ]
then
log_error "The container miserably failed to start or to connect to the internet"
lxc info --show-log $1
$lxc info --show-log $1
return 1
fi
done
sleep 3
LXC_IP=$(lxc exec $1 -- hostname -I | cut -d' ' -f1 | grep -E -o "\<[0-9.]{8,}\>")
LXC_IP=$($lxc exec $1 -- hostname -I | cut -d' ' -f1 | grep -E -o "\<[0-9.]{8,}\>")
}
CLEAN_SWAPFILES() {
# Restart it if needed
if [ "$(lxc info $LXC_NAME | grep Status | awk '{print tolower($2)}')" != "running" ]; then
lxc start $LXC_NAME
if [ "$($lxc info $LXC_NAME | grep Status | awk '{print tolower($2)}')" != "running" ]; then
$lxc start $LXC_NAME
_LXC_START_AND_WAIT $LXC_NAME
fi
lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do swapoff $swapfile; done'
lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do rm -f $swapfile; done'
$lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do swapoff $swapfile; done'
$lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do rm -f $swapfile; done'
}
RUN_INSIDE_LXC() {
lxc exec $LXC_NAME -- "$@"
$lxc exec $LXC_NAME -- "$@"
}

View file

@ -8,7 +8,7 @@ _STUFF_TO_RUN_BEFORE_INITIAL_SNAPSHOT()
{
# Print the version of YunoHost from the LXC container
log_small_title "YunoHost versions"
lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost --version" | tee -a "$full_log"
$lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost --version" | tee -a "$full_log"
log_title "Package linter"
./package_linter/package_linter.py "$package_path" | tee -a "$full_log"
@ -26,13 +26,13 @@ _STUFF_TO_RUN_BEFORE_INITIAL_SNAPSHOT()
log_title "Preinstalling apt dependencies before creating the initial snapshot..."
apt="LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0"
lxc exec $LXC_NAME -t -- /bin/bash -c "$apt update; $apt install $apt_deps" | tee -a "$full_log" >/dev/null
$lxc exec $LXC_NAME -t -- /bin/bash -c "$apt update; $apt install $apt_deps" | tee -a "$full_log" >/dev/null
fi
# Gotta generate the psql password even though apparently it's not even useful anymore these days but it otherwise trigger warnings ~_~
if echo "$apt_deps" | grep -q postgresql
then
lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost tools regen-conf postgresql" | tee -a "$full_log" >/dev/null
$lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost tools regen-conf postgresql" | tee -a "$full_log" >/dev/null
fi
}
@ -42,8 +42,8 @@ _RUN_YUNOHOST_CMD() {
log_debug "Running yunohost $1"
# Copy the package into the container.
lxc exec $LXC_NAME -- rm -rf /app_folder
lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet
$lxc exec $LXC_NAME -- rm -rf /app_folder
$lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet
# --output-as none is to disable the json-like output for some commands like backup create
LXC_EXEC "yunohost --output-as none --debug $1" \
@ -70,7 +70,7 @@ _PREINSTALL () {
sed -i "s/\$SUBDOMAIN/$SUBDOMAIN/g" "$preinstall_script"
sed -i "s/\$PASSWORD/$YUNO_PWD/g" "$preinstall_script"
# Copy the pre-install script into the container.
lxc file push "$preinstall_script" "$LXC_NAME/preinstall.sh"
$lxc file push "$preinstall_script" "$LXC_NAME/preinstall.sh"
# Then execute the script to execute the pre-install commands.
LXC_EXEC "bash /preinstall.sh"
fi
@ -94,7 +94,7 @@ _PREUPGRADE () {
sed -i "s/\$PASSWORD/$YUNO_PWD/g" "$preupgrade_script"
sed -i "s/\$FROM_COMMIT/$commit/g" "$preupgrade_script"
# Copy the pre-upgrade script into the container.
lxc file push "$preupgrade_script" "$LXC_NAME/preupgrade.sh"
$lxc file push "$preupgrade_script" "$LXC_NAME/preupgrade.sh"
# Then execute the script to execute the pre-upgrade commands.
LXC_EXEC "bash /preupgrade.sh"
return $?
@ -357,7 +357,7 @@ Page extract:\n$page_extract" > $TEST_CONTEXT/curl_result
If you see this page, you have failed the test for alias_traversal issue.</body></html>" \
> $TEST_CONTEXT/alias_traversal.html
lxc file push $TEST_CONTEXT/alias_traversal.html $LXC_NAME/var/www/html/alias_traversal.html
$lxc file push $TEST_CONTEXT/alias_traversal.html $LXC_NAME/var/www/html/alias_traversal.html
curl --location --insecure --silent $check_domain$check_path../html/alias_traversal.html \
| grep "title" | grep --quiet "alias_traversal test" \
@ -557,7 +557,7 @@ TEST_PORT_ALREADY_USED () {
echo -e "[Service]\nExecStart=/bin/netcat -l -k -p $check_port\n
[Install]\nWantedBy=multi-user.target" > $TEST_CONTEXT/netcat.service
lxc file push $TEST_CONTEXT/netcat.service $LXC_NAME/etc/systemd/system/netcat.service
$lxc file push $TEST_CONTEXT/netcat.service $LXC_NAME/etc/systemd/system/netcat.service
# Then start this service to block this port.
LXC_EXEC "systemctl enable --now netcat"
@ -621,7 +621,7 @@ TEST_BACKUP_RESTORE () {
[ $ret -eq 0 ] || { main_result=1; break_before_continue; continue; }
# Grab the backup archive into the LXC container, and keep a copy
lxc file pull -r $LXC_NAME/home/yunohost.backup/archives $TEST_CONTEXT/ynh_backups
$lxc file pull -r $LXC_NAME/home/yunohost.backup/archives $TEST_CONTEXT/ynh_backups
# RESTORE
# Try the restore process in 2 times, first after removing the app, second after a restore of the container.
@ -646,7 +646,7 @@ TEST_BACKUP_RESTORE () {
RUN_INSIDE_LXC rm -rf /home/yunohost.backup/archives
# Place the copy of the backup archive in the container.
lxc file push -r $TEST_CONTEXT/ynh_backups/archives $LXC_NAME/home/yunohost.backup/
$lxc file push -r $TEST_CONTEXT/ynh_backups/archives $LXC_NAME/home/yunohost.backup/
_PREINSTALL

View file

@ -69,7 +69,7 @@ run_all_tests() {
cat $TEST_CONTEXT/tests/*.json >> /proc/self/fd/3
# Reset and create a fresh container to work with
check_lxd_setup
check_lxc_setup
LXC_RESET
LXC_CREATE
@ -213,7 +213,7 @@ break_before_continue () {
if [ $interactive -eq 1 ] || [ $interactive_on_errors -eq 1 ] && [ ! $test_result -eq 0 ]
then
echo "To enter a shell on the lxc:"
echo " lxc exec $LXC_NAME bash"
echo " $lxc exec $LXC_NAME bash"
read -p "Press a key to delete the application and continue...." < /dev/tty
fi
}

View file

@ -10,9 +10,9 @@ print_help() {
Usage: package_check.sh [OPTION]... PACKAGE_TO_CHECK
-b, --branch=BRANCH Specify a branch to check.
-a, --arch=ARCH
-d, --dist=DIST
-y, --ynh-branch=BRANCH
-a, --arch=ARCH
-d, --dist=DIST
-y, --ynh-branch=BRANCH
-D, --dry-run Show a JSON representing which tests are going to be ran (meant for debugging)
-i, --interactive Wait for the user to continue before each remove
-e, --interactive-on-errors Wait for the user to continue on errors
@ -22,6 +22,9 @@ print_help() {
images are supposed to be fetch from
devbaseimgs.yunohost.org automatically)
-h, --help Display this help
Pass YNHDEV_BACKEND=incus to use incus instead of lxd.
EOF
exit 0
}