Allow the use of incus instead of LXD. In the future, we could remove LXD support, making the README way shorter (no snap, etc)

lib/build_base_lxc.sh

@@ -2,48 +2,48 @@
 
 function launch_new_lxc()
 {
-    lxc info $LXC_BASE >/dev/null && lxc delete $LXC_BASE --force
+    $lxc info $LXC_BASE >/dev/null && $lxc delete $LXC_BASE --force
 
     if [ $(get_arch) = $ARCH ];
     then
-        lxc launch images:debian/$DIST/$ARCH $LXC_BASE -c security.privileged=true -c security.nesting=true
+        $lxc launch images:debian/$DIST/$ARCH $LXC_BASE -c security.privileged=true -c security.nesting=true
     else
-        lxc image info $LXC_BASE >/dev/null && lxc image delete $LXC_BASE
+        $lxc image info $LXC_BASE >/dev/null && $lxc image delete $LXC_BASE
 
         tmp_dir=$(mktemp -d)
         pushd $tmp_dir
 
-        lxc image export images:debian/$DIST/$ARCH
+        $lxc image export images:debian/$DIST/$ARCH
 
         tar xJf lxd.tar.xz
         local current_arch=$(get_arch)
         sed -i "0,/architecture: $ARCH/s//architecture: $current_arch/" metadata.yaml
         tar cJf lxd.tar.xz metadata.yaml templates
-        lxc image import lxd.tar.xz rootfs.squashfs --alias $LXC_BASE
+        $lxc image import lxd.tar.xz rootfs.squashfs --alias $LXC_BASE
         popd
         rm -rf "$tmp_dir"
 
-        lxc launch $LXC_BASE $LXC_BASE -c security.privileged=true -c security.nesting=true
+        $lxc launch $LXC_BASE $LXC_BASE -c security.privileged=true -c security.nesting=true
     fi
 }
 
 function rebuild_base_lxc()
 {
-    check_lxd_setup
+    check_lxc_setup
 
     launch_new_lxc
     sleep 5
-    
+
-    IN_LXC="lxc exec $LXC_BASE --"
+    IN_LXC="$lxc exec $LXC_BASE --"
-    
+
     INSTALL_SCRIPT="https://install.yunohost.org/$DIST"
     $IN_LXC apt install curl -y
     $IN_LXC /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $YNH_BRANCH"
-    
+
     $IN_LXC systemctl -q stop apt-daily.timer
     $IN_LXC systemctl -q stop apt-daily-upgrade.timer
     $IN_LXC systemctl -q stop apt-daily.service
-    $IN_LXC systemctl -q stop apt-daily-upgrade.service 
+    $IN_LXC systemctl -q stop apt-daily-upgrade.service
     $IN_LXC systemctl -q disable apt-daily.timer
     $IN_LXC systemctl -q disable apt-daily-upgrade.timer
     $IN_LXC systemctl -q disable apt-daily.service
@@ -63,8 +63,8 @@ function rebuild_base_lxc()
 
     $IN_LXC yunohost --version
 
-    lxc stop $LXC_BASE
+    $lxc stop $LXC_BASE
-    lxc image delete $LXC_BASE
+    $lxc image delete $LXC_BASE
-    lxc publish $LXC_BASE --alias $LXC_BASE --public
+    $lxc publish $LXC_BASE --alias $LXC_BASE --public
-    lxc delete $LXC_BASE
+    $lxc delete $LXC_BASE
 }

lib/common.sh

@@ -32,8 +32,14 @@ assert_we_are_connected_to_the_internets() {
 }
 
 assert_we_have_all_dependencies() {
-    for dep in "lxc" "lxd" "lynx" "jq" "python3" "pip3"
+    deps=("lynx" "jq" "python3" "pip3")
-    do
+    if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then
+        deps+=(incus)
+    else
+        deps+=(lxc lxd)
+    fi
+
+    for dep in "${deps[@]}"; do
         which $dep 2>&1 > /dev/null || log_critical "Please install $dep"
     done
 }
@@ -55,6 +61,39 @@ function check_lxd_setup()
         || log_critical "There is no 'lxdbr0' interface... Did you ran 'lxd init' ?"
 }
 
+function check_incus_setup()
+{
+    # Check incus is installed somehow
+    if ! which incus &>/dev/null; then
+        critical "You need to have Incus installed for ynh-dev to be usable from the host machine. Refer to the README to know how to install it."
+    fi
+    if ! id -nG "$(whoami)" | grep -qw "incus-admin"; then
+        critical "You need to be in the incus-admin group!"
+    fi
+
+    ip a | grep -q incusbr0 \
+        || warn "There is no 'incusbr0' interface... Did you ran 'incus admin init' ?"
+
+    set_incus_remote
+}
+
+function set_incus_remote()
+{
+    configured=$(incus remote list -f json | jq 'has("yunohost")')
+    if [[ "$configured" != "true" ]]; then
+        incus remote add yunohost https://devbaseimgs.yunohost.org --public
+    fi
+}
+
+function check_lxc_setup()
+{
+    if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then
+        check_incus_setup
+    else
+        check_lxd_setup
+    fi
+}
+
 #=================================================
 # Logging helpers
 #=================================================

lib/lxc.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 
+# Check for LXC or Incus
+function switch_lxc_incus()
+{
+    if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then
+        lxc=incus
+    else
+        lxc=lxc
+    fi
+}
+
+switch_lxc_incus
+
 #=================================================
 # RUNNING SNAPSHOT
 #=================================================
@@ -7,19 +19,19 @@
 LXC_CREATE () {
     log_info "Launching new LXC $LXC_NAME ..."
     # Check if we can launch container from YunoHost remote image
-    if lxc remote list | grep -q "yunohost" && lxc image list yunohost:$LXC_BASE | grep -q -w $LXC_BASE; then
+    if $lxc remote list | grep -q "yunohost" && $lxc image list yunohost:$LXC_BASE | grep -q -w $LXC_BASE; then
         # Force the usage of the fingerprint because otherwise for some reason lxd won't use the newer version
         # available even though it's aware it exists -_-
-        LXC_BASE_HASH="$(lxc image list yunohost:$LXC_BASE --format json | jq -r '.[].fingerprint')"
+        LXC_BASE_HASH="$($lxc image list yunohost:$LXC_BASE --format json | jq -r '.[].fingerprint')"
-        lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME \
+        $lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME \
             -c security.nesting=true \
             -c security.privileged=true \
             -c limits.memory=80% \
             -c limits.cpu.allowance=80% \
             >>/proc/self/fd/3
     # Check if we can launch container from a local image
-    elif lxc image list $LXC_BASE | grep -q -w $LXC_BASE; then
+    elif $lxc image list $LXC_BASE | grep -q -w $LXC_BASE; then
-        lxc launch $LXC_BASE $LXC_NAME \
+        $lxc launch $LXC_BASE $LXC_NAME \
             -c security.nesting=true \
             -c security.privileged=true \
             -c limits.memory=80% \
@@ -30,12 +42,12 @@ LXC_CREATE () {
     fi
 
     pipestatus="${PIPESTATUS[0]}"
-    location=$(lxc list --format json | jq -e --arg LXC_NAME $LXC_NAME '.[] | select(.name==$LXC_NAME) | .location' | tr -d '"')
+    location=$($lxc list --format json | jq -e --arg LXC_NAME $LXC_NAME '.[] | select(.name==$LXC_NAME) | .location' | tr -d '"')
     [[ "$location" != "none" ]] && log_info "... on $location"
 
     [[ "$pipestatus" -eq 0 ]] || exit 1
 
-    if [[ "$(lxc list $LXC_NAME --format json)" == "[]" ]]
+    if [[ "$($lxc list $LXC_NAME --format json)" == "[]" ]]
     then
         log_critical "Failed to create the new LXC :/"
     fi
@@ -43,7 +55,7 @@ LXC_CREATE () {
     _LXC_START_AND_WAIT $LXC_NAME
     sleep 3
 
-    if ! lxc exec $LXC_NAME -- test -e /etc/yunohost
+    if ! $lxc exec $LXC_NAME -- test -e /etc/yunohost
     then
          log_critical "Failed to run 'test -e /etc/yunohost' on the container ... either the container did not start, or YunoHost doesn't exists yet in the container :/"
     fi
@@ -52,9 +64,9 @@ LXC_CREATE () {
 
     sleep 3
     log_info "Creating initial snapshot $LXC_NAME ..."
-    lxc snapshot $LXC_NAME snap0
+    $lxc snapshot create $LXC_NAME snap0
 
-    if [[ -z "$(lxc list $LXC_NAME --format json | jq '.[].snapshots[] | select(.name=="snap0")')" ]]
+    if [[ -z "$($lxc list $LXC_NAME --format json | jq '.[].snapshots[] | select(.name=="snap0")')" ]]
     then
         log_critical "Failed to create the initial snapshot :/"
     fi
@@ -62,7 +74,7 @@ LXC_CREATE () {
 
 LXC_SNAPSHOT_EXISTS() {
     local snapname=$1
-    lxc list --format json \
+    $lxc list --format json \
         | jq -e --arg LXC_NAME $LXC_NAME --arg snapname $snapname \
         '.[] | select(.name==$LXC_NAME) | .snapshots[] | select(.name==$snapname)' \
             >/dev/null
@@ -87,7 +99,7 @@ CREATE_LXC_SNAPSHOT () {
     if ! LXC_SNAPSHOT_EXISTS "$snapname"
     then
         log_info "(Creating snapshot $snapname ...)"
-        lxc snapshot $LXC_NAME $snapname
+        $lxc snapshot create $LXC_NAME $snapname
     fi
 
     _LXC_START_AND_WAIT $LXC_NAME
@@ -106,7 +118,7 @@ LOAD_LXC_SNAPSHOT () {
     while [[ ${retry_lxc} -lt 10 ]]
     do
         LXC_STOP $LXC_NAME || true
-        lxc restore $LXC_NAME $snapname && break || retry_lxc=$(($retry_lxc+1))
+        $lxc snapshot restore $LXC_NAME $snapname && break || retry_lxc=$(($retry_lxc+1))
         log_warning "Failed to restore snapshot? Retrying in 20 sec ..."
         if [[ ${retry_lxc} -ge 3 ]]
         then
@@ -120,7 +132,7 @@ LOAD_LXC_SNAPSHOT () {
         log_error "Failed to restore snapshot ? The next step may miserably crash because of this ... if this happens to often, maybe restarting the LXD daemon can help ..."
     fi
 
-    lxc start $LXC_NAME
+    $lxc start $LXC_NAME
     _LXC_START_AND_WAIT $LXC_NAME
 }
 
@@ -135,7 +147,7 @@ LXC_EXEC () {
     start_timer
 
     # Execute the command given in argument in the container and log its results.
-    lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- /bin/bash -c "$cmd" | tee -a "$full_log" $current_test_log
+    $lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- /bin/bash -c "$cmd" | tee -a "$full_log" $current_test_log
 
     # Store the return code of the command
     local returncode=${PIPESTATUS[0]}
@@ -151,42 +163,42 @@ LXC_STOP () {
     local container_to_stop=$1
     # (We also use timeout 30 in front of the command because sometime lxc
     # commands can hang forever despite the --timeout >_>...)
-    timeout 30 lxc stop --timeout 15 $container_to_stop 2>/dev/null
+    timeout 30 $lxc stop --timeout 15 $container_to_stop 2>/dev/null
 
     local retry_stop_lxc=0
     while [[ ${retry_stop_lxc} -lt 5 ]]
     do
-        local status="$(lxc list $container_to_stop --format json | jq -r '.[].state.status')"
+        local status="$($lxc list $container_to_stop --format json | jq -r '.[].state.status')"
         if [[ -z "$status" ]] || [[ "$status" == "Stopped" ]] || [[ "$status" == "null" ]]
         then
             break
         fi
         retry_stop_lxc="$(($retry_stop_lxc+1))"
         sleep 10
-        timeout 30 lxc stop --timeout 15 $container_to_stop 2>/dev/null
+        timeout 30 $lxc stop --timeout 15 $container_to_stop 2>/dev/null
     done
 
     if [[ ${retry_stop_lxc} -ge 5 ]]
     then
-        timeout 30 lxc stop --timeout 15 $container_to_stop --force 2>/dev/null
+        timeout 30 $lxc stop --timeout 15 $container_to_stop --force 2>/dev/null
     fi
 }
 
 LXC_RESET () {
     # If the container exists
-    if lxc info $LXC_NAME >/dev/null 2>/dev/null; then
+    if $lxc info $LXC_NAME >/dev/null 2>/dev/null; then
         # Remove swap files before deletting the continer
         CLEAN_SWAPFILES
     fi
 
     LXC_STOP $LXC_NAME
 
-    if lxc info $LXC_NAME >/dev/null 2>/dev/null; then
+    if $lxc info $LXC_NAME >/dev/null 2>/dev/null; then
-        local current_storage=$(lxc list $LXC_NAME --format json --columns b | jq -r '.[].expanded_devices.root.pool')
+        local current_storage=$($lxc list $LXC_NAME --format json --columns b | jq -r '.[].expanded_devices.root.pool')
-        swapoff "$(lxc storage get $current_storage source)/containers/$LXC_NAME/rootfs/swap" 2>/dev/null
+        swapoff "$($lxc storage get $current_storage source)/containers/$LXC_NAME/rootfs/swap" 2>/dev/null
     fi
 
-    lxc delete $LXC_NAME --force 2>/dev/null
+    $lxc delete $LXC_NAME --force 2>/dev/null
 }
 
 
@@ -195,7 +207,7 @@ _LXC_START_AND_WAIT() {
     restart_container()
 	{
         LXC_STOP $1
-		lxc start "$1"
+		$lxc start "$1"
 	}
 
 	# Try to start the container 3 times.
@@ -209,7 +221,7 @@ _LXC_START_AND_WAIT() {
 		# Wait for container to start, we are using systemd to check this,
 		# for the sake of brevity.
 		for j in $(seq 1 10); do
-			if lxc exec "$1" -- timeout 30 systemctl isolate multi-user.target >/dev/null 2>/dev/null; then
+			if $lxc exec "$1" -- timeout 30 systemctl isolate multi-user.target >/dev/null 2>/dev/null; then
 				break
 			fi
 
@@ -225,7 +237,7 @@ _LXC_START_AND_WAIT() {
 
 		# Wait for container to access the internet
 		for j in $(seq 1 10); do
-			if lxc exec "$1" -- timeout 10 curl -s http://wikipedia.org > /dev/null 2>/dev/null; then
+			if $lxc exec "$1" -- timeout 10 curl -s http://wikipedia.org > /dev/null 2>/dev/null; then
 				break
 			fi
 
@@ -249,26 +261,26 @@ _LXC_START_AND_WAIT() {
 		if [ $i -eq $max_try ] && [ $failstart -eq 1 ]
 		then
             log_error "The container miserably failed to start or to connect to the internet"
-            lxc info --show-log $1
+            $lxc info --show-log $1
 			return 1
 		fi
 	done
 
     sleep 3
 
-    LXC_IP=$(lxc exec $1 -- hostname -I | cut -d' ' -f1 | grep -E -o "\<[0-9.]{8,}\>")
+    LXC_IP=$($lxc exec $1 -- hostname -I | cut -d' ' -f1 | grep -E -o "\<[0-9.]{8,}\>")
 }
 
 CLEAN_SWAPFILES() {
     # Restart it if needed
-    if [ "$(lxc info $LXC_NAME | grep Status | awk '{print tolower($2)}')" != "running" ]; then
+    if [ "$($lxc info $LXC_NAME | grep Status | awk '{print tolower($2)}')" != "running" ]; then
-        lxc start $LXC_NAME
+        $lxc start $LXC_NAME
         _LXC_START_AND_WAIT $LXC_NAME
     fi
-    lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do swapoff $swapfile; done'
+    $lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do swapoff $swapfile; done'
-    lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do rm -f $swapfile; done'
+    $lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do rm -f $swapfile; done'
 }
 
 RUN_INSIDE_LXC() {
-    lxc exec $LXC_NAME -- "$@"
+    $lxc exec $LXC_NAME -- "$@"
 }

lib/tests.sh

@@ -8,7 +8,7 @@ _STUFF_TO_RUN_BEFORE_INITIAL_SNAPSHOT()
 {
     # Print the version of YunoHost from the LXC container
     log_small_title "YunoHost versions"
-    lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost --version" | tee -a "$full_log"
+    $lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost --version" | tee -a "$full_log"
 
     log_title "Package linter"
     ./package_linter/package_linter.py "$package_path" | tee -a "$full_log"
@@ -26,13 +26,13 @@ _STUFF_TO_RUN_BEFORE_INITIAL_SNAPSHOT()
         log_title "Preinstalling apt dependencies before creating the initial snapshot..."
 
         apt="LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0"
-        lxc exec $LXC_NAME -t -- /bin/bash -c "$apt update; $apt install $apt_deps" | tee -a "$full_log" >/dev/null
+        $lxc exec $LXC_NAME -t -- /bin/bash -c "$apt update; $apt install $apt_deps" | tee -a "$full_log" >/dev/null
     fi
 
     # Gotta generate the psql password even though apparently it's not even useful anymore these days but it otherwise trigger warnings ~_~
     if echo "$apt_deps" | grep -q postgresql
     then
-        lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost tools regen-conf postgresql" | tee -a "$full_log" >/dev/null
+        $lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost tools regen-conf postgresql" | tee -a "$full_log" >/dev/null
     fi
 }
 
@@ -42,8 +42,8 @@ _RUN_YUNOHOST_CMD() {
     log_debug "Running yunohost $1"
 
     # Copy the package into the container.
-    lxc exec $LXC_NAME -- rm -rf /app_folder
+    $lxc exec $LXC_NAME -- rm -rf /app_folder
-    lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet
+    $lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet
 
     # --output-as none is to disable the json-like output for some commands like backup create
     LXC_EXEC "yunohost --output-as none --debug $1" \
@@ -70,7 +70,7 @@ _PREINSTALL () {
         sed -i "s/\$SUBDOMAIN/$SUBDOMAIN/g" "$preinstall_script"
         sed -i "s/\$PASSWORD/$YUNO_PWD/g" "$preinstall_script"
         # Copy the pre-install script into the container.
-        lxc file push "$preinstall_script" "$LXC_NAME/preinstall.sh"
+        $lxc file push "$preinstall_script" "$LXC_NAME/preinstall.sh"
         # Then execute the script to execute the pre-install commands.
         LXC_EXEC "bash /preinstall.sh"
     fi
@@ -94,7 +94,7 @@ _PREUPGRADE () {
         sed -i "s/\$PASSWORD/$YUNO_PWD/g" "$preupgrade_script"
         sed -i "s/\$FROM_COMMIT/$commit/g" "$preupgrade_script"
         # Copy the pre-upgrade script into the container.
-        lxc file push "$preupgrade_script" "$LXC_NAME/preupgrade.sh"
+        $lxc file push "$preupgrade_script" "$LXC_NAME/preupgrade.sh"
         # Then execute the script to execute the pre-upgrade commands.
         LXC_EXEC "bash /preupgrade.sh"
         return $?
@@ -357,7 +357,7 @@ Page extract:\n$page_extract" > $TEST_CONTEXT/curl_result
     If you see this page, you have failed the test for alias_traversal issue.</body></html>" \
     > $TEST_CONTEXT/alias_traversal.html
 
-    lxc file push $TEST_CONTEXT/alias_traversal.html $LXC_NAME/var/www/html/alias_traversal.html
+    $lxc file push $TEST_CONTEXT/alias_traversal.html $LXC_NAME/var/www/html/alias_traversal.html
 
     curl --location --insecure --silent $check_domain$check_path../html/alias_traversal.html \
         | grep "title" | grep --quiet "alias_traversal test" \
@@ -557,7 +557,7 @@ TEST_PORT_ALREADY_USED () {
     echo -e "[Service]\nExecStart=/bin/netcat -l -k -p $check_port\n
     [Install]\nWantedBy=multi-user.target" > $TEST_CONTEXT/netcat.service
 
-    lxc file push $TEST_CONTEXT/netcat.service $LXC_NAME/etc/systemd/system/netcat.service
+    $lxc file push $TEST_CONTEXT/netcat.service $LXC_NAME/etc/systemd/system/netcat.service
 
     # Then start this service to block this port.
     LXC_EXEC "systemctl enable --now netcat"
@@ -621,7 +621,7 @@ TEST_BACKUP_RESTORE () {
         [ $ret -eq 0 ] || { main_result=1; break_before_continue; continue; }
 
         # Grab the backup archive into the LXC container, and keep a copy
-        lxc file pull -r $LXC_NAME/home/yunohost.backup/archives $TEST_CONTEXT/ynh_backups
+        $lxc file pull -r $LXC_NAME/home/yunohost.backup/archives $TEST_CONTEXT/ynh_backups
 
         # RESTORE
         # Try the restore process in 2 times, first after removing the app, second after a restore of the container.
@@ -646,7 +646,7 @@ TEST_BACKUP_RESTORE () {
                 RUN_INSIDE_LXC rm -rf /home/yunohost.backup/archives
 
                 # Place the copy of the backup archive in the container.
-                lxc file push -r $TEST_CONTEXT/ynh_backups/archives $LXC_NAME/home/yunohost.backup/
+                $lxc file push -r $TEST_CONTEXT/ynh_backups/archives $LXC_NAME/home/yunohost.backup/
 
                 _PREINSTALL
 

lib/tests_coordination.sh

@@ -69,7 +69,7 @@ run_all_tests() {
     cat $TEST_CONTEXT/tests/*.json >> /proc/self/fd/3
 
     # Reset and create a fresh container to work with
-    check_lxd_setup
+    check_lxc_setup
     LXC_RESET
     LXC_CREATE
 
@@ -213,7 +213,7 @@ break_before_continue () {
     if [ $interactive -eq 1 ] || [ $interactive_on_errors -eq 1 ] && [ ! $test_result -eq 0 ]
     then
         echo "To enter a shell on the lxc:"
-        echo "     lxc exec $LXC_NAME bash"
+        echo "     $lxc exec $LXC_NAME bash"
         read -p "Press a key to delete the application and continue...." < /dev/tty
     fi
 }

package_check.sh

@@ -10,9 +10,9 @@ print_help() {
  Usage: package_check.sh [OPTION]... PACKAGE_TO_CHECK
 
     -b, --branch=BRANCH     Specify a branch to check.
-    -a, --arch=ARCH         
+    -a, --arch=ARCH
-    -d, --dist=DIST         
+    -d, --dist=DIST
-    -y, --ynh-branch=BRANCH 
+    -y, --ynh-branch=BRANCH
     -D, --dry-run               Show a JSON representing which tests are going to be ran (meant for debugging)
     -i, --interactive           Wait for the user to continue before each remove
     -e, --interactive-on-errors Wait for the user to continue on errors
@@ -22,6 +22,9 @@ print_help() {
                                 images are supposed to be fetch from
                                 devbaseimgs.yunohost.org automatically)
     -h, --help                  Display this help
+
+    Pass YNHDEV_BACKEND=incus to use incus instead of lxd.
+
 EOF
 exit 0
 }