diff --git a/lib/build_base_lxc.sh b/lib/build_base_lxc.sh index c3ad172..2822afc 100644 --- a/lib/build_base_lxc.sh +++ b/lib/build_base_lxc.sh @@ -2,48 +2,48 @@ function launch_new_lxc() { - lxc info $LXC_BASE >/dev/null && lxc delete $LXC_BASE --force + $lxc info $LXC_BASE >/dev/null && $lxc delete $LXC_BASE --force if [ $(get_arch) = $ARCH ]; then - lxc launch images:debian/$DIST/$ARCH $LXC_BASE -c security.privileged=true -c security.nesting=true + $lxc launch images:debian/$DIST/$ARCH $LXC_BASE -c security.privileged=true -c security.nesting=true else - lxc image info $LXC_BASE >/dev/null && lxc image delete $LXC_BASE + $lxc image info $LXC_BASE >/dev/null && $lxc image delete $LXC_BASE tmp_dir=$(mktemp -d) pushd $tmp_dir - lxc image export images:debian/$DIST/$ARCH + $lxc image export images:debian/$DIST/$ARCH tar xJf lxd.tar.xz local current_arch=$(get_arch) sed -i "0,/architecture: $ARCH/s//architecture: $current_arch/" metadata.yaml tar cJf lxd.tar.xz metadata.yaml templates - lxc image import lxd.tar.xz rootfs.squashfs --alias $LXC_BASE + $lxc image import lxd.tar.xz rootfs.squashfs --alias $LXC_BASE popd rm -rf "$tmp_dir" - lxc launch $LXC_BASE $LXC_BASE -c security.privileged=true -c security.nesting=true + $lxc launch $LXC_BASE $LXC_BASE -c security.privileged=true -c security.nesting=true fi } function rebuild_base_lxc() { - check_lxd_setup + check_lxc_setup launch_new_lxc sleep 5 - - IN_LXC="lxc exec $LXC_BASE --" - + + IN_LXC="$lxc exec $LXC_BASE --" + INSTALL_SCRIPT="https://install.yunohost.org/$DIST" $IN_LXC apt install curl -y $IN_LXC /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $YNH_BRANCH" - + $IN_LXC systemctl -q stop apt-daily.timer $IN_LXC systemctl -q stop apt-daily-upgrade.timer $IN_LXC systemctl -q stop apt-daily.service - $IN_LXC systemctl -q stop apt-daily-upgrade.service + $IN_LXC systemctl -q stop apt-daily-upgrade.service $IN_LXC systemctl -q disable apt-daily.timer $IN_LXC systemctl -q disable apt-daily-upgrade.timer $IN_LXC systemctl -q disable apt-daily.service @@ -63,8 +63,8 @@ function rebuild_base_lxc() $IN_LXC yunohost --version - lxc stop $LXC_BASE - lxc image delete $LXC_BASE - lxc publish $LXC_BASE --alias $LXC_BASE --public - lxc delete $LXC_BASE + $lxc stop $LXC_BASE + $lxc image delete $LXC_BASE + $lxc publish $LXC_BASE --alias $LXC_BASE --public + $lxc delete $LXC_BASE } diff --git a/lib/common.sh b/lib/common.sh index 0d00e89..3118516 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -32,8 +32,14 @@ assert_we_are_connected_to_the_internets() { } assert_we_have_all_dependencies() { - for dep in "lxc" "lxd" "lynx" "jq" "python3" "pip3" - do + deps=("lynx" "jq" "python3" "pip3") + if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then + deps+=(incus) + else + deps+=(lxc lxd) + fi + + for dep in "${deps[@]}"; do which $dep 2>&1 > /dev/null || log_critical "Please install $dep" done } @@ -55,6 +61,39 @@ function check_lxd_setup() || log_critical "There is no 'lxdbr0' interface... Did you ran 'lxd init' ?" } +function check_incus_setup() +{ + # Check incus is installed somehow + if ! which incus &>/dev/null; then + critical "You need to have Incus installed for ynh-dev to be usable from the host machine. Refer to the README to know how to install it." + fi + if ! id -nG "$(whoami)" | grep -qw "incus-admin"; then + critical "You need to be in the incus-admin group!" + fi + + ip a | grep -q incusbr0 \ + || warn "There is no 'incusbr0' interface... Did you ran 'incus admin init' ?" + + set_incus_remote +} + +function set_incus_remote() +{ + configured=$(incus remote list -f json | jq 'has("yunohost")') + if [[ "$configured" != "true" ]]; then + incus remote add yunohost https://devbaseimgs.yunohost.org --public + fi +} + +function check_lxc_setup() +{ + if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then + check_incus_setup + else + check_lxd_setup + fi +} + #================================================= # Logging helpers #================================================= diff --git a/lib/lxc.sh b/lib/lxc.sh index 0595eef..4493f76 100644 --- a/lib/lxc.sh +++ b/lib/lxc.sh @@ -1,5 +1,17 @@ #!/bin/bash +# Check for LXC or Incus +function switch_lxc_incus() +{ + if [[ "${YNHDEV_BACKEND:-}" == "incus" ]]; then + lxc=incus + else + lxc=lxc + fi +} + +switch_lxc_incus + #================================================= # RUNNING SNAPSHOT #================================================= @@ -7,19 +19,19 @@ LXC_CREATE () { log_info "Launching new LXC $LXC_NAME ..." # Check if we can launch container from YunoHost remote image - if lxc remote list | grep -q "yunohost" && lxc image list yunohost:$LXC_BASE | grep -q -w $LXC_BASE; then + if $lxc remote list | grep -q "yunohost" && $lxc image list yunohost:$LXC_BASE | grep -q -w $LXC_BASE; then # Force the usage of the fingerprint because otherwise for some reason lxd won't use the newer version # available even though it's aware it exists -_- - LXC_BASE_HASH="$(lxc image list yunohost:$LXC_BASE --format json | jq -r '.[].fingerprint')" - lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME \ + LXC_BASE_HASH="$($lxc image list yunohost:$LXC_BASE --format json | jq -r '.[].fingerprint')" + $lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME \ -c security.nesting=true \ -c security.privileged=true \ -c limits.memory=80% \ -c limits.cpu.allowance=80% \ >>/proc/self/fd/3 # Check if we can launch container from a local image - elif lxc image list $LXC_BASE | grep -q -w $LXC_BASE; then - lxc launch $LXC_BASE $LXC_NAME \ + elif $lxc image list $LXC_BASE | grep -q -w $LXC_BASE; then + $lxc launch $LXC_BASE $LXC_NAME \ -c security.nesting=true \ -c security.privileged=true \ -c limits.memory=80% \ @@ -30,12 +42,12 @@ LXC_CREATE () { fi pipestatus="${PIPESTATUS[0]}" - location=$(lxc list --format json | jq -e --arg LXC_NAME $LXC_NAME '.[] | select(.name==$LXC_NAME) | .location' | tr -d '"') + location=$($lxc list --format json | jq -e --arg LXC_NAME $LXC_NAME '.[] | select(.name==$LXC_NAME) | .location' | tr -d '"') [[ "$location" != "none" ]] && log_info "... on $location" [[ "$pipestatus" -eq 0 ]] || exit 1 - if [[ "$(lxc list $LXC_NAME --format json)" == "[]" ]] + if [[ "$($lxc list $LXC_NAME --format json)" == "[]" ]] then log_critical "Failed to create the new LXC :/" fi @@ -43,7 +55,7 @@ LXC_CREATE () { _LXC_START_AND_WAIT $LXC_NAME sleep 3 - if ! lxc exec $LXC_NAME -- test -e /etc/yunohost + if ! $lxc exec $LXC_NAME -- test -e /etc/yunohost then log_critical "Failed to run 'test -e /etc/yunohost' on the container ... either the container did not start, or YunoHost doesn't exists yet in the container :/" fi @@ -52,9 +64,9 @@ LXC_CREATE () { sleep 3 log_info "Creating initial snapshot $LXC_NAME ..." - lxc snapshot $LXC_NAME snap0 + $lxc snapshot create $LXC_NAME snap0 - if [[ -z "$(lxc list $LXC_NAME --format json | jq '.[].snapshots[] | select(.name=="snap0")')" ]] + if [[ -z "$($lxc list $LXC_NAME --format json | jq '.[].snapshots[] | select(.name=="snap0")')" ]] then log_critical "Failed to create the initial snapshot :/" fi @@ -62,7 +74,7 @@ LXC_CREATE () { LXC_SNAPSHOT_EXISTS() { local snapname=$1 - lxc list --format json \ + $lxc list --format json \ | jq -e --arg LXC_NAME $LXC_NAME --arg snapname $snapname \ '.[] | select(.name==$LXC_NAME) | .snapshots[] | select(.name==$snapname)' \ >/dev/null @@ -87,7 +99,7 @@ CREATE_LXC_SNAPSHOT () { if ! LXC_SNAPSHOT_EXISTS "$snapname" then log_info "(Creating snapshot $snapname ...)" - lxc snapshot $LXC_NAME $snapname + $lxc snapshot create $LXC_NAME $snapname fi _LXC_START_AND_WAIT $LXC_NAME @@ -106,7 +118,7 @@ LOAD_LXC_SNAPSHOT () { while [[ ${retry_lxc} -lt 10 ]] do LXC_STOP $LXC_NAME || true - lxc restore $LXC_NAME $snapname && break || retry_lxc=$(($retry_lxc+1)) + $lxc snapshot restore $LXC_NAME $snapname && break || retry_lxc=$(($retry_lxc+1)) log_warning "Failed to restore snapshot? Retrying in 20 sec ..." if [[ ${retry_lxc} -ge 3 ]] then @@ -120,7 +132,7 @@ LOAD_LXC_SNAPSHOT () { log_error "Failed to restore snapshot ? The next step may miserably crash because of this ... if this happens to often, maybe restarting the LXD daemon can help ..." fi - lxc start $LXC_NAME + $lxc start $LXC_NAME _LXC_START_AND_WAIT $LXC_NAME } @@ -135,7 +147,7 @@ LXC_EXEC () { start_timer # Execute the command given in argument in the container and log its results. - lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- /bin/bash -c "$cmd" | tee -a "$full_log" $current_test_log + $lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- /bin/bash -c "$cmd" | tee -a "$full_log" $current_test_log # Store the return code of the command local returncode=${PIPESTATUS[0]} @@ -151,42 +163,42 @@ LXC_STOP () { local container_to_stop=$1 # (We also use timeout 30 in front of the command because sometime lxc # commands can hang forever despite the --timeout >_>...) - timeout 30 lxc stop --timeout 15 $container_to_stop 2>/dev/null + timeout 30 $lxc stop --timeout 15 $container_to_stop 2>/dev/null local retry_stop_lxc=0 while [[ ${retry_stop_lxc} -lt 5 ]] do - local status="$(lxc list $container_to_stop --format json | jq -r '.[].state.status')" + local status="$($lxc list $container_to_stop --format json | jq -r '.[].state.status')" if [[ -z "$status" ]] || [[ "$status" == "Stopped" ]] || [[ "$status" == "null" ]] then break fi retry_stop_lxc="$(($retry_stop_lxc+1))" sleep 10 - timeout 30 lxc stop --timeout 15 $container_to_stop 2>/dev/null + timeout 30 $lxc stop --timeout 15 $container_to_stop 2>/dev/null done if [[ ${retry_stop_lxc} -ge 5 ]] then - timeout 30 lxc stop --timeout 15 $container_to_stop --force 2>/dev/null + timeout 30 $lxc stop --timeout 15 $container_to_stop --force 2>/dev/null fi } LXC_RESET () { # If the container exists - if lxc info $LXC_NAME >/dev/null 2>/dev/null; then + if $lxc info $LXC_NAME >/dev/null 2>/dev/null; then # Remove swap files before deletting the continer CLEAN_SWAPFILES fi LXC_STOP $LXC_NAME - if lxc info $LXC_NAME >/dev/null 2>/dev/null; then - local current_storage=$(lxc list $LXC_NAME --format json --columns b | jq -r '.[].expanded_devices.root.pool') - swapoff "$(lxc storage get $current_storage source)/containers/$LXC_NAME/rootfs/swap" 2>/dev/null + if $lxc info $LXC_NAME >/dev/null 2>/dev/null; then + local current_storage=$($lxc list $LXC_NAME --format json --columns b | jq -r '.[].expanded_devices.root.pool') + swapoff "$($lxc storage get $current_storage source)/containers/$LXC_NAME/rootfs/swap" 2>/dev/null fi - lxc delete $LXC_NAME --force 2>/dev/null + $lxc delete $LXC_NAME --force 2>/dev/null } @@ -195,7 +207,7 @@ _LXC_START_AND_WAIT() { restart_container() { LXC_STOP $1 - lxc start "$1" + $lxc start "$1" } # Try to start the container 3 times. @@ -209,7 +221,7 @@ _LXC_START_AND_WAIT() { # Wait for container to start, we are using systemd to check this, # for the sake of brevity. for j in $(seq 1 10); do - if lxc exec "$1" -- timeout 30 systemctl isolate multi-user.target >/dev/null 2>/dev/null; then + if $lxc exec "$1" -- timeout 30 systemctl isolate multi-user.target >/dev/null 2>/dev/null; then break fi @@ -225,7 +237,7 @@ _LXC_START_AND_WAIT() { # Wait for container to access the internet for j in $(seq 1 10); do - if lxc exec "$1" -- timeout 10 curl -s http://wikipedia.org > /dev/null 2>/dev/null; then + if $lxc exec "$1" -- timeout 10 curl -s http://wikipedia.org > /dev/null 2>/dev/null; then break fi @@ -249,26 +261,26 @@ _LXC_START_AND_WAIT() { if [ $i -eq $max_try ] && [ $failstart -eq 1 ] then log_error "The container miserably failed to start or to connect to the internet" - lxc info --show-log $1 + $lxc info --show-log $1 return 1 fi done sleep 3 - LXC_IP=$(lxc exec $1 -- hostname -I | cut -d' ' -f1 | grep -E -o "\<[0-9.]{8,}\>") + LXC_IP=$($lxc exec $1 -- hostname -I | cut -d' ' -f1 | grep -E -o "\<[0-9.]{8,}\>") } CLEAN_SWAPFILES() { # Restart it if needed - if [ "$(lxc info $LXC_NAME | grep Status | awk '{print tolower($2)}')" != "running" ]; then - lxc start $LXC_NAME + if [ "$($lxc info $LXC_NAME | grep Status | awk '{print tolower($2)}')" != "running" ]; then + $lxc start $LXC_NAME _LXC_START_AND_WAIT $LXC_NAME fi - lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do swapoff $swapfile; done' - lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do rm -f $swapfile; done' + $lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do swapoff $swapfile; done' + $lxc exec $LXC_NAME -- bash -c 'for swapfile in $(ls /swap_* 2>/dev/null); do rm -f $swapfile; done' } RUN_INSIDE_LXC() { - lxc exec $LXC_NAME -- "$@" + $lxc exec $LXC_NAME -- "$@" } diff --git a/lib/tests.sh b/lib/tests.sh index 872e475..37dd546 100644 --- a/lib/tests.sh +++ b/lib/tests.sh @@ -8,7 +8,7 @@ _STUFF_TO_RUN_BEFORE_INITIAL_SNAPSHOT() { # Print the version of YunoHost from the LXC container log_small_title "YunoHost versions" - lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost --version" | tee -a "$full_log" + $lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost --version" | tee -a "$full_log" log_title "Package linter" ./package_linter/package_linter.py "$package_path" | tee -a "$full_log" @@ -26,13 +26,13 @@ _STUFF_TO_RUN_BEFORE_INITIAL_SNAPSHOT() log_title "Preinstalling apt dependencies before creating the initial snapshot..." apt="LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0" - lxc exec $LXC_NAME -t -- /bin/bash -c "$apt update; $apt install $apt_deps" | tee -a "$full_log" >/dev/null + $lxc exec $LXC_NAME -t -- /bin/bash -c "$apt update; $apt install $apt_deps" | tee -a "$full_log" >/dev/null fi # Gotta generate the psql password even though apparently it's not even useful anymore these days but it otherwise trigger warnings ~_~ if echo "$apt_deps" | grep -q postgresql then - lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost tools regen-conf postgresql" | tee -a "$full_log" >/dev/null + $lxc exec $LXC_NAME -t -- /bin/bash -c "yunohost tools regen-conf postgresql" | tee -a "$full_log" >/dev/null fi } @@ -42,8 +42,8 @@ _RUN_YUNOHOST_CMD() { log_debug "Running yunohost $1" # Copy the package into the container. - lxc exec $LXC_NAME -- rm -rf /app_folder - lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet + $lxc exec $LXC_NAME -- rm -rf /app_folder + $lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet # --output-as none is to disable the json-like output for some commands like backup create LXC_EXEC "yunohost --output-as none --debug $1" \ @@ -70,7 +70,7 @@ _PREINSTALL () { sed -i "s/\$SUBDOMAIN/$SUBDOMAIN/g" "$preinstall_script" sed -i "s/\$PASSWORD/$YUNO_PWD/g" "$preinstall_script" # Copy the pre-install script into the container. - lxc file push "$preinstall_script" "$LXC_NAME/preinstall.sh" + $lxc file push "$preinstall_script" "$LXC_NAME/preinstall.sh" # Then execute the script to execute the pre-install commands. LXC_EXEC "bash /preinstall.sh" fi @@ -94,7 +94,7 @@ _PREUPGRADE () { sed -i "s/\$PASSWORD/$YUNO_PWD/g" "$preupgrade_script" sed -i "s/\$FROM_COMMIT/$commit/g" "$preupgrade_script" # Copy the pre-upgrade script into the container. - lxc file push "$preupgrade_script" "$LXC_NAME/preupgrade.sh" + $lxc file push "$preupgrade_script" "$LXC_NAME/preupgrade.sh" # Then execute the script to execute the pre-upgrade commands. LXC_EXEC "bash /preupgrade.sh" return $? @@ -357,7 +357,7 @@ Page extract:\n$page_extract" > $TEST_CONTEXT/curl_result If you see this page, you have failed the test for alias_traversal issue." \ > $TEST_CONTEXT/alias_traversal.html - lxc file push $TEST_CONTEXT/alias_traversal.html $LXC_NAME/var/www/html/alias_traversal.html + $lxc file push $TEST_CONTEXT/alias_traversal.html $LXC_NAME/var/www/html/alias_traversal.html curl --location --insecure --silent $check_domain$check_path../html/alias_traversal.html \ | grep "title" | grep --quiet "alias_traversal test" \ @@ -557,7 +557,7 @@ TEST_PORT_ALREADY_USED () { echo -e "[Service]\nExecStart=/bin/netcat -l -k -p $check_port\n [Install]\nWantedBy=multi-user.target" > $TEST_CONTEXT/netcat.service - lxc file push $TEST_CONTEXT/netcat.service $LXC_NAME/etc/systemd/system/netcat.service + $lxc file push $TEST_CONTEXT/netcat.service $LXC_NAME/etc/systemd/system/netcat.service # Then start this service to block this port. LXC_EXEC "systemctl enable --now netcat" @@ -621,7 +621,7 @@ TEST_BACKUP_RESTORE () { [ $ret -eq 0 ] || { main_result=1; break_before_continue; continue; } # Grab the backup archive into the LXC container, and keep a copy - lxc file pull -r $LXC_NAME/home/yunohost.backup/archives $TEST_CONTEXT/ynh_backups + $lxc file pull -r $LXC_NAME/home/yunohost.backup/archives $TEST_CONTEXT/ynh_backups # RESTORE # Try the restore process in 2 times, first after removing the app, second after a restore of the container. @@ -646,7 +646,7 @@ TEST_BACKUP_RESTORE () { RUN_INSIDE_LXC rm -rf /home/yunohost.backup/archives # Place the copy of the backup archive in the container. - lxc file push -r $TEST_CONTEXT/ynh_backups/archives $LXC_NAME/home/yunohost.backup/ + $lxc file push -r $TEST_CONTEXT/ynh_backups/archives $LXC_NAME/home/yunohost.backup/ _PREINSTALL diff --git a/lib/tests_coordination.sh b/lib/tests_coordination.sh index 3facb04..3f06d63 100644 --- a/lib/tests_coordination.sh +++ b/lib/tests_coordination.sh @@ -69,7 +69,7 @@ run_all_tests() { cat $TEST_CONTEXT/tests/*.json >> /proc/self/fd/3 # Reset and create a fresh container to work with - check_lxd_setup + check_lxc_setup LXC_RESET LXC_CREATE @@ -213,7 +213,7 @@ break_before_continue () { if [ $interactive -eq 1 ] || [ $interactive_on_errors -eq 1 ] && [ ! $test_result -eq 0 ] then echo "To enter a shell on the lxc:" - echo " lxc exec $LXC_NAME bash" + echo " $lxc exec $LXC_NAME bash" read -p "Press a key to delete the application and continue...." < /dev/tty fi } diff --git a/package_check.sh b/package_check.sh index 52d3ba2..aa555ce 100755 --- a/package_check.sh +++ b/package_check.sh @@ -10,9 +10,9 @@ print_help() { Usage: package_check.sh [OPTION]... PACKAGE_TO_CHECK -b, --branch=BRANCH Specify a branch to check. - -a, --arch=ARCH - -d, --dist=DIST - -y, --ynh-branch=BRANCH + -a, --arch=ARCH + -d, --dist=DIST + -y, --ynh-branch=BRANCH -D, --dry-run Show a JSON representing which tests are going to be ran (meant for debugging) -i, --interactive Wait for the user to continue before each remove -e, --interactive-on-errors Wait for the user to continue on errors @@ -22,6 +22,9 @@ print_help() { images are supposed to be fetch from devbaseimgs.yunohost.org automatically) -h, --help Display this help + + Pass YNHDEV_BACKEND=incus to use incus instead of lxd. + EOF exit 0 }