Merge 754fd57c84 into 90147c25c9

lib/build_base_lxc.sh

@@ -4,9 +4,14 @@ function launch_new_lxc()
 {
     lxc info $LXC_BASE >/dev/null && lxc delete $LXC_BASE --force
 
+    profile_option=""
+    if [ "$profile" != "" ]; then
+       profile_option="--profile $profile"
+    fi
+
     if [ $(get_arch) = $ARCH ];
     then
-        lxc launch images:debian/$DIST/$ARCH $LXC_BASE -c security.privileged=true -c security.nesting=true
+       lxc launch images:debian/$DIST/$ARCH $LXC_BASE $profile_option -c security.privileged=true -c security.nesting=true
     else
         lxc image info $LXC_BASE >/dev/null && lxc image delete $LXC_BASE
 
@@ -23,7 +28,7 @@ function launch_new_lxc()
         popd
         rm -rf "$tmp_dir"
 
-        lxc launch $LXC_BASE $LXC_BASE -c security.privileged=true -c security.nesting=true
+        lxc launch $LXC_BASE $LXC_BASE $profile_option -c security.privileged=true -c security.nesting=true
     fi
 }
 
@@ -33,17 +38,17 @@ function rebuild_base_lxc()
 
     launch_new_lxc
     sleep 5
-    
+
     IN_LXC="lxc exec $LXC_BASE --"
-    
+
     INSTALL_SCRIPT="https://install.yunohost.org/$DIST"
     $IN_LXC apt install curl -y
     $IN_LXC /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $YNH_BRANCH"
-    
+
     $IN_LXC systemctl -q stop apt-daily.timer
     $IN_LXC systemctl -q stop apt-daily-upgrade.timer
     $IN_LXC systemctl -q stop apt-daily.service
-    $IN_LXC systemctl -q stop apt-daily-upgrade.service 
+    $IN_LXC systemctl -q stop apt-daily-upgrade.service
     $IN_LXC systemctl -q disable apt-daily.timer
     $IN_LXC systemctl -q disable apt-daily-upgrade.timer
     $IN_LXC systemctl -q disable apt-daily.service

lib/common.sh

@@ -21,6 +21,8 @@ LXC_NAME="ynh-appci-$DIST-$ARCH-$YNH_BRANCH-test-${WORKER_ID}"
 
 readonly lock_file="./pcheck-${WORKER_ID}.lock"
 
+LXC_PROFILE_LIST=(yunohost_ramfs yunohost_btrfs yunohost_dir)
+
 #=================================================
 # LXC helpers
 #=================================================
@@ -53,6 +55,12 @@ function check_lxd_setup()
 
     ip a | grep -q lxdbr0 \
         || log_critical "There is no 'lxdbr0' interface... Did you ran 'lxd init' ?"
+
+    # Check the asked profile exists
+    if [[ "$profile" != "" ]] && [[ $(lxc profile list) != *"$profile"* ]]
+    then
+      log_critical "The lxd profile asked, $profile, is not configured. Did you run setup_lxd.sh ?"
+    fi
 }
 
 #=================================================

lib/lxc.sh

@@ -6,12 +6,17 @@
 
 LXC_CREATE () {
     log_info "Launching new LXC $LXC_NAME ..."
+    profile_option=""
+    if [ "$profile" != "" ]; then
+      profile_option="--profile $profile"
+    fi
+
     # Check if we can launch container from YunoHost remote image
     if lxc remote list | grep -q "yunohost" && lxc image list yunohost:$LXC_BASE | grep -q -w $LXC_BASE; then
         # Force the usage of the fingerprint because otherwise for some reason lxd won't use the newer version
         # available even though it's aware it exists -_-
         LXC_BASE_HASH="$(lxc image list yunohost:ynh-appci-bullseye-amd64-stable-base --format json | jq -r '.[].fingerprint')"
-        lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME \
+        lxc launch yunohost:$LXC_BASE_HASH $LXC_NAME $profile_option \
             -c security.nesting=true \
             -c security.privileged=true \
             -c limits.memory=80% \
@@ -19,7 +24,7 @@ LXC_CREATE () {
             >>/proc/self/fd/3
     # Check if we can launch container from a local image
     elif lxc image list $LXC_BASE | grep -q -w $LXC_BASE; then
-        lxc launch $LXC_BASE $LXC_NAME \
+        lxc launch $LXC_BASE $LXC_NAME $profile_option \
             -c security.nesting=true \
             -c security.privileged=true \
             -c limits.memory=80% \

package_check.sh

@@ -10,12 +10,13 @@ print_help() {
  Usage: package_check.sh [OPTION]... PACKAGE_TO_CHECK
 
     -b, --branch=BRANCH     Specify a branch to check.
-    -a, --arch=ARCH         
+    -a, --arch=ARCH
-    -d, --dist=DIST         
+    -d, --dist=DIST
-    -y, --ynh-branch=BRANCH 
+    -y, --ynh-branch=BRANCH
     -i, --interactive           Wait for the user to continue before each remove
     -e, --interactive-on-errors Wait for the user to continue on errors
     -s, --force-stop            Force the stop of running package_check
+    -p, --profile=PROFILE       LXD profile to use when creating the instance
     -r, --rebuild               (Re)Build the base container
                                 (N.B.: you're not supposed to use this option,
                                 images are supposed to be fetch from
@@ -39,6 +40,7 @@ interactive=0
 interactive_on_errors=0
 rebuild=0
 force_stop=0
+profile=""
 
 function parse_args() {
 
@@ -52,6 +54,11 @@ function parse_args() {
             getopts_built_arg+=(-b)
             arguments[$i]=${arguments[$i]//--branch=/}
         fi
+        if [[ "${arguments[$i]}" =~ "--profile=" ]]
+        then
+            getopts_built_arg+=(-p)
+            arguments[$i]=${arguments[$i]//--profile=/}
+        fi
         # For each argument in the array, reduce to short argument for getopts
         arguments[$i]=${arguments[$i]//--interactive/-i}
         arguments[$i]=${arguments[$i]//--rebuild/-r}
@@ -71,7 +78,7 @@ function parse_args() {
                 # Initialize the index of getopts
                 OPTIND=1
                 # Parse with getopts only if the argument begin by -
-                getopts ":b:iresh" parameter || true
+                getopts ":b:iresp:h" parameter || true
                 case $parameter in
                     b)
                         # --branch=branch-name
@@ -98,6 +105,11 @@ function parse_args() {
                         force_stop=1
                         shift_value=1
                         ;;
+                    p)
+                        # --profile=profile
+                        profile="$OPTARG"
+                        shift_value=2
+                        ;;
                     h)
                         # --help
                         print_help
@@ -128,6 +140,28 @@ function parse_args() {
 arguments=("$@")
 parse_args
 
+#=================================================
+# Guess the best lxd profile to apply
+#=================================================
+select_the_best_profile () {
+  if [[ "$profile" == "" ]]
+  then
+      # No profile preselected try to find one automatically
+        for name in "${LXC_PROFILE_LIST[@]}"
+        do
+            if [[ "$profile" == "" ]] && [[ $(lxc profile list --format=compact) == *"$name"* ]]
+            then
+              profile=$name
+            fi
+        done
+  fi
+
+  if [[ "$profile" != "" ]]
+  then
+    echo "Using LXD profile $profile"
+  fi
+}
+
 #=================================================
 # Cleanup / force-stop
 #=================================================
@@ -193,6 +227,8 @@ trap 'exit 2' TERM
 assert_we_are_connected_to_the_internets
 assert_we_have_all_dependencies
 
+select_the_best_profile
+
 if [[ $rebuild == 1 ]]
 then
     rebuild_base_lxc 2>&1 | tee -a "./build_base_lxc.log"
@@ -202,7 +238,13 @@ fi
 self_upgrade
 fetch_or_upgrade_package_linter
 
-readonly TEST_CONTEXT=$(mktemp -d /tmp/package_check.XXXXXX)
+root_tmp_dir=/tmp
+if [[ $profile == "yunohost_ramfs" ]]
+then
+    # Download the application into the ramdisk if it exist
+  root_tmp_dir=/tmp/yunohost_ramfs
+fi
+readonly TEST_CONTEXT=$(mktemp -d $root_tmp_dir/package_check.XXXXXX)
 
 fetch_package_to_test "$path_to_package_to_test"
 run_all_tests

setup_lxd.sh

@@ -0,0 +1,213 @@
+#!/bin/bash
+
+cd $(dirname $(realpath $0))
+source "./lib/common.sh"
+
+print_help() {
+    cat << EOF
+ Usage: setup_lxd.sh [OPTION]...
+
+    -t, --type=TYPE Specify which type of storage to use: ramfs, dir or btrfs
+    -s, --size=SIZE The storage size allocated to ramfs (5G, 10G...)
+    -r, --reset     Removes all the storage and disks created with this script. Be careful as instances will be deleted as well
+    -h, --help                  Display this help
+EOF
+exit 0
+}
+
+
+#=================================================
+# Pase CLI arguments
+#=================================================
+
+# If no arguments provided
+# Print the help and exit
+[ "$#" -eq 0 ] && print_help
+
+type="dir"
+size="10G"
+reset=0
+
+function parse_args() {
+
+    local getopts_built_arg=()
+
+    # Read the array value per value
+    for i in $(seq 0 $(( ${#arguments[@]} -1 )))
+    do
+           if [[ "${arguments[$i]}" =~ "--type=" ]]
+           then
+               getopts_built_arg+=(-t)
+               arguments[$i]=${arguments[$i]//--type=/}
+           fi
+           if [[ "${arguments[$i]}" =~ "--size=" ]]
+           then
+               getopts_built_arg+=(-s)
+               arguments[$i]=${arguments[$i]//--size=/}
+           fi
+     # For each argument in the array, reduce to short argument for getopts
+        arguments[$i]=${arguments[$i]//--reset/-r}
+        arguments[$i]=${arguments[$i]//--help/-h}
+        getopts_built_arg+=("${arguments[$i]}")
+    done
+
+    # Read and parse all the arguments
+    # Use a function here, to use standart arguments $@ and be able to use shift.
+    parse_arg () {
+        while [ $# -ne 0 ]
+        do
+            # If the paramater begins by -, treat it with getopts
+            if [ "${1:0:1}" == "-" ]
+            then
+                # Initialize the index of getopts
+                OPTIND=1
+                # Parse with getopts only if the argument begin by -
+                getopts "t:s:r" parameter || true
+                case $parameter in
+                    t)
+                        # --type
+                        type=$OPTARG
+                        shift_value=2
+                        ;;
+                    s)
+                        # --size
+                        size=$OPTARG
+                        shift_value=2
+                        ;;
+                    r)
+                        # --reset
+                        reset=1
+                        ;;
+                    h)
+                        # --help
+                        print_help
+                        ;;
+                    \?)
+                        echo "Invalid argument: -${OPTARG:-}"
+                        print_help
+                        ;;
+                    :)
+                        echo "-$OPTARG parameter requires an argument."
+                        print_help
+                        ;;
+                esac
+                # Otherwise, it's not an option, it's an operand
+#            else
+#                path_to_package_to_test="$1"
+#                shift_value=1
+            fi
+            # Shift the parameter and its argument
+            shift $shift_value
+        done
+    }
+
+    # Call parse_arg and pass the modified list of args as a array of arguments.
+    parse_arg "${getopts_built_arg[@]}"
+}
+
+function create_lxd_profile () {
+      if [[ $(lxc profile list --format=compact) == *"$profile_name"* ]]
+      then
+        lxc profile delete $profile_name
+      fi
+      lxc profile create $profile_name
+      lxc profile device add $profile_name root disk path=/ pool=$profile_name
+      lxc profile device add $profile_name eth0 nic name=eth0 network=lxdbr0
+      echo "You can use the profile $profile_name (--profile $profile_name) when running package_check.sh"
+}
+
+# Create the asked storage
+function create_storage() {
+  case $type in
+    dir)
+      echo "Creating dir storage for package_check"
+      if [[ $(lxc storage list --format=compact) != *"yunohost_dir"* ]]
+      then
+        lxc storage create yunohost_dir dir
+      fi
+      profile_name=yunohost_dir
+      create_lxd_profile
+      ;;
+    ramfs)
+      echo "Creating RamFS storage"
+
+      if [[ $(lxc profile list --format=compact) == *"yunohost_ramfs"* ]]
+      then
+        lxc profile delete yunohost_ramfs
+      fi
+
+      if [[ $(lxc storage list --format=compact) == *"yunohost_ramfs"* ]]
+      then
+        lxc storage delete yunohost_ramfs
+      fi
+
+      if [[ -e /tmp/yunohost_ramfs ]]
+      then
+        sudo umount /tmp/yunohost_ramfs
+        sudo rm -rf /tmp/yunohost_ramfs
+      fi
+
+      sudo mkdir --parents /tmp/yunohost_ramfs
+      sudo mount -t tmpfs -o size=$size tmpfs /tmp/yunohost_ramfs
+
+      lxc storage create yunohost_ramfs dir source=/tmp/yunohost_ramfs
+
+      profile_name=yunohost_ramfs
+      create_lxd_profile
+      ;;
+    btrfs)
+      echo "Creating btrfs storage for package_check"
+      if [[ $(lxc storage list --format=compact) != *"yunohost_btrfs"* ]]
+      then
+        lxc storage create yunohost_btrfs btrfs
+      fi
+
+      profile_name=yunohost_btrfs
+      create_lxd_profile
+      ;;
+  esac
+}
+
+# Removes all the storage that has been created by this script
+function remove_all_storage() {
+  for name in "${LXC_PROFILE_LIST[@]}"
+  do
+      if [[ $(lxc profile list --format=compact) == *"$name"* ]]
+      then
+        lxc profile delete $name
+      fi
+      if [[ $(lxc storage list --format=compact) == *"$name"* ]]
+      then
+        lxc storage delete $name
+      fi
+  done
+
+  if [[ -e /tmp/yunohost_ramfs ]]
+  then
+    sudo umount /tmp/yunohost_ramfs
+    sudo rm -rf /tmp/yunohost_ramfs
+  fi
+}
+
+arguments=("$@")
+parse_args
+
+#==========================
+# Main code
+#==========================
+
+assert_we_are_connected_to_the_internets
+assert_we_have_all_dependencies
+
+if [[ $reset == 1 ]]
+then
+    remove_all_storage
+    exit 0
+fi
+
+create_storage
+
+
+exit 0
+
+}