Merge pull request #1 from YunoHost/new-version

New version

README.md

@@ -72,13 +72,11 @@ check_interval = 0
 
 ## Using images
 
-Use the field `image` to switch between `before-install`, `before-postinstall` or `after-postinstall` (`after-postinstall` by default) for example:
+Use the field `image` to switch between `before-install` or `after-install` (`after-install` by default) for example:
-- `image: after-postinstall` to use the image after the postinstall of Yunohost
+- `image: after-install` to use the image after the postinstall of Yunohost
 - `image: before-install` to use the image before the installation of YunoHost
-- `...`
 
 ## TODO
 
 - Support more YunoHost Core projects (for now only `yunohost` is supported, not `moulinette`...)
-- Be sure that the runner can run several jobs in parallel (The `rebuild_base_containers` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?).
 - Git pull this repo before running tests to keep these files up-to-date.

auto_upgrade_container.sh

@@ -1,15 +1,15 @@
+#!/usr/bin/env bash
+
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/utils.sh # Get utils functions.
 
 for debian_version in "stretch" "buster"
 do
-    for ynh_version in "stable" "testing" "unstable"
+	for ynh_version in "stable" "testing" "unstable"
-    do
+	do
-        for snapshot in "before-install" "before-postinstall" "after-postinstall"
+		for snapshot in "before-install" "after-install"
-        do
+		do
-            image="yunohost-$debian_version-$ynh_version-$snapshot"
+			update_image $debian_version $ynh_version $snapshot
-
+		done
-            update_image $image
+	done
-        done
-    done
 done

base.sh

@@ -1,18 +1,7 @@
 #!/usr/bin/env bash
 
 # All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_
-#CUSTOM_ENV_CI_DEFAULT_BRANCH=stretch-unstable
+
-#CUSTOM_ENV_CI_JOB_NAME=build1
-#CUSTOM_ENV_CI_BUILD_STAGE=pre-postinstall
-#CUSTOM_ENV_CI_JOB_STAGE=pre-postinstall
-#CUSTOM_ENV_CI_BUILD_NAME=build1
-#CUSTOM_ENV_CI_PROJECT_TITLE=yunohost
-#CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH=linux/amd64
-#CUSTOM_ENV_CI_PROJECT_NAMESPACE=yunohost
-#CUSTOM_ENV_CI_COMMIT_REF_NAME=stretch-unstable
-#CUSTOM_ENV_CI_COMMIT_REF_SLUG=stretch-unstable
-#CUSTOM_ENV_CI_PROJECT_NAME=yunohost
-#CUSTOM_ENV_CI_PROJECT_DIR=/builds/yunohost/yunohost
 CONTAINER_ID="runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID-$CUSTOM_ENV_CI_JOB_ID"
 ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64
 DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH"
@@ -21,16 +10,16 @@ CURRENT_BRANCH="$CUSTOM_ENV_CI_COMMIT_REF_NAME"
 DEBIAN_VERSION=$(echo $CUSTOM_ENV_CI_COMMIT_REF_NAME | cut -d'-' -f1) # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR: stretch-unstable, buster-unstable...
 if [ -z "$DEBIAN_VERSION" ] || [ "$DEBIAN_VERSION" != "stretch" ] && [ "$DEBIAN_VERSION" != "buster" ]
 then
-    DEBIAN_VERSION="$(echo $CUSTOM_ENV_CI_DEFAULT_BRANCH | cut -d'-' -f1)" # stretch-unstable, buster-unstable...
+	DEBIAN_VERSION="$(echo $CUSTOM_ENV_CI_DEFAULT_BRANCH | cut -d'-' -f1)" # stretch-unstable, buster-unstable...
-    echo "Use the default debian version: $DEBIAN_VERSION"
+	echo "Use the default debian version: $DEBIAN_VERSION"
 fi
 SNAPSHOT_NAME="$CUSTOM_ENV_CI_JOB_IMAGE"
 if [ -z "$SNAPSHOT_NAME" ]
 then
-    SNAPSHOT_NAME="after-postinstall"
+	SNAPSHOT_NAME="after-install"
 fi
 PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR"
 PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME"
 
 # For example yunohost-stretch-unstable
-BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"
+BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"

rebuild_all.sh

@@ -1,10 +1,43 @@
+#!/usr/bin/env bash
+
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/utils.sh # Get utils functions.
 
 for debian_version in "stretch" "buster"
 do
-    for ynh_version in "stable" "testing" "unstable"
+
-    do
+	# There is no stable and testing version for Buster at this time.
-        rebuild_base_containers $debian_version $ynh_version "amd64"
+	if [[ "$debian_version" == "buster" ]]
-    done
+	then
+		rebuild_base_containers $debian_version "unstable" "amd64"
+	else
+		rebuild_base_containers $debian_version "stable" "amd64"
+
+		for ynh_version in "testing" "unstable"
+		do
+			for snapshot in "before-install" "after-install"
+			do
+				lxc launch "yunohost-$debian_version-stable-$snapshot" "yunohost-$debian_version-$ynh_version-$snapshot-tmp"
+
+				if [[ "$ynh_version" == "testing" ]]
+				then
+					repo_version="testing"
+				elif [[ "$DISTRIB" == "unstable" ]]
+				then
+					repo_version="testing unstable"
+				fi
+
+				lxc exec "yunohost-$debian_version-$ynh_version-$snapshot-tmp" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`;
+		do
+			sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE
+		done"
+
+				rotate_image "yunohost-$debian_version-$ynh_version-$snapshot-tmp" "yunohost-$debian_version-$ynh_version-$snapshot"
+
+				lxc delete -f "yunohost-$debian_version-$ynh_version-$snapshot-tmp"
+
+				update_image $debian_version $ynh_version $snapshot
+			done
+		done
+	fi
 done

run.sh

@@ -3,52 +3,6 @@
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/base.sh # Get variables from base.
 
-create_sym_link() {
-	local DEST=$1
-	local LINK=$2
-	# Remove current sources if not a symlink
-	lxc exec "$CONTAINER_ID" -- sh -c "[ -L "$LINK" ] || rm -rf $LINK"
-	# Symlink from Git repository
-	lxc exec "$CONTAINER_ID" -- sh -c "ln -sfn $DEST $LINK"
-}
-
-link_moulinette_from_git() {
-	echo "Moulinette: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
-
-	moulinette_dir="/tmp/ci_moulinette"
-	lxc exec "$CONTAINER_ID" -- sh -c "mkdir $moulinette_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/moulinette $moulinette_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "(cd $moulinette_dir; \
-	if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
-	then \
-		git checkout $CURRENT_BRANCH; \
-	else \
-		git checkout $DEFAULT_BRANCH; \
-	fi)"
-
-	create_sym_link "$moulinette_dir/locales" "/usr/share/moulinette/locale"
-	create_sym_link "$moulinette_dir/moulinette" "/usr/lib/python2.7/dist-packages/moulinette"
-}
-
-link_ssowat_from_git() {
-	echo "SSOWAT: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
-
-	ssowat_dir="/tmp/ci_ssowat"
-	lxc exec "$CONTAINER_ID" -- sh -c "mkdir $ssowat_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/ssowat $ssowat_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "(cd $ssowat_dir; \
-	if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
-	then \
-		git checkout $CURRENT_BRANCH; \
-	else \
-		git checkout $DEFAULT_BRANCH; \
-	fi)"
-
-	create_sym_link "$ssowat_dir" "/usr/share/ssowat"
-
-	lxc exec "$CONTAINER_ID" -- sh -c "systemctl reload nginx"
-}
-
 case ${2} in
 	prepare_script)
 		;;
@@ -61,43 +15,7 @@ case ${2} in
 	build_script)
 		case $PROJECT_NAME in
 			yunohost)
-				echo "Link yunohost"
+				# Nothing to do?
-
-				# bin
-				create_sym_link "$PROJECT_DIR/bin/yunohost" "/usr/bin/yunohost"
-				create_sym_link "$PROJECT_DIR/bin/yunohost-api" "/usr/bin/yunohost-api"
-
-				# data
-				create_sym_link "$PROJECT_DIR/data/actionsmap/yunohost.yml" "/usr/share/moulinette/actionsmap/yunohost.yml"
-				create_sym_link "$PROJECT_DIR/data/hooks" "/usr/share/yunohost/hooks"
-				create_sym_link "$PROJECT_DIR/data/templates" "/usr/share/yunohost/templates"
-				create_sym_link "$PROJECT_DIR/data/helpers" "/usr/share/yunohost/helpers"
-				create_sym_link "$PROJECT_DIR/data/helpers.d" "/usr/share/yunohost/helpers.d"
-				create_sym_link "$PROJECT_DIR/data/other" "/usr/share/yunohost/yunohost-config/moulinette"
-				# debian
-				create_sym_link "$PROJECT_DIR/debian/conf/pam/mkhomedir" "/usr/share/pam-configs/mkhomedir"
-
-				# lib
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/ldap.lib.lua" "/usr/lib/metronome/modules/ldap.lib.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_auth_ldap2.lua" "/usr/lib/metronome/modules/mod_auth_ldap2.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_legacyauth.lua" "/usr/lib/metronome/modules/mod_legacyauth.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_storage_ldap.lua" "/usr/lib/metronome/modules/mod_storage_ldap.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/vcard.lib.lua" "/usr/lib/metronome/modules/vcard.lib.lua"
-
-				# src
-				create_sym_link "$PROJECT_DIR/src/yunohost" "/usr/lib/moulinette/yunohost"
-
-				# locales
-				create_sym_link "$PROJECT_DIR/locales" "/usr/lib/moulinette/yunohost/locales"
-
-				# moulinette
-				link_moulinette_from_git
-
-				# ssowat
-				link_ssowat_from_git
-
-				# Run migrations
-				lxc exec "$CONTAINER_ID" -- sh -c "yunohost tools migrations migrate"
 			;;
 		esac
 		;;

utils.sh

@@ -5,7 +5,7 @@ source $current_dir/base.sh # Get variables from base.
 
 clean_containers()
 {
-    local base_image_to_clean=$1
+	local base_image_to_clean=$1
 
 	for image_to_delete in "$base_image_to_clean"{,"-tmp"}
 	do
@@ -15,7 +15,7 @@ clean_containers()
 		fi
 	done
 
-	for image_to_delete in "$base_image_to_clean-"{"before-install","before-postinstall","after-postinstall"}
+	for image_to_delete in "$base_image_to_clean-"{"before-install","after-install"}
 	do
 		if lxc image info $image_to_delete &>/dev/null
 		then
@@ -95,7 +95,7 @@ rotate_image()
 	local alias_image=$2
 
 	# Save the finger print to delete the old image later 
-    local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}')
+	local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}')
 	local should_restart=0
 
 	# If the container is running, stop it
@@ -117,49 +117,80 @@ rotate_image()
 	fi
 }
 
+# These lines are used to extract the dependencies/recommendations from the debian/control file.
+# /!\ There's a high risk of lamentable failure if we change the format of this file
+get_dependencies()
+{
+	local debian_version=$1
+	local ynh_version=$2
+
+	# To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'.
+	YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ")
+	YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
+	MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+/g" | tr "\n" " ")
+	# Same as above, except that all dependencies are in the same line
+	SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$debian_version-$ynh_version/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
+	BUILD_DEPENDENCIES="git-buildpackage postfix python-setuptools python-pip"
+	PIP_PKG="mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html"
+}
 
 rebuild_base_containers()
 {
-    local debian_version=$1
+	local debian_version=$1
-    local ynh_version=$2
+	local ynh_version=$2
-    local arch=$3
+	local arch=$3
-    local base_image_to_rebuild="yunohost-$debian_version-$ynh_version"
+	local base_image_to_rebuild="yunohost-$debian_version-$ynh_version"
 
 	lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild-tmp"
 	
 	wait_container "$base_image_to_rebuild-tmp"
 
 	if [[ "$debian_version" == "buster" ]]
-    then
+	then
-        lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor
+		lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor
-    fi
+	fi
 
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install curl -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes wget curl"
 	# Install Git LFS, git comes pre installed with ubuntu image.
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install git-lfs -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes git-lfs"
 	# Install gitlab-runner binary since we need for cache/artifacts.
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner"
+
+	if [[ "$debian_version" == "buster" ]]
+	then
+		INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/buster-unstable/install_yunohost"
+	else
+		INSTALL_SCRIPT="https://install.yunohost.org"
+	fi
+
+	# Download the YunoHost install script
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh"
+	
+	# Patch the YunoHost install script
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh"
+
+	# Run the YunoHost install script patched
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version"
+
+	get_dependencies $debian_version $ynh_version
+
+	# Pre install dependencies
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "pip install -U $PIP_PKG"
 
 	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install"
 
-    if [[ "$debian_version" == "buster" ]]
+	# Install YunoHost
-    then
-        INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/buster-unstable/install_yunohost"
-    else
-        INSTALL_SCRIPT="https://install.yunohost.org"
-    fi
-	# Install yunohost
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version"
-
+	
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-postinstall"
+	# Run postinstall
-
-	# Running post Install
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns"
 
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-postinstall"
+	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-install"
 
 	lxc stop "$base_image_to_rebuild-tmp"
 
@@ -167,13 +198,16 @@ rebuild_base_containers()
 }
 
 update_image() {
-	local image_to_update=$1
+	local debian_version=$1
+	local ynh_version=$2
+	local snapshot=$3
+	local image_to_update="yunohost-$debian_version-$ynh_version-$snapshot"
 
-    if ! lxc image info "$image_to_update" &>/dev/null
+	if ! lxc image info "$image_to_update" &>/dev/null
-    then
+	then
-        echo "Unable to upgrade image $image_to_update"
+		echo "Unable to upgrade image $image_to_update"
-        return
+		return
-    fi
+	fi
 
 	# Start and run upgrade
 	lxc launch "$image_to_update" "$image_to_update-tmp"
@@ -181,7 +215,12 @@ update_image() {
 	wait_container "$image_to_update-tmp"
 
 	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update"
-	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y"
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade --assume-yes"
+	
+	get_dependencies $debian_version $ynh_version
+
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES"
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "pip install -U $PIP_PKG"
 
 	rotate_image "$image_to_update-tmp" "$image_to_update"