Merge pull request #1 from YunoHost/new-version

New version

README.md

@@ -72,13 +72,11 @@ check_interval = 0
 
 ## Using images
 
-Use the field `image` to switch between `before-install`, `before-postinstall` or `after-postinstall` (`after-postinstall` by default) for example:
+Use the field `image` to switch between `before-install` or `after-install` (`after-install` by default) for example:
-- `image: after-postinstall` to use the image after the postinstall of Yunohost
+- `image: after-install` to use the image after the postinstall of Yunohost
 - `image: before-install` to use the image before the installation of YunoHost
-- `...`
 
 ## TODO
 
 - Support more YunoHost Core projects (for now only `yunohost` is supported, not `moulinette`...)
-- Be sure that the runner can run several jobs in parallel (The `rebuild_base_containers` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?).
 - Git pull this repo before running tests to keep these files up-to-date.

auto_upgrade_container.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/utils.sh # Get utils functions.
 
@@ -5,11 +7,9 @@ for debian_version in "stretch" "buster"
 do
 	for ynh_version in "stable" "testing" "unstable"
 	do
-        for snapshot in "before-install" "before-postinstall" "after-postinstall"
+		for snapshot in "before-install" "after-install"
 		do
-            image="yunohost-$debian_version-$ynh_version-$snapshot"
+			update_image $debian_version $ynh_version $snapshot
-
-            update_image $image
 		done
 	done
 done

base.sh

@@ -1,18 +1,7 @@
 #!/usr/bin/env bash
 
 # All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_
-#CUSTOM_ENV_CI_DEFAULT_BRANCH=stretch-unstable
+
-#CUSTOM_ENV_CI_JOB_NAME=build1
-#CUSTOM_ENV_CI_BUILD_STAGE=pre-postinstall
-#CUSTOM_ENV_CI_JOB_STAGE=pre-postinstall
-#CUSTOM_ENV_CI_BUILD_NAME=build1
-#CUSTOM_ENV_CI_PROJECT_TITLE=yunohost
-#CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH=linux/amd64
-#CUSTOM_ENV_CI_PROJECT_NAMESPACE=yunohost
-#CUSTOM_ENV_CI_COMMIT_REF_NAME=stretch-unstable
-#CUSTOM_ENV_CI_COMMIT_REF_SLUG=stretch-unstable
-#CUSTOM_ENV_CI_PROJECT_NAME=yunohost
-#CUSTOM_ENV_CI_PROJECT_DIR=/builds/yunohost/yunohost
 CONTAINER_ID="runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID-$CUSTOM_ENV_CI_JOB_ID"
 ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64
 DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH"
@@ -27,7 +16,7 @@ fi
 SNAPSHOT_NAME="$CUSTOM_ENV_CI_JOB_IMAGE"
 if [ -z "$SNAPSHOT_NAME" ]
 then
-    SNAPSHOT_NAME="after-postinstall"
+	SNAPSHOT_NAME="after-install"
 fi
 PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR"
 PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME"

rebuild_all.sh

@@ -1,10 +1,43 @@
+#!/usr/bin/env bash
+
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/utils.sh # Get utils functions.
 
 for debian_version in "stretch" "buster"
 do
-    for ynh_version in "stable" "testing" "unstable"
+
+	# There is no stable and testing version for Buster at this time.
+	if [[ "$debian_version" == "buster" ]]
+	then
+		rebuild_base_containers $debian_version "unstable" "amd64"
+	else
+		rebuild_base_containers $debian_version "stable" "amd64"
+
+		for ynh_version in "testing" "unstable"
 		do
-        rebuild_base_containers $debian_version $ynh_version "amd64"
+			for snapshot in "before-install" "after-install"
+			do
+				lxc launch "yunohost-$debian_version-stable-$snapshot" "yunohost-$debian_version-$ynh_version-$snapshot-tmp"
+
+				if [[ "$ynh_version" == "testing" ]]
+				then
+					repo_version="testing"
+				elif [[ "$DISTRIB" == "unstable" ]]
+				then
+					repo_version="testing unstable"
+				fi
+
+				lxc exec "yunohost-$debian_version-$ynh_version-$snapshot-tmp" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`;
+		do
+			sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE
+		done"
+
+				rotate_image "yunohost-$debian_version-$ynh_version-$snapshot-tmp" "yunohost-$debian_version-$ynh_version-$snapshot"
+
+				lxc delete -f "yunohost-$debian_version-$ynh_version-$snapshot-tmp"
+
+				update_image $debian_version $ynh_version $snapshot
 			done
 		done
+	fi
+done

run.sh

@@ -3,52 +3,6 @@
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/base.sh # Get variables from base.
 
-create_sym_link() {
-	local DEST=$1
-	local LINK=$2
-	# Remove current sources if not a symlink
-	lxc exec "$CONTAINER_ID" -- sh -c "[ -L "$LINK" ] || rm -rf $LINK"
-	# Symlink from Git repository
-	lxc exec "$CONTAINER_ID" -- sh -c "ln -sfn $DEST $LINK"
-}
-
-link_moulinette_from_git() {
-	echo "Moulinette: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
-
-	moulinette_dir="/tmp/ci_moulinette"
-	lxc exec "$CONTAINER_ID" -- sh -c "mkdir $moulinette_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/moulinette $moulinette_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "(cd $moulinette_dir; \
-	if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
-	then \
-		git checkout $CURRENT_BRANCH; \
-	else \
-		git checkout $DEFAULT_BRANCH; \
-	fi)"
-
-	create_sym_link "$moulinette_dir/locales" "/usr/share/moulinette/locale"
-	create_sym_link "$moulinette_dir/moulinette" "/usr/lib/python2.7/dist-packages/moulinette"
-}
-
-link_ssowat_from_git() {
-	echo "SSOWAT: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
-
-	ssowat_dir="/tmp/ci_ssowat"
-	lxc exec "$CONTAINER_ID" -- sh -c "mkdir $ssowat_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/ssowat $ssowat_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "(cd $ssowat_dir; \
-	if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
-	then \
-		git checkout $CURRENT_BRANCH; \
-	else \
-		git checkout $DEFAULT_BRANCH; \
-	fi)"
-
-	create_sym_link "$ssowat_dir" "/usr/share/ssowat"
-
-	lxc exec "$CONTAINER_ID" -- sh -c "systemctl reload nginx"
-}
-
 case ${2} in
 	prepare_script)
 		;;
@@ -61,43 +15,7 @@ case ${2} in
 	build_script)
 		case $PROJECT_NAME in
 			yunohost)
-				echo "Link yunohost"
+				# Nothing to do?
-
-				# bin
-				create_sym_link "$PROJECT_DIR/bin/yunohost" "/usr/bin/yunohost"
-				create_sym_link "$PROJECT_DIR/bin/yunohost-api" "/usr/bin/yunohost-api"
-
-				# data
-				create_sym_link "$PROJECT_DIR/data/actionsmap/yunohost.yml" "/usr/share/moulinette/actionsmap/yunohost.yml"
-				create_sym_link "$PROJECT_DIR/data/hooks" "/usr/share/yunohost/hooks"
-				create_sym_link "$PROJECT_DIR/data/templates" "/usr/share/yunohost/templates"
-				create_sym_link "$PROJECT_DIR/data/helpers" "/usr/share/yunohost/helpers"
-				create_sym_link "$PROJECT_DIR/data/helpers.d" "/usr/share/yunohost/helpers.d"
-				create_sym_link "$PROJECT_DIR/data/other" "/usr/share/yunohost/yunohost-config/moulinette"
-				# debian
-				create_sym_link "$PROJECT_DIR/debian/conf/pam/mkhomedir" "/usr/share/pam-configs/mkhomedir"
-
-				# lib
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/ldap.lib.lua" "/usr/lib/metronome/modules/ldap.lib.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_auth_ldap2.lua" "/usr/lib/metronome/modules/mod_auth_ldap2.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_legacyauth.lua" "/usr/lib/metronome/modules/mod_legacyauth.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_storage_ldap.lua" "/usr/lib/metronome/modules/mod_storage_ldap.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/vcard.lib.lua" "/usr/lib/metronome/modules/vcard.lib.lua"
-
-				# src
-				create_sym_link "$PROJECT_DIR/src/yunohost" "/usr/lib/moulinette/yunohost"
-
-				# locales
-				create_sym_link "$PROJECT_DIR/locales" "/usr/lib/moulinette/yunohost/locales"
-
-				# moulinette
-				link_moulinette_from_git
-
-				# ssowat
-				link_ssowat_from_git
-
-				# Run migrations
-				lxc exec "$CONTAINER_ID" -- sh -c "yunohost tools migrations migrate"
 			;;
 		esac
 		;;

utils.sh

@@ -15,7 +15,7 @@ clean_containers()
 		fi
 	done
 
-	for image_to_delete in "$base_image_to_clean-"{"before-install","before-postinstall","after-postinstall"}
+	for image_to_delete in "$base_image_to_clean-"{"before-install","after-install"}
 	do
 		if lxc image info $image_to_delete &>/dev/null
 		then
@@ -117,6 +117,22 @@ rotate_image()
 	fi
 }
 
+# These lines are used to extract the dependencies/recommendations from the debian/control file.
+# /!\ There's a high risk of lamentable failure if we change the format of this file
+get_dependencies()
+{
+	local debian_version=$1
+	local ynh_version=$2
+
+	# To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'.
+	YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ")
+	YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
+	MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+/g" | tr "\n" " ")
+	# Same as above, except that all dependencies are in the same line
+	SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$debian_version-$ynh_version/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
+	BUILD_DEPENDENCIES="git-buildpackage postfix python-setuptools python-pip"
+	PIP_PKG="mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html"
+}
 
 rebuild_base_containers()
 {
@@ -135,15 +151,13 @@ rebuild_base_containers()
 	fi
 
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install curl -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes wget curl"
 	# Install Git LFS, git comes pre installed with ubuntu image.
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install git-lfs -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes git-lfs"
 	# Install gitlab-runner binary since we need for cache/artifacts.
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner"
-
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install"
 
 	if [[ "$debian_version" == "buster" ]]
 	then
@@ -151,15 +165,32 @@ rebuild_base_containers()
 	else
 		INSTALL_SCRIPT="https://install.yunohost.org"
 	fi
-	# Install yunohost
+
+	# Download the YunoHost install script
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh"
+	
+	# Patch the YunoHost install script
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh"
+
+	# Run the YunoHost install script patched
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version"
+
+	get_dependencies $debian_version $ynh_version
+
+	# Pre install dependencies
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "pip install -U $PIP_PKG"
+
+	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install"
+
+	# Install YunoHost
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version"
 	
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-postinstall"
+	# Run postinstall
-
-	# Running post Install
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns"
 
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-postinstall"
+	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-install"
 
 	lxc stop "$base_image_to_rebuild-tmp"
 
@@ -167,7 +198,10 @@ rebuild_base_containers()
 }
 
 update_image() {
-	local image_to_update=$1
+	local debian_version=$1
+	local ynh_version=$2
+	local snapshot=$3
+	local image_to_update="yunohost-$debian_version-$ynh_version-$snapshot"
 
 	if ! lxc image info "$image_to_update" &>/dev/null
 	then
@@ -181,7 +215,12 @@ update_image() {
 	wait_container "$image_to_update-tmp"
 
 	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update"
-	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y"
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade --assume-yes"
+	
+	get_dependencies $debian_version $ynh_version
+
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES"
+	lxc exec "$image_to_update-tmp" -- /bin/bash -c "pip install -U $PIP_PKG"
 
 	rotate_image "$image_to_update-tmp" "$image_to_update"