no more pre-installed image

auto_upgrade_container.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/utils.sh # Get utils functions.
 
@@ -5,11 +7,8 @@ for debian_version in "stretch" "buster"
 do
     for ynh_version in "stable" "testing" "unstable"
     do
-        for snapshot in "before-install" "before-postinstall" "after-postinstall"
+        image="yunohost-$debian_version-$ynh_version"
-        do
-            image="yunohost-$debian_version-$ynh_version-$snapshot"
 
-            update_image $image
+        update_image $image
-        done
     done
 done

base.sh

@@ -1,18 +1,7 @@
 #!/usr/bin/env bash
 
 # All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_
-#CUSTOM_ENV_CI_DEFAULT_BRANCH=stretch-unstable
+
-#CUSTOM_ENV_CI_JOB_NAME=build1
-#CUSTOM_ENV_CI_BUILD_STAGE=pre-postinstall
-#CUSTOM_ENV_CI_JOB_STAGE=pre-postinstall
-#CUSTOM_ENV_CI_BUILD_NAME=build1
-#CUSTOM_ENV_CI_PROJECT_TITLE=yunohost
-#CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH=linux/amd64
-#CUSTOM_ENV_CI_PROJECT_NAMESPACE=yunohost
-#CUSTOM_ENV_CI_COMMIT_REF_NAME=stretch-unstable
-#CUSTOM_ENV_CI_COMMIT_REF_SLUG=stretch-unstable
-#CUSTOM_ENV_CI_PROJECT_NAME=yunohost
-#CUSTOM_ENV_CI_PROJECT_DIR=/builds/yunohost/yunohost
 CONTAINER_ID="runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID-$CUSTOM_ENV_CI_JOB_ID"
 ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64
 DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH"
@@ -33,4 +22,8 @@ PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR"
 PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME"
 
 # For example yunohost-stretch-unstable
-BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"
+BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"
+
+
+YNH_DEPENDENCIES="debhelper python-psutil python-requests python-dnspython python-openssl python-miniupnpc python-dbus python-jinja2 python-toml python-packaging apt apt-transport-https nginx nginx-extras php-fpm php-ldap php-intl mariadb-server php-mysql php-mysqlnd openssh-server iptables fail2ban dnsutils bind9utils openssl ca-certificates netcat-openbsd iproute2 slapd ldap-utils sudo-ldap libnss-ldapd unscd libpam-ldapd dnsmasq avahi-daemon libnss-mdns resolvconf libnss-myhostname postfix postfix-ldap postfix-policyd-spf-perl postfix-pcre dovecot-core dovecot-ldap dovecot-lmtpd dovecot-managesieved dovecot-antispam rspamd opendkim-tools postsrsd procmail mailutils redis-server git curl wget cron unzip lsb-release haveged fake-hwclock equivs lsof whois python-publicsuffix"
+BUILD_DEPENDENCIES="git-buildpackage postfix python-setuptools python-pip"

rebuild_all.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/utils.sh # Get utils functions.
 

run.sh

@@ -3,52 +3,6 @@
 current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $current_dir/base.sh # Get variables from base.
 
-create_sym_link() {
-	local DEST=$1
-	local LINK=$2
-	# Remove current sources if not a symlink
-	lxc exec "$CONTAINER_ID" -- sh -c "[ -L "$LINK" ] || rm -rf $LINK"
-	# Symlink from Git repository
-	lxc exec "$CONTAINER_ID" -- sh -c "ln -sfn $DEST $LINK"
-}
-
-link_moulinette_from_git() {
-	echo "Moulinette: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
-
-	moulinette_dir="/tmp/ci_moulinette"
-	lxc exec "$CONTAINER_ID" -- sh -c "mkdir $moulinette_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/moulinette $moulinette_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "(cd $moulinette_dir; \
-	if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
-	then \
-		git checkout $CURRENT_BRANCH; \
-	else \
-		git checkout $DEFAULT_BRANCH; \
-	fi)"
-
-	create_sym_link "$moulinette_dir/locales" "/usr/share/moulinette/locale"
-	create_sym_link "$moulinette_dir/moulinette" "/usr/lib/python2.7/dist-packages/moulinette"
-}
-
-link_ssowat_from_git() {
-	echo "SSOWAT: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
-
-	ssowat_dir="/tmp/ci_ssowat"
-	lxc exec "$CONTAINER_ID" -- sh -c "mkdir $ssowat_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/ssowat $ssowat_dir"
-	lxc exec "$CONTAINER_ID" -- sh -c "(cd $ssowat_dir; \
-	if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
-	then \
-		git checkout $CURRENT_BRANCH; \
-	else \
-		git checkout $DEFAULT_BRANCH; \
-	fi)"
-
-	create_sym_link "$ssowat_dir" "/usr/share/ssowat"
-
-	lxc exec "$CONTAINER_ID" -- sh -c "systemctl reload nginx"
-}
-
 case ${2} in
 	prepare_script)
 		;;
@@ -61,40 +15,7 @@ case ${2} in
 	build_script)
 		case $PROJECT_NAME in
 			yunohost)
-				echo "Link yunohost"
+				echo "Running migrations yunohost"
-
-				# bin
-				create_sym_link "$PROJECT_DIR/bin/yunohost" "/usr/bin/yunohost"
-				create_sym_link "$PROJECT_DIR/bin/yunohost-api" "/usr/bin/yunohost-api"
-
-				# data
-				create_sym_link "$PROJECT_DIR/data/actionsmap/yunohost.yml" "/usr/share/moulinette/actionsmap/yunohost.yml"
-				create_sym_link "$PROJECT_DIR/data/hooks" "/usr/share/yunohost/hooks"
-				create_sym_link "$PROJECT_DIR/data/templates" "/usr/share/yunohost/templates"
-				create_sym_link "$PROJECT_DIR/data/helpers" "/usr/share/yunohost/helpers"
-				create_sym_link "$PROJECT_DIR/data/helpers.d" "/usr/share/yunohost/helpers.d"
-				create_sym_link "$PROJECT_DIR/data/other" "/usr/share/yunohost/yunohost-config/moulinette"
-				# debian
-				create_sym_link "$PROJECT_DIR/debian/conf/pam/mkhomedir" "/usr/share/pam-configs/mkhomedir"
-
-				# lib
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/ldap.lib.lua" "/usr/lib/metronome/modules/ldap.lib.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_auth_ldap2.lua" "/usr/lib/metronome/modules/mod_auth_ldap2.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_legacyauth.lua" "/usr/lib/metronome/modules/mod_legacyauth.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_storage_ldap.lua" "/usr/lib/metronome/modules/mod_storage_ldap.lua"
-				create_sym_link "$PROJECT_DIR/lib/metronome/modules/vcard.lib.lua" "/usr/lib/metronome/modules/vcard.lib.lua"
-
-				# src
-				create_sym_link "$PROJECT_DIR/src/yunohost" "/usr/lib/moulinette/yunohost"
-
-				# locales
-				create_sym_link "$PROJECT_DIR/locales" "/usr/lib/moulinette/yunohost/locales"
-
-				# moulinette
-				link_moulinette_from_git
-
-				# ssowat
-				link_ssowat_from_git
 
 				# Run migrations
 				lxc exec "$CONTAINER_ID" -- sh -c "yunohost tools migrations migrate"

utils.sh

@@ -5,7 +5,7 @@ source $current_dir/base.sh # Get variables from base.
 
 clean_containers()
 {
-    local base_image_to_clean=$1
+	local base_image_to_clean=$1
 
 	for image_to_delete in "$base_image_to_clean"{,"-tmp"}
 	do
@@ -15,13 +15,10 @@ clean_containers()
 		fi
 	done
 
-	for image_to_delete in "$base_image_to_clean-"{"before-install","before-postinstall","after-postinstall"}
+	if lxc image info $base_image_to_clean &>/dev/null
-	do
+	then
-		if lxc image info $image_to_delete &>/dev/null
+		lxc image delete $base_image_to_clean
-		then
+	fi
-			lxc image delete $image_to_delete
-		fi
-	done
 }
 
 wait_container()
@@ -95,7 +92,7 @@ rotate_image()
 	local alias_image=$2
 
 	# Save the finger print to delete the old image later 
-    local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}')
+	local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}')
 	local should_restart=0
 
 	# If the container is running, stop it
@@ -120,46 +117,59 @@ rotate_image()
 
 rebuild_base_containers()
 {
-    local debian_version=$1
+	local debian_version=$1
-    local ynh_version=$2
+	local ynh_version=$2
-    local arch=$3
+	local arch=$3
-    local base_image_to_rebuild="yunohost-$debian_version-$ynh_version"
+	local base_image_to_rebuild="yunohost-$debian_version-$ynh_version"
 
 	lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild-tmp"
 	
 	wait_container "$base_image_to_rebuild-tmp"
 
 	if [[ "$debian_version" == "buster" ]]
-    then
+	then
-        lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor
+		lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor
-    fi
+	fi
 
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install curl -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes wget curl"
 	# Install Git LFS, git comes pre installed with ubuntu image.
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install git-lfs -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes git-lfs"
 	# Install gitlab-runner binary since we need for cache/artifacts.
 	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash"
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner"
 
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install"
+	# Add yunohost repo
-
+	local CUSTOMDEB="deb http://forge.yunohost.org/debian/ stretch stable"
-    if [[ "$debian_version" == "buster" ]]
+	if [[ "$ynh_version" == "stable" ]] ; then
-    then
+        CUSTOMDEB="$CUSTOMDEB"
-        INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/buster-unstable/install_yunohost"
+    elif [[ "$ynh_version" == "testing" ]] ; then
-    else
+        CUSTOMDEB="$CUSTOMDEB testing"
-        INSTALL_SCRIPT="https://install.yunohost.org"
+    elif [[ "$ynh_version" == "unstable" ]] ; then
+        CUSTOMDEB="$CUSTOMDEB testing unstable"
     fi
-	# Install yunohost
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version"
 
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-postinstall"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "echo \"$CUSTOMDEB\" > /etc/apt/sources.list.d/yunohost.list"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "wget -O- https://forge.yunohost.org/yunohost.asc -q | apt-key add -qq - >/dev/null 2>&1"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update"
 
-	# Running post Install
+	# Patch install dependencies
-	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "touch /var/log/auth.log"
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "
+if ! id avahi > /dev/null 2>&1; then
+	avahi_id=$((500 + RANDOM % 500))
+	while cut -d ':' -f 3 /etc/passwd | grep -q \$avahi_id
+	do
+		avahi_id=$((500 + RANDOM % 500))
+	done
+	adduser --disabled-password  --quiet --system --home /var/run/avahi-daemon --no-create-home --gecos \"Avahi mDNS daemon\" --group avahi --uid \$avahi_id
+fi"
 
-	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-postinstall"
+	# Pre install dependencies
+	lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes $YNH_DEPENDENCIES $BUILD_DEPENDENCIES"
+
+	rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild"
 
 	lxc stop "$base_image_to_rebuild-tmp"
 
@@ -169,11 +179,11 @@ rebuild_base_containers()
 update_image() {
 	local image_to_update=$1
 
-    if ! lxc image info "$image_to_update" &>/dev/null
+	if ! lxc image info "$image_to_update" &>/dev/null
-    then
+	then
-        echo "Unable to upgrade image $image_to_update"
+		echo "Unable to upgrade image $image_to_update"
-        return
+		return
-    fi
+	fi
 
 	# Start and run upgrade
 	lxc launch "$image_to_update" "$image_to_update-tmp"