Merge pull request #1 from YunoHost/new-version

New version
This commit is contained in:
Kayou 2020-05-22 22:08:53 +02:00 committed by GitHub
commit 7072c26e72
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 125 additions and 148 deletions

View file

@ -72,13 +72,11 @@ check_interval = 0
## Using images
Use the field `image` to switch between `before-install`, `before-postinstall` or `after-postinstall` (`after-postinstall` by default) for example:
- `image: after-postinstall` to use the image after the postinstall of Yunohost
Use the field `image` to switch between `before-install` or `after-install` (`after-install` by default) for example:
- `image: after-install` to use the image after the postinstall of Yunohost
- `image: before-install` to use the image before the installation of YunoHost
- `...`
## TODO
- Support more YunoHost Core projects (for now only `yunohost` is supported, not `moulinette`...)
- Be sure that the runner can run several jobs in parallel (The `rebuild_base_containers` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?).
- Git pull this repo before running tests to keep these files up-to-date.

View file

@ -1,15 +1,15 @@
#!/usr/bin/env bash
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $current_dir/utils.sh # Get utils functions.
for debian_version in "stretch" "buster"
do
for ynh_version in "stable" "testing" "unstable"
do
for snapshot in "before-install" "before-postinstall" "after-postinstall"
do
image="yunohost-$debian_version-$ynh_version-$snapshot"
update_image $image
done
done
for ynh_version in "stable" "testing" "unstable"
do
for snapshot in "before-install" "after-install"
do
update_image $debian_version $ynh_version $snapshot
done
done
done

21
base.sh
View file

@ -1,18 +1,7 @@
#!/usr/bin/env bash
# All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_
#CUSTOM_ENV_CI_DEFAULT_BRANCH=stretch-unstable
#CUSTOM_ENV_CI_JOB_NAME=build1
#CUSTOM_ENV_CI_BUILD_STAGE=pre-postinstall
#CUSTOM_ENV_CI_JOB_STAGE=pre-postinstall
#CUSTOM_ENV_CI_BUILD_NAME=build1
#CUSTOM_ENV_CI_PROJECT_TITLE=yunohost
#CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH=linux/amd64
#CUSTOM_ENV_CI_PROJECT_NAMESPACE=yunohost
#CUSTOM_ENV_CI_COMMIT_REF_NAME=stretch-unstable
#CUSTOM_ENV_CI_COMMIT_REF_SLUG=stretch-unstable
#CUSTOM_ENV_CI_PROJECT_NAME=yunohost
#CUSTOM_ENV_CI_PROJECT_DIR=/builds/yunohost/yunohost
CONTAINER_ID="runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID-$CUSTOM_ENV_CI_JOB_ID"
ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64
DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH"
@ -21,16 +10,16 @@ CURRENT_BRANCH="$CUSTOM_ENV_CI_COMMIT_REF_NAME"
DEBIAN_VERSION=$(echo $CUSTOM_ENV_CI_COMMIT_REF_NAME | cut -d'-' -f1) # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR: stretch-unstable, buster-unstable...
if [ -z "$DEBIAN_VERSION" ] || [ "$DEBIAN_VERSION" != "stretch" ] && [ "$DEBIAN_VERSION" != "buster" ]
then
DEBIAN_VERSION="$(echo $CUSTOM_ENV_CI_DEFAULT_BRANCH | cut -d'-' -f1)" # stretch-unstable, buster-unstable...
echo "Use the default debian version: $DEBIAN_VERSION"
DEBIAN_VERSION="$(echo $CUSTOM_ENV_CI_DEFAULT_BRANCH | cut -d'-' -f1)" # stretch-unstable, buster-unstable...
echo "Use the default debian version: $DEBIAN_VERSION"
fi
SNAPSHOT_NAME="$CUSTOM_ENV_CI_JOB_IMAGE"
if [ -z "$SNAPSHOT_NAME" ]
then
SNAPSHOT_NAME="after-postinstall"
SNAPSHOT_NAME="after-install"
fi
PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR"
PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME"
# For example yunohost-stretch-unstable
BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"
BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION"

View file

@ -1,10 +1,43 @@
#!/usr/bin/env bash
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $current_dir/utils.sh # Get utils functions.
for debian_version in "stretch" "buster"
do
for ynh_version in "stable" "testing" "unstable"
do
rebuild_base_containers $debian_version $ynh_version "amd64"
done
# There is no stable and testing version for Buster at this time.
if [[ "$debian_version" == "buster" ]]
then
rebuild_base_containers $debian_version "unstable" "amd64"
else
rebuild_base_containers $debian_version "stable" "amd64"
for ynh_version in "testing" "unstable"
do
for snapshot in "before-install" "after-install"
do
lxc launch "yunohost-$debian_version-stable-$snapshot" "yunohost-$debian_version-$ynh_version-$snapshot-tmp"
if [[ "$ynh_version" == "testing" ]]
then
repo_version="testing"
elif [[ "$DISTRIB" == "unstable" ]]
then
repo_version="testing unstable"
fi
lxc exec "yunohost-$debian_version-$ynh_version-$snapshot-tmp" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`;
do
sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE
done"
rotate_image "yunohost-$debian_version-$ynh_version-$snapshot-tmp" "yunohost-$debian_version-$ynh_version-$snapshot"
lxc delete -f "yunohost-$debian_version-$ynh_version-$snapshot-tmp"
update_image $debian_version $ynh_version $snapshot
done
done
fi
done

84
run.sh
View file

@ -3,52 +3,6 @@
current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $current_dir/base.sh # Get variables from base.
create_sym_link() {
local DEST=$1
local LINK=$2
# Remove current sources if not a symlink
lxc exec "$CONTAINER_ID" -- sh -c "[ -L "$LINK" ] || rm -rf $LINK"
# Symlink from Git repository
lxc exec "$CONTAINER_ID" -- sh -c "ln -sfn $DEST $LINK"
}
link_moulinette_from_git() {
echo "Moulinette: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
moulinette_dir="/tmp/ci_moulinette"
lxc exec "$CONTAINER_ID" -- sh -c "mkdir $moulinette_dir"
lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/moulinette $moulinette_dir"
lxc exec "$CONTAINER_ID" -- sh -c "(cd $moulinette_dir; \
if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
then \
git checkout $CURRENT_BRANCH; \
else \
git checkout $DEFAULT_BRANCH; \
fi)"
create_sym_link "$moulinette_dir/locales" "/usr/share/moulinette/locale"
create_sym_link "$moulinette_dir/moulinette" "/usr/lib/python2.7/dist-packages/moulinette"
}
link_ssowat_from_git() {
echo "SSOWAT: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise"
ssowat_dir="/tmp/ci_ssowat"
lxc exec "$CONTAINER_ID" -- sh -c "mkdir $ssowat_dir"
lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/ssowat $ssowat_dir"
lxc exec "$CONTAINER_ID" -- sh -c "(cd $ssowat_dir; \
if git ls-remote --heads | grep -q $CURRENT_BRANCH; \
then \
git checkout $CURRENT_BRANCH; \
else \
git checkout $DEFAULT_BRANCH; \
fi)"
create_sym_link "$ssowat_dir" "/usr/share/ssowat"
lxc exec "$CONTAINER_ID" -- sh -c "systemctl reload nginx"
}
case ${2} in
prepare_script)
;;
@ -61,43 +15,7 @@ case ${2} in
build_script)
case $PROJECT_NAME in
yunohost)
echo "Link yunohost"
# bin
create_sym_link "$PROJECT_DIR/bin/yunohost" "/usr/bin/yunohost"
create_sym_link "$PROJECT_DIR/bin/yunohost-api" "/usr/bin/yunohost-api"
# data
create_sym_link "$PROJECT_DIR/data/actionsmap/yunohost.yml" "/usr/share/moulinette/actionsmap/yunohost.yml"
create_sym_link "$PROJECT_DIR/data/hooks" "/usr/share/yunohost/hooks"
create_sym_link "$PROJECT_DIR/data/templates" "/usr/share/yunohost/templates"
create_sym_link "$PROJECT_DIR/data/helpers" "/usr/share/yunohost/helpers"
create_sym_link "$PROJECT_DIR/data/helpers.d" "/usr/share/yunohost/helpers.d"
create_sym_link "$PROJECT_DIR/data/other" "/usr/share/yunohost/yunohost-config/moulinette"
# debian
create_sym_link "$PROJECT_DIR/debian/conf/pam/mkhomedir" "/usr/share/pam-configs/mkhomedir"
# lib
create_sym_link "$PROJECT_DIR/lib/metronome/modules/ldap.lib.lua" "/usr/lib/metronome/modules/ldap.lib.lua"
create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_auth_ldap2.lua" "/usr/lib/metronome/modules/mod_auth_ldap2.lua"
create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_legacyauth.lua" "/usr/lib/metronome/modules/mod_legacyauth.lua"
create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_storage_ldap.lua" "/usr/lib/metronome/modules/mod_storage_ldap.lua"
create_sym_link "$PROJECT_DIR/lib/metronome/modules/vcard.lib.lua" "/usr/lib/metronome/modules/vcard.lib.lua"
# src
create_sym_link "$PROJECT_DIR/src/yunohost" "/usr/lib/moulinette/yunohost"
# locales
create_sym_link "$PROJECT_DIR/locales" "/usr/lib/moulinette/yunohost/locales"
# moulinette
link_moulinette_from_git
# ssowat
link_ssowat_from_git
# Run migrations
lxc exec "$CONTAINER_ID" -- sh -c "yunohost tools migrations migrate"
# Nothing to do?
;;
esac
;;

103
utils.sh
View file

@ -5,7 +5,7 @@ source $current_dir/base.sh # Get variables from base.
clean_containers()
{
local base_image_to_clean=$1
local base_image_to_clean=$1
for image_to_delete in "$base_image_to_clean"{,"-tmp"}
do
@ -15,7 +15,7 @@ clean_containers()
fi
done
for image_to_delete in "$base_image_to_clean-"{"before-install","before-postinstall","after-postinstall"}
for image_to_delete in "$base_image_to_clean-"{"before-install","after-install"}
do
if lxc image info $image_to_delete &>/dev/null
then
@ -95,7 +95,7 @@ rotate_image()
local alias_image=$2
# Save the finger print to delete the old image later
local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}')
local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}')
local should_restart=0
# If the container is running, stop it
@ -117,49 +117,80 @@ rotate_image()
fi
}
# These lines are used to extract the dependencies/recommendations from the debian/control file.
# /!\ There's a high risk of lamentable failure if we change the format of this file
get_dependencies()
{
local debian_version=$1
local ynh_version=$2
# To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'.
YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ")
YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+/g" | tr "\n" " ")
# Same as above, except that all dependencies are in the same line
SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$debian_version-$ynh_version/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ")
BUILD_DEPENDENCIES="git-buildpackage postfix python-setuptools python-pip"
PIP_PKG="mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html"
}
rebuild_base_containers()
{
local debian_version=$1
local ynh_version=$2
local arch=$3
local base_image_to_rebuild="yunohost-$debian_version-$ynh_version"
local debian_version=$1
local ynh_version=$2
local arch=$3
local base_image_to_rebuild="yunohost-$debian_version-$ynh_version"
lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild-tmp"
wait_container "$base_image_to_rebuild-tmp"
if [[ "$debian_version" == "buster" ]]
then
lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor
fi
then
lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor
fi
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install curl -y"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes wget curl"
# Install Git LFS, git comes pre installed with ubuntu image.
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install git-lfs -y"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes git-lfs"
# Install gitlab-runner binary since we need for cache/artifacts.
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner"
if [[ "$debian_version" == "buster" ]]
then
INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/buster-unstable/install_yunohost"
else
INSTALL_SCRIPT="https://install.yunohost.org"
fi
# Download the YunoHost install script
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh"
# Patch the YunoHost install script
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh"
# Run the YunoHost install script patched
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version"
get_dependencies $debian_version $ynh_version
# Pre install dependencies
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES"
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "pip install -U $PIP_PKG"
rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install"
if [[ "$debian_version" == "buster" ]]
then
INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/buster-unstable/install_yunohost"
else
INSTALL_SCRIPT="https://install.yunohost.org"
fi
# Install yunohost
# Install YunoHost
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version"
rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-postinstall"
# Running post Install
# Run postinstall
lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns"
rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-postinstall"
rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-install"
lxc stop "$base_image_to_rebuild-tmp"
@ -167,13 +198,16 @@ rebuild_base_containers()
}
update_image() {
local image_to_update=$1
local debian_version=$1
local ynh_version=$2
local snapshot=$3
local image_to_update="yunohost-$debian_version-$ynh_version-$snapshot"
if ! lxc image info "$image_to_update" &>/dev/null
then
echo "Unable to upgrade image $image_to_update"
return
fi
if ! lxc image info "$image_to_update" &>/dev/null
then
echo "Unable to upgrade image $image_to_update"
return
fi
# Start and run upgrade
lxc launch "$image_to_update" "$image_to_update-tmp"
@ -181,7 +215,12 @@ update_image() {
wait_container "$image_to_update-tmp"
lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update"
lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y"
lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade --assume-yes"
get_dependencies $debian_version $ynh_version
lxc exec "$image_to_update-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES"
lxc exec "$image_to_update-tmp" -- /bin/bash -c "pip install -U $PIP_PKG"
rotate_image "$image_to_update-tmp" "$image_to_update"