diff --git a/README.md b/README.md index a9fadc4..c60edad 100644 --- a/README.md +++ b/README.md @@ -72,13 +72,11 @@ check_interval = 0 ## Using images -Use the field `image` to switch between `before-install`, `before-postinstall` or `after-postinstall` (`after-postinstall` by default) for example: -- `image: after-postinstall` to use the image after the postinstall of Yunohost +Use the field `image` to switch between `before-install` or `after-install` (`after-install` by default) for example: +- `image: after-install` to use the image after the postinstall of Yunohost - `image: before-install` to use the image before the installation of YunoHost -- `...` ## TODO - Support more YunoHost Core projects (for now only `yunohost` is supported, not `moulinette`...) -- Be sure that the runner can run several jobs in parallel (The `rebuild_base_containers` function in `prepare.sh` script can't be run in parallel, should we run a pre-prepare script manually to download and prepare lxc envs?). - Git pull this repo before running tests to keep these files up-to-date. diff --git a/auto_upgrade_container.sh b/auto_upgrade_container.sh index cf966a3..ed58bd6 100755 --- a/auto_upgrade_container.sh +++ b/auto_upgrade_container.sh @@ -1,15 +1,15 @@ +#!/usr/bin/env bash + current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/utils.sh # Get utils functions. for debian_version in "stretch" "buster" do - for ynh_version in "stable" "testing" "unstable" - do - for snapshot in "before-install" "before-postinstall" "after-postinstall" - do - image="yunohost-$debian_version-$ynh_version-$snapshot" - - update_image $image - done - done + for ynh_version in "stable" "testing" "unstable" + do + for snapshot in "before-install" "after-install" + do + update_image $debian_version $ynh_version $snapshot + done + done done \ No newline at end of file diff --git a/base.sh b/base.sh index 98ff9b0..bf6f3ed 100755 --- a/base.sh +++ b/base.sh @@ -1,18 +1,7 @@ #!/usr/bin/env bash # All Variables here: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference, strating with CUSTOM_ENV_ -#CUSTOM_ENV_CI_DEFAULT_BRANCH=stretch-unstable -#CUSTOM_ENV_CI_JOB_NAME=build1 -#CUSTOM_ENV_CI_BUILD_STAGE=pre-postinstall -#CUSTOM_ENV_CI_JOB_STAGE=pre-postinstall -#CUSTOM_ENV_CI_BUILD_NAME=build1 -#CUSTOM_ENV_CI_PROJECT_TITLE=yunohost -#CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH=linux/amd64 -#CUSTOM_ENV_CI_PROJECT_NAMESPACE=yunohost -#CUSTOM_ENV_CI_COMMIT_REF_NAME=stretch-unstable -#CUSTOM_ENV_CI_COMMIT_REF_SLUG=stretch-unstable -#CUSTOM_ENV_CI_PROJECT_NAME=yunohost -#CUSTOM_ENV_CI_PROJECT_DIR=/builds/yunohost/yunohost + CONTAINER_ID="runner-$CUSTOM_ENV_CI_RUNNER_ID-project-$CUSTOM_ENV_CI_PROJECT_ID-concurrent-$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID-$CUSTOM_ENV_CI_JOB_ID" ARCH="$(echo $CUSTOM_ENV_CI_RUNNER_EXECUTABLE_ARCH | cut -d'/' -f2)" # linux/amd64 DEFAULT_BRANCH="$CUSTOM_ENV_CI_DEFAULT_BRANCH" @@ -21,16 +10,16 @@ CURRENT_BRANCH="$CUSTOM_ENV_CI_COMMIT_REF_NAME" DEBIAN_VERSION=$(echo $CUSTOM_ENV_CI_COMMIT_REF_NAME | cut -d'-' -f1) # CUSTOM_ENV_CI_COMMIT_REF_NAME is the target branch of the MR: stretch-unstable, buster-unstable... if [ -z "$DEBIAN_VERSION" ] || [ "$DEBIAN_VERSION" != "stretch" ] && [ "$DEBIAN_VERSION" != "buster" ] then - DEBIAN_VERSION="$(echo $CUSTOM_ENV_CI_DEFAULT_BRANCH | cut -d'-' -f1)" # stretch-unstable, buster-unstable... - echo "Use the default debian version: $DEBIAN_VERSION" + DEBIAN_VERSION="$(echo $CUSTOM_ENV_CI_DEFAULT_BRANCH | cut -d'-' -f1)" # stretch-unstable, buster-unstable... + echo "Use the default debian version: $DEBIAN_VERSION" fi SNAPSHOT_NAME="$CUSTOM_ENV_CI_JOB_IMAGE" if [ -z "$SNAPSHOT_NAME" ] then - SNAPSHOT_NAME="after-postinstall" + SNAPSHOT_NAME="after-install" fi PROJECT_DIR="$CUSTOM_ENV_CI_PROJECT_DIR" PROJECT_NAME="$CUSTOM_ENV_CI_PROJECT_NAME" # For example yunohost-stretch-unstable -BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION" \ No newline at end of file +BASE_IMAGE="yunohost-$DEBIAN_VERSION-$CURRENT_VERSION" diff --git a/rebuild_all.sh b/rebuild_all.sh index 47372a1..45b3ab4 100755 --- a/rebuild_all.sh +++ b/rebuild_all.sh @@ -1,10 +1,43 @@ +#!/usr/bin/env bash + current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/utils.sh # Get utils functions. for debian_version in "stretch" "buster" do - for ynh_version in "stable" "testing" "unstable" - do - rebuild_base_containers $debian_version $ynh_version "amd64" - done + + # There is no stable and testing version for Buster at this time. + if [[ "$debian_version" == "buster" ]] + then + rebuild_base_containers $debian_version "unstable" "amd64" + else + rebuild_base_containers $debian_version "stable" "amd64" + + for ynh_version in "testing" "unstable" + do + for snapshot in "before-install" "after-install" + do + lxc launch "yunohost-$debian_version-stable-$snapshot" "yunohost-$debian_version-$ynh_version-$snapshot-tmp" + + if [[ "$ynh_version" == "testing" ]] + then + repo_version="testing" + elif [[ "$DISTRIB" == "unstable" ]] + then + repo_version="testing unstable" + fi + + lxc exec "yunohost-$debian_version-$ynh_version-$snapshot-tmp" -- /bin/bash -c "for FILE in \`ls /etc/apt/sources.list /etc/apt/sources.list.d/*\`; + do + sed -i 's@^deb http://forge.yunohost.org.*@& $repo_version@' \$FILE + done" + + rotate_image "yunohost-$debian_version-$ynh_version-$snapshot-tmp" "yunohost-$debian_version-$ynh_version-$snapshot" + + lxc delete -f "yunohost-$debian_version-$ynh_version-$snapshot-tmp" + + update_image $debian_version $ynh_version $snapshot + done + done + fi done \ No newline at end of file diff --git a/run.sh b/run.sh index 1ccc3bd..33a538e 100755 --- a/run.sh +++ b/run.sh @@ -3,52 +3,6 @@ current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $current_dir/base.sh # Get variables from base. -create_sym_link() { - local DEST=$1 - local LINK=$2 - # Remove current sources if not a symlink - lxc exec "$CONTAINER_ID" -- sh -c "[ -L "$LINK" ] || rm -rf $LINK" - # Symlink from Git repository - lxc exec "$CONTAINER_ID" -- sh -c "ln -sfn $DEST $LINK" -} - -link_moulinette_from_git() { - echo "Moulinette: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise" - - moulinette_dir="/tmp/ci_moulinette" - lxc exec "$CONTAINER_ID" -- sh -c "mkdir $moulinette_dir" - lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/moulinette $moulinette_dir" - lxc exec "$CONTAINER_ID" -- sh -c "(cd $moulinette_dir; \ - if git ls-remote --heads | grep -q $CURRENT_BRANCH; \ - then \ - git checkout $CURRENT_BRANCH; \ - else \ - git checkout $DEFAULT_BRANCH; \ - fi)" - - create_sym_link "$moulinette_dir/locales" "/usr/share/moulinette/locale" - create_sym_link "$moulinette_dir/moulinette" "/usr/lib/python2.7/dist-packages/moulinette" -} - -link_ssowat_from_git() { - echo "SSOWAT: using the $CURRENT_BRANCH branch if it exists, $DEFAULT_BRANCH otherwise" - - ssowat_dir="/tmp/ci_ssowat" - lxc exec "$CONTAINER_ID" -- sh -c "mkdir $ssowat_dir" - lxc exec "$CONTAINER_ID" -- sh -c "git clone https://github.com/YunoHost/ssowat $ssowat_dir" - lxc exec "$CONTAINER_ID" -- sh -c "(cd $ssowat_dir; \ - if git ls-remote --heads | grep -q $CURRENT_BRANCH; \ - then \ - git checkout $CURRENT_BRANCH; \ - else \ - git checkout $DEFAULT_BRANCH; \ - fi)" - - create_sym_link "$ssowat_dir" "/usr/share/ssowat" - - lxc exec "$CONTAINER_ID" -- sh -c "systemctl reload nginx" -} - case ${2} in prepare_script) ;; @@ -61,43 +15,7 @@ case ${2} in build_script) case $PROJECT_NAME in yunohost) - echo "Link yunohost" - - # bin - create_sym_link "$PROJECT_DIR/bin/yunohost" "/usr/bin/yunohost" - create_sym_link "$PROJECT_DIR/bin/yunohost-api" "/usr/bin/yunohost-api" - - # data - create_sym_link "$PROJECT_DIR/data/actionsmap/yunohost.yml" "/usr/share/moulinette/actionsmap/yunohost.yml" - create_sym_link "$PROJECT_DIR/data/hooks" "/usr/share/yunohost/hooks" - create_sym_link "$PROJECT_DIR/data/templates" "/usr/share/yunohost/templates" - create_sym_link "$PROJECT_DIR/data/helpers" "/usr/share/yunohost/helpers" - create_sym_link "$PROJECT_DIR/data/helpers.d" "/usr/share/yunohost/helpers.d" - create_sym_link "$PROJECT_DIR/data/other" "/usr/share/yunohost/yunohost-config/moulinette" - # debian - create_sym_link "$PROJECT_DIR/debian/conf/pam/mkhomedir" "/usr/share/pam-configs/mkhomedir" - - # lib - create_sym_link "$PROJECT_DIR/lib/metronome/modules/ldap.lib.lua" "/usr/lib/metronome/modules/ldap.lib.lua" - create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_auth_ldap2.lua" "/usr/lib/metronome/modules/mod_auth_ldap2.lua" - create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_legacyauth.lua" "/usr/lib/metronome/modules/mod_legacyauth.lua" - create_sym_link "$PROJECT_DIR/lib/metronome/modules/mod_storage_ldap.lua" "/usr/lib/metronome/modules/mod_storage_ldap.lua" - create_sym_link "$PROJECT_DIR/lib/metronome/modules/vcard.lib.lua" "/usr/lib/metronome/modules/vcard.lib.lua" - - # src - create_sym_link "$PROJECT_DIR/src/yunohost" "/usr/lib/moulinette/yunohost" - - # locales - create_sym_link "$PROJECT_DIR/locales" "/usr/lib/moulinette/yunohost/locales" - - # moulinette - link_moulinette_from_git - - # ssowat - link_ssowat_from_git - - # Run migrations - lxc exec "$CONTAINER_ID" -- sh -c "yunohost tools migrations migrate" + # Nothing to do? ;; esac ;; diff --git a/utils.sh b/utils.sh index 46ac7ad..2dbb5af 100755 --- a/utils.sh +++ b/utils.sh @@ -5,7 +5,7 @@ source $current_dir/base.sh # Get variables from base. clean_containers() { - local base_image_to_clean=$1 + local base_image_to_clean=$1 for image_to_delete in "$base_image_to_clean"{,"-tmp"} do @@ -15,7 +15,7 @@ clean_containers() fi done - for image_to_delete in "$base_image_to_clean-"{"before-install","before-postinstall","after-postinstall"} + for image_to_delete in "$base_image_to_clean-"{"before-install","after-install"} do if lxc image info $image_to_delete &>/dev/null then @@ -95,7 +95,7 @@ rotate_image() local alias_image=$2 # Save the finger print to delete the old image later - local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}') + local finger_print_to_delete=$(lxc image info "$alias_image" | grep Fingerprint | awk '{print $2}') local should_restart=0 # If the container is running, stop it @@ -117,49 +117,80 @@ rotate_image() fi } +# These lines are used to extract the dependencies/recommendations from the debian/control file. +# /!\ There's a high risk of lamentable failure if we change the format of this file +get_dependencies() +{ + local debian_version=$1 + local ynh_version=$2 + + # To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'. + YUNOHOST_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v moulinette | grep -v ssowat | tr "\n" " ") + YUNOHOST_RECOMMENDS=$(curl https://raw.githubusercontent.com/YunoHost/yunohost/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") + MOULINETTE_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/moulinette/$debian_version-$ynh_version/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+/g" | tr "\n" " ") + # Same as above, except that all dependencies are in the same line + SSOWAT_DEPENDENCIES=$(curl https://raw.githubusercontent.com/YunoHost/ssowat/$debian_version-$ynh_version/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | tr "\n" " ") + BUILD_DEPENDENCIES="git-buildpackage postfix python-setuptools python-pip" + PIP_PKG="mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html" +} rebuild_base_containers() { - local debian_version=$1 - local ynh_version=$2 - local arch=$3 - local base_image_to_rebuild="yunohost-$debian_version-$ynh_version" + local debian_version=$1 + local ynh_version=$2 + local arch=$3 + local base_image_to_rebuild="yunohost-$debian_version-$ynh_version" lxc launch images:debian/$debian_version/$arch "$base_image_to_rebuild-tmp" wait_container "$base_image_to_rebuild-tmp" if [[ "$debian_version" == "buster" ]] - then - lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor - fi + then + lxc config set "$base_image_to_rebuild-tmp" security.nesting true # Need this for buster because it is using apparmor + fi lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get update" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install curl -y" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes wget curl" # Install Git LFS, git comes pre installed with ubuntu image. lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install git-lfs -y" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes git-lfs" # Install gitlab-runner binary since we need for cache/artifacts. lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl -s https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash" - lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install gitlab-runner -y" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "apt-get install --assume-yes gitlab-runner" + + if [[ "$debian_version" == "buster" ]] + then + INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/buster-unstable/install_yunohost" + else + INSTALL_SCRIPT="https://install.yunohost.org" + fi + + # Download the YunoHost install script + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT > install.sh" + + # Patch the YunoHost install script + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "sed -i -E 's/(step\s+restart_services)/#\1/' install.sh" + + # Run the YunoHost install script patched + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "cat install.sh | bash -s -- -a -d $ynh_version" + + get_dependencies $debian_version $ynh_version + + # Pre install dependencies + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "pip install -U $PIP_PKG" rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-install" - if [[ "$debian_version" == "buster" ]] - then - INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/buster-unstable/install_yunohost" - else - INSTALL_SCRIPT="https://install.yunohost.org" - fi - # Install yunohost + # Install YunoHost lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $ynh_version" - - rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-before-postinstall" - - # Running post Install + + # Run postinstall lxc exec "$base_image_to_rebuild-tmp" -- /bin/bash -c "yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns" - rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-postinstall" + rotate_image "$base_image_to_rebuild-tmp" "$base_image_to_rebuild-after-install" lxc stop "$base_image_to_rebuild-tmp" @@ -167,13 +198,16 @@ rebuild_base_containers() } update_image() { - local image_to_update=$1 + local debian_version=$1 + local ynh_version=$2 + local snapshot=$3 + local image_to_update="yunohost-$debian_version-$ynh_version-$snapshot" - if ! lxc image info "$image_to_update" &>/dev/null - then - echo "Unable to upgrade image $image_to_update" - return - fi + if ! lxc image info "$image_to_update" &>/dev/null + then + echo "Unable to upgrade image $image_to_update" + return + fi # Start and run upgrade lxc launch "$image_to_update" "$image_to_update-tmp" @@ -181,7 +215,12 @@ update_image() { wait_container "$image_to_update-tmp" lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get update" - lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade -y" + lxc exec "$image_to_update-tmp" -- /bin/bash -c "apt-get upgrade --assume-yes" + + get_dependencies $debian_version $ynh_version + + lxc exec "$image_to_update-tmp" -- /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes -o Dpkg::Options::=\"--force-confold\" install --assume-yes $YUNOHOST_DEPENDENCIES $YUNOHOST_RECOMMENDS $MOULINETTE_DEPENDENCIES $SSOWAT_DEPENDENCIES $BUILD_DEPENDENCIES" + lxc exec "$image_to_update-tmp" -- /bin/bash -c "pip install -U $PIP_PKG" rotate_image "$image_to_update-tmp" "$image_to_update"