mirror of
https://github.com/YunoHost/lxd_img_builder.git
synced 2024-09-03 19:56:55 +02:00
349 lines
16 KiB
Bash
Executable file
349 lines
16 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
set -eu
|
|
|
|
readonly CMD=${1:-help}
|
|
readonly RELEASE=${2:-stable}
|
|
readonly DEBIAN_VERSION=${3:-bullseye}
|
|
readonly ARCH=${4:-$(dpkg --print-architecture)}
|
|
readonly CONTAINER=$RELEASE-$DEBIAN_VERSION-$ARCH
|
|
readonly IN_CONTAINER="incus exec $CONTAINER --"
|
|
[[ "$DEBIAN_VERSION" == "bullseye" ]] && gitbranch="dev" || gitbranch="$DEBIAN_VERSION"
|
|
|
|
function help()
|
|
{
|
|
cat << EOF
|
|
|
|
Usage: ./image_builder [command] [additional args...]
|
|
|
|
The 'base' image is after yunohost install script, but before postinstall. It is fetched by ynh-dev for local development.
|
|
The 'appci' image is after postinstall, with a first user named 'package_checker' as expected by the CI (package_check)
|
|
|
|
Commands:
|
|
|
|
- rebuild [RELEASE] [DEBIAN_VERSION] Rebuild the base and appci image for RELEASE on DEBIAN_VERSION
|
|
- update_appci [RELEASE] [DEBIAN_VERSION] Update the appci image for RELEASE on DEBIAN_VERSION
|
|
- rebuild_build_and_lint [RELEASE] [DEBIAN_VERSION] Rebuild the special 'build-and-lint' image for core CI
|
|
|
|
Arguments:
|
|
|
|
- RELEASE stable, testing or unstable
|
|
- DEBIAN_VERSION bullseye, or bookworm
|
|
|
|
EOF
|
|
}
|
|
|
|
function main()
|
|
{
|
|
|
|
KNOWN_COMMANDS=$(declare -F | awk '{print $3}')
|
|
if [ "$CMD" == "help" ] || [ "$CMD" == "--help" ]
|
|
then
|
|
help
|
|
elif grep -q -w "$1" <<< "$KNOWN_COMMANDS"
|
|
then
|
|
cmd="$1"
|
|
set -x
|
|
$cmd
|
|
else
|
|
echo "Unknown command '$1', check --help to list available commands"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function _publish_as()
|
|
{
|
|
local shortname="$1"
|
|
local alias="ynh-$shortname-$DEBIAN_VERSION-$ARCH-$RELEASE-base"
|
|
|
|
# Save the finger print to delete the old image later
|
|
#local finger_print_to_delete=$(incus image info "$alias" | grep Fingerprint | awk '{print $2}')
|
|
local should_restart=0
|
|
|
|
# If the container is running, stop it
|
|
if [ "$(incus info $CONTAINER | grep Status | awk '{print tolower($2)}')" = "running" ]
|
|
then
|
|
should_restart=1
|
|
incus stop "$CONTAINER"
|
|
fi
|
|
|
|
# Create image before install
|
|
incus publish "$CONTAINER" --alias "$alias" --reuse --public "os=YunoHost" "ynh-release=$RELEASE" "release=${DEBIAN_VERSION^}" "architecture=$ARCH" "stage=ynh-$shortname" "description=YunoHost $DEBIAN_VERSION $RELEASE ynh-$shortname $ARCH ($(date '+%Y%m%d'))"
|
|
|
|
# Remove old image
|
|
#incus image delete "$finger_print_to_delete"
|
|
|
|
if [ $should_restart = 1 ]
|
|
then
|
|
incus start "$CONTAINER"
|
|
sleep 5
|
|
# 240501: fix because the container was not getting an IP
|
|
$IN_CONTAINER dhclient eth0
|
|
fi
|
|
}
|
|
|
|
function rebuild_build_and_lint()
|
|
{
|
|
incus info $CONTAINER >/dev/null && incus delete $CONTAINER --force
|
|
incus launch images:debian/$DEBIAN_VERSION/$ARCH $CONTAINER
|
|
sleep 5
|
|
$IN_CONTAINER dhclient eth0
|
|
|
|
# Needed to build and access artefacts on core CI ...
|
|
incus file push ./gitlab-runner-light.deb $CONTAINER/root/
|
|
$IN_CONTAINER /bin/bash -c "apt-get update"
|
|
$IN_CONTAINER /bin/bash -c "DEBIAN_FRONTEND=noninteractive apt-get --assume-yes install ca-certificates git curl --no-install-recommends"
|
|
$IN_CONTAINER /bin/bash -c "dpkg -i /root/gitlab-runner-light.deb"
|
|
$IN_CONTAINER /bin/bash -c "rm /root/gitlab-runner-light.deb"
|
|
|
|
# This is for
|
|
# a) building .debs
|
|
TOOLING_APT_DEPENDENCIES="devscripts build-essential debhelper dpkg-dev dh-python wget hub"
|
|
TOOLING_APT_DEPENDENCIES+=" python3 python3-all python3-yaml python3-jinja2 python3-pip python-is-python3"
|
|
$IN_CONTAINER /bin/bash -c "apt-get update"
|
|
$IN_CONTAINER /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install $TOOLING_APT_DEPENDENCIES --no-install-recommends"
|
|
$IN_CONTAINER /bin/bash -c "apt-get clean"
|
|
|
|
# b) running tox, black, mypy, flake8, i18n string consistency check, bot sending PRs (actually this one is 'hub' in apt dependency right before)
|
|
if [[ $DEBIAN_VERSION == "bullseye" ]]
|
|
then
|
|
TOOLING_PIP_DEPENDENCIES='pyOpenSSL "tox==4.0.0" ansi2html toml "black>=22.12" jinja2 "packaging<22"'
|
|
else
|
|
TOOLING_PIP_DEPENDENCIES='pyOpenSSL "tox>=4.17" ansi2html toml "black>=24.4" jinja2 --break-system-packages'
|
|
fi
|
|
$IN_CONTAINER /bin/bash -c "PIP_NO_CACHE_DIR=1 PIP_PROGRESS_BAR='off' python3 -m pip install -U $TOOLING_PIP_DEPENDENCIES"
|
|
|
|
TOOLING_PIP_DEPENDENCIES='types-ipaddress types-enum34 types-cryptography types-toml types-requests types-PyYAML types-pyOpenSSL types-mock'
|
|
[[ $DEBIAN_VERSION == "bullseye" ]] || TOOLING_PIP_DEPENDENCIES+=" --break-system-packages"
|
|
$IN_CONTAINER /bin/bash -c "PIP_NO_CACHE_DIR=1 PIP_PROGRESS_BAR='off' python3 -m pip install -U $TOOLING_PIP_DEPENDENCIES"
|
|
|
|
# Disable mandb because zzzzz: https://stackoverflow.com/questions/69974259/fully-disable-man-db
|
|
$IN_CONTAINER /bin/bash -c "mv /usr/bin/mandb /usr/bin/mandb-OFF"
|
|
$IN_CONTAINER /bin/bash -c "cp -p /bin/true /usr/bin/mandb"
|
|
$IN_CONTAINER /bin/bash -c "rm -r /var/cache/man"
|
|
$IN_CONTAINER /bin/bash -c "rm /var/lib/man-db/auto-update"
|
|
$IN_CONTAINER /bin/bash -c "apt-mark hold man-db"
|
|
|
|
# Other tricks to save up space (at least 100MB or even 200MB wtf?!)
|
|
# https://stackoverflow.com/questions/59794891/how-does-debian-differ-from-debian-slim
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/doc"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/info"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/i18n"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/locale"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/man"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /var/lib/apt/lists/*"
|
|
$IN_CONTAINER /bin/bash -c "apt remove vim --purge --autoremove || true"
|
|
|
|
# Disable apt annoyances
|
|
$IN_CONTAINER systemctl -q disable apt-daily.timer --now
|
|
$IN_CONTAINER systemctl -q disable apt-daily-upgrade.timer --now
|
|
$IN_CONTAINER systemctl -q disable apt-daily.service --now
|
|
$IN_CONTAINER systemctl -q disable apt-daily-upgrade.service --now
|
|
$IN_CONTAINER rm -f /etc/cron.daily/apt-compat
|
|
$IN_CONTAINER cp /bin/true /usr/lib/apt/apt.systemd.daily
|
|
|
|
###########################################################################
|
|
_publish_as "build-and-lint"
|
|
###########################################################################
|
|
|
|
incus stop $CONTAINER
|
|
incus delete $CONTAINER
|
|
}
|
|
|
|
function _dependencies_to_preinstall()
|
|
{
|
|
curl https://raw.githubusercontent.com/YunoHost/yunohost/$gitbranch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v "moulinette\|ssowat\|yunohost-portal"
|
|
curl https://raw.githubusercontent.com/YunoHost/yunohost/$gitbranch/debian/control 2> /dev/null | sed -n '/^Recommends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g" | grep -v "yunohost-admin"
|
|
curl https://raw.githubusercontent.com/YunoHost/moulinette/$gitbranch/debian/control 2> /dev/null | sed -n '/^Depends:/,/^\w/{//!p}' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g"
|
|
# Same as above, except that all dependencies are in the same line
|
|
curl https://raw.githubusercontent.com/YunoHost/ssowat/$gitbranch/debian/control 2> /dev/null | grep '^Depends:' | sed 's/Depends://' | sed -e "s/,//g" -e "s/[(][^)]*[)]//g" -e "s/ | \S\+//g"
|
|
}
|
|
|
|
function rebuild()
|
|
{
|
|
incus info $CONTAINER >/dev/null && incus delete $CONTAINER --force
|
|
incus launch images:debian/$DEBIAN_VERSION/$ARCH $CONTAINER -c security.privileged=true -c security.nesting=true
|
|
|
|
sleep 5
|
|
$IN_CONTAINER dhclient eth0
|
|
|
|
# Needed to build and access artefacts on core CI ...
|
|
incus file push ./gitlab-runner-light.deb $CONTAINER/root/
|
|
$IN_CONTAINER /bin/bash -c "apt update"
|
|
$IN_CONTAINER /bin/bash -c "DEBIAN_FRONTEND=noninteractive apt-get --assume-yes install ca-certificates git curl"
|
|
$IN_CONTAINER /bin/bash -c "dpkg -i /root/gitlab-runner-light.deb"
|
|
$IN_CONTAINER /bin/bash -c "rm /root/gitlab-runner-light.deb"
|
|
|
|
local INSTALL_SCRIPT="https://raw.githubusercontent.com/YunoHost/install_script/main/$DEBIAN_VERSION"
|
|
|
|
# Download the YunoHost install script
|
|
$IN_CONTAINER /bin/bash -c "curl $INSTALL_SCRIPT > install.sh"
|
|
|
|
# Disable the install of yunohost itself, because we need this for the core CI
|
|
$IN_CONTAINER /bin/bash -c "sed -i -E 's/(step\s+install_yunohost_packages)/#\1/' install.sh"
|
|
$IN_CONTAINER /bin/bash -c "sed -i -E 's/(^\s+install_yunohost_packages)/#\1/' install.sh"
|
|
|
|
# Trick to disable restarting the service during install
|
|
$IN_CONTAINER /bin/bash -c "sed -i -E 's/(step\s+restart_services)/echo skip restart service #\1/' install.sh"
|
|
$IN_CONTAINER /bin/bash -c "echo exit 101 > /usr/sbin/policy-rc.d"
|
|
$IN_CONTAINER /bin/bash -c "chmod +x /usr/sbin/policy-rc.d"
|
|
|
|
# Actual install of everything...except yunohost itself
|
|
$IN_CONTAINER /bin/bash -c "cat install.sh | bash -s -- -a -d $RELEASE"
|
|
|
|
# To extract the dependencies, we want to retrieve the lines between "^Dependencies:" and the new line that doesn't start with a space (exclusively) . Then, we remove ",", then we remove the version specifiers "(>= X.Y)", then we add simple quotes to packages when there is a pipe (or) 'php-mysql|php-mysqlnd'.
|
|
$IN_CONTAINER /bin/bash -c "apt update"
|
|
$IN_CONTAINER /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install python3-all $(_dependencies_to_preinstall | tr '\n' ' ')"
|
|
$IN_CONTAINER /bin/bash -c "apt clean"
|
|
|
|
$IN_CONTAINER /bin/bash -c "rm /usr/sbin/policy-rc.d"
|
|
|
|
$IN_CONTAINER systemctl -q disable apt-daily.timer --now
|
|
$IN_CONTAINER systemctl -q disable apt-daily-upgrade.timer --now
|
|
$IN_CONTAINER systemctl -q disable apt-daily.service --now
|
|
$IN_CONTAINER systemctl -q disable apt-daily-upgrade.service --now
|
|
$IN_CONTAINER rm -f /etc/cron.daily/apt-compat
|
|
$IN_CONTAINER cp /bin/true /usr/lib/apt/apt.systemd.daily
|
|
|
|
# Disable mandb because zzzzz: https://stackoverflow.com/questions/69974259/fully-disable-man-db
|
|
$IN_CONTAINER /bin/bash -c "mv /usr/bin/mandb /usr/bin/mandb-OFF"
|
|
$IN_CONTAINER /bin/bash -c "cp -p /bin/true /usr/bin/mandb"
|
|
$IN_CONTAINER /bin/bash -c "rm -r /var/cache/man"
|
|
$IN_CONTAINER /bin/bash -c "rm /var/lib/man-db/auto-update"
|
|
$IN_CONTAINER /bin/bash -c "apt-mark hold man-db"
|
|
|
|
# Other tricks to save up space (at least 100MB or even 200MB wtf?!)
|
|
# https://stackoverflow.com/questions/59794891/how-does-debian-differ-from-debian-slim
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/doc"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/info"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/i18n"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/locale"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /usr/share/man"
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /var/lib/apt/lists/*"
|
|
$IN_CONTAINER /bin/bash -c "apt remove vim --purge --autoremove || true"
|
|
|
|
# Disable services that are useless in the vast majority of cases to try to improve perfs
|
|
$IN_CONTAINER systemctl -q disable rspamd --now || true
|
|
$IN_CONTAINER systemctl -q disable dovecot --now || true
|
|
$IN_CONTAINER systemctl -q disable postsrsd --now || true
|
|
$IN_CONTAINER systemctl -q disable metronome --now || true
|
|
$IN_CONTAINER systemctl -q disable fake-hwclock.service --now || true
|
|
$IN_CONTAINER systemctl -q disable haveged.service --now || true
|
|
$IN_CONTAINER systemctl -q disable unattended-upgrades.service --now || true
|
|
$IN_CONTAINER systemctl -q disable e2scrub_all.timer --now || true
|
|
$IN_CONTAINER systemctl -q disable logrotate.timer --now || true
|
|
$IN_CONTAINER systemctl -q disable phpsessionclean.timer --now || true
|
|
$IN_CONTAINER systemctl -q disable systemd-tmpfiles-clean.timer --now || true
|
|
|
|
# FIXME: where does this comes from x_x / why
|
|
$IN_CONTAINER sed -i 's/worker_processes.*;/worker_processes 4;/g' /etc/nginx/nginx.conf
|
|
|
|
$IN_CONTAINER reboot 0
|
|
sleep 5
|
|
|
|
###########################################################################
|
|
_publish_as "before-install"
|
|
###########################################################################
|
|
|
|
# Publish ynh-dev image
|
|
YUNOHOST_PACKAGES="yunohost yunohost-admin"
|
|
if [[ $DEBIAN_VERSION == "bookworm" ]]; then
|
|
YUNOHOST_PACKAGES+=" yunohost-portal"
|
|
fi
|
|
# Do not install vim (in recommends), just to save up space...
|
|
YUNOHOST_PACKAGES+="vim-"
|
|
$IN_CONTAINER /bin/bash -c "apt update"
|
|
$IN_CONTAINER /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install $YUNOHOST_PACKAGES"
|
|
$IN_CONTAINER /bin/bash -c "apt clean"
|
|
$IN_CONTAINER systemctl -q disable yunohost-api --now
|
|
$IN_CONTAINER systemctl -q disable yunoprompt --now
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /var/lib/apt/lists/*"
|
|
|
|
###########################################################################
|
|
_publish_as "dev"
|
|
###########################################################################
|
|
|
|
local YUNO_PWD="SomeSuperStrongPassword"
|
|
local DOMAIN="domain.tld"
|
|
local SUBDOMAIN="sub.$DOMAIN"
|
|
local TEST_USER="package_checker"
|
|
local TEST_USER_DISPLAY=${TEST_USER//"_"/""}
|
|
|
|
# Disable password strength check
|
|
$IN_CONTAINER yunohost tools postinstall --domain $DOMAIN --password $YUNO_PWD --username $TEST_USER --fullname "$TEST_USER_DISPLAY"
|
|
|
|
$IN_CONTAINER /bin/bash -c "echo 'admin_strength: -1' >> /etc/yunohost/settings.yml"
|
|
$IN_CONTAINER /bin/bash -c "echo 'user_strength: -1' >> /etc/yunohost/settings.yml"
|
|
|
|
$IN_CONTAINER yunohost domain add $SUBDOMAIN
|
|
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /var/lib/apt/lists/*"
|
|
|
|
###########################################################################
|
|
_publish_as "appci"
|
|
###########################################################################
|
|
|
|
CORE_TESTS_APT_DEPENDENCIES="python3-pip"
|
|
CORE_TESTS_PIP_DEPENCENDIES='mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock "packaging<22"'
|
|
|
|
if [[ "$DEBIAN_VERSION" == "bookworm" ]]
|
|
then
|
|
# We add php8.2-cli, mariadb-client and mariadb-server to the dependencies for test_app_resources
|
|
CORE_TESTS_APT_DEPENDENCIES+=" php8.2-cli mariadb-client mariadb-server"
|
|
CORE_TESTS_PIP_DEPENCENDIES+=" --break-system-packages"
|
|
fi
|
|
|
|
$IN_CONTAINER /bin/bash -c "apt-get update"
|
|
$IN_CONTAINER /bin/bash -c "DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt-get --assume-yes install --no-install-recommends $CORE_TESTS_APT_DEPENDENCIES"
|
|
$IN_CONTAINER /bin/bash -c "apt-get clean"
|
|
$IN_CONTAINER /bin/bash -c "PIP_NO_CACHE_DIR=1 PIP_PROGRESS_BAR='off' python3 -m pip install -U $CORE_TESTS_PIP_DEPENCENDIES"
|
|
|
|
$IN_CONTAINER /bin/bash -c "rm -rf /var/lib/apt/lists/*"
|
|
|
|
###########################################################################
|
|
_publish_as "core-tests"
|
|
###########################################################################
|
|
|
|
incus stop $CONTAINER
|
|
incus delete $CONTAINER
|
|
}
|
|
|
|
function update_appci()
|
|
{
|
|
local BASE="ynh-dev-$DEBIAN_VERSION-$ARCH-$RELEASE-base"
|
|
incus launch $BASE $CONTAINER -c security.privileged=true -c security.nesting=true
|
|
|
|
sleep 3
|
|
|
|
echo "nameserver 8.8.8.8" | $IN_CONTAINER tee /etc/resolv.conf
|
|
|
|
sleep 3
|
|
|
|
$IN_CONTAINER ping -c3 deb.debian.org || exit 1
|
|
|
|
$IN_CONTAINER apt-get update
|
|
$IN_CONTAINER apt-get dist-upgrade -y
|
|
|
|
local YUNO_PWD="SomeSuperStrongPassword"
|
|
local DOMAIN="domain.tld"
|
|
local SUBDOMAIN="sub.$DOMAIN"
|
|
local TEST_USER="package_checker"
|
|
local TEST_USER_DISPLAY=${TEST_USER//"_"/""}
|
|
|
|
# Disable password strength check
|
|
$IN_CONTAINER yunohost tools postinstall --domain $DOMAIN --password $YUNO_PWD --username $TEST_USER --fullname "$TEST_USER_DISPLAY"
|
|
|
|
$IN_CONTAINER /bin/bash -c "echo 'admin_strength: -1' >> /etc/yunohost/settings.yml"
|
|
$IN_CONTAINER /bin/bash -c "echo 'user_strength: -1' >> /etc/yunohost/settings.yml"
|
|
|
|
$IN_CONTAINER yunohost domain add $SUBDOMAIN
|
|
|
|
$IN_CONTAINER yunohost --version
|
|
|
|
|
|
incus stop $CONTAINER
|
|
_publish_as "appci"
|
|
incus delete $CONTAINER
|
|
}
|
|
|
|
main $@
|