ynh-dev/ynh-dev
Salamandar b8c47886a5
Merge pull request #71 from Salamandar/incus
Allow the use of incus instead of LXD.
2024-03-09 18:40:36 +01:00

591 lines
22 KiB
Bash
Executable file

#!/usr/bin/env bash
function show_usage() {
cat <<EOF
${BLUE}On the host, to manage boxes${NORMAL}
${BLUE}============================${NORMAL}
start [DIST] [NAME] [YNH_BRANCH] (Create and) starts a box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default)
attach [DIST] [NAME] [YNH_BRANCH] Attach an already started box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default)
destroy [DIST] [NAME] [YNH_BRANCH] Destroy the ynh-dev box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default)
rebuild [DIST] [NAME] [YNH_BRANCH] Rebuild a fresh, up-to-date box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default)
${BLUE}Inside the dev instance${NORMAL}
${BLUE}=======================${NORMAL}
ip Give the ip of the guest container
use-git [PKG] Use Git repositories from dev environment path
test [PKG] Deploy, update and run tests for some packages
Tests for single modules and functions can ran with
e.g. ./ynh-dev test yunohost/appurl:urlavailable
dev Watch python files and restart yunohost-api and yunohost-portal-api when changes occur
catalog
build Rebuild the custom catalog
add Add the custom catalog in Yunohost catalog list
override Override default catalog with the custom catalog
reset Reset the catalog list to Yunohost's default
rebuild-api-doc Rebuild the API swagger doc
EOF
}
function main()
{
local ACTION="$1"
local ARGUMENTS="${@:2}"
[ -z "$ACTION" ] && show_usage && exit 0
case "${ACTION}" in
help|-h|--help) show_usage $ARGUMENTS ;;
start|--start) start_ynhdev $ARGUMENTS ;;
attach|--attach) attach_ynhdev $ARGUMENTS ;;
destroy|--destroy) destroy_ynhdev $ARGUMENTS ;;
rebuild|--rebuild) rebuild_ynhdev $ARGUMENTS ;;
ip|--ip) show_vm_ip $ARGUMENTS ;;
use-git|--use-git) use_git $ARGUMENTS ;;
dev|--dev) dev $ARGUMENTS ;;
test|--test) run_tests $ARGUMENTS ;;
catalog|--catalog) catalog $ARGUMENTS ;;
rebuild-api-doc|--rebuild-api-doc) rebuild_api_doc $ARGUMENTS ;;
*) critical "Unknown action ${ACTION}." ;;
esac
}
##################################################################
# Misc helpers #
##################################################################
readonly NORMAL=$(printf '\033[0m')
readonly BOLD=$(printf '\033[1m')
readonly faint=$(printf '\033[2m')
readonly UNDERLINE=$(printf '\033[4m')
readonly NEGATIVE=$(printf '\033[7m')
readonly RED=$(printf '\033[31m')
readonly GREEN=$(printf '\033[32m')
readonly ORANGE=$(printf '\033[33m')
readonly BLUE=$(printf '\033[34m')
readonly YELLOW=$(printf '\033[93m')
readonly WHITE=$(printf '\033[39m')
function success()
{
local msg=${1}
echo "[${BOLD}${GREEN} OK ${NORMAL}] ${msg}"
}
function info()
{
local msg=${1}
echo "[${BOLD}${BLUE}INFO${NORMAL}] ${msg}"
}
function warn()
{
local msg=${1}
echo "[${BOLD}${ORANGE}WARN${NORMAL}] ${msg}" 2>&1
}
function error()
{
local msg=${1}
echo "[${BOLD}${RED}FAIL${NORMAL}] ${msg}" 2>&1
}
function critical()
{
local msg=${1}
echo "[${BOLD}${RED}CRIT${NORMAL}] ${msg}" 2>&1
exit 1
}
function assert_inside_vm() {
[ -d /etc/yunohost ] || critical "There's no YunoHost in there. Are you sure that you are inside the container ?"
}
function assert_yunohost_is_installed() {
[ -e /etc/yunohost/installed ] || critical "YunoHost is not yet properly installed. Rerun this after post-install."
}
function create_sym_link() {
local DEST=$1
local LINK=$2
# Remove current sources if not a symlink
[ -L "$LINK" ] || rm -rf $LINK
# Symlink from Git repository
ln -sfn $DEST $LINK
}
function prepare_cache_and_deps() {
local DEV_PATH="$1"
local CACHE_PATH="$2"
mkdir -p "$CACHE_PATH"
# create_sym_link "$DEV_PATH/.env" "$CACHE_PATH/.env"
create_sym_link "$CACHE_PATH/node_modules" "$DEV_PATH/node_modules"
create_sym_link "$DEV_PATH/package.json" "$CACHE_PATH/package.json"
create_sym_link "$DEV_PATH/yarn.lock" "$CACHE_PATH/yarn.lock"
# Vite require node v14 to parse modern syntax
local DISTRO="$(lsb_release -s -c)"
local YARN=$([ "$DISTRO" == "bullseye" ] && echo "yarnpkg" || echo "yarn")
if [ "$DISTRO" == "bullseye" ]
then
if [[ ! $(node -v) == v14* ]]
then
info "Installing node v14..."
KEYRING=/usr/share/keyrings/nodesource.gpg
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor | sudo tee "$KEYRING" >/dev/null
gpg --no-default-keyring --keyring "$KEYRING" --list-keys
VERSION=node_14.x
echo "deb [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | sudo tee /etc/apt/sources.list.d/nodesource.list
apt update
apt install nodejs -y
# to return to nodejs debian version
# apt purge nodejs && rm -r /etc/apt/sources.list.d/nodesource.list && apt install nodejs
export NODE_PATH=/usr/lib/nodejs:/usr/share/nodejs
fi
fi
# install yarn if not already
if [[ $(dpkg-query -W -f='${Status}' "$YARN" 2>/dev/null | grep -c "ok installed") -eq 0 ]];
then
info "Installing yarn…"
apt update
apt install "$YARN"
fi
pushd "$CACHE_PATH"
# Install dependencies with yarn forced to lock file versions (equivalent to `npm ci`)
info "Installing dependencies ... (this may take a while)"
"$YARN" install --frozen-lockfile
popd
}
##################################################################
# Actions #
##################################################################
function check_incus_setup()
{
# Check incus is installed somehow
if ! which incus &>/dev/null; then
critical "You need to have Incus installed for ynh-dev to be usable from the host machine. Refer to the README to know how to install it."
fi
if ! id -nG "$(whoami)" | grep -qw "incus-admin"; then
critical "You need to be in the incus-admin group!"
fi
ip a | grep -q incusbr0 \
|| warn "There is no 'incusbr0' interface... Did you ran 'incus admin init' ?"
set_incus_remote
}
function set_incus_remote()
{
configured=$(incus remote list -f json | jq 'has("yunohost")')
if [[ "$configured" != "true" ]]; then
incus remote add yunohost https://devbaseimgs.yunohost.org --public
fi
}
function start_ynhdev()
{
check_incus_setup
local DIST=${1:-bookworm}
local YNH_BRANCH=${3:-unstable}
local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH}
if ! incus info $BOX &>/dev/null
then
if ! incus image info $BOX-base &>/dev/null
then
LXC_BASE="ynh-dev-$DIST-amd64-$YNH_BRANCH-base"
incus launch yunohost:$LXC_BASE $BOX -c security.nesting=true -c security.privileged=true \
|| critical "Failed to launch the container ?"
else
incus launch $BOX-base $BOX -c security.nesting=true -c security.privileged=true
fi
incus config device add $BOX ynhdev-shared-folder disk path=/ynh-dev source="$(readlink -f $(pwd))"
info "Now attaching to the container"
else
info "Attaching to existing container"
fi
attach_ynhdev "$@"
}
function attach_ynhdev()
{
check_setup
local DIST=${1:-bookworm}
local YNH_BRANCH=${3:-unstable}
local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH}
incus start $BOX 2>/dev/null || true
incus exec $BOX --cwd /ynh-dev -- /bin/bash
}
function destroy_ynhdev()
{
check_setup
local DIST=${1:-bookworm}
local YNH_BRANCH=${3:-unstable}
local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH}
incus stop $BOX
incus delete $BOX
}
function rebuild_ynhdev()
{
check_setup
local DIST=${1:-bookworm}
local YNH_BRANCH=${3:-unstable}
local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH}
set -x
incus info $BOX-rebuild >/dev/null && incus delete $BOX-rebuild --force
incus launch images:debian/$DIST/amd64 $BOX-rebuild -c security.nesting=true -c security.privileged=true
sleep 5
incus exec $BOX-rebuild -- apt install curl -y
INSTALL_SCRIPT="https://install.yunohost.org/$DIST"
incus exec $BOX-rebuild -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $YNH_BRANCH"
incus stop $BOX-rebuild
incus publish $BOX-rebuild --alias $BOX-base
set +x
}
function show_vm_ip()
{
assert_inside_vm
hostname --all-ip-addresses | tr ' ' '\n'
}
function use_git()
{
assert_inside_vm
local PACKAGES=("$@")
if [ "$PACKAGES" = "" ]
then
PACKAGES=('moulinette' 'ssowat' 'yunohost' 'yunohost-admin')
fi
for i in "${!PACKAGES[@]}";
do
case ${PACKAGES[i]} in
ssowat)
create_sym_link "/ynh-dev/ssowat" "/usr/share/ssowat"
local ssowat_conf_file="/etc/nginx/conf.d/ssowat.conf"
if ! grep -q lua_package_path $ssowat_conf_file; then
local current_ssowatconf=$(cat $ssowat_conf_file)
echo "lua_package_path '/ynh-dev/ZeroBraneStudio/lualibs/?/?.lua;/ynh-dev/ZeroBraneStudio/lualibs/?.lua;;';" > $ssowat_conf_file
echo "lua_package_cpath '/ynh-dev/ZeroBraneStudio/bin/linux/x64/clibs/?.so;;';" >> $ssowat_conf_file
echo "$current_ssowatconf" >> $ssowat_conf_file
fi
if [ ! -d "/ynh-dev/ZeroBraneStudio" ]; then
info "If you want to debug ssowat, you can clone https://github.com/pkulchenko/ZeroBraneStudio into the ynh-dev directory of your host,"
info "open it, and open a file at the root of the ssowat project in ynh-dev directory, click on \"Project -> Project Directory -> Set From Current File\"."
info "You can start the remote debugger with \"Project -> Start Debugger Server\"."
info "Add the line \"require(\"mobdebug\").start('THE_IP_OF_YOUR_HOST_IN_THE_CONTAINER_NETWORK')\" at the top of the file access.lua and reload nginx in your container with \"systemctl reload nginx\"."
info "After that you should be able to debug ssowat \o/. The debugger should pause the first time it reaches the line \"require(\"mobdebug\").start('...')\", but you can add breakpoints where needed."
info "More info here: http://notebook.kulchenko.com/zerobrane/debugging-openresty-nginx-lua-scripts-with-zerobrane-studio and here: https://github.com/pkulchenko/MobDebug."
fi
success "Now using Git repository for SSOwat"
;;
moulinette)
create_sym_link "/ynh-dev/moulinette/locales" "/usr/share/moulinette/locale"
create_sym_link "/ynh-dev/moulinette/moulinette" "/usr/lib/python3/dist-packages/moulinette"
success "Now using Git repository for Moulinette"
;;
yunohost)
for FILE in $(ls /ynh-dev/yunohost/bin/)
do
create_sym_link "/ynh-dev/yunohost/bin/$FILE" "/usr/bin/$FILE"
done
for FILE in $(ls /ynh-dev/yunohost/conf/metronome/modules/)
do
create_sym_link "/ynh-dev/yunohost/conf/metronome/modules/$FILE" "/usr/lib/metronome/modules/$FILE"
done
for FILE in $(ls /ynh-dev/yunohost/share/)
do
create_sym_link "/ynh-dev/yunohost/share/$FILE" "/usr/share/yunohost/$FILE"
done
create_sym_link "/ynh-dev/yunohost/hooks" "/usr/share/yunohost/hooks"
create_sym_link "/ynh-dev/yunohost/helpers" "/usr/share/yunohost/helpers.d"
create_sym_link "/ynh-dev/yunohost/conf" "/usr/share/yunohost/conf"
create_sym_link "/ynh-dev/yunohost/locales" "/usr/share/yunohost/locales"
create_sym_link "/ynh-dev/yunohost/src" "/usr/lib/python3/dist-packages/yunohost"
python3 "/ynh-dev/yunohost/doc/generate_bash_completion.py"
create_sym_link "/ynh-dev/yunohost/doc/bash-completion.sh" "/etc/bash_completion.d/yunohost"
success "Now using Git repository for YunoHost"
;;
yunohost-admin)
local DEV_PATH="/ynh-dev/yunohost-admin/app"
local CACHE_PATH="/var/cache/ynh-dev/yunohost-admin"
create_sym_link "/ynh-dev/yunohost-admin/app/.env" "/var/cache/ynh-dev/yunohost-admin/.env"
prepare_cache_and_deps "$DEV_PATH" "$CACHE_PATH"
cd "$CACHE_PATH"
# Inject container ip in .env file
# Used by vite to expose itself on network and proxy api requests.
IP=$(hostname -I | tr ' ' '\n' | grep "\.")
echo "VITE_IP=$IP" > .env
# Allow port 8080 in config file or else the dev server will stop working after postinstall
if [[ ! -e /etc/yunohost/installed ]]
then
python3 - <<EOF
import os, yaml
setting_file = "/etc/yunohost/firewall.yml"
assert os.path.exists(setting_file), "Firewall yaml file %s does not exists ?" % setting_file
with open(setting_file) as f:
settings = yaml.load(f)
if 8080 not in settings["ipv4"]["TCP"]:
settings["ipv4"]["TCP"].append(8080)
with open(setting_file, "w") as f:
yaml.safe_dump(settings, f, default_flow_style=False)
EOF
fi
local DISTRO="$(lsb_release -s -c)"
local YARN=$([ "$DISTRO" == "bullseye" ] && echo "yarnpkg" || echo "yarn")
cd "$DEV_PATH"
info "Now running dev server"
"$YARN" dev --host
;;
yunohost-admin-build)
if [[ ! -e "/usr/share/yunohost/admin-bkp" ]]
then
info "Backuping base yunohost-admin sources"
mv /usr/share/yunohost/admin /usr/share/yunohost/admin-bkp
fi
local DISTRO="$(lsb_release -s -c)"
local YARN=$([ "$DISTRO" == "bullseye" ] && echo "yarnpkg" || echo "yarn")
cd /ynh-dev/yunohost-admin/app
"$YARN" build
create_sym_link "/ynh-dev/yunohost-admin/app/dist" "/usr/share/yunohost/admin"
IP=$(hostname -I | tr ' ' '\n' | grep "\.")
success "App builded and available at https://$IP/yunohost/admin"
;;
yunohost-portal)
assert_yunohost_is_installed
# open firewall port 3000 (dev server) and 24678 (dev server websocket)
yunohost firewall allow TCP 3000 -4 --no-reload
yunohost firewall allow TCP 24678 -4
local DEV_PATH="/ynh-dev/yunohost-portal"
local CACHE_PATH="/var/cache/ynh-dev/yunohost-portal"
if [[ ! -e "$DEV_PATH/.env" ]];
then
local IP=$(hostname -I | tr ' ' '\n' | grep "\.")
local MAIN_DOMAIN=$(yunohost domain main-domain | cut -d " " -f2)
critical "There's no 'yunohost-portal/.env' file.
Based on your current main domain (but you can use any domain added on your YunoHost instance) the file should look like:
NUXT_PUBLIC_API_IP=\"$MAIN_DOMAIN\"
If not already, add your instance's IP into '/etc/yunohost/.portal-api-allowed-cors-origins' to avoid CORS issues and make sure to add a redirection in your host's '/etc/hosts' which, based on your instance ip and main domain, would be:
$IP $MAIN_DOMAIN"
fi
prepare_cache_and_deps "$DEV_PATH" "$CACHE_PATH"
cd "$DEV_PATH"
info "Now running dev server"
yarn dev --host
;;
yunohost-portal-build)
assert_yunohost_is_installed
local DEV_PATH="/ynh-dev/yunohost-portal"
local CACHE_PATH="/var/cache/ynh-dev/yunohost-portal"
local SOURCE_PATH="/usr/share/yunohost/portal"
if [[ ! -e "$SOURCE_PATH-bkp" ]]
then
info "Backuping system yunohost-portal sources…"
mv "$SOURCE_PATH" "$SOURCE_PATH-bkp"
fi
prepare_cache_and_deps "$DEV_PATH" "$CACHE_PATH"
cd "$DEV_PATH"
yarn generate
create_sym_link "$DEV_PATH/.output/public" "$SOURCE_PATH"
local IP=$(hostname -I | tr ' ' '\n' | grep "\.")
local MAIN_DOMAIN=$(yunohost domain main-domain | cut -d " " -f2)
success "App builded and available at http://$MAIN_DOMAIN/yunohost/sso or http://$IP/yunohost/sso"
;;
*)
error "Invalid package '${PACKAGES[i]}': correct arguments are 'yunohost', 'ssowat', 'moulinette', 'yunohost-admin' or nothing for all"
;;
esac
done
}
function dev()
{
assert_inside_vm
which inotifywait &>/dev/null || critical "You should first run: apt install inotify-tools"
info "Now monitoring for changes in python files, restarting yunohost-api and yunohost-portal-api when changes occur!"
journalctl --no-pager --no-hostname -u yunohost-api -u yunohost-portal-api -n 0 -f &
JOURNALCTL_PID=$!
trap "kill $JOURNALCTL_PID; exit" SIGINT
while ! inotifywait -q -e modify /usr/lib/python3/dist-packages/{moulinette,yunohost}/{*.py,*/*.py}
do
info "Restarting services"
systemctl restart yunohost-api yunohost-portal-api
done
}
function run_tests()
{
assert_inside_vm
local PACKAGES="$@"
for PACKAGE in $PACKAGES;
do
TEST_FUNCTION=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $3}')
TEST_MODULE=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $2}')
PACKAGE=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $1}')
case $PACKAGE in
yunohost)
# Pytest and tests dependencies
if ! type "pytest" > /dev/null 2>&1; then
info "> Installing pytest ..."
apt-get install python3-pip -y
pip3 install pytest pytest-sugar --break-system-packages
fi
for DEP in "pytest-mock requests-mock mock"
do
if [ -z "$(pip3 show $DEP)" ]; then
info "Installing $DEP with pip3"
pip3 install $DEP --break-system-packages
fi
done
# Apps for test
cd /ynh-dev/yunohost/src/tests
[ -d "apps" ] || git clone https://github.com/YunoHost/test_apps ./apps
cd apps
git pull > /dev/null 2>&1
# Run tests
info "Running tests for YunoHost"
[ -e "/etc/yunohost/installed" ] || critical "You should run postinstallation before running tests :s."
if [[ -z "$TEST_MODULE" ]]
then
cd /ynh-dev/yunohost/
py.test tests
cd /ynh-dev/yunohost/src
py.test tests
else
cd /ynh-dev/yunohost/src
if [[ -z "$TEST_FUNCTION" ]]
then
py.test tests/test_"$TEST_MODULE".py
else
py.test tests/test_"$TEST_MODULE".py::test_"$TEST_FUNCTION"
fi
fi
;;
esac
done
}
function catalog()
{
assert_inside_vm
local ACTION="$1"
local CUSTOM_APPS_FOLDER=${2:-"/ynh-dev/custom-catalog"}
local CUSTOM_CAT_PATH="${CUSTOM_APPS_FOLDER}/catalog.json"
local CACHE_FOLDER="/var/cache/yunohost/repo"
cd /ynh-dev/custom-catalog/
case "${ACTION}" in
build)
info "Rebuilding custom app catalog"
python3 -c "from catalog_manager import build; build(folder='${CUSTOM_APPS_FOLDER}')" && success "Successfully build custom catalog list in '${CUSTOM_CAT_PATH}'"
;;
add)
info "Injecting custom catalog in YunoHost catalog list"
create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json"
python3 -c "from catalog_manager import add; add()" && success "Custom catalog '${CUSTOM_CAT_PATH}' added to catalog list"
;;
override)
info "Overriding default catalog by custom one"
create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json"
python3 -c "from catalog_manager import override; override()" && success "Default catalog is now overrided by '$CUSTOM_CAT_PATH'"
;;
reset)
info "Reseting to YunoHost default catalog list"
[ -e "$CACHE_FOLDER/custom.json" ] && rm "$CACHE_FOLDER/custom.json"
python3 -c "from catalog_manager import reset; reset()" || exit 1
success "Returned to default"
;;
*)
critical "Unknown catalog action '${ACTION}'."
;;
esac
}
function rebuild_api_doc()
{
test -d yunohost || critical "Expected to find a 'yunohost' folder ?"
cd yunohost/doc
if ! test -d swagger
then
# Download swagger ui
info "Downloading swagger UI"
curl -L https://github.com/swagger-api/swagger-ui/archive/refs/tags/v4.15.5.tar.gz | tar -xvz swagger-ui-4.15.5/dist/;
mv swagger-ui-4.15.5/dist/ swagger
rmdir swagger-ui-4.15.5
fi
info "Rebuild swagger json/js according to actionsmap"
python3 generate_api_doc.py
success "You should now open yunohost/doc/api.html using your favorite browser"
}
main $@