From 3bee2e99838b8835c88d19ffc0d9f33027ef22ab Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Thu, 6 Apr 2023 06:07:54 +0000 Subject: [PATCH 01/16] Upgrade to v2023.04.05 --- manifest.json | 2 +- scripts/_common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest.json b/manifest.json index e830e79..1187c98 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "CI runner of YunoHost", "fr": "Runner d'intégration continue de YunoHost" }, - "version": "2023.02.12~ynh1", + "version": "2023.04.05~ynh1", "url": "https://github.com/YunoHost/yunorunner", "upstream": { "license": "GPL-3.0-or-later", diff --git a/scripts/_common.sh b/scripts/_common.sh index 16d4bd7..ac4384b 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -9,7 +9,7 @@ pkg_dependencies="python3-venv python3-dev python3-pip sqlite3 wkhtmltopdf optip yunorunner_repository="https://github.com/YunoHost/yunorunner" -yunorunner_release="50ae066741b8a7e845b691eac1045406b1ee7f16" +yunorunner_release="52ef23a2cb37cb4fe13debca58eb589bb2f4d927" #================================================= # PERSONAL HELPERS From 5f77fcc122c7a16118563b08237b8a9ec7951dc0 Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Thu, 6 Apr 2023 06:07:59 +0000 Subject: [PATCH 02/16] Auto-update README --- README.md | 2 +- README_fr.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 01cead0..3d5c146 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in CI runner of YunoHost -**Shipped version:** 2023.02.12~ynh1 +**Shipped version:** 2023.04.05~ynh1 ## Screenshots diff --git a/README_fr.md b/README_fr.md index d1095d4..883a18e 100644 --- a/README_fr.md +++ b/README_fr.md @@ -18,7 +18,7 @@ Si vous n’avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) po Runner d'intégration continue de YunoHost -**Version incluse :** 2023.02.12~ynh1 +**Version incluse :** 2023.04.05~ynh1 ## Captures d’écran From 33ef3ea8737607b4fd0fd30f7258a66c1664b03d Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 7 Apr 2023 00:15:58 +0200 Subject: [PATCH 03/16] Run the upstream finish_install script --- manifest.json | 18 ++++++++++++++++++ scripts/install | 11 +++++++++++ scripts/upgrade | 9 +++++++++ 3 files changed, 38 insertions(+) diff --git a/manifest.json b/manifest.json index 1187c98..6c99a23 100644 --- a/manifest.json +++ b/manifest.json @@ -37,6 +37,24 @@ "name": "path", "type": "path", "default": "/ci" + }, + { + "name": "", + "type": "select", + "choices": ["auto", "manual"] + "default": "manual" + "ask": { + "en": "Should the jobs be automatically run from the apps list?" + } + }, + { + "name": "cluster", + "type": "select", + "choices": ["cluster", "no"] + "default": "no" + "ask": { + "en": "Should an LXD cluster be created with this server as first node?" + } } ] } diff --git a/scripts/install b/scripts/install index 7a8381d..41212ac 100644 --- a/scripts/install +++ b/scripts/install @@ -25,6 +25,8 @@ ynh_abort_if_errors domain=$YNH_APP_ARG_DOMAIN path_url=$YNH_APP_ARG_PATH +mode=$YNH_APP_ARG_MODE +cluster=$YNH_APP_ARG_CLUSTER app=$YNH_APP_INSTANCE_NAME @@ -126,6 +128,15 @@ ynh_add_config --template="yunorunner.config.py" --destination="$final_path/conf chmod 400 "$final_path/config.py" chown $app:$app "$final_path/config.py" +#================================================= +# FINISH INSTALL +#================================================= +ynh_script_progression --message="Run Yunorunner's finish_install script..." + +pushd "$final_path" + ./finish_install.sh $mode $cluster +popd + #================================================= # SETUP SYSTEMD #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 17a41ac..0f5eed8 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -151,6 +151,15 @@ pushd $final_path venv/bin/pip install -r requirements-frozen.txt popd +#================================================= +# FINISH INSTALL +#================================================= +ynh_script_progression --message="Run Yunorunner's finish_install script..." + +pushd "$final_path" + ./finish_install.sh +popd + #================================================= # UPDATE A CONFIG FILE #================================================= From 6bceb7bb0ebc0de09e58135d1ac2f81fd6564fbd Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 7 Apr 2023 00:18:21 +0200 Subject: [PATCH 04/16] Fix manifest --- manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest.json b/manifest.json index 6c99a23..8fcb6fa 100644 --- a/manifest.json +++ b/manifest.json @@ -41,7 +41,7 @@ { "name": "", "type": "select", - "choices": ["auto", "manual"] + "choices": ["auto", "manual"], "default": "manual" "ask": { "en": "Should the jobs be automatically run from the apps list?" @@ -50,7 +50,7 @@ { "name": "cluster", "type": "select", - "choices": ["cluster", "no"] + "choices": ["cluster", "no"], "default": "no" "ask": { "en": "Should an LXD cluster be created with this server as first node?" From 38754e7615d146543f1f31d3e4f874a0e66173be Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 7 Apr 2023 00:19:23 +0200 Subject: [PATCH 05/16] =?UTF-8?q?Fix=20manifest=C2=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest.json b/manifest.json index 8fcb6fa..30b2a36 100644 --- a/manifest.json +++ b/manifest.json @@ -42,7 +42,7 @@ "name": "", "type": "select", "choices": ["auto", "manual"], - "default": "manual" + "default": "manual", "ask": { "en": "Should the jobs be automatically run from the apps list?" } @@ -51,7 +51,7 @@ "name": "cluster", "type": "select", "choices": ["cluster", "no"], - "default": "no" + "default": "no", "ask": { "en": "Should an LXD cluster be created with this server as first node?" } From 6838bfe41c7e060658f2082ccce08980524c5d92 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 7 Apr 2023 00:23:03 +0200 Subject: [PATCH 06/16] Fix check_process --- check_process | 2 ++ 1 file changed, 2 insertions(+) diff --git a/check_process b/check_process index 1571f8c..5f69854 100644 --- a/check_process +++ b/check_process @@ -4,6 +4,8 @@ ; Manifest domain="domain.tld" path="/path" + mode="manual" + cluster="no" ; Checks pkg_linter=1 setup_sub_dir=1 From 1b49dec4442f29b343d590fa0f9b9246b298908f Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 7 Apr 2023 11:19:12 +0200 Subject: [PATCH 07/16] Fix manifest again --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 30b2a36..6342d4d 100644 --- a/manifest.json +++ b/manifest.json @@ -39,7 +39,7 @@ "default": "/ci" }, { - "name": "", + "name": "mode", "type": "select", "choices": ["auto", "manual"], "default": "manual", From eaf56818bcf887354f63af634e5c1ec2588b9f7e Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 7 Apr 2023 13:14:00 +0200 Subject: [PATCH 08/16] Update install --- scripts/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install b/scripts/install index 41212ac..de405c5 100644 --- a/scripts/install +++ b/scripts/install @@ -134,7 +134,7 @@ chown $app:$app "$final_path/config.py" ynh_script_progression --message="Run Yunorunner's finish_install script..." pushd "$final_path" - ./finish_install.sh $mode $cluster + ./maintenance/finish_install.sh $mode $cluster popd #================================================= From fedeb805fa1a56b0c0289f22e9a30ee69db3cc04 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 7 Apr 2023 13:14:47 +0200 Subject: [PATCH 09/16] Update upgrade --- scripts/upgrade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upgrade b/scripts/upgrade index 0f5eed8..faf7b42 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -157,7 +157,7 @@ popd ynh_script_progression --message="Run Yunorunner's finish_install script..." pushd "$final_path" - ./finish_install.sh + ./maintenance/finish_install.sh popd #================================================= From 67b5f637147039d0bf7e5927e8cc4818164e1a0e Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 12 Apr 2023 20:57:37 +0200 Subject: [PATCH 10/16] Introduce personal-ci or official-infra contexts --- check_process | 3 +- conf/yunorunner.config.py | 6 +- manifest.json | 11 +++- scripts/_common.sh | 122 ++++++++++++++++++++++++++++++++++++++ scripts/install | 17 +++++- scripts/upgrade | 9 --- 6 files changed, 150 insertions(+), 18 deletions(-) diff --git a/check_process b/check_process index 5f69854..7f5c3c4 100644 --- a/check_process +++ b/check_process @@ -1,9 +1,8 @@ ;; Test complet - ; pre-install - sudo git clone --quiet https://github.com/YunoHost/CI_package_check /home/CI_package_check ; Manifest domain="domain.tld" path="/path" + context="personal-ci" mode="manual" cluster="no" ; Checks diff --git a/conf/yunorunner.config.py b/conf/yunorunner.config.py index eb77e8e..354c7fe 100644 --- a/conf/yunorunner.config.py +++ b/conf/yunorunner.config.py @@ -1,10 +1,10 @@ BASE_URL = "https://__DOMAIN____PATH__" PORT = __PORT__ PACKAGE_CHECK_DIR = "__FINALPATH__/package_check/" -MONITOR_APPS_LIST = False -MONITOR_GIT = False +MONITOR_APPS_LIST = __AUTO__ +MONITOR_GIT = __AUTO__ MONITOR_ONLY_GOOD_QUALITY_APPS = False -MONTHLY_JOBS = False +MONTHLY_JOBS = __AUTO__ WORKER_COUNT = 1 YNH_BRANCH = "stable" DIST = "bullseye" diff --git a/manifest.json b/manifest.json index 30b2a36..7f1e2f7 100644 --- a/manifest.json +++ b/manifest.json @@ -39,7 +39,16 @@ "default": "/ci" }, { - "name": "", + "name": "context", + "type": "select", + "choices": ["personal-ci", "official-infra"], + "default": "personal-ci", + "ask": { + "en": "Leave it to 'personal-ci'. If set to 'official-infra', Yunorunner will take over your server and disable vital services to run alone." + } + }, + { + "name": "mode", "type": "select", "choices": ["auto", "manual"], "default": "manual", diff --git a/scripts/_common.sh b/scripts/_common.sh index ac4384b..48b39a5 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -15,6 +15,128 @@ yunorunner_release="52ef23a2cb37cb4fe13debca58eb589bb2f4d927" # PERSONAL HELPERS #================================================= +function tweak_yunohost() { + + # Idk why this is needed but wokay I guess >_> + echo -e "\n127.0.0.1 $domain #CI_APP" >> /etc/hosts + + ynh_print_info "Disabling unecessary services to save up RAM..." + for SERVICE in mysql php7.4-fpm metronome rspamd dovecot postfix redis-server postsrsd yunohost-api avahi-daemon + do + systemctl stop $SERVICE + systemctl disable $SERVICE --quiet + done + + yunohost app makedefault -d "$domain" $app + +} + +function setup_lxd() { + if ! yunohost app list --output-as json --quiet | jq -e '.apps[] | select(.id == "lxd")' >/dev/null + then + ynh_script_progression --message="Installing LXD... (this make take a long time!" + yunohost app install --force https://github.com/YunoHost-Apps/lxd_ynh + fi + + mkdir .lxd + pushd .lxd + + ynh_print_info "Configuring lxd..." + + if [ "$lxd_cluster" == "cluster" ] + then + local free_space=$(df --output=avail / | sed 1d) + local btrfs_size=$(( $free_space * 90 / 100 / 1024 / 1024 )) + local lxc_network=$((1 + $RANDOM % 254)) + + yunohost firewall allow TCP 8443 + cat >./preseed.conf <> "/etc/cron.d/yunorunner" << EOF +# self-upgrade every night +0 3 * * * root "$YUNORUNNER_HOME/maintenance/self_upgrade.sh" >> "$YUNORUNNER_HOME/maintenance/self_upgrade.log" 2>&1 +EOF +} + +# ========================= +# Main stuff +# ========================= + +# Add permission to the user for the entire yunorunner home because it'll be the one running the tests (as a non-root user) +chown -R $ci_user $YUNORUNNER_HOME + +echo "Done!" +echo " " +echo "N.B. : If you want to enable Matrix notification, you should look at " +echo "the instructions inside lib/chat_notify.sh to deploy matrix-commander" +echo "" +echo "You may also want to tweak the 'config' file to run test with a different branch / arch" +echo "" +echo "When you're ready to start the CI, run: systemctl restart $ci_user" + #================================================= # EXPERIMENTAL HELPERS #================================================= diff --git a/scripts/install b/scripts/install index 41212ac..7ff5cea 100644 --- a/scripts/install +++ b/scripts/install @@ -25,6 +25,7 @@ ynh_abort_if_errors domain=$YNH_APP_ARG_DOMAIN path_url=$YNH_APP_ARG_PATH +context=$YNH_APP_ARG_CONTEXT mode=$YNH_APP_ARG_MODE cluster=$YNH_APP_ARG_CLUSTER @@ -89,6 +90,8 @@ pushd "$final_path" ynh_exec_as $app git remote add origin "$yunorunner_repository" ynh_exec_as $app git fetch --quiet --depth=1 origin "$yunorunner_release" ynh_exec_as $app git reset --quiet --hard FETCH_HEAD + setup_lxd + git clone https://github.com/YunoHost/package_check "./package_check" popd chmod 750 "$final_path" @@ -123,6 +126,12 @@ popd #================================================= ynh_script_progression --message="Adding a config file..." +if $mode = "auto"; then + auto = "True" +else + auto = "False" +fi + ynh_add_config --template="yunorunner.config.py" --destination="$final_path/config.py" chmod 400 "$final_path/config.py" @@ -133,9 +142,11 @@ chown $app:$app "$final_path/config.py" #================================================= ynh_script_progression --message="Run Yunorunner's finish_install script..." -pushd "$final_path" - ./finish_install.sh $mode $cluster -popd +if [ $context != "personal-ci" ] && [ ${PACKAGE_CHECK_EXEC:-0} -ne 0 ]; then + tweak_yunohost +fi + +add_cron_jobs #================================================= # SETUP SYSTEMD diff --git a/scripts/upgrade b/scripts/upgrade index 0f5eed8..17a41ac 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -151,15 +151,6 @@ pushd $final_path venv/bin/pip install -r requirements-frozen.txt popd -#================================================= -# FINISH INSTALL -#================================================= -ynh_script_progression --message="Run Yunorunner's finish_install script..." - -pushd "$final_path" - ./finish_install.sh -popd - #================================================= # UPDATE A CONFIG FILE #================================================= From 526b80024c196d9f3429bff9b70b43c7e37ac3a4 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 12 Apr 2023 22:09:46 +0200 Subject: [PATCH 11/16] Fix lxd config --- scripts/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 48b39a5..201a0fe 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -43,7 +43,7 @@ function setup_lxd() { ynh_print_info "Configuring lxd..." - if [ "$lxd_cluster" == "cluster" ] + if [ "$cluster" == "cluster" ] then local free_space=$(df --output=avail / | sed 1d) local btrfs_size=$(( $free_space * 90 / 100 / 1024 / 1024 )) From 403f279ce8bc1ea8942d5d9c4b1d371670ed58d6 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 12 Apr 2023 22:31:41 +0200 Subject: [PATCH 12/16] Fix test --- scripts/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install b/scripts/install index 7ff5cea..88b015f 100644 --- a/scripts/install +++ b/scripts/install @@ -126,7 +126,7 @@ popd #================================================= ynh_script_progression --message="Adding a config file..." -if $mode = "auto"; then +if [ $mode = "auto" ]; then auto = "True" else auto = "False" From 1a74c54e84928c7b24e32228f24b3f9a14b6918c Mon Sep 17 00:00:00 2001 From: tituspijean Date: Thu, 20 Jul 2023 23:49:43 +0200 Subject: [PATCH 13/16] Update scripts/install Co-authored-by: Alexandre Aubin --- scripts/install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install b/scripts/install index 88b015f..5445d22 100644 --- a/scripts/install +++ b/scripts/install @@ -127,9 +127,9 @@ popd ynh_script_progression --message="Adding a config file..." if [ $mode = "auto" ]; then - auto = "True" + auto="True" else - auto = "False" + auto="False" fi ynh_add_config --template="yunorunner.config.py" --destination="$final_path/config.py" From 24d06bc012c556932187f06ba461b127aab6dd93 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Sun, 23 Jul 2023 09:02:32 +0000 Subject: [PATCH 14/16] Suggestions from review, improve documentation --- doc/POST_INSTALL.md | 6 ++++++ doc/PRE_INSTALL.md | 2 ++ manifest.json | 4 ++-- scripts/_common.sh | 16 ---------------- 4 files changed, 10 insertions(+), 18 deletions(-) create mode 100644 doc/POST_INSTALL.md create mode 100644 doc/PRE_INSTALL.md diff --git a/doc/POST_INSTALL.md b/doc/POST_INSTALL.md new file mode 100644 index 0000000..074585b --- /dev/null +++ b/doc/POST_INSTALL.md @@ -0,0 +1,6 @@ +If you want to enable Matrix notification, you should look at +the instructions inside `__FINAL_PATH__lib/chat_notify.sh` to deploy matrix-commander. + +You may also want to tweak the `__FINAL_PATH__/config.py` file to run test with a different branch or architecture. + +When you're ready to start the CI, restart its service via the webadmin or run `systemctl restart __APP__`. diff --git a/doc/PRE_INSTALL.md b/doc/PRE_INSTALL.md new file mode 100644 index 0000000..65be331 --- /dev/null +++ b/doc/PRE_INSTALL.md @@ -0,0 +1,2 @@ +YunoRunner will install LXD, for containers management. +Make sure your system has virtualization capability, and enough resources. diff --git a/manifest.json b/manifest.json index 7f1e2f7..37af0ce 100644 --- a/manifest.json +++ b/manifest.json @@ -53,7 +53,7 @@ "choices": ["auto", "manual"], "default": "manual", "ask": { - "en": "Should the jobs be automatically run from the apps list?" + "en": "Should the jobs be automatically run from the apps list? In manual mode, Yunorunner will expect to be triggered by the `ciclic` command or webhooks (like ci-apps-dev). In auto mode, all apps of the catalog will be scheduled to be tested at least once a month and upon any change in their designated main branch (like ci-apps)." } }, { @@ -62,7 +62,7 @@ "choices": ["cluster", "no"], "default": "no", "ask": { - "en": "Should an LXD cluster be created with this server as first node?" + "en": "Should an LXD cluster be created with this server as first node? (cluster mode is experimental)" } } ] diff --git a/scripts/_common.sh b/scripts/_common.sh index 201a0fe..0ea18f3 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -121,22 +121,6 @@ function add_cron_jobs() { EOF } -# ========================= -# Main stuff -# ========================= - -# Add permission to the user for the entire yunorunner home because it'll be the one running the tests (as a non-root user) -chown -R $ci_user $YUNORUNNER_HOME - -echo "Done!" -echo " " -echo "N.B. : If you want to enable Matrix notification, you should look at " -echo "the instructions inside lib/chat_notify.sh to deploy matrix-commander" -echo "" -echo "You may also want to tweak the 'config' file to run test with a different branch / arch" -echo "" -echo "When you're ready to start the CI, run: systemctl restart $ci_user" - #================================================= # EXPERIMENTAL HELPERS #================================================= From 35fab13ddb90d07075d83f21731393bf8e186569 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Sun, 23 Jul 2023 10:07:39 +0000 Subject: [PATCH 15/16] Fix _common.sh --- scripts/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 0ea18f3..2137aa1 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -117,7 +117,7 @@ function add_cron_jobs() { # Cron tasks cat >> "/etc/cron.d/yunorunner" << EOF # self-upgrade every night -0 3 * * * root "$YUNORUNNER_HOME/maintenance/self_upgrade.sh" >> "$YUNORUNNER_HOME/maintenance/self_upgrade.log" 2>&1 +0 3 * * * root "$final_path/maintenance/self_upgrade.sh" >> "$final_path/maintenance/self_upgrade.log" 2>&1 EOF } From d8b5ca5b6c5034d75773c71909121ce0b3f3e86d Mon Sep 17 00:00:00 2001 From: tituspijean Date: Sun, 23 Jul 2023 17:32:05 +0000 Subject: [PATCH 16/16] Tweak lxd init to support already-installed lxd --- scripts/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 2137aa1..dd5a08b 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -100,7 +100,7 @@ EOF rm ./preseed.conf lxc config set core.https_address [::] else - lxd init --auto --storage-backend=dir + lxd init --auto #--storage-backend=dir fi popd