diff --git a/analyze_yunohost_app.sh b/analyze_yunohost_app.sh new file mode 100755 index 0000000..685310c --- /dev/null +++ b/analyze_yunohost_app.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +if [ "${0:0:1}" == "/" ]; then script_dir="$(dirname "$0")"; else script_dir="$(echo $PWD/$(dirname "$0" | cut -d '.' -f2) | sed 's@/$@@')"; fi + +cd $script_dir + +if [ $# -ne 4 ] +then + echo "This script need to take in argument the package which be tested, the name of the test, and the ID of the worker." + exit 1 +fi + +mkdir -p /var/run/yunorunner/locks/ + +worker_id="$4" +lock_yunorunner="/var/run/yunorunner/locks/${worker_id}.lock" +lock_package_check="./package_check/pcheck-${worker_id}.lock" + +# 10800 sec / 60 = 180 min = 3 hours +TIMEOUT="10800" + +BASE_URL="$(cat "./config.py" | tr -d ' ' | grep "^BASE_URL=" | cut --delimiter="=" --fields=2)" +ynh_branch="$(cat "./config.py" | tr -d ' ' | grep "^YNH_BRANCH=" | cut --delimiter="=" --fields=2)" +arch="$(dpkg --print-architecture)" +dist="$(cat "./config.py" | tr -d ' ' | grep "^DIST=" | cut --delimiter="=" --fields=2)" + +# Enable chat notifications only on main CI +if [[ "$ynh_branch" == "stable" ]] && [[ "$arch" == "amd64" ]] && [[ -e "./maintenance/chat_notify.sh" ]] +then + chat_notify="./maintenance/chat_notify.sh" +else + chat_notify="true" # 'true' is a dummy program that won't do anything +fi + +#================================================= +# Delay the beginning of this script, to prevent concurrent executions +#================================================= + +# Get 3 ramdom digit. To build a value between 001 and 999 +milli_sleep=$(head --lines=20 /dev/urandom | tr --complement --delete '0-9' | head --bytes=3) +# And wait for this value in millisecond +sleep "5.$milli_sleep" + +#============================ +# Check / take the lock +#============================= + +if [ -e $lock_yunorunner ] +then + lock_yunorunner_PID="$(cat $lock_yunorunner)" + if [ -n "$lock_yunorunner_PID" ] + then + # We check that the corresponding PID is still running AND that the PPid is not 1 .. + # If the PPid is 1, it tends to indicate that a previous analyseCI is still running and was not killed, and therefore got adopted by init. + # This typically happens when the job is cancelled / restarted .. though we should have a better way of handling cancellation from yunorunner directly :/ + if ps --pid $lock_yunorunner_PID | grep --quiet $lock_yunorunner_PID && [[ $(grep PPid /proc/${lock_yunorunner_PID}/status | awk '{print $2}') != "1" ]] + then + echo -e "\e[91m\e[1m!!! Another analyseCI process is currently using the lock $lock_yunorunner !!!\e[0m" + "$chat_notify" "CI miserably crashed because another process is using the lock" + sleep 10 + exit 1 + fi + fi + [[ $(grep PPid /proc/$(lock_yunorunner_PID)/status | awk '{print $2}') != "1" ]] && { echo "Killing stale analyseCI process ..."; kill -s SIGTERM $lock_yunorunner_PID; sleep 30; } + echo "Removing stale lock" + rm -f $lock_yunorunner +fi + +echo "$$" > $lock_yunorunner + +#============================ +# Cleanup after exit/kill +#============================= + +function cleanup() +{ + rm $lock_yunorunner + + if [ -n "$package_check_pid" ] + then + kill -s SIGTERM $package_check_pid + WORKER_ID="$worker_id" ARCH="$arch" DIST="$dist" YNH_BRANCH="$ynh_branch" "./package_check/package_check.sh" --force-stop + fi +} + +trap cleanup EXIT +trap 'exit 2' TERM + +#============================ +# Test parameters +#============================= + +repo="$1" +test_name="$2" +job_id="$3" + +# Keep only the repositery +repo=$(echo $repo | cut --delimiter=';' --fields=1) +app="$(echo $test_name | awk '{print $1}')" + +test_full_log=${app}_${arch}_${ynh_branch}_complete.log +test_json_results=${app}_${arch}_${ynh_branch}_results.json +test_url="$BASE_URL/job/$job_id" + +# Make sure /usr/local/bin is in the path, because that's where the lxc/lxd bin lives +export PATH=$PATH:/usr/local/bin + +#================================================= +# Timeout handling utils +#================================================= + +function watchdog() { + local package_check_pid=$1 + # Start a loop while package check is working + while ps --pid $package_check_pid | grep --quiet $package_check_pid + do + sleep 10 + + if [ -e $lock_package_check ] + then + lock_timestamp="$(stat -c %Y $lock_package_check)" + current_timestamp="$(date +%s)" + if [[ "$(($current_timestamp - $lock_timestamp))" -gt "$TIMEOUT" ]] + then + kill -s SIGTERM $package_check_pid + rm -f $lock_package_check + force_stop "Package check aborted, timeout reached ($(( $TIMEOUT / 60 )) min)." + return 1 + fi + fi + done + + if [ ! -e "./package_check/results-$worker_id.json" ] + then + force_stop "It looks like package_check did not finish properly ... on $test_url" + return 1 + fi +} + +function force_stop() { + local message="$1" + + echo -e "\e[91m\e[1m!!! $message !!!\e[0m" + + "$chat_notify" "While testing $app: $message" + + WORKER_ID="$worker_id" ARCH="$arch" DIST="$dist" YNH_BRANCH="$ynh_branch" "./package_check/package_check.sh" --force-stop +} + +#================================================= +# The actual testing ... +#================================================= + +# Exec package check according to the architecture +echo "$(date) - Starting a test for $app on architecture $arch distribution $dist with yunohost $ynh_branch" + +rm -f "./package_check/Complete-$worker_id.log" +rm -f "./package_check/results-$worker_id.json" + +# Here we use a weird trick with 'script -qefc' +# The reason is that : +# if running the command in background (with &) the corresponding command *won't be in a tty* (not sure exactly) +# therefore later, the command lxc exec -t *won't be in a tty* (despite the -t) and command outputs will appear empty... +# Instead, with the magic of script -qefc we can pretend to be in a tty somehow... +# Adapted from https://stackoverflow.com/questions/32910661/pretend-to-be-a-tty-in-bash-for-any-command +cmd="WORKER_ID=$worker_id ARCH=$arch DIST=$dist YNH_BRANCH=$ynh_branch nice --adjustment=10 './package_check/package_check.sh' '$repo' 2>&1" +script -qefc "$cmd" & + +watchdog $! || exit 1 + +# Copy the complete log +cp "./package_check/Complete-$worker_id.log" "./results/logs/$test_full_log" +cp "./package_check/results-$worker_id.json" "./results/logs/$test_json_results" +rm -f "./package_check/Complete-$worker_id.log" +rm -f "./package_check/results-$worker_id.json" +mkdir -p "./summary/" +[ ! -e "./package_check/summary.png" ] || cp "./package_check/summary.png" "./summary/${job_id}.png" + +if [ -n "$BASE_URL" ] +then + full_log_path="$BASE_URL/logs/$test_full_log" +else + full_log_path="$(pwd)/logs/$test_full_log" +fi + +echo "The complete log for this application was duplicated and is accessible at $full_log_path" + +echo "" +echo "-------------------------------------------" +echo "" + +#================================================= +# Check / update level of the app +#================================================= + +public_result_list="./results/logs/list_level_${ynh_branch}_$arch.json" +[ -s "$public_result_list" ] || echo "{}" > "$public_result_list" + +# Check that we have a valid json... +jq -e '' "./results/logs/$test_json_results" >/dev/null 2>/dev/null && bad_json="false" || bad_json="true" + +# Get new level and previous level +app_level="$(jq -r ".level" "./logs/$test_json_results")" +previous_level="$(jq -r ".$app" "$public_result_list")" + +# We post message on chat if we're running for tests on stable/amd64 +if [ "$bad_json" == "true" ] || [ "$app_level" -eq 0 ]; then + message="Application $app completely failed the continuous integration tests" +elif [ -z "$previous_level" ]; then + message="Application $app rises from level (unknown) to level $app_level" +elif [ $app_level -gt $previous_level ]; then + message="Application $app rises from level $previous_level to level $app_level" +elif [ $app_level -lt $previous_level ]; then + message="Application $app goes down from level $previous_level to level $app_level" +elif [ $app_level -ge 6 ]; then + # Dont notify anything, reduce CI flood on app chatroom + message="" +else + message="Application $app stays at level $app_level" +fi + +# Send chat notification +if [[ -n "$message" ]] +then + message+=" on $test_url" + echo $message + "$chat_notify" "$message" +fi + +# Update/add the results from package_check in the public result list +if [ "$bad_json" == "false" ] +then + jq --argfile results "./results/logs/$test_json_results" ".\"$app\"=\$results" $public_result_list > $public_result_list.new + mv $public_result_list.new $public_result_list +fi + +# Annnd we're done ! +echo "$(date) - Test completed" + +[ "$app_level" -gt 5 ] && exit 0 || exit 1 diff --git a/ciclic b/ciclic index f55f1e6..ae2efd5 100644 --- a/ciclic +++ b/ciclic @@ -14,23 +14,6 @@ except ImportError: DOMAIN = "localhost:" + str(PORT) -def require_token(): - if os.path.exists("token") and open("token").read().strip(): - return - - print("You need a token to be able to uses this command tool for security reasons, please refer to the README on how to add one https://github.com/YunoHost/yunorunner") - try: - token = input("Token: ").strip() - except KeyboardInterrupt: - print() - token = None - - if not token: - print("Error: you need to provide a valid token") - sys.exit(1) - - open("token", "w").write(token) - def request_api(path, domain, verb, data={}, check_return_code=True): assert verb in ("get", "post", "put", "delete") @@ -41,7 +24,7 @@ def request_api(path, domain, verb, data={}, check_return_code=True): response = getattr(requests, verb)( "http%s://%s/api/%s" % ("s" if https else "", domain, path), - headers={"X-Token": open("token", "r").read().strip()}, + headers={"X-Token": open(".admin_token", "r").read().strip()}, json=data, ) @@ -145,5 +128,4 @@ def restart(job_id, domain=DOMAIN): if __name__ == '__main__': - require_token() argh.dispatch_commands([add, list_, delete, stop, restart, app_list, shell]) diff --git a/instances b/instances deleted file mode 100644 index f8b657a..0000000 --- a/instances +++ /dev/null @@ -1,4 +0,0 @@ -bearnaise https://ci-apps.yunohost.org/ -samourai https://ci-apps-arm.yunohost.org/ -samourai https://ci-apps-unstable.yunohost.org/ -root@ci-stretch.nohost.me https://ci-stretch.nohost.me/ diff --git a/maintenance/chat_notify.sh b/maintenance/chat_notify.sh new file mode 100755 index 0000000..38581a7 --- /dev/null +++ b/maintenance/chat_notify.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# +# To install before using: +# +# MCHOME="/opt/matrix-commander" +# MCARGS="-c $MCHOME/credentials.json --store $MCHOME/store" +# mkdir -p "$MCHOME/venv" +# python3 -m venv "$MCHOME/venv" +# source "$MCHOME/venv/bin/activate" +# pip3 install matrix-commander +# chmod 700 "$MCHOME" +# matrix-commander $MCARGS --login password # < NB here this is literally 'password' as authentication method, the actual password will be asked by a prompt +# matrix-commander $MCARGS --room-join '#yunohost-apps:matrix.org' +# +# groupadd matrixcommander +# usermod -a -G matrixcommander yunorunner +# chgrp -R matrixcommander $MCHOME +# chmod -R 770 $MCHOME +# + +MCHOME="/opt/matrix-commander/" +MCARGS="-c $MCHOME/credentials.json --store $MCHOME/store" +timeout 10 "$MCHOME/venv/bin/matrix-commander" $MCARGS -m "$@" --room 'yunohost-apps' diff --git a/maintenance/finish_install.sh b/maintenance/finish_install.sh new file mode 100755 index 0000000..e3457ca --- /dev/null +++ b/maintenance/finish_install.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +cd "$(dirname $(realpath $0))" + +if (( $# < 1 )) +then + cat << EOF +Usage: ./finish_install.sh [auto|manual] [cluster] + +1st argument is the CI type (scheduling strategy): + - auto means job will automatically be scheduled by yunorunner from apps.json etc. + - manual means job will be scheduled manually (e.g. via webhooks or yunorunner ciclic) + +2nd argument is to build the first node of an lxc cluster + - lxd cluster will be created with the current server + - some.domain.tld will be the cluster hostname and SecretAdminPasswurzd! the trust password to join the cluster + +EOF + exit 1 +fi + +YUNORUNNER_HOME="/var/www/yunorunner" +if [ $(pwd) != "$YUNORUNNER_HOME" ] +then + echo "This script should be ran from $YUNORUNNER_HOME" + exit 1 +fi + +ci_type=$3 +lxd_cluster=$4 + +# User which execute the CI software. +ci_user=yunorunner + +echo_bold () { + echo -e "\e[1m$1\e[0m" +} + +# ----------------------------------------------------------------- + + +function tweak_yunohost() { + + #echo_bold "> Setting up Yunohost..." + #local DIST="bullseye" + #local INSTALL_SCRIPT="https://install.yunohost.org/$DIST" + #curl $INSTALL_SCRIPT | bash -s -- -a + + #echo_bold "> Running yunohost postinstall" + #yunohost tools postinstall --domain $domain --password $yuno_pwd + + # What is it used for :| ... + #echo_bold "> Create Yunohost CI user" + #local ynh_ci_user=ynhci + #yunohost user create --firstname "$ynh_ci_user" --domain "$domain" --lastname "$ynh_ci_user" "$ynh_ci_user" --password $yuno_pwd + + # Idk why this is needed but wokay I guess >_> + echo -e "\n127.0.0.1 $domain #CI_APP" >> /etc/hosts + + echo_bold "> Disabling unecessary services to save up RAM" + for SERVICE in mysql php7.4-fpm metronome rspamd dovecot postfix redis-server postsrsd yunohost-api avahi-daemon + do + systemctl stop $SERVICE + systemctl disable $SERVICE --quiet + done +} + +function tweak_yunorunner() { + echo_bold "> Tweaking YunoRunner..." + + + #if ! yunohost app list --output-as json --quiet | jq -e '.apps[] | select(.id == "yunorunner")' >/dev/null + #then + # yunohost app install --force https://github.com/YunoHost-Apps/yunorunner_ynh -a "domain=$domain&path=/$ci_path" + #fi + domain=$(yunohost app setting yunorunner domain) + ci_path=$(yunohost app setting yunorunner path) + port=$(yunohost app setting yunorunner port) + + # Stop YunoRunner + # to be started manually by the admin after the CI_package_check install + # finishes + systemctl stop $ci_user + + # Remove the original database, in order to rebuilt it with the new config. + rm -f $YUNORUNNER_HOME/db.sqlite + + # For automatic / "main" CI we want to auto schedule jobs using the app list + if [ $ci_type == "auto" ] + then + cat >/var/www/yunorunner/config.py </var/www/yunorunner/config.py </dev/null + then + yunohost app install --force https://github.com/YunoHost-Apps/lxd_ynh + fi + + echo_bold "> Configuring lxd..." + + if [ "$lxd_cluster" == "cluster" ] + then + local free_space=$(df --output=avail / | sed 1d) + local btrfs_size=$(( $free_space * 90 / 100 / 1024 / 1024 )) + local lxc_network=$((1 + $RANDOM % 254)) + + yunohost firewall allow TCP 8443 + cat >./preseed.conf < Configuring the CI..." + + # Cron tasks + cat >> "/etc/cron.d/yunorunner" << EOF +# self-upgrade every night +0 3 * * * root "$YUNORUNNER_HOME/maintenance/self_upgrade.sh" >> "$YUNORUNNER_HOME/maintenance/self_upgrade.log" 2>&1 + +# Update app list +0 20 * * 5 root "$YUNORUNNER_HOME/maintenance/update_level_apps.sh" >> "$YUNORUNNER_HOME/maintenance/update_level_apps.log" 2>&1 + +# Update badges +0 1 * * * root "$YUNORUNNER_HOME/maintenance/update_badges.sh" >> "$YUNORUNNER_HOME/maintenance/update_badges.log" 2>&1 +EOF +} + +# ========================= +# Main stuff +# ========================= + +#git clone https://github.com/YunoHost/package_check "./package_check" +#install_dependencies + +[ -e /usr/bin/yunohost ] || { echo "YunoHost is not installed"; exit; } +[ -e /etc/yunohost/apps/yunorunner ] || { echo "Yunorunner is not installed on YunoHost"; exit; } + +tweak_yunohost +tweak_yunorunner +setup_lxd +add_cron_jobs + +echo "Done!" +echo " " +echo "N.B. : If you want to enable Matrix notification, you should look at " +echo "the instructions inside lib/chat_notify.sh to deploy matrix-commander" +echo "" +echo "You may also want to tweak the 'config' file to run test with a different branch / arch" +echo "" +echo "When you're ready to start the CI, run: systemctl restart $ci_user" diff --git a/maintenance/self_upgrade.sh b/maintenance/self_upgrade.sh new file mode 100755 index 0000000..b16f03e --- /dev/null +++ b/maintenance/self_upgrade.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# This script is designed to be used in a cron file + +#================================================= +# Grab the script directory +#================================================= + +if [ "${0:0:1}" == "/" ]; then script_dir="$(dirname "$0")"; else script_dir="$(echo $PWD/$(dirname "$0" | cut -d '.' -f2) | sed 's@/$@@')"; fi + +cd $script_dir/.. + +# We only self-upgrade if we're in a git repo on master branch +# (which should correspond to production contexts) +[[ -d ".git" ]] || exit + +[[ $(git rev-parse --abbrev-ref HEAD) == "master" ]] || exit + +git fetch origin --quiet + +# If already up to date, don't do anything else +[[ $(git rev-parse HEAD) == $(git rev-parse origin/master) ]] && exit + +git reset --hard origin/master --quiet diff --git a/maintenance/update_badges.sh b/maintenance/update_badges.sh new file mode 100755 index 0000000..f7b9fb2 --- /dev/null +++ b/maintenance/update_badges.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +#================================================= +# Grab the script directory +#================================================= + +if [ "${0:0:1}" == "/" ]; then script_dir="$(dirname "$0")"; else script_dir="$(echo $PWD/$(dirname "$0" | cut -d '.' -f2) | sed 's@/$@@')"; fi + +#================================================= +# Get the list and check for any modifications +#================================================= + +# Get the apps list from app.yunohost.org +wget -nv https://app.yunohost.org/default/v3/apps.json -O "$script_dir/apps.json" + +do_update=1 +if [ -e "$script_dir/apps.json.md5" ] +then + if md5sum --check --status "$script_dir/apps.json.md5" + then + echo "No changes into the app list since the last execution." + do_update=0 + fi +fi + +if [ $do_update -eq 1 ] +then + md5sum "$script_dir/apps.json" > "$script_dir/apps.json.md5" + + #================================================= + # Update badges for all apps + #================================================= + + # Parse each app into the list + while read app + do + # Get the status for this app + state=$(jq --raw-output ".apps[\"$app\"] | .state" "$script_dir/apps.json") + level=$(jq --raw-output ".apps[\"$app\"] | .level" "$script_dir/apps.json") + if [[ "$state" == "working" ]] + then + if [[ "$level" == "null" ]] || [[ "$level" == "?" ]] + then + state="just-got-added-to-catalog" + elif [[ "$level" == "0" ]] || [[ "$level" == "-1" ]] + then + state="broken" + fi + fi + + # Get the maintained status for this app + maintained=$(jq --raw-output ".apps[\"$app\"] | .antifeatures | .[]" "$script_dir/apps.json" | grep -q 'package-not-maintained' && echo unmaintained || echo maintained) + + cp "$script_dir/../badges/$state.svg" "$script_dir/../badges/${app}.status.svg" + cp "$script_dir/../badges/$maintained.svg" "$script_dir/../badges/${app}.maintain.svg" + + # List all apps from the list, by getting manifest ID. + done <<< "$(jq --raw-output ".apps[] | .manifest.id" "$script_dir/apps.json")" +fi diff --git a/maintenance/update_level_apps.sh b/maintenance/update_level_apps.sh new file mode 100755 index 0000000..1555012 --- /dev/null +++ b/maintenance/update_level_apps.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Récupère le dossier du script +if [ "${0:0:1}" == "/" ]; then script_dir="$(dirname "$0")"; else script_dir="$(echo $PWD/$(dirname "$0" | cut -d '.' -f2) | sed 's@/$@@')"; fi + +# Only run this on the true main, official CI, prevent instances from packagers to push weird stuff +grep -q "BASE_URL.*=.*ci-apps.yunohost.org" "$script_dir/../config" || exit 0 + +# Supprime le précédent clone de YunoHost/apps +rm -r "$script_dir/../../apps" + +# Se place dans le dossier du dépot git pour le script python +tmp_dir=$(mktemp -d) +git clone -q git@github.com:YunoHost/apps.git "$tmp_dir" +cd $tmp_dir + +# Créer une nouvelle branche pour commiter les changements +git checkout -b modify_level + +public_result_list="$script_dir/results/logs/list_level_stable_amd64.json" +majorregressions="" +minorregressions="" +improvements="" + +# For each app in the result file +for APP in $(jq -r 'keys[]' "$public_result_list") +do + # Check if the app is still in the list + if ! jq -r 'keys[]' "apps.json" | grep -qw $APP; then + continue + fi + + current_level="$(jq -r ".\"$APP\".level" apps.json)" + # Get the level from the stable+amd64 tests + new_level="$(jq -r ".\"$APP\".level" "$public_result_list")" + + if [[ "$current_level" != "null" ]] && [[ "$new_level" -lt "$current_level" ]] + then + if [[ "$new_level" -le 4 ]] && [[ "$current_level" -gt 4 ]] + then + majorregressions+=" - $APP $current_level -> $new_level | https://ci-apps.yunohost.org/ci/apps/$APP/latestjob\n" + else + minorregressions+=" - $APP $current_level -> $new_level | https://ci-apps.yunohost.org/ci/apps/$APP/latestjob\n" + fi + elif [[ "$new_level" != "$current_level" ]] + then + improvements+=" - $APP $current_level -> $new_level | https://ci-apps.yunohost.org/ci/apps/$APP/latestjob\n" + fi + + # Inject the new level value to apps.json + jq --sort-keys --indent 4 --arg APP $APP --argjson new_level $new_level '.[$APP].level=$new_level' apps.json > apps.json.new + mv apps.json.new apps.json +done + +# Affiche les changements (2 lignes de contexte suffisent à voir l'app) +# git diff -U2 --raw +# Ajoute les modifications des listes au prochain commit +git add --all *.json +git commit -q -m "Update app levels according to CI results$(echo -e "\n\n### Major Regressions\n$majorregressions\n\n### Minor Regressions\n$minorregressions\n\n### Improvements\n$improvements")" + +# Git doit être configuré sur la machine. +# git config --global user.email "MAIL..." +# git config --global user.name "yunohost-bot" +# ssh-keygen -t rsa -f $HOME/.ssh/github -P '' Pour créer une clé ssh sans passphrase +# Host github.com +# IdentityFile ~/.ssh/github +# Dans le config ssh +# Et la clé doit être enregistrée dans le compte github de yunohost-bot +git push -q -u origin modify_level diff --git a/migrates b/migrates deleted file mode 100644 index c89eb09..0000000 --- a/migrates +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE {tableName} ADD COLUMN COLNew {type}; - -ALTER TABLE Repo ADD COLUMN state string; -ALTER TABLE Repo ADD COLUMN random_job_day int; diff --git a/results/badges/get_badges.sh b/results/badges/get_badges.sh new file mode 100755 index 0000000..650c338 --- /dev/null +++ b/results/badges/get_badges.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Get the path of this script +script_dir="$(dirname "$(realpath "$0")")" + +wget -q https://upload.wikimedia.org/wikipedia/commons/1/1d/No_image.svg -O "$script_dir/maintained.svg" +wget -q https://img.shields.io/badge/Status-Package%20not%20maintained-red.svg -O "$script_dir/unmaintained.svg" + +wget -q https://img.shields.io/badge/Status-working-brightgreen.svg -O "$script_dir/working.svg" +wget -q https://img.shields.io/badge/Status-Just%20got%20added%20to%20catalog-yellowgreen.svg -O "$script_dir/just-got-added-to-catalog.svg" +wget -q https://img.shields.io/badge/Status-In%20progress-orange.svg -O "$script_dir/inprogress.svg" +wget -q https://img.shields.io/badge/Status-Not%20working-red.svg -O "$script_dir/notworking.svg" +wget -q https://img.shields.io/badge/Status-Broken-red.svg -O "$script_dir/broken.svg" diff --git a/results/logs/.gitkeep b/results/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/results/summary/empty.png b/results/summary/empty.png new file mode 100644 index 0000000..2c4d743 Binary files /dev/null and b/results/summary/empty.png differ diff --git a/run.py b/run.py index 67b9bcb..67cfae8 100644 --- a/run.py +++ b/run.py @@ -10,6 +10,7 @@ import asyncio import traceback import itertools import tracemalloc +import string import hmac import hashlib @@ -40,6 +41,10 @@ from playhouse.shortcuts import model_to_dict from models import Repo, Job, db, Worker from schedule import always_relaunch, once_per_day +# This is used by ciclic +admin_token = ''.join(random.choices(string.ascii_lowercase + string.digits, k=32)) +open(".admin_token", "w").write(admin_token) + try: asyncio_all_tasks = asyncio.all_tasks except AttributeError as e: @@ -425,7 +430,7 @@ async def jobs_dispatcher(): async def run_job(worker, job): - path_to_analyseCI = app.config.PATH_TO_ANALYZER + path_to_analyzer = app.config.PATH_TO_ANALYZER await broadcast({ "action": "update_job", @@ -435,11 +440,11 @@ async def run_job(worker, job): # fake stupid command, whould run CI instead task_logger.info(f"Starting job '{job.name}' #{job.id}...") - cwd = os.path.split(path_to_analyseCI)[0] + cwd = os.path.split(path_to_analyzer)[0] arguments = f' {job.url_or_path} "{job.name}" {job.id} {worker.id}' - task_logger.info(f"Launch command: /bin/bash " + path_to_analyseCI + arguments) + task_logger.info(f"Launch command: /bin/bash " + path_to_analyzer + arguments) try: - command = await asyncio.create_subprocess_shell("/bin/bash " + path_to_analyseCI + arguments, + command = await asyncio.create_subprocess_shell("/bin/bash " + path_to_analyzer + arguments, cwd=cwd, # default limit is not enough in some situations limit=(2 ** 16) ** 10, @@ -777,22 +782,11 @@ def require_token(): 'to access the API, please ' 'refer to the README'}, 403) - if not os.path.exists("tokens"): - api_logger.warning("No tokens available and a user is trying " - "to access the API") - return response.json({'status': 'invalide token'}, 403) - - async with aiofiles.open('tokens', mode='r') as file: - tokens = await file.read() - tokens = {x.strip() for x in tokens.split("\n") if x.strip()} - token = request.headers["X-Token"].strip() - if token not in tokens: - api_logger.warning(f"someone tried to access the API using " - "the {token} but it's not a valid token in " - "the 'tokens' file") - return response.json({'status': 'invalide token'}, 403) + if not await hmac.compare_digest(token, admin_token): + api_logger.warning("someone tried to access the API using an invalid admin token") + return response.json({'status': 'invalid token'}, 403) result = await f(request, *args, **kwargs) return result @@ -1265,7 +1259,7 @@ def main(config="./config.py"): "BASE_URL": "", "PORT": 4242, "DEBUG": False, - "PATH_TO_ANALYZER": "/home/CI_package_check/analyseCI.sh", + "PATH_TO_ANALYZER": "./analyze_yunohost_app.sh", "MONITOR_APPS_LIST": False, "MONITOR_GIT": False, "MONITOR_ONLY_GOOD_QUALITY_APPS": False,