From ed7be850051c0e56b5655a5ed8f47b28e704f9df Mon Sep 17 00:00:00 2001
From: Alexandre Aubin <alex.aubin@mailoo.org>
Date: Mon, 8 Mar 2021 20:27:30 +0100
Subject: [PATCH] Add a check that there isn't a shitload of warnings in the
 script outputs

---
 lib/analyze_test_results.py |  3 +++
 lib/lxc.sh                  |  2 +-
 lib/tests_coordination.sh   | 17 ++++++++++++++---
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/lib/analyze_test_results.py b/lib/analyze_test_results.py
index 4edddf0..dd0a115 100644
--- a/lib/analyze_test_results.py
+++ b/lib/analyze_test_results.py
@@ -91,13 +91,16 @@ def level_5(tests):
     """
     Linter returned no errors (= main_result is success)
     and no alias/path traversal issue detected during tests
+    and not too many warnings in log outputs
     """
 
     alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests)
+    too_many_warnings = any(t["results"].get("too_many_warnings") for t in tests)
 
     linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
 
     return not alias_traversal_detected \
+        and not too_many_warnings \
         and linter_tests != [] \
         and linter_tests[0]["results"]["main_result"] == "success"
 
diff --git a/lib/lxc.sh b/lib/lxc.sh
index e8f2d38..705074f 100644
--- a/lib/lxc.sh
+++ b/lib/lxc.sh
@@ -91,7 +91,7 @@ LXC_EXEC () {
     start_timer
 
     # Execute the command given in argument in the container and log its results.
-    lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- $cmd | tee -a "$complete_log"
+    lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- $cmd | tee -a "$complete_log" $current_test_log
 
     # Store the return code of the command
     local returncode=${PIPESTATUS[0]}
diff --git a/lib/tests_coordination.sh b/lib/tests_coordination.sh
index 14d0260..0583461 100644
--- a/lib/tests_coordination.sh
+++ b/lib/tests_coordination.sh
@@ -239,6 +239,7 @@ run_all_tests() {
 
     mkdir -p $TEST_CONTEXT/tests
     mkdir -p $TEST_CONTEXT/results
+    mkdir -p $TEST_CONTEXT/logs
 
     readonly app_id="$(jq -r .id $package_path/manifest.json)"
 
@@ -284,6 +285,7 @@ run_all_tests() {
     for testfile in $(ls $TEST_CONTEXT/tests/*.json);
     do
         TEST_LAUNCHER $testfile
+        current_test_number=$((current_test_number+1))
     done
 
     # Print the final results of the tests
@@ -311,7 +313,9 @@ TEST_LAUNCHER () {
     current_test_id=$(basename $testfile | cut -d. -f1)
     current_test_infos="$TEST_CONTEXT/tests/$current_test_id.json"
     current_test_results="$TEST_CONTEXT/results/$current_test_id.json"
+    current_test_log="$TEST_CONTEXT/logs/$current_test_id.log"
     echo "{}" > $current_test_results
+    echo "" > $current_test_log
 
     local test_type=$(jq -r '.test_type' $testfile)
     local test_arg=$(jq -r '.test_arg' $testfile)
@@ -319,6 +323,16 @@ TEST_LAUNCHER () {
     # Execute the test
     $test_type $test_arg
 
+    # Check that the number of warning ain't higher than a treshold
+    local n_warnings=$(grep --extended-regexp '^[0-9]+\s+.{1,15}WARNING' $current_test_log | wc -l)
+    # (we ignore this test for upgrade from older commits to avoid having to patch older commits for this)
+    if [ "$n_warnings" -gt 250 ] && [ "$test_type" != "TEST_UPGRADE" -o "$test_arg" == "" ]
+    then
+        log_error "There's a shitload of warnings in the output ! If those warnings are coming from some app build step and ain't actual warnings, please redirect them to the standard output instead of the error output ...!"
+        log_report_test_failed
+        SET_RESULT "failure" too_many_warnings
+    fi
+
     [ $? -eq 0 ] && SET_RESULT "success" main_result || SET_RESULT "failure" main_result
 
     local test_duration=$(echo $(( $(date +%s) - $global_start_timer )))
@@ -385,9 +399,6 @@ start_test () {
     total_number_of_test=$(ls $TEST_CONTEXT/tests/*.json | wc -l)
 
     log_title " [Test $current_test_number/$total_number_of_test] $current_test_serie$1"
-
-    # Increment the value of the current test
-    current_test_number=$((current_test_number+1))
 }
 
 there_is_an_install_type() {