Add a check that there isn't a shitload of warnings in the script outputs

lib/analyze_test_results.py

@@ -91,13 +91,16 @@ def level_5(tests):
     """
     Linter returned no errors (= main_result is success)
     and no alias/path traversal issue detected during tests
+    and not too many warnings in log outputs
     """
 
     alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests)
+    too_many_warnings = any(t["results"].get("too_many_warnings") for t in tests)
 
     linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
 
     return not alias_traversal_detected \
+        and not too_many_warnings \
         and linter_tests != [] \
         and linter_tests[0]["results"]["main_result"] == "success"
 

lib/lxc.sh

@@ -91,7 +91,7 @@ LXC_EXEC () {
     start_timer
 
     # Execute the command given in argument in the container and log its results.
-    lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- $cmd | tee -a "$complete_log"
+    lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- $cmd | tee -a "$complete_log" $current_test_log
 
     # Store the return code of the command
     local returncode=${PIPESTATUS[0]}

lib/tests_coordination.sh

@@ -239,6 +239,7 @@ run_all_tests() {
 
     mkdir -p $TEST_CONTEXT/tests
     mkdir -p $TEST_CONTEXT/results
+    mkdir -p $TEST_CONTEXT/logs
 
     readonly app_id="$(jq -r .id $package_path/manifest.json)"
 
@@ -284,6 +285,7 @@ run_all_tests() {
     for testfile in $(ls $TEST_CONTEXT/tests/*.json);
     do
         TEST_LAUNCHER $testfile
+        current_test_number=$((current_test_number+1))
     done
 
     # Print the final results of the tests
@@ -311,7 +313,9 @@ TEST_LAUNCHER () {
     current_test_id=$(basename $testfile | cut -d. -f1)
     current_test_infos="$TEST_CONTEXT/tests/$current_test_id.json"
     current_test_results="$TEST_CONTEXT/results/$current_test_id.json"
+    current_test_log="$TEST_CONTEXT/logs/$current_test_id.log"
     echo "{}" > $current_test_results
+    echo "" > $current_test_log
 
     local test_type=$(jq -r '.test_type' $testfile)
     local test_arg=$(jq -r '.test_arg' $testfile)
@@ -319,6 +323,16 @@ TEST_LAUNCHER () {
     # Execute the test
     $test_type $test_arg
 
+    # Check that the number of warning ain't higher than a treshold
+    local n_warnings=$(grep --extended-regexp '^[0-9]+\s+.{1,15}WARNING' $current_test_log | wc -l)
+    # (we ignore this test for upgrade from older commits to avoid having to patch older commits for this)
+    if [ "$n_warnings" -gt 250 ] && [ "$test_type" != "TEST_UPGRADE" -o "$test_arg" == "" ]
+    then
+        log_error "There's a shitload of warnings in the output ! If those warnings are coming from some app build step and ain't actual warnings, please redirect them to the standard output instead of the error output ...!"
+        log_report_test_failed
+        SET_RESULT "failure" too_many_warnings
+    fi
+
     [ $? -eq 0 ] && SET_RESULT "success" main_result || SET_RESULT "failure" main_result
 
     local test_duration=$(echo $(( $(date +%s) - $global_start_timer )))
@@ -385,9 +399,6 @@ start_test () {
     total_number_of_test=$(ls $TEST_CONTEXT/tests/*.json | wc -l)
 
     log_title " [Test $current_test_number/$total_number_of_test] $current_test_serie$1"
-
-    # Increment the value of the current test
-    current_test_number=$((current_test_number+1))
 }
 
 there_is_an_install_type() {