diff --git a/lib/analyze_test_results.py b/lib/analyze_test_results.py
index cc06dd7..8ad9a08 100644
--- a/lib/analyze_test_results.py
+++ b/lib/analyze_test_results.py
@@ -43,6 +43,9 @@ def test_notes(test):
     if test['results'].get("install_dir_permissions"):
         yield '<style=danger>Unsafe install dir permissions</style>'
 
+    if test['results'].get("file_manually_modified"):
+        yield '<style=danger>Config file overwritten / manually modified</style>'
+
 
 levels = []
 
@@ -161,6 +164,7 @@ def level_7(tests):
     unsafe_install_dir_perms = any(t["results"].get("install_dir_permissions") for t in tests_on_which_to_check_for_runtime_warnings)
     alias_traversal = any(t["results"].get("alias_traversal") for t in tests_on_which_to_check_for_runtime_warnings)
     witness = any(t["results"].get("witness") for t in tests_on_which_to_check_for_runtime_warnings)
+    file_manually_modified = any(t["results"].get("file_manually_modified") for t in tests_on_which_to_check_for_runtime_warnings)
 
     return all(t["results"]["main_result"] == "success" for t in tests) \
         and linter_tests != [] \
@@ -168,6 +172,7 @@ def level_7(tests):
         and not alias_traversal \
         and not too_many_warnings \
         and not unsafe_install_dir_perms \
+        and not file_manually_modified \
         and "App.qualify_for_level_7" in linter_tests[0]["results"]["success"]
 
 
diff --git a/lib/tests_coordination.sh b/lib/tests_coordination.sh
index 01c82e2..2d0e42e 100644
--- a/lib/tests_coordination.sh
+++ b/lib/tests_coordination.sh
@@ -132,7 +132,15 @@ TEST_LAUNCHER () {
     # which should not happen during tests because no human modified the file ...
     if grep -q --extended-regexp 'has been manually modified since the installation or last upgrade. So it has been duplicated' $current_test_log
     then
-        log_error "Apparently the log is telling that 'some file got manually modified' ... which should not happen, considering that no human modified the file ... ! Maybe you need to check what's happening with ynh_store_file_checksum and ynh_backup_if_checksum_is_different between install and upgrade."
+        log_error "Apparently the log is telling that 'some file got manually modified' ... which should not happen, considering that no human modified the file ... ! This is usually symptomatic of something that modified a conf file after installing it with ynh_add_config. Maybe usigin ynh_store_file_checksum can help, or maybe the issue is more subtle!"
+        if [[ "$test_type" == "TEST_UPGRADE" ]] && [[ "$test_arg" == "" ]]
+        then
+            SET_RESULT "failure" file_manually_modified
+        fi
+        if [[ "$test_type" == "TEST_BACKUP_RESTORE" ]]
+        then
+            SET_RESULT "failure" file_manually_modified
+        fi
     fi
 
     # Check that the number of warning ain't higher than a treshold