Moar fixes + reimplement summary/level analysis

lib/analyze_test_results.py

@@ -1,284 +1,223 @@
+import sys
+import json
+import os
 
-# Levels
+def load_tests(test_folder):
 
-#0  Broken
+    for test in sorted(os.listdir(test_folder + "/tests")):
-#1  Installable
-#2  Installable in all situations
-#3  Can be updated
-#4  Backup and restore support
-#5  Clean
-#6  Open to contributions from the community
-#7  Successfully pass all functional tests and linter tests
-#8  Maintained and long-term good quality
-#9  High quality app
-#10 Package assessed as perfect
 
-# Linter stuff:
+        j = json.load(open(test_folder + "/tests/" + test))
+        j["id"] = os.path.basename(test).split(".")[0]
+        j["results"] = json.load(open(test_folder + "/results/" + j["id"] + ".json"))
+        yield j
 
 
-#    # Check we qualify for level 6, 7, 8
+levels = []
-#    # Linter will have a warning called "app_in_github_org" if app ain't in the
+
-#    # yunohost-apps org...
+
-#    if ! cat "./temp_linter_result.json" | jq ".warning" | grep -q "app_in_github_org"
+def level(level_, descr):
-#    then
+    def decorator(f):
-#        local pass_level_6="true"
+        f.descr = descr
-#    fi
+        f.level = level_
-#    if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_7"
+        levels.insert(level_, f)
-#    then
+        return f
-#        local pass_level_7="true"
+    return decorator
-#    fi
+
-#    if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_8"
+
-#    then
+###############################################################################
-#        local pass_level_8="true"
+
-#    fi
+
-#
+@level(0, "Broken")
-#    # If there are any critical errors, we'll force level 0
+def level_0(tests):
-#    if [[ -n "$(cat "./temp_linter_result.json" | jq ".critical" | grep -v '\[\]')" ]]
+    return True
-#    then
+
-#        local pass_level_0="false"
+
-#        # If there are any regular errors, we'll cap to 4
+@level(1, "Installable in at least one scenario")
-#    elif [[ -n "$(cat "./temp_linter_result.json" | jq ".error" | grep -v '\[\]')" ]]
+def level_1(tests):
-#    then
+    """
-#        local pass_level_4="false"
+    Any install test succeded
-#        # Otherwise, test pass (we'll display a warning depending on if there are
+    And there are no critical issues in the linter
-#        # any remaning warnings or not)
+    """
-#    else
+
-#        if [[ -n "$(cat "./temp_linter_result.json" | jq ".warning" | grep -v '\[\]')" ]]
+    linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
-#        then
+    install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
-#            log_report_test_warning
+    witness_missing_detected = any(t["results"].get("witness") for t in tests)
-#        else
+
-#            log_report_test_success
+    return linter_tests != [] \
-#        fi
+        and linter_tests[0]["results"]["critical"] == [] \
-#        local pass_level_4="true"
+        and not witness_missing_detected \
-#    fi
+        and any(t["results"]["main_result"] == "success" for t in install_tests)
+
+
+@level(2, "Installable in all scenarios")
+def level_2(tests):
+    """
+    All install tests succeeded (and at least one test was made)
+    """
+
+    install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
+
+    return install_tests != [] \
+        and all(t["results"]["main_result"] == "success" for t in install_tests)
+
+
+@level(3, "Can be upgraded")
+def level_3(tests):
+    """
+    All upgrade tests succeeded (and at least one test was made)
+    """
+
+    upgrade_tests = [t for t in tests if t["test_type"] == "TEST_UPGRADE"]
+
+    return upgrade_tests != [] \
+        and all(t["results"]["main_result"] == "success" for t in upgrade_tests)
+
+
+@level(4, "Can be backup/restored")
+def level_4(tests):
+    """
+    All backup/restore tests succeded (and at least one test was made)
+    """
+
+    backup_tests = [t for t in tests if t["test_type"] == "TEST_BACKUP"]
+
+    return backup_tests != [] \
+        and all(t["results"]["main_result"] == "success" for t in backup_tests)
+
+
+@level(5, "No linter errors")
+def level_5(tests):
+    """
+    Linter returned no errors (= main_result is success)
+    and no alias/path traversal issue detected during tests
+    """
+
+    alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests)
+
+    linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
+
+    return not alias_traversal_detected \
+        and linter_tests != [] \
+        and linter_tests[0]["results"]["main_result"] == "success"
+
+
+@level(6, "App is in a community-operated git org")
+def level_6(tests):
+    """
+    The app is in the Yunohost-Apps organization
+    (the linter will report a warning named "is_in_github_org" if it's not)
+    """
+
+    linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
+
+    return linter_tests != [] \
+        and "is_in_github_org" not in linter_tests[0]["results"]["warning"]
+
+
+@level(7, "Pass all tests + no linter warnings")
+def level_7(tests):
+    """
+    All tests succeeded + no warning in linter (that part is tested by the
+    # linter which will report a "qualify_for_level_7" in successes)
+    """
+
+    linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
+
+    return all(t["results"]["main_result"] == "success" for t in tests) \
+        and linter_tests != [] \
+        and "qualify_for_level_7" in linter_tests[0]["results"]["success"]
+
+
+@level(8, "Maintained and long-term good quality")
+def level_8(tests):
+    """
+    App is maintained and long-term good quality (this is tested by the linter
+    which will report a "qualify_for_level_8")
+    """
+
+    linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
+
+    return linter_tests != [] \
+        and "qualify_for_level_8" in linter_tests[0]["results"]["success"]
+
+
+@level(9, "Flagged high-quality in app catalog")
+def level_9(tests):
+    """
+    App is flagged high-quality in the app catalog (this is tested by the linter
+    which will rpeort a "qualify_for_level_9")
+    """
+    linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
+
+    return linter_tests != [] \
+        and "qualify_for_level_9" in linter_tests[0]["results"]["success"]
+
+
+tests = list(load_tests(sys.argv[1]))
+
+test_types = {
+    "PACKAGE_LINTER": "Package linter",
+    "TEST_INSTALL": "Install",
+    "TEST_UPGRADE": "Upgrade",
+    "TEST_BACKUP_RESTORE": "Backup/restore",
+    "TEST_CHANGE_URL": "Change url",
+    "TEST_PORT_ALREADY_USED": "Port already used",
+    "ACTIONS_CONFIG_PANEL": "Config/panel"
+}
+
+OK = ' \033[1m\033[92mOK\033[0m '
+FAIL = '\033[91mfail\033[0m'
+
+print("Tests summary")
+print("=============")
+for test in tests:
+    test_display_name = test_types[test["test_type"]]
+    if test["test_arg"]:
+        test_display_name += " (%s)" % test["test_arg"][:8]
+    test_display_name += ":"
+
+    result = OK if test["results"]["main_result"] == "success" else FAIL
+    print("{test: <25}{result}".format(test=test_display_name, result=result))
+
+print()
+print("Level results")
+print("=============")
+
+stop_global_level_bump = False
+
+global_level = level_0
+
+for level in levels[1:]:
+    level.passed = level(tests)
+
+    if not level.passed:
+        stop_global_level_bump = True
+
+    if not stop_global_level_bump:
+        global_level = level
+        display = OK
+    else:
+        display = " ok " if level.passed else ""
+
+    print("Level {i} {descr: <40} {result}".format(i=level.level,
+        descr="(%s)"%level.descr[:38], result=display))
+
+print()
+print("\033[1mGlobal level for this application: %s (%s)\033[0m" % (global_level.level, global_level.descr))
+print()
+
+
+summary = {
+    "tests": [{
+        "test_type": t["test_type"],
+        "test_arg": t["test_arg"],
+        "main_result": t["results"]["main_result"]
+    } for t in tests],
+    "levels": {level.level: level.passed for level in levels[1:]},
+    "global_level": global_level.level
+}
+
+sys.stderr.write(json.dumps(summary))
 
 
 
-
-
-
-
-
-
-
-
-
-
-#
-#    local test_serie_id=$1
-#    source $TEST_CONTEXT/$test_serie_id/results
-#
-#    # Print the test result
-#    print_result () {
-#
-#        # Print the result of this test
-#        # we use printf to force the length to 30 (filled with space)
-#        testname=$(printf %-30.30s "$1:")
-#        if [ $2 -eq 1 ]
-#        then
-#            echo "$testname ${BOLD}${GREEN}SUCCESS${NORMAL}"
-#        elif [ $2 -eq -1 ]
-#        then
-#            echo "$testname ${BOLD}${RED}FAIL${NORMAL}"
-#        else
-#            echo "$testname Not evaluated."
-#        fi
-#    }
-#
-#    # Print the result for each test
-#    echo -e "\n\n"
-#    print_result "Package linter" $RESULT_linter
-#    print_result "Install (root)" $RESULT_check_root
-#    print_result "Install (subpath)" $RESULT_check_subdir
-#    print_result "Install (no url)" $RESULT_check_nourl
-#    print_result "Install (private)" $RESULT_check_private
-#    print_result "Install (multi-instance)" $RESULT_check_multi_instance
-#    print_result "Upgrade" $RESULT_check_upgrade
-#    print_result "Backup" $RESULT_check_backup
-#    print_result "Restore" $RESULT_check_restore
-#    print_result "Change URL" $RESULT_change_url
-#    print_result "Port already used" $RESULT_check_port
-#    print_result "Actions and config-panel" $RESULT_action_config_panel
-#
-#    # Determine the level for this app
-#
-#    # Each level can has 5 different values
-#    # 0    -> If this level can't be validated
-#    # 1    -> If this level is forced. Even if the tests fails
-#    # 2    -> Indicates the tests had previously validated this level
-#    # auto -> This level has not a value yet.
-#    # na   -> This level will not be checked, but it'll be ignored in the final sum
-#
-#    # Set default values for level, if they're empty.
-#    test -n "${level[1]}" || level[1]=auto
-#    test -n "${level[2]}" || level[2]=auto
-#    test -n "${level[3]}" || level[3]=auto
-#    test -n "${level[4]}" || level[4]=auto
-#    test -n "${level[5]}" || level[5]=auto
-#    test -n "${level[5]}" || level[5]=auto
-#    test -n "${level[6]}" || level[6]=auto
-#    test -n "${level[7]}" || level[7]=auto
-#    test -n "${level[8]}" || level[8]=auto
-#    test -n "${level[9]}" || level[9]=0
-#    test -n "${level[10]}" || level[10]=0
-#
-#    pass_level_1() {
-#	 # FIXME FIXME #FIXME
-#	 return 0
-#    }
-#
-#    pass_level_2() {
-#        # -> The package can be install and remove in all tested configurations.
-#        # Validated if none install failed
-#        [ $RESULT_check_subdir -ne -1 ] && \
-#        [ $RESULT_check_root -ne -1 ] && \
-#        [ $RESULT_check_private -ne -1 ] && \
-#        [ $RESULT_check_multi_instance -ne -1 ]
-#    }
-#
-#    pass_level_3() {
-#        # -> The package can be upgraded from the same version.
-#        # Validated if the upgrade is ok. Or if the upgrade has been not tested but already validated before.
-#        [ $RESULT_check_upgrade -eq 1 ] || \
-#        ( [ $RESULT_check_upgrade -ne -1 ] && \
-#        [ "${level[3]}" == "2" ] )
-#    }
-#
-#    pass_level_4() {
-#        # -> The package can be backup and restore without error
-#        # Validated if backup and restore are ok. Or if backup and restore have been not tested but already validated before.
-#        ( [ $RESULT_check_backup -eq 1 ] && \
-#        [ $RESULT_check_restore -eq 1 ] ) || \
-#        ( [ $RESULT_check_backup -ne -1 ] && \
-#        [ $RESULT_check_restore -ne -1 ] && \
-#        [ "${level[4]}" == "2" ] )
-#    }
-#
-#    pass_level_5() {
-#        # -> The package have no error with package linter
-#        # -> The package does not have any alias_traversal error
-#        # Validated if Linter is ok. Or if Linter has been not tested but already validated before.
-#        [ $RESULT_alias_traversal -ne 1 ] && \
-#        ([ $RESULT_linter -ge 1 ] || \
-#        ( [ $RESULT_linter -eq 0 ] && \
-#        [ "${level[5]}" == "2" ] ) )
-#    }
-#
-#    pass_level_6() {
-#        # -> The package can be backup and restore without error
-#        # This is from the linter, tests if app is the Yunohost-apps organization
-#        [ $RESULT_linter_level_6 -eq 1 ] || \
-#        ([ $RESULT_linter_level_6 -eq 0 ] && \
-#        [ "${level[6]}" == "2" ] )
-#    }
-#
-#    pass_level_7() {
-#        # -> None errors in all tests performed
-#        # Validated if none errors is happened.
-#        [ $RESULT_check_subdir -ne -1 ] && \
-#        [ $RESULT_check_upgrade -ne -1 ] && \
-#        [ $RESULT_check_private -ne -1 ] && \
-#        [ $RESULT_check_multi_instance -ne -1 ] && \
-#        [ $RESULT_check_port -ne -1 ] && \
-#        [ $RESULT_check_backup -ne -1 ] && \
-#        [ $RESULT_check_restore -ne -1 ] && \
-#        [ $RESULT_change_url -ne -1 ] && \
-#        [ $RESULT_action_config_panel -ne -1 ] && \
-#        ([ $RESULT_linter_level_7 -ge 1 ] ||
-#        ([ $RESULT_linter_level_7 -eq 0 ] && \
-#        [ "${level[8]}" == "2" ] ))
-#    }
-#
-#    pass_level_8() {
-#        # This happens in the linter
-#        # When writing this, defined as app being maintained + long term quality (=
-#        # certain amount of time level 5+ in the last year)
-#        [ $RESULT_linter_level_8 -ge 1 ] || \
-#        ([ $RESULT_linter_level_8 -eq 0 ] && \
-#        [ "${level[8]}" == "2" ] )
-#    }
-#
-#    pass_level_9() {
-#        list_url="https://raw.githubusercontent.com/YunoHost/apps/master/apps.json"
-#        curl --silent $list_url | jq ".[\"$app_id\"].high_quality" | grep -q "true"
-#    }
-#
-#    # Check if the level can be changed
-#    level_can_change () {
-#        # If the level is set at auto, it's waiting for a change
-#        # And if it's set at 2, its value can be modified by a new result
-#        [ "${level[$1]}" == "auto" ] || [ "${level[$1]}" -eq 2 ]
-#    }
-#
-#    if level_can_change 1; then pass_level_1 && level[1]=2 || level[1]=0; fi
-#    if level_can_change 2; then pass_level_2 && level[2]=2 || level[2]=0; fi
-#    if level_can_change 3; then pass_level_3 && level[3]=2 || level[3]=0; fi
-#    if level_can_change 4; then pass_level_4 && level[4]=2 || level[4]=0; fi
-#    if level_can_change 5; then pass_level_5 && level[5]=2 || level[5]=0; fi
-#    if level_can_change 6; then pass_level_6 && level[6]=2 || level[6]=0; fi
-#    if level_can_change 7; then pass_level_7 && level[7]=2 || level[7]=0; fi
-#    if level_can_change 8; then pass_level_8 && level[8]=2 || level[8]=0; fi
-#    if level_can_change 9; then pass_level_9 && level[9]=2 || level[9]=0; fi
-#
-#    # Level 10 has no definition yet
-#    level[10]=0
-#
-#    # Initialize the global level
-#    global_level=0
-#
-#    # Calculate the final level
-#    for i in `seq 1 10`
-#    do
-#
-#        # If there is a level still at 'auto', it's a mistake.
-#        if [ "${level[i]}" == "auto" ]
-#        then
-#            # So this level will set at 0.
-#            level[i]=0
-#
-#        # If the level is at 1 or 2. The global level will be set at this level
-#        elif [ "${level[i]}" -ge 1 ]
-#        then
-#            global_level=$i
-#
-#            # But, if the level is at 0, the loop stop here
-#            # Like that, the global level rise while none level have failed
-#        else
-#            break
-#        fi
-#    done
-#
-#    # If some witness files was missing, it's a big error ! So, the level fall immediately at 0.
-#    if [ $RESULT_witness -eq 1 ]
-#    then
-#        log_error "Some witness files has been deleted during those tests ! It's a very bad thing !"
-#        global_level=0
-#    fi
-#
-#    # If the package linter returned a critical error, the app is flagged as broken / level 0
-#    if [ $RESULT_linter_broken -eq 1 ]
-#    then
-#        log_error "The package linter reported a critical failure ! App is considered broken !"
-#        global_level=0
-#    fi
-#
-#    if [ $RESULT_alias_traversal -eq 1 ]
-#    then
-#        log_error "Issue alias_traversal was detected ! Please see here https://github.com/YunoHost/example_ynh/pull/45 to fix that."
-#    fi
-#
-#    # Then, print the levels
-#    # Print the global level
-#    verbose_level=$(grep "^$global_level " "./levels.list" | cut -c4-)
-#
-#    log_info "Level of this application: $global_level ($verbose_level)"
-#
-#    # And print the value for each level
-#    for i in `seq 1 10`
-#    do
-#        display="0"
-#        if [ "${level[$i]}" -ge 1 ]; then
-#            display="1"
-#        fi
-#        echo -e "\t   Level $i: $display"
-#    done

lib/common.sh

@@ -27,7 +27,7 @@ clean_exit () {
 
     LXC_RESET
 
-    [ -n "$TEST_CONTEXT" ] rm -rf "$TEST_CONTEXT"
+    [ -n "$TEST_CONTEXT" ] && rm -rf "$TEST_CONTEXT"
     rm -f "$lock_file"
 
     exit $1
@@ -78,7 +78,7 @@ readonly WHITE=$(printf '\033[39m')
 
 function log_title()
 {
-    cat << EOF
+    cat << EOF | tee -a /proc/self/fd/3
 ${BOLD}
  ===================================
  $1
@@ -89,51 +89,51 @@ EOF
 
 function log_small_title()
 {
-    echo -e "\n${BOLD} > ${1}${NORMAL}\n"
+    echo -e "\n${BOLD} > ${1}${NORMAL}\n" | tee -a /proc/self/fd/3
 }
 
 
 function log_debug()
 {
-    echo "$1" >&3
+    echo "$1" >> /proc/self/fd/3
 }
 
 function log_info()
 {
-    echo "${1}"
+    echo "${1}" | tee -a /proc/self/fd/3
 }
 
 function log_success()
 {
-    echo "${BOLD}${GREEN}Success: ${1}${NORMAL}"
+    echo "${BOLD}${GREEN}Success: ${1}${NORMAL}" | tee -a /proc/self/fd/3
 }
 
 function log_warning()
 {
-    echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}"
+    echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}" | tee -a /proc/self/fd/3
 }
 
 function log_error()
 {
-    echo "${BOLD}${RED}Error: ${1}${NORMAL}"
+    echo "${BOLD}${RED}Error: ${1}${NORMAL}" | tee -a /proc/self/fd/3
 }
 
 function log_critical()
 {
-    echo "${BOLD}${RED}Critical: ${1}${NORMAL}"
+    echo "${BOLD}${RED}Critical: ${1}${NORMAL}" | tee -a /proc/self/fd/3
     clean_exit 1
 }
 
 function log_report_test_success () {
-    echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n"
+    echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n" | tee -a /proc/self/fd/3
 }
 
 function log_report_test_warning () {
-    echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n"
+    echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n" | tee -a /proc/self/fd/3
 }
 
 function log_report_test_failed () {
-    echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n"
+    echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n" | tee -a /proc/self/fd/3
 }
 
 #=================================================

lib/tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 #=================================================
-# Logistic helpers
+# "Low-level" logistic helpers
 #=================================================
 
 _RUN_YUNOHOST_CMD() {
@@ -258,19 +258,24 @@ PACKAGE_LINTER () {
     # Execute package linter and linter_result gets the return code of the package linter
     ./package_linter/package_linter.py "$package_path" | tee -a "$complete_log"
     ./package_linter/package_linter.py "$package_path" --json | tee -a "$complete_log" > $current_test_results
+
+    return $?
 }
 
 TEST_INSTALL () {
-    # Try to install in a sub path, on root or without url access
-    # $1 = install type
 
     local install_type=$1
+
+    # This is a separate case ... at least from an hystorical point of view ...
+    # but it helpers for semantic that the test is a "TEST_INSTALL" ...
+    [ "$install_type" = "multi"   ] && { _TEST_MULTI_INSTANCE; return $?; }
+
     local check_path="/"
     local is_public="1"
-    [ "$install_type" = "subdir"   ] && { start_test "Installation in a sub path";      local check_path=/path; }
+    [ "$install_type" = "subdir"  ] && { start_test "Installation in a sub path";      local check_path=/path; }
-    [ "$install_type" = "root"     ] && { start_test "Installation on the root";                                }
+    [ "$install_type" = "root"    ] && { start_test "Installation on the root";                                }
-    [ "$install_type" = "nourl"    ] && { start_test "Installation without url access"; local check_path="";    }
+    [ "$install_type" = "nourl"   ] && { start_test "Installation without url access"; local check_path="";    }
-    [ "$install_type" = "private"  ] && { start_test "Installation in private mode";    local is_public="0";    }
+    [ "$install_type" = "private" ] && { start_test "Installation in private mode";    local is_public="0";    }
     local snapname=snap_${install_type}install
 
     LOAD_LXC_SNAPSHOT snap0
@@ -298,6 +303,29 @@ TEST_INSTALL () {
     return $?
 }
 
+_TEST_MULTI_INSTANCE () {
+
+    start_test "Multi-instance installations"
+
+    # Check if an install have previously work
+    at_least_one_install_succeeded || return 1
+
+    local check_path=$(default_install_path)
+
+    LOAD_LXC_SNAPSHOT snap0
+
+    log_small_title "First installation: path=$DOMAIN$check_path" \
+        &&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \
+        && log_small_title "Second installation: path=$SUBDOMAIN$check_path" \
+        && _INSTALL_APP "path=$check_path" \
+        && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \
+        && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2 \
+        && _REMOVE_APP \
+        && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2
+
+    return $?
+}
+
 TEST_UPGRADE () {
 
     local commit=$1
@@ -351,27 +379,6 @@ TEST_UPGRADE () {
     return $?
 }
 
-TEST_MULTI_INSTANCE () {
-
-    start_test "Multi-instance installations"
-
-    # Check if an install have previously work
-    at_least_one_install_succeeded || return 1
-
-    local check_path=$(default_install_path)
-
-    LOAD_LXC_SNAPSHOT snap0
-
-    log_small_title "First installation: path=$DOMAIN$check_path" \
-        &&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \
-        && log_small_title "Second installation: path=$SUBDOMAIN$check_path" \
-        &&_INSTALL_APP "path=$check_path" \
-        && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \
-        && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2
-
-    return $?
-}
-
 TEST_PORT_ALREADY_USED () {
 
     start_test "Port already used"
@@ -464,6 +471,7 @@ TEST_BACKUP_RESTORE () {
 
             # Place the copy of the backup archive in the container.
             sudo lxc file push -r ./ynh_backups $LXC_NAME/home/yunohost.backup/archives/
+            RUN_INSIDE_LXC ls -l /home/yunohost.backup/archives/
 
             log_small_title "Restore on a clean YunoHost system..."
         fi

lib/tests_coordination.sh

@@ -7,7 +7,7 @@ source lib/witness.sh
 complete_log="./Complete.log"
 
 # Purge some log files
-> "$complete_log"
+rm -f "$complete_log" && touch "$complete_log"
 
 # Redirect fd 3 (=debug steam) to complete log
 exec 3>>$complete_log
@@ -16,9 +16,6 @@ exec 3>>$complete_log
 # Parse the check_process and generate jsons that describe tests to run
 #=======================================================================
 
-# Parse the check_process only if it's exist
-check_process="$package_path/check_process"
-
 # Extract a section found between $1 and $2 from the file $3
 extract_check_process_section () {
     local source_file="${3:-$check_process}"
@@ -156,11 +153,11 @@ parse_check_process() {
         fi
 
         is_test_enabled pkg_linter     && add_test "PACKAGE_LINTER"
-        is_test_enabled setup_sub_dir  && add_test "TEST_INSTALL" "subdir"
         is_test_enabled setup_root     && add_test "TEST_INSTALL" "root"
+        is_test_enabled setup_sub_dir  && add_test "TEST_INSTALL" "subdir"
         is_test_enabled setup_nourl    && add_test "TEST_INSTALL" "nourl"
         is_test_enabled setup_private  && add_test "TEST_INSTALL" "private"
-        is_test_enabled multi_instance && add_test "TEST_MULTI_INSTANCE"
+        is_test_enabled multi_instance && add_test "TEST_INSTALL" "multi"
         is_test_enabled backup_restore && add_test "TEST_BACKUP_RESTORE"
 
         # Upgrades
@@ -220,15 +217,15 @@ guess_test_configuration() {
     local install_args=$(python "./lib/manifest_parsing.py" "$package_path/manifest.json" | cut -d ':' -f1,2 | tr ':' '=' | tr '\n' '&')
 
     add_test "PACKAGE_LINTER"
-    add_test "TEST_INSTALL subdir"
+    add_test "TEST_INSTALL" "root"
-    add_test "TEST_INSTALL root"
+    add_test "TEST_INSTALL" "subdir"
     if echo $install_args | grep -q "is_public="
     then
         add_test "TEST_INSTALL" "private"
     fi
     if grep multi_instance "$package_path/manifest.json" | grep -q true
     then
-        add_test "TEST_MULTI_INSTANCE"
+        add_test "TEST_INSTALL" "multi"
     fi
     add_test "TEST_BACKUP_RESTORE"
     add_test "TEST_UPGRADE"
@@ -243,6 +240,9 @@ run_all_tests() {
     mkdir -p $TEST_CONTEXT/tests
     mkdir -p $TEST_CONTEXT/results
 
+    # Parse the check_process only if it's exist
+    check_process="$package_path/check_process"
+
     [ -e "$check_process" ] \
         && parse_check_process \
         || guess_test_configuration
@@ -281,7 +281,7 @@ run_all_tests() {
     done
 
     # Print the final results of the tests
-    # FIXME COMPUTE_RESULTS_SUMMARY $test_serie_id
+    python3 lib/analyze_test_results.py $TEST_CONTEXT 2>$TEST_CONTEXT/summary.json
 
     # Restore the started time for the timer
     starttime=$complete_start_timer