Rework test results handling to match new CI format

app/app.py

@@ -2,25 +2,13 @@ from datetime import datetime
 from flask import render_template, make_response, Blueprint
 from .models.pr import PullRequest
 from .models.appcatalog import App
-from .models.appci import AppCI, AppCIBranch
+from .models.appci import AppCI, AppCIBranch, test_categories
 from .models.unlistedapps import UnlistedApp
 from .settings import SITE_ROOT
 import json
 import os
 
 main = Blueprint('main', __name__, url_prefix=SITE_ROOT)
-tests = [ "Package linter",
-          "Installation",
-          "Installation in a sub path",
-          "Installation on the root",
-          "Installation in private mode",
-          "Multi-instance installations",
-          "Upgrade",
-          "Backup",
-          "Restore",
-          "Change URL",
-          "Port already used",
-]
 
 def sort_test_results(results):
 
@@ -65,7 +53,7 @@ def appci_branch(branch):
 
     app_results = sort_test_results(branch.most_recent_tests_per_app())
 
-    return render_template("appci_branch.html", tests=tests,
+    return render_template("appci_branch.html", test_categories=test_categories,
                                                 branch=branch,
                                                 app_results=app_results)
 
@@ -86,7 +74,7 @@ def appci_app(app):
     else:
         history = []
 
-    return render_template("appci_app.html", tests=tests,
+    return render_template("appci_app.html", test_categories=test_categories,
                                              app=app,
                                              branch_results=branch_results,
                                              history=history)

app/models/appcatalog.py

@@ -56,22 +56,23 @@ class AppCatalog():
             known_app.public_level = app.get("level", None)
 
             if "github" in known_app.repo:
-                issues_and_prs = g.issues(known_app)
 
                 known_app.public_commit = app["git"]["revision"]
                 known_app.master_commit = g.commit(known_app, "master")
                 known_app.public_commit_date = g.commit_date(known_app, known_app.public_commit)
                 known_app.master_commit_date = g.commit_date(known_app, known_app.master_commit)
                 known_app.testing_pr = g.testing_pr(known_app)
-                known_app.opened_issues = issues_and_prs["nb_issues"]
+
-                known_app.opened_prs = issues_and_prs["nb_prs"]
+                #issues_and_prs = g.issues(known_app)
+                #known_app.opened_issues = issues_and_prs["nb_issues"]
+                #known_app.opened_prs = issues_and_prs["nb_prs"]
 
             else:
                 known_app.public_commit = "???"
                 known_app.master_commit = "???"
                 known_app.testing_pr = None
-                known_app.opened_issues = 0
+                #known_app.opened_issues = 0
-                known_app.opened_prs = 0
+                #known_app.opened_prs = 0
 
             try:
                 db.session.commit()
@@ -96,8 +97,8 @@ class App(db.Model):
     public_commit_date = db.Column(db.DateTime, nullable=True)
     master_commit_date = db.Column(db.DateTime, nullable=True)
     testing_pr = db.Column(db.PickleType, default=None)
-    opened_issues = db.Column(db.Integer, default=-1)
+    #opened_issues = db.Column(db.Integer, default=-1)
-    opened_prs = db.Column(db.Integer, default=-1)
+    #opened_prs = db.Column(db.Integer, default=-1)
 
     long_term_good_quality = db.Column(db.Boolean)
     long_term_broken = db.Column(db.Boolean)
@@ -140,6 +141,9 @@ class Github():
         self.user = GITHUB_USER
         self.token = GITHUB_TOKEN
 
+        print(self.user)
+        print(self.token)
+
     def request(self, uri, autoretry=True):
 
         r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json()
@@ -185,6 +189,7 @@ class Github():
         j = self.request('repos/{}/git/refs/heads/{}'.format(repo, ref))
         if not "object" in j:
             print('Failed to fetch repos/{}/git/refs/heads/{}'.format(repo, ref))
+            print(j)
             return "???"
         return j["object"]["sha"]
 

app/models/appci.py

@@ -20,14 +20,14 @@ class AppCIBranch(db.Model):
 
     def init():
         yield AppCIBranch(name='stable',
-                          arch="x86",
+                          arch="amd64",
                           branch="stable",
                           display_name='Stable (x86)',
                           url='https://ci-apps.yunohost.org/ci/logs/list_level_stable_amd64.json',
                           url_per_app='https://ci-apps.yunohost.org/ci/apps/{}/')
 
         yield AppCIBranch(name='unstable',
-                          arch="x86",
+                          arch="amd64",
                           branch="unstable",
                           display_name='Unstable (x86)',
                           url='https://ci-apps-unstable.yunohost.org/ci/logs/list_level_unstable_amd64.json',
@@ -56,6 +56,13 @@ class AppCIBranch(db.Model):
                                   results = { t:None for t in AppCI.tests })
 
 
+test_categories = []
+def test_category(category_name):
+    def decorator(func):
+        test_categories.append((func.__name__, category_name, func))
+        return func
+    return decorator
+
 class AppCIResult(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
@@ -76,6 +83,66 @@ class AppCIResult(db.Model):
     def __repr__(self):
         return '<AppCIResult %s>' % self.date
 
+    def __init__(self, infos):
+
+        self.app = App.query.filter_by(name=infos["app"]).first()
+        self.branch = AppCIBranch.query.filter_by(arch=infos["architecture"], branch=infos["yunohost_branch"]).first()
+        self.level = infos["level"]
+        self.commit = infos["commit"]
+        self.date = datetime.datetime.fromtimestamp(infos["timestamp"])
+        self.results = { category: result for category, result in list(self.analyze_test_categories(infos["tests"])) }
+
+    def analyze_test_categories(self, raw_results):
+
+        for category_id, category_display, is_in_category in test_categories:
+
+            relevant_tests = [test for test in raw_results if is_in_category(test)]
+
+            if not relevant_tests:
+                yield (category_id, None)
+            else:
+                yield (category_id, all(test["main_result"] == "success" for test in relevant_tests))
+
+    @test_category("Linter")
+    def package_linter(test):
+        return test["test_type"] == "PACKAGE_LINTER"
+
+    @test_category("Install on domain's root")
+    def install_root(test):
+        return test["test_type"] == "TEST_INSTALL" and test["test_arg"] == "root"
+
+    @test_category("Install on domain subpath")
+    def install_subpath(test):
+        return test["test_type"] == "TEST_INSTALL" and test["test_arg"] == "subdir"
+
+    @test_category("Install with no url")
+    def install_nourl(test):
+        return test["test_type"] == "TEST_INSTALL" and test["test_arg"] == "nourl"
+
+    @test_category("Install in private mode")
+    def install_private(test):
+        return test["test_type"] == "TEST_INSTALL" and test["test_arg"] == "private"
+
+    @test_category("Install multi-instance")
+    def install_multi(test):
+        return test["test_type"] == "TEST_INSTALL" and test["test_arg"] == "multi"
+
+    @test_category("Upgrade (same version)")
+    def upgrade_same_version(test):
+        return test["test_type"] == "TEST_UPGRADE" and test["test_arg"] == ""
+
+    @test_category("Upgrade (older versions)")
+    def upgrade_older_versions(test):
+        return test["test_type"] == "TEST_UPGRADE" and test["test_arg"] != ""
+
+    @test_category("Backup / restore")
+    def backup_restore(test):
+        return test["test_type"] == "TEST_BACKUP_RESTORE"
+
+    @test_category("Change url")
+    def change_url(test):
+        return test["test_type"] == "TEST_CHANGE_URL"
+
     def init():
         pass
 
@@ -94,23 +161,6 @@ class AppCIResult(db.Model):
 
 class AppCI():
 
-    tests = [ "Package linter",
-              "Installation",
-              "Deleting",
-              "Installation in a sub path",
-              "Deleting from a sub path",
-              "Installation on the root",
-              "Deleting from root",
-              "Upgrade",
-              "Installation in private mode",
-              "Installation in public mode",
-              "Multi-instance installations",
-              "Malformed path",
-              "Port already used",
-              "Backup",
-              "Restore",
-              "Change URL" ]
-
     def update():
 
         cibranches = AppCIBranch.query.all()
@@ -119,27 +169,17 @@ class AppCI():
         for cibranch in cibranches:
             print("> Fetching current CI results for C.I. branch {}".format(cibranch.name))
             try:
-                result_json = requests.get(cibranch.url).text
+                results = requests.get(cibranch.url).json()
             except:
                 print("Failed to fetch %s" % cibranch.url)
                 continue
-            cleaned_json = [ line for line in result_json.split("\n") if "test_name" in line ]
-            cleaned_json = [ line.replace('"level": ?,', '"level": null,') for line in cleaned_json ]
-            cleaned_json = "[" + ''.join(cleaned_json)[:-1] + "]"
-            cleaned_json = cleaned_json.replace("Binary", '"?"')
-            j = json.loads(cleaned_json)
-            for test_summary in j:
-                if test_summary["app"] is None:
-                     print("No app to parse in test_summary ? : %s" % test_summary)
-                     continue
 
-                if (test_summary["arch"], test_summary["branch"]) != (cibranch.arch, cibranch.branch):
+            import pdb; pdb.set_trace()
+            for app, test_summary in results.items():
+
+                if (test_summary["architecture"], test_summary["yunohost_branch"]) != (cibranch.arch, cibranch.branch):
                     continue
 
-                test_results = { }
-                for test, result in zip(AppCI.tests, test_summary["detailled_success"]):
-                    test_results[test] = bool(int(result)) if result in [ "1", "0" ] else None
-
                 app = App.query.filter_by(name=test_summary["app"]).first()
                 if app is None:
                     print("Couldnt found corresponding app object for %s, skipping" % test_summary["app"])
@@ -157,12 +197,8 @@ class AppCI():
                     existing_test = None
 
                 if not existing_test:
-                        print("New record for app %s" % str(app))
+                    print("New record for app %s" % str(app))
-                        results = AppCIResult(app = app,
+                    results = AppCIResult(test_summary)
-                                              branch = cibranch,
+                    db.session.add(results)
-                                              level = test_summary["level"],
-                                              date = date,
-                                              results = test_results)
-                        db.session.add(results)
 
         db.session.commit()

app/templates/appci_app.html

@@ -16,8 +16,8 @@
                     <tr>
                         <th class="ci-app-row-title"></th>
                         <th class="ci-app-test-title"><div>Level</div></th>
-                        {% for test in tests %}
+                        {% for test in test_categories %}
-                         <th class="ci-app-test-title"><div><span>{{ test }}</span></div></th>
+                         <th class="ci-app-test-title"><div><span>{{ test[1] }}</span></div></th>
                         {% endfor %}
                          <th class="ci-app-test-title"></th>
                     </tr>
@@ -33,10 +33,10 @@
                     </td>
                     <td class="ci-app-level" value="{{ result.level }}">
                         <div title="Level"><strong>{{ result.level if result.level >= 0 else "?" }}</strong></div></td>
-                    {% for test in tests %}
+                    {% for test in test_categories %}
-                    {% set r = result.results[test] %}
+                    {% set r = result.results[test[0]] %}
                         <td class="ci-app-test-result">
-                            <div title="{{ test }}" value="{{ r }}"></div>
+                            <div title="{{ test[1] }}" value="{{ r }}"></div>
                         </td>
                     {% endfor %}
                     <td class="ci-app-test-info">

app/templates/appci_branch.html

@@ -15,8 +15,8 @@
                     <tr>
                         <th class="ci-app-row-title"></th>
                         <th class="ci-app-test-title ci-app-level"><div>Quality level</div></th>
-                        {% for test in tests %}
+                        {% for test in test_categories %}
-                         <th class="ci-app-test-title"><div><span>{{ test }}</span></div></th>
+                         <th class="ci-app-test-title"><div><span>{{ test[1] }}</span></div></th>
                         {% endfor %}
                          <th class="ci-app-test-title"></th>
                          <th></th>
@@ -46,10 +46,10 @@
                         <span class="ml-1 oi oi-arrow-thick-top" title="Improvement" aria-hidden="true" style="color: limegreen;"></span>
                         {% endif %}
                     </div></td>
-                    {% for test in tests %}
+                    {% for test in test_categories %}
-                    {% set r = result.results[test] %}
+                    {% set r = result.results[test[0]] %}
                         <td class="ci-app-test-result">
-                            <div title="{{ test }}" value="{{ r }}"></div>
+                            <div title="{{ test[1] }}" value="{{ r }}"></div>
                         </td>
                     {% endfor %}
                     <td class="ci-app-test-info px-0">
@@ -94,8 +94,8 @@ window.onload = function () {
           startAngle: -90,
           //innerRadius: 60,
           indexLabelFontSize: 17,
-          indexLabel: "{label} - #percent%",
+          indexLabel: "{label} - {y}%",
-          toolTipContent: "<b>{label}:</b> {y} (#percent%)",
+          toolTipContent: "<b>{label}:</b> {y}",
           dataPoints: [
               { y: $(".ci-app-level[value=-1]").length, label: "Unknown", color: "#cccccc" },
               { y: $(".ci-app-level[value=0]").length,  label: "Level 0", color: "#d9534f" },