From dfbd7250d21b1d1176a6033dae43ddeaad06a086 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 5 Nov 2020 22:23:31 +0100 Subject: [PATCH] [yolo] Add level graph for individual apps --- app/app.py | 21 ++-- app/scripts/appListsHistory/script.py | 149 +++++++------------------- app/templates/appci_app.html | 52 +++++++++ 3 files changed, 103 insertions(+), 119 deletions(-) diff --git a/app/app.py b/app/app.py index 56636c1..1b97ac9 100644 --- a/app/app.py +++ b/app/app.py @@ -6,6 +6,7 @@ from .models.appci import AppCI, AppCIBranch from .models.unlistedapps import UnlistedApp from .settings import SITE_ROOT import json +import os main = Blueprint('main', __name__, url_prefix=SITE_ROOT) @@ -55,7 +56,7 @@ def appci_branch(branch): tests = AppCI.tests.copy() if "Malformed path" in tests: - tests.remove("Malformed path") + tests.remove("Malformed path") return render_template("appci_branch.html", tests=tests, branch=branch, @@ -74,10 +75,18 @@ def appci_app(app): tests = AppCI.tests.copy() if "Malformed path" in tests: - tests.remove("Malformed path") + tests.remove("Malformed path") + + history_file = "./app/scripts/appListsHistory/per_app/history_%s.json" % app + if os.path.exists(history_file): + history = json.loads(open("./app/scripts/appListsHistory/count_history.json").read()) + else: + history = [] + return render_template("appci_app.html", tests=tests, app=app, - branch_results=branch_results) + branch_results=branch_results, + history=history) @main.route('/appci/compare/...') @@ -193,14 +202,14 @@ def app_maintainer_dash(maintainer=None): def testings(): apps = App.query.filter(App.testing_pr!=None).all() - + def daysAgo(date): return (datetime.now() - date).days for app in apps: app.testing_pr["created_ago"] = daysAgo(app.testing_pr["created_at"]) app.testing_pr["updated_ago"] = daysAgo(app.testing_pr["updated_at"]) - + apps = sorted(apps, key=lambda app: (-app.testing_pr["created_ago"], -app.testing_pr["updated_ago"], app.name.lower())) - + return render_template("testings.html", apps=apps) diff --git a/app/scripts/appListsHistory/script.py b/app/scripts/appListsHistory/script.py index 074e6e4..aaeaa81 100644 --- a/app/scripts/appListsHistory/script.py +++ b/app/scripts/appListsHistory/script.py @@ -2,13 +2,7 @@ import json import os from datetime import datetime -from feedgen.feed import FeedGenerator -import jinja2 -state_to_value = { "working":1, - "inprogress": 0, - "notworking": -1, - } def _time_points_until_today(): @@ -32,8 +26,10 @@ def _time_points_until_today(): date = datetime(year, month, day) + time_points_until_today = list(_time_points_until_today()) + def get_lists_history(): os.system("rm -rf ./.work") @@ -61,113 +57,18 @@ def get_lists_history(): # Save it json.dump(merged, open('./.work/merged_lists.json.%s' % t.strftime("%y-%m-%d"), 'w')) -def diffs(): - - # Iterate over pairs of date : (t0,t1), (t1,t2), ... - dates = time_points_until_today - for d1, d2 in zip(dates[:-1], dates[1:]): - - print("Analyzing %s ... %s" % (d1.strftime("%y-%m-%d"), d2.strftime("%y-%m-%d"))) - - # Load corresponding json - f1 = json.loads(open("./.work/merged_lists.json.%s" % d1.strftime("%y-%m-%d")).read()) - f2 = json.loads(open("./.work/merged_lists.json.%s" % d2.strftime("%y-%m-%d")).read()) - - for key in f1: - f1[key]["name"] = key - - for key in f2: - f2[key]["name"] = key - - keys_f1 = set(f1.keys()) - keys_f2 = set(f2.keys()) - - removed = [ f1[k] for k in keys_f1 - keys_f2 ] - added = [ f2[k] for k in keys_f2 - keys_f1 ] - keys_inboth = keys_f1 & keys_f2 - - state_changes = [] - level_changes = [] - updates = [] - for key in keys_inboth: - - changes = [] - - # FIXME : this mechanism aint relevant anymore since - # the norm is to use HEAD now... - commit1 = f1[key].get("revision", None) - commit2 = f2[key].get("revision", None) - if commit1 != commit2: - changes.append("updated") - - state1 = f1[key].get("state", None) - state2 = f2[key].get("state", None) - if state1 != state2: - changes.append(("state", state1, state2)) - - level1 = f1[key].get("level", None) - level2 = f2[key].get("level", None) - if level1 != level2: - changes.append(("level", level1, level2)) - - if level1 != level2 or state1 != state2: - level1_value = level1 if level1 else 0 - level2_value = level2 if level2 else 0 - if level1_value < level2_value: - changes.append("improvement") - elif level1_value > level2_value: - changes.append("regression") - else: - state1_value = state_to_value.get(state1,-1) - state2_value = state_to_value.get(state2,-1) - if state1_value < state2_value: - changes.append("improvement") - elif state1_value > state2_value: - changes.append("regression") - else: - changes.append("same") - - if changes: - updates.append((f2[key], changes)) - - yield { "begin": d1, - "end": d2, - "new": sorted(added, key=lambda a:a["name"]), - "improvements": sorted([a for a in updates if "improvement" in a[1]], key=lambda a:a[0]["name"]), - "updates": sorted([a for a in updates if "same" in a[1]], key=lambda a:a[0]["name"]), - "regressions": sorted([a for a in updates if "regression" in a[1]], key=lambda a:a[0]["name"]), - "removed": sorted(removed, key=lambda a:a["name"]), - } - - -def make_rss_feed(): - - fg = FeedGenerator() - fg.id('https://github.com/YunoHost/Apps/') - fg.title('App Lists news') - fg.author( {'name':'YunoHost'} ) - fg.language('en') - - for diff in diffs(): - fe = fg.add_entry() - fe.id('https://github.com/YunoHost/Apps/#'+diff["end"].strftime("%y-%m-%d")) - fe.title('Changes between %s and %s' % (diff["begin"].strftime("%b %d"), diff["end"].strftime("%b %d"))) - fe.link(href='https://github.com/YunoHost/apps/commits/master/community.json') - fe.content(jinja2.Template(open("rss_template.html").read()).render(data=diff), type="html") - fe._FeedEntry__atom_updated = diff["end"] - - print('Changes between %s and %s' % (diff["begin"].strftime("%b %d"), diff["end"].strftime("%b %d"))) - open("tmp.html", "w").write(jinja2.Template(open("rss_template.html").read()).render(data=diff)) - - fg.atom_file('atom.xml') def make_count_summary(): states = ["official", "working", "inprogress", "notworking"] history = [] - per_state = { state: [] for state in states } - per_level = { "level-%s"%i: [] for i in range(0,8) } + last_time_point = time_points_until_today[-1] + json_at_last_time_point = json.loads(open("./.work/merged_lists.json.%s" % last_time_point.strftime("%y-%m-%d")).read()) + relevant_apps_to_track = [app + for app, infos in json_at_last_time_point.items() + if infos.get("state") in ["working", "official"]] + history_per_app = {app: [] for app in relevant_apps_to_track} for d in time_points_until_today: @@ -180,17 +81,39 @@ def make_count_summary(): summary = {} summary["date"] = d_label for state in states: - summary[state] = len([ k for k, infos in j.items() if infos["state"] == state ]) + summary[state] = len([k for k, infos in j.items() if infos["state"] == state]) - for level in range(0,9): - summary["level-%s"%level] = len([ k for k, infos in j.items() \ - if infos["state"] in ["working", "official"] \ - and infos.get("level", None) == level ]) + for level in range(0, 9): + summary["level-%s" % level] = len([k for k, infos in j.items() + if infos["state"] in ["working", "official"] + and infos.get("level", None) == level]) history.append(summary) + for app in relevant_apps_to_track: + + infos = j.get(app, {}) + + if not infos or infos.get("state") not in ["working", "official"]: + level = -1 + else: + level = infos.get("level", -1) + try: + level = int(level) + except: + level = -1 + + history_per_app[app].append({ + "date": d_label, + "level": infos.get("level", -1) + }) + json.dump(history, open('count_history.json', 'w')) + os.system("mkdir -p per_app/") + for app in relevant_apps_to_track: + json.dump(history_per_app[app], open('per_app/history_%s.json' % app, 'w')) + + get_lists_history() -make_rss_feed() make_count_summary() diff --git a/app/templates/appci_app.html b/app/templates/appci_app.html index 22ac0eb..0d80ba7 100644 --- a/app/templates/appci_app.html +++ b/app/templates/appci_app.html @@ -125,4 +125,56 @@ +
+
+
+
+
+ +{% endblock %} + + + + + {% endblock %}