import os import time import json import requests import datetime from .. import db class AppList(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(64), unique=True, nullable=False) url = db.Column(db.String(128), nullable=False) state_for_ci = db.Column(db.String(32), nullable=False) def __repr__(self): return '' % self.name def init(): yield AppList(name='official', url="https://raw.githubusercontent.com/YunoHost/apps/master/official.json", state_for_ci='validated') yield AppList(name='community', url="https://raw.githubusercontent.com/YunoHost/apps/master/community.json", state_for_ci='working') def update(self): g = Github() raw_apps = json.loads(requests.get(self.url).text).values() apps_for_ci = [ app for app in raw_apps if app["state"] == self.state_for_ci ] for app in apps_for_ci: app['url'] = app["url"].strip('/') name = os.path.basename(app["url"]).replace("_ynh", "") # Try to find an app for this name known_app = App.query.filter_by(name=name).first() # If this app is not already registered... if not known_app: print("Adding new app {}".format(name)) known_app = App(name=name, repo=app["url"], list=self, public_commit=app["revision"]) db.session.add(known_app) else: print("Updating already known app {}".format(name)) if "github" in known_app.repo: issues_and_prs = g.issues(known_app) known_app.public_commit = app["revision"] known_app.master_commit = g.commit(known_app, "master") known_app.testing_diff = g.diff(known_app, "master", "testing")["ahead_by"] known_app.opened_issues = issues_and_prs["nb_issues"] known_app.opened_prs = issues_and_prs["nb_prs"] known_app.public_vs_master_time_diff = \ (g.commit_date(known_app, known_app.master_commit) - g.commit_date(known_app, known_app.public_commit)).days else: known_app.public_commit = "???" known_app.master_commit = "???" known_app.testing_diff = -1 known_app.opened_issues = 0 known_app.opened_prs = 0 known_app.public_vs_master_time_diff = 0 try: db.session.commit() except Exception as e: print(e) class App(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(64), unique=True, nullable=False) repo = db.Column(db.String(128), unique=True, nullable=False) list = db.relationship(AppList, backref='apps', lazy=True, uselist=False) list_id = db.Column(db.ForeignKey(AppList.id)) # 'Status info' stuff public_commit = db.Column(db.String(64), nullable=False) master_commit = db.Column(db.String(64), nullable=False) testing_diff = db.Column(db.Integer, default=-1) opened_issues = db.Column(db.Integer, default=-1) opened_prs = db.Column(db.Integer, default=-1) public_vs_master_time_diff = db.Column(db.Integer, default=9999) def __repr__(self): return '' % self.name def init(): pass def most_recent_tests_per_branch(self): branches = AppCIBranch.query.all() for branch in branches: most_recent_test = AppCIResult.query \ .filter_by(branch = branch) \ .filter_by(app = self) \ .order_by('date desc') \ .first() if most_recent_test: yield most_recent_test else: yield AppCIResult(app = self, branch = branch, level = None, date = datetime.datetime.fromtimestamp(0), results = [ None for t in AppCI.tests ]) class AppCIBranch(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(64), unique=True, nullable=False) arch = db.Column(db.String(64), nullable=False) branch = db.Column(db.String(64), nullable=False) display_name = db.Column(db.String(64), unique=True, nullable=False) url = db.Column(db.String(128), nullable=False) console_uri = db.Column(db.String(128), nullable=False) def __repr__(self): return '' % self.name def init(): yield AppCIBranch(name='stable', arch="x86", branch="stable", display_name='Stable (x86)', url='https://ci-apps.yunohost.org/ci/logs/list_level_stable.json', console_uri='/job/{} ({})/lastBuild/consoleText') yield AppCIBranch(name='arm', arch="arm", branch="stable", display_name='Stable (ARM)', url='https://ci-apps-arm.yunohost.org/ci/logs/list_level_stable.json', console_uri='/job/{} ({}) (~ARM~)/lastBuild/consoleText') def last_build_url(self, app): return self.url + self.console_uri.format(app.name, app.list.name.title()) def most_recent_tests_per_app(self): apps = App.query.all() for app in apps: most_recent_test = AppCIResult.query \ .filter_by(branch = self) \ .filter_by(app = app) \ .order_by('date desc') \ .first() if most_recent_test: yield most_recent_test else: yield AppCIResult(app = app, branch = self, level = None, date = datetime.datetime.fromtimestamp(0), results = [ None for t in AppCI.tests ]) class AppCIResult(db.Model): id = db.Column(db.Integer, primary_key=True) app = db.relationship(App, backref='tests', lazy=True, uselist=False) branch = db.relationship(AppCIBranch, backref='tests', lazy=True, uselist=False) app_id = db.Column(db.ForeignKey(App.id)) branch_id = db.Column(db.ForeignKey(AppCIBranch.id)) results = db.Column(db.PickleType) date = db.Column(db.DateTime, nullable=True) level = db.Column(db.Integer, nullable=True) url = db.Column(db.String(128), nullable=True) commit = db.Column(db.String(64), nullable=True) def __repr__(self): return '' % self.date def init(): pass def score(self): s_dict = { True: +1, False: -1, None: 0 } return sum([ s_dict[result] for result in self.results ]) class AppCI(): tests = [ "Package linter", "Installation", "Deleting", "Upgrade", "Backup", "Restore", "Change URL", "Installation in a sub path", "Deleting from a sub path", "Installation on the root", "Deleting from root", "Installation in private mode", "Installation in public mode", "Multi-instance installations", "Malformed path", "Port already used" ] def update(): applists = AppList.query.all() # Updating applists... for applist in applists: applist.update() cibranches = AppCIBranch.query.all() def symbol_to_bool(s): return bool(int(s)) if s in [ "1", "0" ] else None # Scrap jenkins for cibranch in cibranches: print("> Fetching current CI results for C.I. branch {}".format(cibranch.name)) result_json = requests.get(cibranch.url).text cleaned_json = [ line for line in result_json.split("\n") if "test_name" in line ] cleaned_json = [ line.replace('"level": ?,', '"level": null,') for line in cleaned_json ] cleaned_json = "[" + ''.join(cleaned_json)[:-1] + "]" j = json.loads(cleaned_json) for test_summary in j: if (test_summary["arch"], test_summary["branch"]) != (cibranch.arch, cibranch.branch): continue results = AppCIResult(app = App.query.filter_by(name=test_summary["app"]).first(), branch = cibranch, level = test_summary["level"], date = datetime.datetime.fromtimestamp(test_summary["timestamp"]), results = [ symbol_to_bool(s) for s in test_summary["detailled_success"] ]) db.session.add(results) db.session.commit() class Github(): def __init__(self): from ..settings import GITHUB_USER, GITHUB_TOKEN self.user = GITHUB_USER self.token = GITHUB_TOKEN def request(self, uri, autoretry=True): r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() if "message" in r and r["message"] == "Not Found" and autoretry: time.sleep(30) r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() if "message" in r and r["message"] == "Not Found": print('https://api.github.com/{}'.format(uri)) return {} return r def diff(self, app, ref, commit): repo = app.repo.replace("https://github.com/", "") j = self.request('repos/{}/compare/{}...{}'.format(repo, ref, commit), autoretry=False) return { "ahead_by": j.get("ahead_by", -1), "behind_by": j.get("behind_by", -1) } def issues(self, app): repo = app.repo.replace("https://github.com/", "") j = self.request('repos/{}/issues'.format(repo)) nb_issues = len([ i for i in j if not "pull_request" in i.keys() ]) nb_prs = len([ i for i in j if "pull_request" in i.keys() ]) return { "nb_issues": nb_issues, "nb_prs": nb_prs } def commit(self, app, ref): repo = app.repo.replace("https://github.com/", "") j = self.request('repos/{}/git/refs/heads/{}'.format(repo, ref)) if not "object" in j: print('Failed to fetch repos/{}/git/refs/heads/{}'.format(repo, ref)) return "???" return j["object"]["sha"] def commit_date(self, app, sha): repo = app.repo.replace("https://github.com/", "") github_date = self.request('repos/{}/commits/{}'.format(repo, sha))["commit"]["author"]["date"] parsed_date = datetime.datetime.strptime(github_date, "%Y-%m-%dT%H:%M:%SZ") return parsed_date