import re import os import time import json import requests import datetime from .. import db class AppList(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(64), unique=True, nullable=False) url = db.Column(db.String(128), nullable=False) state_for_ci = db.Column(db.String(32), nullable=False) def __repr__(self): return '' % self.name def init(): yield AppList(name='official', url="https://app.yunohost.org/official.json", state_for_ci='validated') yield AppList(name='community', url="https://app.yunohost.org/community.json", state_for_ci='working') def update(): applists = AppList.query.all() for applist in applists: applist.update_list() def update_list(self): g = Github() raw_apps = json.loads(requests.get(self.url).text) for name in sorted(raw_apps.keys()): app = raw_apps[name] app['url'] = app["git"]["url"].strip('/') # Try to find an app for this name known_app = App.query.filter_by(name=name).first() # If this app is not already registered... if not known_app: print("Adding new app {}".format(name)) known_app = App(name=name, repo=app["url"], list=self, public_commit=app["git"]["revision"]) db.session.add(known_app) else: print("Updating already known app {}".format(name)) maintainers_info = app["manifest"].get("maintainer", app["manifest"].get("developer", None)) if maintainers_info is None: known_app.maintainers = [ ] if isinstance(maintainers_info, dict): if maintainers_info["name"] == "-" or maintainers_info["name"] == "": known_app.maintainers = [ ] else: known_app.maintainers = re.split(", | et ", maintainers_info["name"]) if isinstance(maintainers_info, list): known_app.maintainers = [ maintainer["name"] for maintainer in maintainers_info ] known_app.maintained = app.get("maintained", True) # Quick and dirty fix to convert maintained state to boolean if known_app.maintained == "orphaned": known_app.maintained = False elif isinstance(known_app.maintained, str): known_app.maintained = True known_app.state = app["state"] known_app.ci_enabled = app["state"] == self.state_for_ci known_app.public_level = app.get("level", None) if "github" in known_app.repo: issues_and_prs = g.issues(known_app) known_app.public_commit = app["git"]["revision"] known_app.master_commit = g.commit(known_app, "master") known_app.public_commit_date = g.commit_date(known_app, known_app.public_commit) known_app.master_commit_date = g.commit_date(known_app, known_app.master_commit) known_app.testing_pr = g.testing_pr(known_app) known_app.opened_issues = issues_and_prs["nb_issues"] known_app.opened_prs = issues_and_prs["nb_prs"] else: known_app.public_commit = "???" known_app.master_commit = "???" known_app.testing_pr = None known_app.opened_issues = 0 known_app.opened_prs = 0 try: db.session.commit() except Exception as e: print(e) class App(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(64), unique=True, nullable=False) repo = db.Column(db.String(128), unique=True, nullable=False) maintainers = db.Column(db.PickleType) maintained = db.Column(db.Boolean, nullable=False) state = db.Column(db.String(64), nullable=False) ci_enabled = db.Column(db.Boolean, nullable=False) public_level = db.Column(db.Integer, default=-1, nullable=True) list = db.relationship(AppList, backref='apps', lazy=True, uselist=False) list_id = db.Column(db.ForeignKey(AppList.id)) # 'Status info' stuff public_commit = db.Column(db.String(64), nullable=False) master_commit = db.Column(db.String(64), nullable=False) public_commit_date = db.Column(db.DateTime, nullable=True) master_commit_date = db.Column(db.DateTime, nullable=True) testing_pr = db.Column(db.PickleType, default=None) opened_issues = db.Column(db.Integer, default=-1) opened_prs = db.Column(db.Integer, default=-1) def __repr__(self): return '' % self.name def init(): pass @property def public_vs_master_time_diff(self): return (self.public_commit_date - self.master_commit_date).days def most_recent_tests_per_branch(self): from app.models.appci import AppCIBranch, AppCIResult, AppCI branches = AppCIBranch.query.all() for branch in branches: most_recent_test = AppCIResult.query \ .filter_by(branch = branch) \ .filter_by(app = self) \ .order_by(AppCIResult.date.desc()) \ .first() if most_recent_test: yield most_recent_test else: yield AppCIResult(app = self, branch = branch, level = None, date = datetime.datetime.fromtimestamp(0), results = { t:None for t in AppCI.tests }) class Github(): def __init__(self): from ..settings import GITHUB_USER, GITHUB_TOKEN self.user = GITHUB_USER self.token = GITHUB_TOKEN def request(self, uri, autoretry=True): r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() if "message" in r and r["message"] == "Not Found" and autoretry: time.sleep(30) r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() if "message" in r and r["message"] == "Not Found": print('https://api.github.com/{}'.format(uri)) return {} return r def issues(self, app): repo = app.repo.replace("https://github.com/", "") j = self.request('repos/{}/issues'.format(repo)) nb_issues = len([ i for i in j if not "pull_request" in i.keys() ]) nb_prs = len([ i for i in j if "pull_request" in i.keys() ]) return { "nb_issues": nb_issues, "nb_prs": nb_prs } def testing_pr(self, app): repo = app.repo.replace("https://github.com/", "") owner = repo.split("/")[0] j = self.request('repos/{}/pulls?head={}:testing&base=master'.format(repo,owner)) if len(j) == 0: return None else: j = j[0] return { "number": j["number"], "created_at": datetime.datetime.strptime(j["created_at"], "%Y-%m-%dT%H:%M:%SZ"), "updated_at": datetime.datetime.strptime(j["updated_at"], "%Y-%m-%dT%H:%M:%SZ"), } def commit(self, app, ref): repo = app.repo.replace("https://github.com/", "") j = self.request('repos/{}/git/refs/heads/{}'.format(repo, ref)) if not "object" in j: print('Failed to fetch repos/{}/git/refs/heads/{}'.format(repo, ref)) return "???" return j["object"]["sha"] def commit_date(self, app, sha): repo = app.repo.replace("https://github.com/", "") try: github_date = self.request('repos/{}/commits/{}'.format(repo, sha))["commit"]["author"]["date"] except: print("Error parsing date...") github_date = "1970-01-01T00:00:00Z" parsed_date = datetime.datetime.strptime(github_date, "%Y-%m-%dT%H:%M:%SZ") return parsed_date