diff --git a/app/models/appci.py b/app/models/appci.py index 3fe116a..44c2c6b 100644 --- a/app/models/appci.py +++ b/app/models/appci.py @@ -1,142 +1,9 @@ -import re -import os -import time import json import requests import datetime from .. import db - - -class AppList(db.Model): - - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String(64), unique=True, nullable=False) - url = db.Column(db.String(128), nullable=False) - state_for_ci = db.Column(db.String(32), nullable=False) - - def __repr__(self): - return '' % self.name - - def init(): - yield AppList(name='official', - url="https://app.yunohost.org/official.json", - state_for_ci='validated') - yield AppList(name='community', - url="https://app.yunohost.org/community.json", - state_for_ci='working') - - def update(self): - - g = Github() - - raw_apps = json.loads(requests.get(self.url).text).values() - apps = [ app for app in raw_apps ] - - for app in apps: - - app['url'] = app["git"]["url"].strip('/') - name = os.path.basename(app["url"]).replace("_ynh", "") - - # Try to find an app for this name - known_app = App.query.filter_by(name=name).first() - - # If this app is not already registered... - if not known_app: - print("Adding new app {}".format(name)) - known_app = App(name=name, - repo=app["url"], - list=self, - public_commit=app["git"]["revision"]) - db.session.add(known_app) - else: - print("Updating already known app {}".format(name)) - - maintainers_info = app["manifest"].get("maintainer", app["manifest"].get("developer", None)) - if maintainers_info is None: - known_app.maintainers = [ ] - if isinstance(maintainers_info, dict): - if maintainers_info["name"] == "-" or maintainers_info["name"] == "": - known_app.maintainers = [ ] - else: - known_app.maintainers = re.split(", | et ", maintainers_info["name"]) - if isinstance(maintainers_info, list): - known_app.maintainers = [ maintainer["name"] for maintainer in maintainers_info ] - - known_app.maintained = app.get("maintained", True) - known_app.ci_enabled = app["state"] == self.state_for_ci - known_app.public_level = app.get("level", None) - - if "github" in known_app.repo: - issues_and_prs = g.issues(known_app) - - known_app.public_commit = app["git"]["revision"] - known_app.master_commit = g.commit(known_app, "master") - known_app.testing_diff = g.diff(known_app, "master", "testing")["ahead_by"] - known_app.opened_issues = issues_and_prs["nb_issues"] - known_app.opened_prs = issues_and_prs["nb_prs"] - - known_app.public_vs_master_time_diff = \ - (g.commit_date(known_app, known_app.master_commit) - - g.commit_date(known_app, known_app.public_commit)).days - else: - known_app.public_commit = "???" - known_app.master_commit = "???" - known_app.testing_diff = -1 - known_app.opened_issues = 0 - known_app.opened_prs = 0 - known_app.public_vs_master_time_diff = 0 - - try: - db.session.commit() - except Exception as e: - print(e) - - -class App(db.Model): - - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String(64), unique=True, nullable=False) - repo = db.Column(db.String(128), unique=True, nullable=False) - maintainers = db.Column(db.PickleType) - maintained = db.Column(db.Boolean, nullable=False) - ci_enabled = db.Column(db.Boolean, nullable=False) - public_level = db.Column(db.Integer, default=-1, nullable=True) - - list = db.relationship(AppList, backref='apps', lazy=True, uselist=False) - list_id = db.Column(db.ForeignKey(AppList.id)) - - # 'Status info' stuff - public_commit = db.Column(db.String(64), nullable=False) - master_commit = db.Column(db.String(64), nullable=False) - testing_diff = db.Column(db.Integer, default=-1) - opened_issues = db.Column(db.Integer, default=-1) - opened_prs = db.Column(db.Integer, default=-1) - public_vs_master_time_diff = db.Column(db.Integer, default=9999) - - def __repr__(self): - return '' % self.name - - def init(): - pass - - def most_recent_tests_per_branch(self): - - branches = AppCIBranch.query.all() - for branch in branches: - most_recent_test = AppCIResult.query \ - .filter_by(branch = branch) \ - .filter_by(app = self) \ - .order_by('date desc') \ - .first() - if most_recent_test: - yield most_recent_test - else: - yield AppCIResult(app = self, - branch = branch, - level = None, - date = datetime.datetime.fromtimestamp(0), - results = [ None for t in AppCI.tests ]) +from app.models.applists import App class AppCIBranch(db.Model): @@ -237,12 +104,6 @@ class AppCI(): def update(): - applists = AppList.query.all() - - # Updating applists... - for applist in applists: - applist.update() - cibranches = AppCIBranch.query.all() def symbol_to_bool(s): @@ -268,60 +129,3 @@ class AppCI(): db.session.add(results) db.session.commit() - - -class Github(): - - def __init__(self): - - from ..settings import GITHUB_USER, GITHUB_TOKEN - - self.user = GITHUB_USER - self.token = GITHUB_TOKEN - - def request(self, uri, autoretry=True): - - r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() - if "message" in r and r["message"] == "Not Found" and autoretry: - time.sleep(30) - r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() - if "message" in r and r["message"] == "Not Found": - print('https://api.github.com/{}'.format(uri)) - return {} - return r - - def diff(self, app, ref, commit): - - repo = app.repo.replace("https://github.com/", "") - j = self.request('repos/{}/compare/{}...{}'.format(repo, ref, commit), autoretry=False) - - return { "ahead_by": j.get("ahead_by", -1), - "behind_by": j.get("behind_by", -1) } - - def issues(self, app): - - repo = app.repo.replace("https://github.com/", "") - j = self.request('repos/{}/issues'.format(repo)) - - nb_issues = len([ i for i in j if not "pull_request" in i.keys() ]) - nb_prs = len([ i for i in j if "pull_request" in i.keys() ]) - - return { "nb_issues": nb_issues, - "nb_prs": nb_prs } - - def commit(self, app, ref): - - repo = app.repo.replace("https://github.com/", "") - j = self.request('repos/{}/git/refs/heads/{}'.format(repo, ref)) - if not "object" in j: - print('Failed to fetch repos/{}/git/refs/heads/{}'.format(repo, ref)) - return "???" - return j["object"]["sha"] - - - def commit_date(self, app, sha): - - repo = app.repo.replace("https://github.com/", "") - github_date = self.request('repos/{}/commits/{}'.format(repo, sha))["commit"]["author"]["date"] - parsed_date = datetime.datetime.strptime(github_date, "%Y-%m-%dT%H:%M:%SZ") - return parsed_date diff --git a/app/models/applists.py b/app/models/applists.py new file mode 100644 index 0000000..8d587ba --- /dev/null +++ b/app/models/applists.py @@ -0,0 +1,202 @@ +import re +import os +import time +import json +import requests +import datetime + +from .. import db + + +class AppList(db.Model): + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(64), unique=True, nullable=False) + url = db.Column(db.String(128), nullable=False) + state_for_ci = db.Column(db.String(32), nullable=False) + + def __repr__(self): + return '' % self.name + + def init(): + yield AppList(name='official', + url="https://app.yunohost.org/official.json", + state_for_ci='validated') + yield AppList(name='community', + url="https://app.yunohost.org/community.json", + state_for_ci='working') + + def update(): + + applists = AppList.query.all() + for applist in applists: + applist.update_list() + + def update_list(self): + + g = Github() + + raw_apps = json.loads(requests.get(self.url).text).values() + apps = [ app for app in raw_apps ] + + for app in apps: + + app['url'] = app["git"]["url"].strip('/') + name = os.path.basename(app["url"]).replace("_ynh", "") + + # Try to find an app for this name + known_app = App.query.filter_by(name=name).first() + + # If this app is not already registered... + if not known_app: + print("Adding new app {}".format(name)) + known_app = App(name=name, + repo=app["url"], + list=self, + public_commit=app["git"]["revision"]) + db.session.add(known_app) + else: + print("Updating already known app {}".format(name)) + + maintainers_info = app["manifest"].get("maintainer", app["manifest"].get("developer", None)) + if maintainers_info is None: + known_app.maintainers = [ ] + if isinstance(maintainers_info, dict): + if maintainers_info["name"] == "-" or maintainers_info["name"] == "": + known_app.maintainers = [ ] + else: + known_app.maintainers = re.split(", | et ", maintainers_info["name"]) + if isinstance(maintainers_info, list): + known_app.maintainers = [ maintainer["name"] for maintainer in maintainers_info ] + + known_app.maintained = app.get("maintained", True) + known_app.ci_enabled = app["state"] == self.state_for_ci + known_app.public_level = app.get("level", None) + + if "github" in known_app.repo: + issues_and_prs = g.issues(known_app) + + known_app.public_commit = app["git"]["revision"] + known_app.master_commit = g.commit(known_app, "master") + known_app.testing_diff = g.diff(known_app, "master", "testing")["ahead_by"] + known_app.opened_issues = issues_and_prs["nb_issues"] + known_app.opened_prs = issues_and_prs["nb_prs"] + + known_app.public_vs_master_time_diff = \ + (g.commit_date(known_app, known_app.master_commit) - + g.commit_date(known_app, known_app.public_commit)).days + else: + known_app.public_commit = "???" + known_app.master_commit = "???" + known_app.testing_diff = -1 + known_app.opened_issues = 0 + known_app.opened_prs = 0 + known_app.public_vs_master_time_diff = 0 + + try: + db.session.commit() + except Exception as e: + print(e) + + +class App(db.Model): + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(64), unique=True, nullable=False) + repo = db.Column(db.String(128), unique=True, nullable=False) + maintainers = db.Column(db.PickleType) + maintained = db.Column(db.Boolean, nullable=False) + ci_enabled = db.Column(db.Boolean, nullable=False) + public_level = db.Column(db.Integer, default=-1, nullable=True) + + list = db.relationship(AppList, backref='apps', lazy=True, uselist=False) + list_id = db.Column(db.ForeignKey(AppList.id)) + + # 'Status info' stuff + public_commit = db.Column(db.String(64), nullable=False) + master_commit = db.Column(db.String(64), nullable=False) + testing_diff = db.Column(db.Integer, default=-1) + opened_issues = db.Column(db.Integer, default=-1) + opened_prs = db.Column(db.Integer, default=-1) + public_vs_master_time_diff = db.Column(db.Integer, default=9999) + + def __repr__(self): + return '' % self.name + + def init(): + pass + + def most_recent_tests_per_branch(self): + + from appci import AppCIBranch, AppCIResult, AppCI + branches = AppCIBranch.query.all() + for branch in branches: + most_recent_test = AppCIResult.query \ + .filter_by(branch = branch) \ + .filter_by(app = self) \ + .order_by('date desc') \ + .first() + if most_recent_test: + yield most_recent_test + else: + yield AppCIResult(app = self, + branch = branch, + level = None, + date = datetime.datetime.fromtimestamp(0), + results = [ None for t in AppCI.tests ]) + +class Github(): + + def __init__(self): + + from ..settings import GITHUB_USER, GITHUB_TOKEN + + self.user = GITHUB_USER + self.token = GITHUB_TOKEN + + def request(self, uri, autoretry=True): + + r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() + if "message" in r and r["message"] == "Not Found" and autoretry: + time.sleep(30) + r = requests.get('https://api.github.com/{}'.format(uri), auth=(self.user, self.token)).json() + if "message" in r and r["message"] == "Not Found": + print('https://api.github.com/{}'.format(uri)) + return {} + return r + + def diff(self, app, ref, commit): + + repo = app.repo.replace("https://github.com/", "") + j = self.request('repos/{}/compare/{}...{}'.format(repo, ref, commit), autoretry=False) + + return { "ahead_by": j.get("ahead_by", -1), + "behind_by": j.get("behind_by", -1) } + + def issues(self, app): + + repo = app.repo.replace("https://github.com/", "") + j = self.request('repos/{}/issues'.format(repo)) + + nb_issues = len([ i for i in j if not "pull_request" in i.keys() ]) + nb_prs = len([ i for i in j if "pull_request" in i.keys() ]) + + return { "nb_issues": nb_issues, + "nb_prs": nb_prs } + + def commit(self, app, ref): + + repo = app.repo.replace("https://github.com/", "") + j = self.request('repos/{}/git/refs/heads/{}'.format(repo, ref)) + if not "object" in j: + print('Failed to fetch repos/{}/git/refs/heads/{}'.format(repo, ref)) + return "???" + return j["object"]["sha"] + + + def commit_date(self, app, sha): + + repo = app.repo.replace("https://github.com/", "") + github_date = self.request('repos/{}/commits/{}'.format(repo, sha))["commit"]["author"]["date"] + parsed_date = datetime.datetime.strptime(github_date, "%Y-%m-%dT%H:%M:%SZ") + return parsed_date diff --git a/manage.py b/manage.py index c7ff3fc..f1d7a6c 100755 --- a/manage.py +++ b/manage.py @@ -17,11 +17,14 @@ class Update(Command): def run(self, args=None): - valid_what = ["appci", "pr", "appobservatory"] + valid_what = ["applists", "appci", "pr", "appobservatory"] what = args[0] if args else None assert what in valid_what, "Please specify what to update among %s" % ', '.join(valid_what) - if what == "appci": + if what == "applists": + from app.models.applists import AppList + AppList.update() + elif what == "appci": from app.models.appci import AppCI AppCI.update() elif what == "pr": @@ -42,6 +45,7 @@ class Nuke(Command): def run(self): + import app.models.applists import app.models.appci import app.models.pr import app.models.unlistedapps @@ -67,6 +71,7 @@ class Init(Command): for submodule in submodules: stuff.extend([submodule.__dict__.get(s) for s in dir(submodule)]) models = [s for s in stuff if isinstance(s, type(db.Model))] + models = set(models) for model in models: objs = model.init()