From 90b4a06e64b5c755e0d186a6a925f72184821450 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 3 Sep 2021 18:03:58 +0200 Subject: [PATCH 01/11] Handle github-actions PRs to trigger CI --- run.py | 70 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/run.py b/run.py index f1ff037..077e34a 100644 --- a/run.py +++ b/run.py @@ -1021,36 +1021,50 @@ async def github(request): # We expect issue comments (issue = also PR in github stuff...) # - *New* comments # - On issue/PRs which are still open - if hook_type != "issue_comment" \ - or hook_infos["action"] != "created" \ - or hook_infos["issue"]["state"] != "open" \ - or "pull_request" not in hook_infos["issue"]: + if hook_type == "issue_comment": + if hook_infos["action"] != "created" \ + or hook_infos["issue"]["state"] != "open" \ + or "pull_request" not in hook_infos["issue"]: + # Nothing to do but success anyway (204 = No content) + abort(204, "Nothing to do") + + # Check the comment contains proper keyword trigger + body = hook_infos["comment"]["body"].strip()[:100].lower() + triggers = ["!testme", "!gogogadgetoci", "By the power of systemd, I invoke The Great App CI to test this Pull Request!"] + if not any(trigger.lower() in body for trigger in triggers): + # Nothing to do but success anyway (204 = No content) + abort(204, "Nothing to do") + + # We only accept this from people which are member of the org + # https://docs.github.com/en/rest/reference/orgs#check-organization-membership-for-a-user + # We need a token an we can't rely on "author_association" because sometimes, users are members in Private, + # which is not represented in the original webhook + async def is_user_in_organization(user): + token = open("./github_bot_token").read().strip() + async with aiohttp.ClientSession(headers={"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"}) as session: + resp = await session.get(f"https://api.github.com/orgs/YunoHost-Apps/members/{user}") + return resp.status == 204 + + if not await is_user_in_organization(hook_infos["comment"]["user"]["login"]): + # Unauthorized + abort(403, "Unauthorized") + type = "issue" + + elif hook_type == "pull_request": + if hook_infos["action"] != "opened": # Nothing to do but success anyway (204 = No content) - abort(204, "Nothing to do") - - # Check the comment contains proper keyword trigger - body = hook_infos["comment"]["body"].strip()[:100].lower() - triggers = ["!testme", "!gogogadgetoci", "By the power of systemd, I invoke The Great App CI to test this Pull Request!"] - if not any(trigger.lower() in body for trigger in triggers): - # Nothing to do but success anyway (204 = No content) - abort(204, "Nothing to do") - - # We only accept this from people which are member of the org - # https://docs.github.com/en/rest/reference/orgs#check-organization-membership-for-a-user - # We need a token an we can't rely on "author_association" because sometimes, users are members in Private, - # which is not represented in the original webhook - async def is_user_in_organization(user): - token = open("./github_bot_token").read().strip() - async with aiohttp.ClientSession(headers={"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"}) as session: - resp = await session.get(f"https://api.github.com/orgs/YunoHost-Apps/members/{user}") - return resp.status == 204 - - if not await is_user_in_organization(hook_infos["comment"]["user"]["login"]): - # Unauthorized - abort(403, "Unauthorized") + abort(204, "Nothing to do") + # We only accept PRs that are created by github-action bot + if hook_infos["pull_request"]["user"]["login"] != "github-actions[bot]": + # Unauthorized + abort(403, "Unauthorized") + type = "pull_request" + else: + # Nothing to do but success anyway (204 = No content) + abort(204, "Nothing to do") # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) - pr_infos_url = hook_infos["issue"]["pull_request"]["url"] + pr_infos_url = hook_infos[type]["url"] async with aiohttp.ClientSession() as session: async with session.get(pr_infos_url) as resp: pr_infos = await resp.json() @@ -1076,7 +1090,7 @@ async def github(request): async def comment(body): - comments_url = hook_infos["issue"]["comments_url"] + comments_url = hook_infos[type]["comments_url"] token = open("./github_bot_token").read().strip() async with aiohttp.ClientSession(headers={"Authorization": f"token {token}"}) as session: From 84c6eb1a2adfc84c22ecffc163227999e54d6b0c Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 3 Sep 2021 19:08:22 +0200 Subject: [PATCH 02/11] Fix pr_infos_url retrieval --- run.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/run.py b/run.py index 077e34a..09f2848 100644 --- a/run.py +++ b/run.py @@ -1048,7 +1048,8 @@ async def github(request): if not await is_user_in_organization(hook_infos["comment"]["user"]["login"]): # Unauthorized abort(403, "Unauthorized") - type = "issue" + # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) + pr_infos_url = hook_infos["issue"]["pull_request"]["url"] elif hook_type == "pull_request": if hook_infos["action"] != "opened": @@ -1058,13 +1059,13 @@ async def github(request): if hook_infos["pull_request"]["user"]["login"] != "github-actions[bot]": # Unauthorized abort(403, "Unauthorized") - type = "pull_request" + # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) + pr_infos_url = hook_infos["pull_request"]["url"] + else: # Nothing to do but success anyway (204 = No content) abort(204, "Nothing to do") - # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) - pr_infos_url = hook_infos[type]["url"] async with aiohttp.ClientSession() as session: async with session.get(pr_infos_url) as resp: pr_infos = await resp.json() @@ -1089,8 +1090,10 @@ async def github(request): # Answer with comment with link+badge for the job async def comment(body): - - comments_url = hook_infos[type]["comments_url"] + if hook_type == "issue_comment": + comments_url = hook_infos["issue"]["comments_url"] + else: + comments_url = hook_infos["pull_request"]["comments_url"] token = open("./github_bot_token").read().strip() async with aiohttp.ClientSession(headers={"Authorization": f"token {token}"}) as session: From 6c15f5aba75433fda3b5f841d744c3b2b50b4b38 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Sat, 4 Sep 2021 00:13:01 +0200 Subject: [PATCH 03/11] Add check for branch to start with ci-auto-update --- run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 09f2848..3b34294 100644 --- a/run.py +++ b/run.py @@ -1056,7 +1056,8 @@ async def github(request): # Nothing to do but success anyway (204 = No content) abort(204, "Nothing to do") # We only accept PRs that are created by github-action bot - if hook_infos["pull_request"]["user"]["login"] != "github-actions[bot]": + if hook_infos["pull_request"]["user"]["login"] != "github-actions[bot]" \ + or not hook_infos["pull_request"]["head"]["ref"].startswith("ci-auto-update-"): # Unauthorized abort(403, "Unauthorized") # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) From 50eb3104296181a8cbda95ff33f537a1d0c7faa5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Sep 2021 15:32:22 +0200 Subject: [PATCH 04/11] PR webhooks: Return 204 if PR gets created by somebody else than github-actions --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 3b34294..f9e4893 100644 --- a/run.py +++ b/run.py @@ -1059,7 +1059,7 @@ async def github(request): if hook_infos["pull_request"]["user"]["login"] != "github-actions[bot]" \ or not hook_infos["pull_request"]["head"]["ref"].startswith("ci-auto-update-"): # Unauthorized - abort(403, "Unauthorized") + abort(204, "Nothing to do") # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) pr_infos_url = hook_infos["pull_request"]["url"] From 0e87f07e8bcb1f3aee055a694f3c7198e22b4019 Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 22 Sep 2021 16:27:24 +0200 Subject: [PATCH 05/11] Bump requirements --- requirements-frozen.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 3f7fc62..a964a28 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -5,6 +5,7 @@ async-timeout==3.0.1 attrs==18.2.0 certifi==2018.11.29 chardet==3.0.4 +charset-normalizer==2.0.6 httptools==0.1.0 idna==2.8 idna-ssl==1.1.0 @@ -12,14 +13,14 @@ Jinja2==3.0.1 MarkupSafe==2.0.1 multidict==5.1.0 peewee==3.14.4 -pkg-resources==0.0.0 -requests==2.25.1 -sanic==21.3.4 +pkg_resources==0.0.0 +requests==2.26.0 +sanic==21.6.2 sanic-jinja2==0.10.0 -sanic-routing==0.6.2 +sanic-routing==0.7.1 typing-extensions==3.10.0.0 ujson==4.0.2 urllib3==1.26.5 uvloop==0.14.0 -websockets==8.1 +websockets==10.0 yarl==1.3.0 From b891ab065ead5e2f90e2744f08c43e4c1d3bf4fb Mon Sep 17 00:00:00 2001 From: tituspijean Date: Mon, 4 Oct 2021 07:32:10 +0200 Subject: [PATCH 06/11] Update badge and summary URLs --- run.py | 2 ++ templates/job.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 0affdb0..e7a680e 100644 --- a/run.py +++ b/run.py @@ -945,6 +945,7 @@ async def html_job(request, job_id): job_url = app.config.BASE_URL + app.url_for("html_job", job_id=job.id) badge_url = app.config.BASE_URL + app.url_for("api_badge_job", job_id=job.id) shield_badge_url = f"https://img.shields.io/endpoint?url={badge_url}" + summary_url = app.config.BASE_URL + "/summary/" + str(job.id) + ".png" return { "job": job, @@ -952,6 +953,7 @@ async def html_job(request, job_id): 'job_url': job_url, 'badge_url': badge_url, 'shield_badge_url': shield_badge_url, + 'summary_url': summary_url, 'relative_path_to_root': '../', 'path': request.path } diff --git a/templates/job.html b/templates/job.html index 70ce1ef..a7824fa 100644 --- a/templates/job.html +++ b/templates/job.html @@ -103,7 +103,7 @@ // Clipboard API not available return } - const text = "[![Test Badge](https://img.shields.io/endpoint?url=https://ci.pijean.ovh/ci/api/job/<{ job.id }>/badge)](https://ci.pijean.ovh/ci/job/<{ job.id }>)" + const text = "[![Test Badge](https://img.shields.io/endpoint?url=<{ badge_url }>)](<{ job_url }>)\n[![](<{ summary_url }>)](<{ job_url }>)" try { await navigator.clipboard.writeText(text) } catch (err) { From 6b70f46789e4f995ddf1b23c2ce7860fa3574882 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 12 Oct 2021 18:06:59 +0200 Subject: [PATCH 07/11] handle multi worker --- run.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/run.py b/run.py index 8b41d32..622ab1f 100644 --- a/run.py +++ b/run.py @@ -351,11 +351,41 @@ async def launch_monthly_job(): await create_job(repo.name, repo.url) +async def ensure_workers_count(): + if Worker.select().count() < app.config.WORKER_COUNT: + for _ in range(app.config.WORKER_COUNT - Worker.select().count()): + Worker.create(state="available") + elif Worker.select().count() > app.config.WORKER_COUNT: + workers_to_remove = Worker.select().count() - app.config.WORKER_COUNT + workers = Worker.select().where(Worker.state == "available") + for worker in workers: + if workers_to_remove == 0: + break + worker.delete_instance() + workers_to_remove -= 1 + + jobs_to_stop = workers_to_remove + for job_id in jobs_in_memory_state: + if jobs_to_stop == 0: + break + await stop_job(job_id) + jobs_to_stop -= 1 + job = Job.select().where(Job.id == job_id)[0] + job.state = "scheduled" + job.log = "" + job.save() + + workers = Worker.select().where(Worker.state == "available") + for worker in workers: + if workers_to_remove == 0: + break + worker.delete_instance() + workers_to_remove -= 1 + + @always_relaunch(sleep=3) async def jobs_dispatcher(): - if Worker.select().count() == 0: - for i in range(1): - Worker.create(state="available") + await ensure_workers_count() workers = Worker.select().where(Worker.state == "available") @@ -400,7 +430,7 @@ async def run_job(worker, job): task_logger.info(f"Starting job '{job.name}' #{job.id}...") cwd = os.path.split(path_to_analyseCI)[0] - arguments = f' {job.url_or_path} "{job.name}" {job.id}' + arguments = f' {job.url_or_path} "{job.name}" {job.id} {worker.id}' task_logger.info(f"Launch command: /bin/bash " + path_to_analyseCI + arguments) try: command = await asyncio.create_subprocess_shell("/bin/bash " + path_to_analyseCI + arguments, @@ -1173,6 +1203,7 @@ def main(config="./config.py"): "MONITOR_GIT": False, "MONITOR_ONLY_GOOD_QUALITY_APPS": False, "MONTHLY_JOBS": False, + "WORKER_COUNT": 1, } app.config.update_config(default_config) From 0395d4349e538632f3425c4f46922aaf797ea35b Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 13 Oct 2021 16:10:10 +0200 Subject: [PATCH 08/11] Restart jobs when the server stop --- run.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/run.py b/run.py index 622ab1f..926dd8e 100644 --- a/run.py +++ b/run.py @@ -1192,6 +1192,37 @@ def format_frame(f): return dict([(k, str(getattr(f, k))) for k in keys]) +@app.listener("before_server_start") +async def listener_before_server_start(*args, **kwargs): + task_logger.info("before_server_start") + reset_pending_jobs() + reset_busy_workers() + merge_jobs_on_startup() + + set_random_day_for_monthy_job() + + +@app.listener("after_server_start") +async def listener_after_server_start(*args, **kwargs): + task_logger.info("after_server_start") + + +@app.listener("before_server_stop") +async def listener_before_server_stop(*args, **kwargs): + task_logger.info("before_server_stop") + + +@app.listener("after_server_stop") +async def listener_after_server_stop(*args, **kwargs): + task_logger.info("after_server_stop") + for job_id in jobs_in_memory_state: + await stop_job(job_id) + job = Job.select().where(Job.id == job_id)[0] + job.state = "scheduled" + job.log = "" + job.save() + + def main(config="./config.py"): default_config = { @@ -1213,12 +1244,6 @@ def main(config="./config.py"): print(f"Error: analyzer script doesn't exist at '{app.config.PATH_TO_ANALYZER}'. Please fix the configuration in {config}") sys.exit(1) - reset_pending_jobs() - reset_busy_workers() - merge_jobs_on_startup() - - set_random_day_for_monthy_job() - if app.config.MONITOR_APPS_LIST: app.add_task(monitor_apps_lists(monitor_git=app.config.MONITOR_GIT, monitor_only_good_quality_apps=app.config.MONITOR_ONLY_GOOD_QUALITY_APPS)) From 0353307716a600e22741acb12d3e8d879762e4ec Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 14 Oct 2021 17:40:35 +0200 Subject: [PATCH 09/11] Create a custom json dumps for datetime convertion/fix ciclic list --- run.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/run.py b/run.py index 926dd8e..0fd4b20 100644 --- a/run.py +++ b/run.py @@ -82,10 +82,20 @@ LOGGING_CONFIG_DEFAULTS["formatters"] = { }, } + +def datetime_to_epoch_json_converter(o): + if isinstance(o, datetime): + return o.strftime('%s') + +# define a custom json dumps to convert datetime +def my_json_dumps(o): + json.dumps(o, default=datetime_to_epoch_json_converter) + + task_logger = logging.getLogger("task") api_logger = logging.getLogger("api") -app = Sanic(__name__) +app = Sanic(__name__, dumps=my_json_dumps) app.static('/static', './static/') loader = FileSystemLoader(os.path.abspath(os.path.dirname(__file__)) + '/templates', encoding='utf8') @@ -110,11 +120,6 @@ subscriptions = defaultdict(list) jobs_in_memory_state = {} -def datetime_to_epoch_json_converter(o): - if isinstance(o, datetime): - return o.strftime('%s') - - async def wait_closed(self): """ Wait until the connection is closed. @@ -513,7 +518,7 @@ async def broadcast(message, channels): for ws in ws_list: try: - await ws.send(json.dumps(message, default=datetime_to_epoch_json_converter)) + await ws.send(my_json_dumps(message)) except ConnectionClosed: dead_ws.append(ws) @@ -605,16 +610,16 @@ async def ws_index(request, websocket): first_chunck = next(data) - await websocket.send(json.dumps({ + await websocket.send(my_json_dumps({ "action": "init_jobs", "data": first_chunck, # send first chunk - }, default=datetime_to_epoch_json_converter)) + })) for chunk in data: - await websocket.send(json.dumps({ + await websocket.send(my_json_dumps({ "action": "init_jobs_stream", "data": chunk, - }, default=datetime_to_epoch_json_converter)) + })) await websocket.wait_closed() @@ -631,10 +636,10 @@ async def ws_job(request, websocket, job_id): subscribe(websocket, f"job-{job.id}") - await websocket.send(json.dumps({ + await websocket.send(my_json_dumps({ "action": "init_job", "data": model_to_dict(job), - }, default=datetime_to_epoch_json_converter)) + })) await websocket.wait_closed() @@ -731,10 +736,10 @@ async def ws_apps(request, websocket): repos = sorted(repos, key=lambda x: x["name"]) - await websocket.send(json.dumps({ + await websocket.send(my_json_dumps({ "action": "init_apps", "data": repos, - }, default=datetime_to_epoch_json_converter)) + })) await websocket.wait_closed() @@ -749,10 +754,10 @@ async def ws_app(request, websocket, app_name): subscribe(websocket, f"app-jobs-{app.url}") job = list(Job.select().where(Job.url_or_path == app.url).order_by(-Job.id).dicts()) - await websocket.send(json.dumps({ + await websocket.send(my_json_dumps({ "action": "init_jobs", "data": job, - }, default=datetime_to_epoch_json_converter)) + })) await websocket.wait_closed() @@ -1150,7 +1155,7 @@ async def github(request): token = open("./github_bot_token").read().strip() async with aiohttp.ClientSession(headers={"Authorization": f"token {token}"}) as session: - async with session.post(comments_url, data=json.dumps({"body": body}, default=datetime_to_epoch_json_converter)) as resp: + async with session.post(comments_url, data=my_json_dumps({"body": body})) as resp: api_logger.info("Added comment %s" % resp.json()["html_url"]) catchphrases = ["Alrighty!", "Fingers crossed!", "May the CI gods be with you!", ":carousel_horse:", ":rocket:", ":sunflower:", "Meow :cat2:", ":v:", ":stuck_out_tongue_winking_eye:" ] From 4a15903d3d9432d8b9973accf30e45b1950b413a Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 14 Oct 2021 17:41:17 +0200 Subject: [PATCH 10/11] add a try catch to avoid a dumb error --- run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 0fd4b20..0d2a3ff 100644 --- a/run.py +++ b/run.py @@ -523,7 +523,10 @@ async def broadcast(message, channels): dead_ws.append(ws) for to_remove in dead_ws: - ws_list.remove(to_remove) + try: + ws_list.remove(to_remove) + except ValueError: + pass def subscribe(ws, channel): subscriptions[channel].append(ws) From 3de369eaf6da2e55148afccde1966c17345d6dae Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 14 Oct 2021 19:24:03 +0200 Subject: [PATCH 11/11] fix the custom json dumps function --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 0d2a3ff..3134a01 100644 --- a/run.py +++ b/run.py @@ -89,7 +89,7 @@ def datetime_to_epoch_json_converter(o): # define a custom json dumps to convert datetime def my_json_dumps(o): - json.dumps(o, default=datetime_to_epoch_json_converter) + return json.dumps(o, default=datetime_to_epoch_json_converter) task_logger = logging.getLogger("task")