From 0f05bba6ec46d814e537a13300837d31b84b0726 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 13 Jan 2021 01:34:16 +0100 Subject: [PATCH 1/7] Simplify create_job, specifying the arch/branch through a comment and job name was a super ugly hack not used anymore... --- run.py | 72 ++++++++++++++++------------------------------------------ 1 file changed, 20 insertions(+), 52 deletions(-) diff --git a/run.py b/run.py index 0631a33..54fe93f 100644 --- a/run.py +++ b/run.py @@ -159,57 +159,32 @@ def set_random_day_for_monthy_job(): repo.save() -async def create_job(app_id, app_list_name, repo, job_command_last_part): - if isinstance(job_command_last_part, str): - job_name = f"{app_id} " + job_command_last_part +async def create_job(app_id, repo_url, job_comment=""): + job_name = f"{app_id}" + if job_comment: + job_name += f" ({job_comment})" - # avoid scheduling twice - if Job.select().where(Job.name == job_name, Job.state == "scheduled").count() > 0: - task_logger.info(f"a job for '{job_name} is already scheduled, don't add another one") - return + # avoid scheduling twice + if Job.select().where(Job.name == job_name, Job.state == "scheduled").count() > 0: + task_logger.info(f"a job for '{job_name} is already scheduled, not adding another one") + return - job = Job.create( - name=job_name, - url_or_path=repo.url, - state="scheduled", - ) + job = Job.create( + name=job_name, + url_or_path=repo_url, + state="scheduled", + ) - await broadcast({ - "action": "new_job", - "data": model_to_dict(job), - }, "jobs") - - else: - for i in job_command_last_part: - job_name = f"{app_id}" + i - - # avoid scheduling twice - if Job.select().where(Job.name == job_name, Job.state == "scheduled").count() > 0: - task_logger.info(f"a job for '{job_name} is already scheduled, don't add another one") - continue - - job = Job.create( - name=job_name, - url_or_path=repo.url, - state="scheduled", - ) - - await broadcast({ - "action": "new_job", - "data": model_to_dict(job), - }, "jobs") + await broadcast({ + "action": "new_job", + "data": model_to_dict(job), + }, "jobs") @always_relaunch(sleep=60 * 5) async def monitor_apps_lists(type="stable", dont_monitor_git=False): "parse apps lists every hour or so to detect new apps" - job_command_last_part = "" - if type == "arm": - job_command_last_part = " (~ARM~)" - elif type == "testing-unstable": - job_command_last_part = [" (testing)", " (unstable)"] - # only support github for now :( async def get_master_commit_sha(url): command = await asyncio.create_subprocess_shell(f"git ls-remote {url} master", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) @@ -275,7 +250,7 @@ async def monitor_apps_lists(type="stable", dont_monitor_git=False): repo.save() repo_is_updated = True - await create_job(app_id, app_list_name, repo, job_command_last_part) + await create_job(app_id, repo.url) repo_state = "working" if app_data["state"] in ("working", "validated") else "other_than_working" @@ -313,7 +288,7 @@ async def monitor_apps_lists(type="stable", dont_monitor_git=False): }, "apps") if not dont_monitor_git: - await create_job(app_id, app_list_name, repo, job_command_last_part) + await create_job(app_id, repo.url) await asyncio.sleep(3) @@ -351,18 +326,11 @@ async def monitor_apps_lists(type="stable", dont_monitor_git=False): @once_per_day async def launch_monthly_job(type): - # XXX DRY - job_command_last_part = "" - if type == "arm": - job_command_last_part = " (~ARM~)" - elif type == "testing-unstable": - job_command_last_part = [" (testing)", " (unstable)"] - today = date.today().day for repo in Repo.select().where(Repo.random_job_day == today): task_logger.info(f"Launch monthly job for {repo.name} on day {today} of the month ") - await create_job(repo.name, repo.app_list, repo, job_command_last_part) + await create_job(repo.name, repo.url) @always_relaunch(sleep=3) From d97747eed06b1f75305a1be547984d9c6f52269e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 13 Jan 2021 01:44:39 +0100 Subject: [PATCH 2/7] Add github webhooks endpoints to create a job from a comment on a PR --- run.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 54fe93f..912e3d7 100644 --- a/run.py +++ b/run.py @@ -11,6 +11,9 @@ import traceback import itertools import tracemalloc +import hmac +import hashlib + from datetime import datetime, date from collections import defaultdict from functools import wraps @@ -25,7 +28,7 @@ from websockets.exceptions import ConnectionClosed from websockets import WebSocketCommonProtocol from sanic import Sanic, response -from sanic.exceptions import NotFound +from sanic.exceptions import NotFound, abort from sanic.log import LOGGING_CONFIG_DEFAULTS from sanic.response import json @@ -944,6 +947,88 @@ async def monitor(request): }) +@app.route("/github", methods=["GET"]) +async def github_get(request): + return response.text( + "You aren't supposed to go on this page using a browser, it's for webhooks push instead." + ) + + +@app.route("/github", methods=["POST"]) +async def github(request): + + # Abort directly if no secret opened + # (which also allows to only enable this feature if + # we define the webhook secret) + if not os.path.exists("./github_webhook_secret"): + abort(403) + + # Only SHA1 is supported + header_signature = request.headers.get("X-Hub-Signature") + if header_signature is None: + print("no header X-Hub-Signature") + abort(403) + + sha_name, signature = header_signature.split("=") + if sha_name != "sha1": + print("signing algo isn't sha1, it's '%s'" % sha_name) + abort(501) + + secret = open("./github_webhook_secret", "r").read().strip() + # HMAC requires the key to be bytes, but data is string + mac = hmac.new(secret.encode(), msg=request.body, digestmod=hashlib.sha1) + + if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): + abort(403) + + hook_type = request.headers.get("X-Github-Event") + hook_infos = request.json + + # We expect issue comments (issue = also PR in github stuff...) + # - *New* comments + # - On issue/PRs which are still open + if hook_type != "issue_comment" \ + or hook_infos["action"] != "created" \ + or hook_infos["issue"]["state"] != "open": \ + # idk what code we want to return + abort(400) + + # Check the comment contains proper keyword trigger + body = hook_infos["comment"]["body"].strip()[:100].lower() + triggers = ["!testme", "!gogogadgetoci", "By the power of systemd, I invoke The Great App CI to test this Pull Request!"] + if not any(trigger.lower() in body for trigger in triggers): + # idk what code we want to return + abort(403) + + # We only accept this from people which are member/owner of the org/repo + # https://docs.github.com/en/free-pro-team@latest/graphql/reference/enums#commentauthorassociation + if hook_infos["comment"]["author_association"] not in ["MEMBER", "OWNER"]: + # idk what code we want to return + abort(403) + + # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) + pr_infos_url = hook_infos["issue"]["pull_request"]["url"] + async with aiohttp.ClientSession() as session: + async with session.get(pr_infos_url) as resp: + pr_infos = await resp.json() + + branch_name = pr_infos["head"]["ref"] + repo = pr_infos["head"]["repo"]["html_url"] + url_to_test = f"{repo}/tree/{branch_name}" + app_id = pr_infos["base"]["repo"]["name"].rstrip("") + if app_id.endswith("_ynh"): + app_id = app_id[:-len("_ynh")] + + pr_id = str(pr_infos["number"]) + + # Add the job for the corresponding app (with the branch url) + await create_job(app_id, url_to_test, job_comment=f"PR #{pr_id}, {branch_name}") + + # TODO : write a comment back using yunobot with a jenkins-like badge + link to the created job + + return response.text("ok") + + def show_coro(c): data = { 'txt': str(c), From 19f26497e7e73c22a60ccdeae31ab331ac8c1c9e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 14 Jan 2021 15:09:27 +0100 Subject: [PATCH 3/7] Make sure the issue is a pull request Co-authored-by: Kayou --- run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 912e3d7..9d2b0ed 100644 --- a/run.py +++ b/run.py @@ -989,7 +989,8 @@ async def github(request): # - On issue/PRs which are still open if hook_type != "issue_comment" \ or hook_infos["action"] != "created" \ - or hook_infos["issue"]["state"] != "open": \ + or hook_infos["issue"]["state"] != "open" \ + or "pull_request" not in hook_infos["issue"]: # idk what code we want to return abort(400) From 49d6b8640fa7f9696c804de738b545a196ae8d9d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 14 Jan 2021 14:55:42 +0100 Subject: [PATCH 4/7] Add badge endpoint --- run.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/run.py b/run.py index 9d2b0ed..029df82 100644 --- a/run.py +++ b/run.py @@ -882,6 +882,34 @@ async def api_restart_job(request, job_id): return response.text("ok") +# Meant to interface with https://shields.io/endpoint +@app.route("/api/job//badge", methods=['GET']) +async def api_badge_job(request, job_id): + + job = Job.select().where(Job.id == job_id) + + if job.count() == 0: + raise NotFound(f"Error: no job with the id '{job_id}'") + + job = job[0] + + state_to_color = { + 'scheduled': 'lightgrey', + 'runnning': 'blue', + 'done': 'brightgreen', + 'failure': 'red', + 'error': 'red', + 'canceled': 'yellow', + } + + return response.json({ + "schemaVersion": 1, + "label": "tests", + "message": job.state, + "color": state_to_color[job.state] + }) + + @app.route('/job/') @jinja.template('job.html') async def html_job(request, job_id): From f05f0ed432d179a71c5ef4f61b03cf25c6e8baf7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 14 Jan 2021 15:23:58 +0100 Subject: [PATCH 5/7] Improve logging, return codes --- run.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/run.py b/run.py index 029df82..711ad4e 100644 --- a/run.py +++ b/run.py @@ -989,25 +989,27 @@ async def github(request): # (which also allows to only enable this feature if # we define the webhook secret) if not os.path.exists("./github_webhook_secret"): + api_logger.info(f"Received a webhook but no ./github_webhook_secret file exists ... ignoring") abort(403) # Only SHA1 is supported header_signature = request.headers.get("X-Hub-Signature") if header_signature is None: - print("no header X-Hub-Signature") + api_logger.info("Received a webhook but there's no header X-Hub-Signature") abort(403) sha_name, signature = header_signature.split("=") if sha_name != "sha1": - print("signing algo isn't sha1, it's '%s'" % sha_name) - abort(501) + api_logger.info("Received a webhook but signing algo isn't sha1, it's '%s'" % sha_name) + abort(501, "Signing algorightm is not sha1 ?!") secret = open("./github_webhook_secret", "r").read().strip() # HMAC requires the key to be bytes, but data is string mac = hmac.new(secret.encode(), msg=request.body, digestmod=hashlib.sha1) if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): - abort(403) + api_logger.info(f"Received a webhook but signature authentication failed (is the secret properly configured?)") + abort(403, "Bad signature ?!") hook_type = request.headers.get("X-Github-Event") hook_infos = request.json @@ -1019,21 +1021,21 @@ async def github(request): or hook_infos["action"] != "created" \ or hook_infos["issue"]["state"] != "open" \ or "pull_request" not in hook_infos["issue"]: - # idk what code we want to return - abort(400) + # Nothing to do but success anyway (204 = No content) + abort(204, "Nothing to do") # Check the comment contains proper keyword trigger body = hook_infos["comment"]["body"].strip()[:100].lower() triggers = ["!testme", "!gogogadgetoci", "By the power of systemd, I invoke The Great App CI to test this Pull Request!"] if not any(trigger.lower() in body for trigger in triggers): - # idk what code we want to return - abort(403) + # Nothing to do but success anyway (204 = No content) + abort(204, "Nothing to do") # We only accept this from people which are member/owner of the org/repo # https://docs.github.com/en/free-pro-team@latest/graphql/reference/enums#commentauthorassociation if hook_infos["comment"]["author_association"] not in ["MEMBER", "OWNER"]: - # idk what code we want to return - abort(403) + # Unauthorized + abort(403, "Unauthorized") # Fetch the PR infos (yeah they ain't in the initial infos we get @_@) pr_infos_url = hook_infos["issue"]["pull_request"]["url"] From f2049e68074510502beb525fe787a8c2cfee74c4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 14 Jan 2021 21:37:22 +0100 Subject: [PATCH 6/7] [yolowip] Answer with a comment with a badge+link to job created --- run.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/run.py b/run.py index 711ad4e..f9e5789 100644 --- a/run.py +++ b/run.py @@ -183,6 +183,8 @@ async def create_job(app_id, repo_url, job_comment=""): "data": model_to_dict(job), }, "jobs") + return job.id + @always_relaunch(sleep=60 * 5) async def monitor_apps_lists(type="stable", dont_monitor_git=False): @@ -988,8 +990,8 @@ async def github(request): # Abort directly if no secret opened # (which also allows to only enable this feature if # we define the webhook secret) - if not os.path.exists("./github_webhook_secret"): - api_logger.info(f"Received a webhook but no ./github_webhook_secret file exists ... ignoring") + if not os.path.exists("./github_webhook_secret") or not os.path.exists("./github_bot_token"): + api_logger.info(f"Received a webhook but no ./github_webhook_secret or ./github_bot_token file exists ... ignoring") abort(403) # Only SHA1 is supported @@ -1052,10 +1054,31 @@ async def github(request): pr_id = str(pr_infos["number"]) - # Add the job for the corresponding app (with the branch url) - await create_job(app_id, url_to_test, job_comment=f"PR #{pr_id}, {branch_name}") + # Create the job for the corresponding app (with the branch url) - # TODO : write a comment back using yunobot with a jenkins-like badge + link to the created job + job_id = await create_job(app_id, url_to_test, job_comment=f"PR #{pr_id}, {branch_name}") + + # Answer with comment with link+badge for the job + + def comment(body): + + comments_url = hook_infos["comments_url"] + + token = open("./github_bot_token").read().strip() + with requests.Session() as s: + s.headers.update({"Authorization": f"token {token}"}) + r = s.post(comments_url, json.dumps({"body": body})) + + api_logger.info("Added comment %s" % json.loads(r.text)["html_url"]) + + catchphrases = ["Alrighty!", "Fingers crossed!", "May the CI gods be with you!", ":carousel_horse:", ":rocket:", ":sunflower:", ":cat2:", ":v:", ":stuck_out_tongue_winking_eye:" ] + catchphrase = random.choice(catchphrases) + job_url = request.url_for("html_job", job_id=job_id) + badge_url = request.url_for("api_badge_job", job_id=job_id) + shield_badge_url = f"https://img.shields.io/endpoint?url={badge_url}" + + body = "{catchphrase}\n![{shield_badge_url}]({job_url})" + comment(body) return response.text("ok") From 36dd615ed4660b65ead12bce1159f906e0545888 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 14 Jan 2021 23:02:56 +0100 Subject: [PATCH 7/7] Misc fixes after testing --- run.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/run.py b/run.py index f9e5789..3b7d468 100644 --- a/run.py +++ b/run.py @@ -183,7 +183,7 @@ async def create_job(app_id, repo_url, job_comment=""): "data": model_to_dict(job), }, "jobs") - return job.id + return job @always_relaunch(sleep=60 * 5) @@ -897,7 +897,7 @@ async def api_badge_job(request, job_id): state_to_color = { 'scheduled': 'lightgrey', - 'runnning': 'blue', + 'running': 'blue', 'done': 'brightgreen', 'failure': 'red', 'error': 'red', @@ -1056,29 +1056,33 @@ async def github(request): # Create the job for the corresponding app (with the branch url) - job_id = await create_job(app_id, url_to_test, job_comment=f"PR #{pr_id}, {branch_name}") + api_logger.info("Scheduling a new job from comment on a PR") + job = await create_job(app_id, url_to_test, job_comment=f"PR #{pr_id}, {branch_name}") + + if not job: + abort(204, "Corresponding job already scheduled") # Answer with comment with link+badge for the job - def comment(body): + async def comment(body): - comments_url = hook_infos["comments_url"] + comments_url = hook_infos["issue"]["comments_url"] token = open("./github_bot_token").read().strip() - with requests.Session() as s: - s.headers.update({"Authorization": f"token {token}"}) - r = s.post(comments_url, json.dumps({"body": body})) + async with aiohttp.ClientSession(headers={"Authorization": f"token {token}"}) as session: + async with session.post(comments_url, data=ujson.dumps({"body": body})) as resp: + api_logger.info("Added comment %s" % resp.json()["html_url"]) - api_logger.info("Added comment %s" % json.loads(r.text)["html_url"]) - - catchphrases = ["Alrighty!", "Fingers crossed!", "May the CI gods be with you!", ":carousel_horse:", ":rocket:", ":sunflower:", ":cat2:", ":v:", ":stuck_out_tongue_winking_eye:" ] + catchphrases = ["Alrighty!", "Fingers crossed!", "May the CI gods be with you!", ":carousel_horse:", ":rocket:", ":sunflower:", "Meow :cat2:", ":v:", ":stuck_out_tongue_winking_eye:" ] catchphrase = random.choice(catchphrases) - job_url = request.url_for("html_job", job_id=job_id) - badge_url = request.url_for("api_badge_job", job_id=job_id) + # Dirty hack with base_url passed from cmd argument because we can't use request.url_for because Sanic < 20.x + job_url = app.config.base_url + app.url_for("html_job", job_id=job.id) + badge_url = app.config.base_url + app.url_for("api_badge_job", job_id=job.id) shield_badge_url = f"https://img.shields.io/endpoint?url={badge_url}" - body = "{catchphrase}\n![{shield_badge_url}]({job_url})" - comment(body) + body = f"{catchphrase}\n[![Test Badge]({shield_badge_url})]({job_url})" + api_logger.info(body) + await comment(body) return response.text("ok") @@ -1108,8 +1112,7 @@ def format_frame(f): return dict([(k, str(getattr(f, k))) for k in keys]) -@argh.arg('-t', '--type', choices=['stable', 'arm', 'testing-unstable', 'dev'], default="stable") -def main(path_to_analyseCI, ssl=False, keyfile_path="/etc/yunohost/certs/ci-apps.yunohost.org/key.pem", certfile_path="/etc/yunohost/certs/ci-apps.yunohost.org/crt.pem", type="stable", dont_monitor_apps_list=False, dont_monitor_git=False, no_monthly_jobs=False, port=4242, debug=False): +def main(path_to_analyseCI, ssl=False, keyfile_path="/etc/yunohost/certs/ci-apps.yunohost.org/key.pem", certfile_path="/etc/yunohost/certs/ci-apps.yunohost.org/crt.pem", type="stable", dont_monitor_apps_list=False, dont_monitor_git=False, no_monthly_jobs=False, port=4242, base_url="", debug=False): if not os.path.exists(path_to_analyseCI): print(f"Error: analyseCI script doesn't exist at '{path_to_analyseCI}'") sys.exit(1) @@ -1121,6 +1124,7 @@ def main(path_to_analyseCI, ssl=False, keyfile_path="/etc/yunohost/certs/ci-apps set_random_day_for_monthy_job() app.config.path_to_analyseCI = path_to_analyseCI + app.config.base_url = base_url if not dont_monitor_apps_list: app.add_task(monitor_apps_lists(type=type,