From 9b664f5706c4efbb4a273f1040948a14ca3f5f0f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 13 Jun 2021 04:50:21 +0200 Subject: [PATCH 1/3] [mod] use with tempfile.TemporaryDirectory --- tools/README-generator/webhook.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index e33f9942..5f52c7bc 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -1,6 +1,6 @@ import subprocess import os -import shutil +import tempfile from github_webhook import Webhook from flask import Flask @@ -43,8 +43,7 @@ def on_push(data): repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] - folder = subprocess.check_output(["mktemp", "-d"]).decode('utf-8').strip() - try: + with tempfile.TemporaryDirectory() as folder: git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) @@ -56,9 +55,7 @@ def on_push(data): git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) git(["push", "origin", branch, "--quiet"], in_folder=folder) - finally: - if os.path.exists(folder): - shutil.rmtree(folder) + if __name__ == "__main__": app.run(host="0.0.0.0", port=8123) From 19fb5931adb04ff9a71b145e4ca817198e1798ab Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 13 Jun 2021 05:12:34 +0200 Subject: [PATCH 2/3] [mod] port to sanic for perfs --- tools/README-generator/requirements.txt | 4 +- tools/README-generator/webhook.py | 64 ++++++++++++++++++------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/tools/README-generator/requirements.txt b/tools/README-generator/requirements.txt index 3e3ec9d5..88cd6e2e 100644 --- a/tools/README-generator/requirements.txt +++ b/tools/README-generator/requirements.txt @@ -1,4 +1,2 @@ -argparse jinja2 -github-webhook==1.0.4 -gunicorn==20.1.0 +sanic diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index 5f52c7bc..b78b36df 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -1,15 +1,19 @@ -import subprocess import os +import hmac +import shlex +import hashlib +import asyncio import tempfile -from github_webhook import Webhook -from flask import Flask +from sanic import Sanic +from sanic.response import text +from sanic.exceptions import abort + from make_readme import generate_READMEs -app = Flask(__name__) +app = Sanic(__name__) github_webhook_secret = open("github_webhook_secret", "r").read().strip() -webhook = Webhook(app, endpoint="/github", secret=github_webhook_secret) login = open("login").read().strip() token = open("token").read().strip() @@ -22,39 +26,63 @@ my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot" my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" -def git(cmd, in_folder=None): +async def git(cmd, in_folder=None): if not isinstance(cmd, list): cmd = cmd.split() if in_folder: cmd = ["-C", in_folder] + cmd cmd = ["git"] + cmd - return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + cmd = " ".join(map(shlex.quote, cmd)) + command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) + data = await command.stdout.read() + return data.decode().strip() -@app.route("/github") -def main_route(): - return "You aren't supposed to go on this page using a browser, it's for webhooks push instead." +@app.route("/github", methods=["GET"]) +def main_route(request): + return text("You aren't supposed to go on this page using a browser, it's for webhooks push instead.") -@webhook.hook() -def on_push(data): +@app.route("/github", methods=["POST"]) +async def on_push(request): + header_signature = request.headers.get("X-Hub-Signature") + if header_signature is None: + print("no header X-Hub-Signature") + abort(403) + + sha_name, signature = header_signature.split("=") + if sha_name != "sha1": + print("signing algo isn't sha1, it's '%s'" % sha_name) + abort(501) + + # HMAC requires the key to be bytes, but data is string + mac = hmac.new(github_webhook_secret.encode(), msg=request.body, digestmod=hashlib.sha1) + + if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): + abort(403) + + data = request.json repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] with tempfile.TemporaryDirectory() as folder: - git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) + await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) - git(["add", "README*.md"], in_folder=folder) + await git(["add", "README*.md"], in_folder=folder) - diff_not_empty = bool(subprocess.check_output(["git", "diff", "HEAD", "--compact-summary"], cwd=folder).strip().decode("utf-8")) + diff_not_empty = await asyncio.create_subprocess_shell(" ".join(["git", "diff", "HEAD", "--compact-summary"]), cwd=folder, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) + diff_not_empty = await diff_not_empty.stdout.read() + diff_not_empty = diff_not_empty.decode().strip() if not diff_not_empty: - return + return text("nothing to do") - git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) - git(["push", "origin", branch, "--quiet"], in_folder=folder) + await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) + await git(["push", "origin", branch, "--quiet"], in_folder=folder) + + return text("ok") if __name__ == "__main__": From 93da394be6a9871e6e61c225d838bfe8b7a52ed7 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 14 Jun 2021 16:34:26 +0200 Subject: [PATCH 3/3] [mod] add some debug logging --- tools/README-generator/webhook.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index b78b36df..a9165e63 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -34,6 +34,7 @@ async def git(cmd, in_folder=None): cmd = ["-C", in_folder] + cmd cmd = ["git"] + cmd cmd = " ".join(map(shlex.quote, cmd)) + print(cmd) command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) data = await command.stdout.read() return data.decode().strip() @@ -67,6 +68,8 @@ async def on_push(request): repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] + print(f"{repository} -> branch '{branch}'") + with tempfile.TemporaryDirectory() as folder: await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) @@ -77,6 +80,7 @@ async def on_push(request): diff_not_empty = await diff_not_empty.stdout.read() diff_not_empty = diff_not_empty.decode().strip() if not diff_not_empty: + print("nothing to do") return text("nothing to do") await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder)