From 5a83fd222f90d9df9887905ce742f5834f139f81 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 May 2021 18:14:58 +0200 Subject: [PATCH 1/4] Add draft for autoreadme webhook service --- tools/README-generator/README.md | 38 ++++++++++++++++++ tools/README-generator/__init__.py | 0 tools/README-generator/nginx.conf | 17 ++++++++ tools/README-generator/requirements.txt | 1 + tools/README-generator/webhook.py | 53 +++++++++++++++++++++++++ tools/README-generator/webhook.service | 16 ++++++++ 6 files changed, 125 insertions(+) create mode 100644 tools/README-generator/README.md create mode 100644 tools/README-generator/__init__.py create mode 100644 tools/README-generator/nginx.conf create mode 100755 tools/README-generator/webhook.py create mode 100644 tools/README-generator/webhook.service diff --git a/tools/README-generator/README.md b/tools/README-generator/README.md new file mode 100644 index 00000000..5da33ffa --- /dev/null +++ b/tools/README-generator/README.md @@ -0,0 +1,38 @@ +# Auto-README generation + +### Initial install + +``` +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +### Use on a single app + +``` +source venv/bin/activate +./make_readme.py /path/to/app +``` + +Then the README.md in the app folder will be updated + +### Launch webhook service for auto update + +Configure the webhook on github + +Also need to allow the bot to push on all repos + +Configure nginx to reverse proxy on port 80123 (or whichever port you set in the systemd config) + +```bash +echo "github_webhook_secret" > github_webhook_secret +echo "the_bot_login" > login +echo "the_bot_token" > token +``` + +Add the webhook.service to systemd config, then start it: + +```bash +systemctl start the_webhook_service +``` diff --git a/tools/README-generator/__init__.py b/tools/README-generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/README-generator/nginx.conf b/tools/README-generator/nginx.conf new file mode 100644 index 00000000..4c1374a9 --- /dev/null +++ b/tools/README-generator/nginx.conf @@ -0,0 +1,17 @@ +location / { + + # Force usage of https + if ($scheme = http) { + rewrite ^ https://$server_name$request_uri? permanent; + } + + client_max_body_size 100M; + + proxy_pass http://127.0.0.1:80123; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + # preserve client IP + proxy_set_header X-Forwarded-For $remote_addr; +} diff --git a/tools/README-generator/requirements.txt b/tools/README-generator/requirements.txt index 908d867d..8f402cbd 100644 --- a/tools/README-generator/requirements.txt +++ b/tools/README-generator/requirements.txt @@ -1,2 +1,3 @@ argparse jinja2 +github-webhook==1.0.4 diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py new file mode 100755 index 00000000..bfdcbe0f --- /dev/null +++ b/tools/README-generator/webhook.py @@ -0,0 +1,53 @@ +import subprocess +import os + +from github_webhook import Webhook +from flask import Flask +from make_readme import generate_READMEs + +app = Flask(__name__) # Standard Flask app +webhook = Webhook(app) # Defines '/postreceive' endpoint + +webhook.secret = open("github_webhook_secret", "r").read().strip() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" + +login = open("login").read().strip() +token = open("token").read().strip() + +def git(cmd, in_folder=None): + + if not isinstance(cmd, list): + cmd = cmd.split() + if in_folder: + cmd = ["-C", in_folder] + cmd + cmd = ["git"] + cmd + return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + + +@app.route("/") +def main_route(): + return "You aren't supposed to go on this page using a browser, it's for webhooks push instead." + + +@webhook.hook() +def on_push(data): + + repository = data["repository"]["full_name"] + branch = data["ref"].split("/", 2)[2] + + folder = subprocess.check_output(["mktemp", "-d"]) + git(f"clone https://{login}:{token}@github.com/{repository} --single-branch --branch {branch} {folder}") + generate_READMEs(folder) + + diff_not_empty = bool(subprocess.check_output(f"cd {folder} && git diff HEAD --compact-summary", shell=True).strip().decode("utf-8")) + if not diff_not_empty: + return + + git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) + git(f"push fork origin {branch} --quiet", in_folder=folder) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=80123) diff --git a/tools/README-generator/webhook.service b/tools/README-generator/webhook.service new file mode 100644 index 00000000..08296b8f --- /dev/null +++ b/tools/README-generator/webhook.service @@ -0,0 +1,16 @@ +[Unit] +Description=Auto-README webhook gunicorn daemon +After=network.target + +[Service] +PIDFile=/run/gunicorn/autoreadme_webhook-pid +User=autoreadme_webhook +Group=autoreadme_webhook +WorkingDirectory=__PATH_TO_README_GENERATOR__ +ExecStart=__PATH_TO_README_GENERATOR__/venv/bin/gunicorn -w 4 -b 127.0.0.1:80123 webhook:app +ExecReload=/bin/kill -s HUP $MAINPID +ExecStop=/bin/kill -s TERM $MAINPID +PrivateTmp=true + +[Install] +WantedBy=multi-user.target From 41e500b9c63065304d74f0973394683ca3915c88 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 May 2021 20:12:00 +0200 Subject: [PATCH 2/4] Misc fixes --- tools/README-generator/README.md | 2 +- tools/README-generator/make_readme.py | 10 +++++++--- tools/README-generator/nginx.conf | 2 +- tools/README-generator/requirements.txt | 1 + tools/README-generator/webhook.py | 13 ++++++++----- tools/README-generator/webhook.service | 2 +- 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tools/README-generator/README.md b/tools/README-generator/README.md index 5da33ffa..260e2fe2 100644 --- a/tools/README-generator/README.md +++ b/tools/README-generator/README.md @@ -23,7 +23,7 @@ Configure the webhook on github Also need to allow the bot to push on all repos -Configure nginx to reverse proxy on port 80123 (or whichever port you set in the systemd config) +Configure nginx to reverse proxy on port 8123 (or whichever port you set in the systemd config) ```bash echo "github_webhook_secret" > github_webhook_secret diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index ffbc8f49..9dac1ea8 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -12,15 +12,19 @@ def generate_READMEs(app_path): if not os.path.exists(app_path): raise Exception("App path provided doesn't exists ?!") + manifest = json.load(open(os.path.join(app_path, "manifest.json"))) + upstream = manifest.get("upstream", {}) + + if not upstream and not os.path.exists(os.path.join(app_path, "doc", "DISCLAIMER.md")): + print("There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet.") + return + env = Environment(loader=FileSystemLoader('./templates')) for lang, lang_suffix in [("en", ""), ("fr", "_fr")]: template = env.get_template(f'README{lang_suffix}.md.j2') - manifest = json.load(open(os.path.join(app_path, "manifest.json"))) - upstream = manifest.get("upstream", {}) - if os.path.exists(os.path.join(app_path, "doc", "screenshots")): screenshots = os.listdir(os.path.join(app_path, "doc", "screenshots")) if ".gitkeep" in screenshots: diff --git a/tools/README-generator/nginx.conf b/tools/README-generator/nginx.conf index 4c1374a9..3dd620ce 100644 --- a/tools/README-generator/nginx.conf +++ b/tools/README-generator/nginx.conf @@ -7,7 +7,7 @@ location / { client_max_body_size 100M; - proxy_pass http://127.0.0.1:80123; + proxy_pass http://127.0.0.1:8123; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; diff --git a/tools/README-generator/requirements.txt b/tools/README-generator/requirements.txt index 8f402cbd..3e3ec9d5 100644 --- a/tools/README-generator/requirements.txt +++ b/tools/README-generator/requirements.txt @@ -1,3 +1,4 @@ argparse jinja2 github-webhook==1.0.4 +gunicorn==20.1.0 diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index bfdcbe0f..b47393c3 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -10,11 +10,12 @@ webhook = Webhook(app) # Defines '/postreceive' endpoint webhook.secret = open("github_webhook_secret", "r").read().strip() +login = open("login").read().strip() +token = open("token").read().strip() + my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" -login = open("login").read().strip() -token = open("token").read().strip() def git(cmd, in_folder=None): @@ -38,16 +39,18 @@ def on_push(data): branch = data["ref"].split("/", 2)[2] folder = subprocess.check_output(["mktemp", "-d"]) - git(f"clone https://{login}:{token}@github.com/{repository} --single-branch --branch {branch} {folder}") + git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) + git(["add", "README*.md"], in_folder=folder) + diff_not_empty = bool(subprocess.check_output(f"cd {folder} && git diff HEAD --compact-summary", shell=True).strip().decode("utf-8")) if not diff_not_empty: return git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) - git(f"push fork origin {branch} --quiet", in_folder=folder) + git(["push", "fork", "origin", branch, "--quiet"], in_folder=folder) if __name__ == "__main__": - app.run(host="0.0.0.0", port=80123) + app.run(host="0.0.0.0", port=8123) diff --git a/tools/README-generator/webhook.service b/tools/README-generator/webhook.service index 08296b8f..e4783c74 100644 --- a/tools/README-generator/webhook.service +++ b/tools/README-generator/webhook.service @@ -7,7 +7,7 @@ PIDFile=/run/gunicorn/autoreadme_webhook-pid User=autoreadme_webhook Group=autoreadme_webhook WorkingDirectory=__PATH_TO_README_GENERATOR__ -ExecStart=__PATH_TO_README_GENERATOR__/venv/bin/gunicorn -w 4 -b 127.0.0.1:80123 webhook:app +ExecStart=__PATH_TO_README_GENERATOR__/venv/bin/gunicorn -w 4 -b 127.0.0.1:8123 webhook:app ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID PrivateTmp=true From a91a81fb06efc443f0ea89f679bb893a7ce77702 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 May 2021 20:17:02 +0200 Subject: [PATCH 3/4] Use /github as endpoint --- tools/README-generator/nginx.conf | 2 +- tools/README-generator/webhook.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/README-generator/nginx.conf b/tools/README-generator/nginx.conf index 3dd620ce..529d59f6 100644 --- a/tools/README-generator/nginx.conf +++ b/tools/README-generator/nginx.conf @@ -1,4 +1,4 @@ -location / { +location /github { # Force usage of https if ($scheme = http) { diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index b47393c3..6f599155 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -5,10 +5,10 @@ from github_webhook import Webhook from flask import Flask from make_readme import generate_READMEs -app = Flask(__name__) # Standard Flask app -webhook = Webhook(app) # Defines '/postreceive' endpoint +app = Flask(__name__) -webhook.secret = open("github_webhook_secret", "r").read().strip() +github_webhook_secret = open("github_webhook_secret", "r").read().strip() +webhook = Webhook(app, endpoint="/github", secret=github_webhook_secret) login = open("login").read().strip() token = open("token").read().strip() From afaee6454d5eadb0c3e8c7713212470d5a18cb9d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 May 2021 22:37:22 +0200 Subject: [PATCH 4/4] Misc fixes after tests on the battlefield --- tools/README-generator/webhook.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index 6f599155..e33f9942 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -1,5 +1,6 @@ import subprocess import os +import shutil from github_webhook import Webhook from flask import Flask @@ -15,6 +16,10 @@ token = open("token").read().strip() my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" +my_env["GIT_AUTHOR_NAME"] = "Yunohost-Bot" +my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" +my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot" +my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" def git(cmd, in_folder=None): @@ -27,7 +32,7 @@ def git(cmd, in_folder=None): return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") -@app.route("/") +@app.route("/github") def main_route(): return "You aren't supposed to go on this page using a browser, it's for webhooks push instead." @@ -38,19 +43,22 @@ def on_push(data): repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] - folder = subprocess.check_output(["mktemp", "-d"]) - git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) - generate_READMEs(folder) + folder = subprocess.check_output(["mktemp", "-d"]).decode('utf-8').strip() + try: + git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) + generate_READMEs(folder) - git(["add", "README*.md"], in_folder=folder) + git(["add", "README*.md"], in_folder=folder) - diff_not_empty = bool(subprocess.check_output(f"cd {folder} && git diff HEAD --compact-summary", shell=True).strip().decode("utf-8")) - if not diff_not_empty: - return - - git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) - git(["push", "fork", "origin", branch, "--quiet"], in_folder=folder) + diff_not_empty = bool(subprocess.check_output(["git", "diff", "HEAD", "--compact-summary"], cwd=folder).strip().decode("utf-8")) + if not diff_not_empty: + return + git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) + git(["push", "origin", branch, "--quiet"], in_folder=folder) + finally: + if os.path.exists(folder): + shutil.rmtree(folder) if __name__ == "__main__": app.run(host="0.0.0.0", port=8123)