Merge pull request #1159 from YunoHost/autoreadme-webhook

Auto-update-README webhook

tools/README-generator/README.md

@@ -0,0 +1,38 @@
+# Auto-README generation
+
+### Initial install
+
+```
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Use on a single app
+
+```
+source venv/bin/activate
+./make_readme.py /path/to/app
+```
+
+Then the README.md in the app folder will be updated
+
+### Launch webhook service for auto update
+
+Configure the webhook on github
+
+Also need to allow the bot to push on all repos
+
+Configure nginx to reverse proxy on port 8123 (or whichever port you set in the systemd config)
+
+```bash
+echo "github_webhook_secret" > github_webhook_secret
+echo "the_bot_login" > login
+echo "the_bot_token" > token
+```
+
+Add the webhook.service to systemd config, then start it:
+
+```bash
+systemctl start the_webhook_service 
+```

tools/README-generator/make_readme.py

@@ -12,15 +12,19 @@ def generate_READMEs(app_path):
     if not os.path.exists(app_path):
         raise Exception("App path provided doesn't exists ?!")
 
+    manifest = json.load(open(os.path.join(app_path, "manifest.json")))
+    upstream = manifest.get("upstream", {})
+
+    if not upstream and not os.path.exists(os.path.join(app_path, "doc", "DISCLAIMER.md")):
+        print("There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet.")
+        return
+
     env = Environment(loader=FileSystemLoader('./templates'))
 
     for lang, lang_suffix in [("en", ""), ("fr", "_fr")]:
 
         template = env.get_template(f'README{lang_suffix}.md.j2')
 
-        manifest = json.load(open(os.path.join(app_path, "manifest.json")))
-        upstream = manifest.get("upstream", {})
-
         if os.path.exists(os.path.join(app_path, "doc", "screenshots")):
             screenshots = os.listdir(os.path.join(app_path, "doc", "screenshots"))
             if ".gitkeep" in screenshots:

tools/README-generator/nginx.conf

@@ -0,0 +1,17 @@
+location /github {
+
+  # Force usage of https
+  if ($scheme = http) {
+    rewrite ^ https://$server_name$request_uri? permanent;
+  }
+
+  client_max_body_size 100M;
+
+  proxy_pass http://127.0.0.1:8123;
+  proxy_http_version 1.1;
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection "Upgrade";
+
+  # preserve client IP
+  proxy_set_header X-Forwarded-For $remote_addr;
+}

tools/README-generator/requirements.txt

@@ -1,2 +1,4 @@
 argparse
 jinja2
+github-webhook==1.0.4
+gunicorn==20.1.0

tools/README-generator/webhook.py

@@ -0,0 +1,64 @@
+import subprocess
+import os
+import shutil
+
+from github_webhook import Webhook
+from flask import Flask
+from make_readme import generate_READMEs
+
+app = Flask(__name__)
+
+github_webhook_secret = open("github_webhook_secret", "r").read().strip()
+webhook = Webhook(app, endpoint="/github", secret=github_webhook_secret)
+
+login = open("login").read().strip()
+token = open("token").read().strip()
+
+my_env = os.environ.copy()
+my_env["GIT_TERMINAL_PROMPT"] = "0"
+my_env["GIT_AUTHOR_NAME"] = "Yunohost-Bot"
+my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org"
+my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot"
+my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org"
+
+
+def git(cmd, in_folder=None):
+
+    if not isinstance(cmd, list):
+        cmd = cmd.split()
+    if in_folder:
+        cmd = ["-C", in_folder] + cmd
+    cmd = ["git"] + cmd
+    return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8")
+
+
+@app.route("/github")
+def main_route():
+    return "You aren't supposed to go on this page using a browser, it's for webhooks push instead."
+
+
+@webhook.hook()
+def on_push(data):
+
+    repository = data["repository"]["full_name"]
+    branch = data["ref"].split("/", 2)[2]
+
+    folder = subprocess.check_output(["mktemp", "-d"]).decode('utf-8').strip()
+    try:
+        git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder])
+        generate_READMEs(folder)
+
+        git(["add", "README*.md"], in_folder=folder)
+
+        diff_not_empty = bool(subprocess.check_output(["git", "diff", "HEAD", "--compact-summary"], cwd=folder).strip().decode("utf-8"))
+        if not diff_not_empty:
+            return
+
+        git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder)
+        git(["push", "origin", branch, "--quiet"], in_folder=folder)
+    finally:
+        if os.path.exists(folder):
+            shutil.rmtree(folder)
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=8123)

tools/README-generator/webhook.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Auto-README webhook gunicorn daemon
+After=network.target
+
+[Service]
+PIDFile=/run/gunicorn/autoreadme_webhook-pid
+User=autoreadme_webhook
+Group=autoreadme_webhook
+WorkingDirectory=__PATH_TO_README_GENERATOR__
+ExecStart=__PATH_TO_README_GENERATOR__/venv/bin/gunicorn -w 4 -b 127.0.0.1:8123 webhook:app
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target