From 66320bf7bea3bad12638ef56c1f39fc3b180cc57 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 1 Sep 2017 02:06:07 +0200 Subject: [PATCH] Adding github webhooks stuff --- init.sh | 20 ++++- webhooks/logs/.gitkeep | 0 webhooks/requirements.txt | 5 ++ webhooks/server.py | 176 ++++++++++++++++++++++++++++++++++++++ webhooks/service | 77 +++++++++++++++++ 5 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 webhooks/logs/.gitkeep create mode 100644 webhooks/requirements.txt create mode 100644 webhooks/server.py create mode 100644 webhooks/service diff --git a/init.sh b/init.sh index 4a83d10..475eec7 100755 --- a/init.sh +++ b/init.sh @@ -1,5 +1,6 @@ apt-get install nginx pbuilder reprepro rebuildd gawk sendxmpp -y +apt-get install python-virtualenv python3-pip -y VINAIGRETTE_HOME="/home/vinaigrette" @@ -16,10 +17,10 @@ git clone https://github.com/yunohost/ssowat git clone https://github.com/yunohost/moulinette cd yunohost -git symbolic-ref refs/heads/jessie-stable refs/heads/stable -git symbolic-ref refs/heads/jessie-testing refs/heads/testing -git symbolic-ref refs/heads/jessie-unstable refs/heads/unstable -git symbolic-ref refs/heads/stretch-unstable refs/heads/stretch +git checkout stable && git symbolic-ref refs/heads/jessie-stable refs/heads/stable +git checkout testing && git symbolic-ref refs/heads/jessie-testing refs/heads/testing +git checkout unstable && git symbolic-ref refs/heads/jessie-unstable refs/heads/unstable +git checkout stretch && git symbolic-ref refs/heads/stretch-unstable refs/heads/stretch cd .. mkdir -p /var/www/repo/debian/conf/ @@ -42,3 +43,14 @@ echo "127.0.0.1 $REPO_URL" >> /etc/hosts service nginx reload rebuildd init + + +cd $VINAIGRETTE_HOME/webhooks +virtualenv venv +source venv/bin/activate +pip install -r requirements.txt + +cp $VINAIGRETTE_HOME/webhooks/service /etc/init.d/github-webhook +systemctl daemon-reload +updated-rc.d github-webhook defaults +github-webhook diff --git a/webhooks/logs/.gitkeep b/webhooks/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/webhooks/requirements.txt b/webhooks/requirements.txt new file mode 100644 index 0000000..3d07731 --- /dev/null +++ b/webhooks/requirements.txt @@ -0,0 +1,5 @@ +bottle==0.12.9 +distribute>=0.6.24 +flup==1.0.3.dev20161029 +wsgiref==0.1.2 +urllib>=1.21.1 diff --git a/webhooks/server.py b/webhooks/server.py new file mode 100644 index 0000000..93367f8 --- /dev/null +++ b/webhooks/server.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +import os +import glob +import json +import time +import shutil +import tarfile +import tempfile +import threading +import subprocess +import collections +import urllib.request +from queue import Queue + +from bottle import route, request, run + +DISTRIBUTION = "jessie" +BRANCHES=['stable', 'testing', 'unstable'] + +# -- Variables + +# The backend to use for running the server +# See: http://bottlepy.org/docs/0.12/deployment.html#switching-the-server-backend +# Note: flup allows to run as FastCGI process +SERVER_BACKEND = 'flup' + +# Host and port on which the server will listen +HOST = '127.0.0.1' +PORT = 9908 + +# Path to the script to build a package +BUILD_CMD = '/home/vinaigrette/scripts/build_deb' + +# Directory where build logs are stored +BUILD_LOG_DIR = '/home/vinaigrette/webhooks/logs' + +# Number of worker threads +WORKERS = 1 + +# -- Types and methods + +# Represent a new release of a package +PackageRelease = collections.namedtuple('PackageRelease', [ + 'name', 'tarball', 'tag', 'branch', 'pid', +]) + + +def package_files(tar, pkgname, callback=None): + """Return the members of a package tarball safely.""" + if not callable(callback): + callback = lambda m: None + for member in tar: + if member.name.startswith(pkgname): + callback(member.name) + yield member + else: + callback("nope: " + member.name) + + +def build(pkg): + """Build the given PackageRelease.""" + with open(os.path.join( + BUILD_LOG_DIR, "{0}.log".format(pkg.pid)), 'w') as logfile: + log = lambda m: print(m, file=logfile) + log("== Building {0} ==".format(pkg)) + + # create a temporary directory for the package + pkg_dir = tempfile.mkdtemp() + + # download and extract tarball + log(":: Downloading and extract tarball to {0}...".format(pkg_dir)) + tarball_stream = urllib.request.urlopen(pkg.tarball) + tarball = tarfile.open(fileobj=tarball_stream, mode="r|gz") + for member in tarball: + if member.name[0] not in ['/', '.']: + try: + _, member.name = member.name.split('/', 1) + except ValueError: + continue + log(member.name) + tarball.extract(member, pkg_dir) + else: + log("ignoring: {0}".format(member.name)) + tarball.close() + + # build the package + log("\n:: Building the Debian package...") + logfile.flush() + retcode = subprocess.call([ + BUILD_CMD, '-d', pkg.branch, + '-c', DISTRIBUTION, '.' + ], cwd=pkg_dir, stdout=logfile, stderr=subprocess.STDOUT, + ) + + # cleaning... + shutil.rmtree(pkg_dir) + return retcode + + +def worker(): + while True: + item = queue.get() + if item is None: + break + build(item) + queue.task_done() + + +# -- Web app and routes + +@route("/", method=['GET', 'POST']) +def index(): + if request.method == 'GET': + return ' Yolo ' + + if request.method == 'POST': + + # Answer to GitHub events + if request.headers.get('X-GitHub-Event') == "ping": + return json.dumps({'msg': 'Hi!'}) + + # Only accept create event: + # https://developer.github.com/v3/activity/events/types/#releaseevent + if request.headers.get('X-GitHub-Event') != "release": + return json.dumps({'msg': "wrong event type"}) + + payload = request.json + if not payload: + return json.dumps({'msg': "invalid payload"}) + + # Only handle tag creation (create event also triggers for branch creation) + if payload['action'] != "published": + return json.dumps({'msg': "uninteresting action"}) + + # Validate tag and branch + tag = payload['release']['tag_name'] + if not tag: + return json.dumps({'msg': "invalid tag"}) + branch = payload['release']['target_commitish'] + if branch not in BRANCHES: + return json.dumps({'msg': "uninteresting branch"}) + + # Init new package release and put it in the queue + package = PackageRelease( + name=payload['repository']['name'].lower(), + tarball=payload['release']['tarball_url'], + tag=tag, branch=branch, pid=int(time.time()), + ) + queue.put(package, False) + + # Return the pid to be able to track building + return "Queue id: {0}".format(package.pid) + + +# -- Main program + +if __name__ == '__main__': + # Create the queue and the workers + queue = Queue() + threads = [] + for i in range(WORKERS): + t = threading.Thread(target=worker) + t.start() + threads.append(t) + + # Run the Web server + run(server=SERVER_BACKEND, host=HOST, port=PORT) + + # Block until all tasks are done + queue.join() + + # Stop workers + for i in range(WORKERS): + queue.put(None) + for t in threads: + t.join() diff --git a/webhooks/service b/webhooks/service new file mode 100644 index 0000000..a017ecc --- /dev/null +++ b/webhooks/service @@ -0,0 +1,77 @@ +#! /bin/bash +# github-webhook init script +# +### BEGIN INIT INFO +# Provides: github-webhook +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# Should-Start: $network +# Should-Stop: $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: rebuild daemon +# Description: daemon providing rebuild system +# for Debian packages +### END INIT INFO + +PATH=/home/vinaigrette/scripts/webhooks/venv/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +DAEMON=/home/vinaitrette/webhooks/server.py +NAME=github-webhook +DESC="GitHub WebHook daemon" + +GWH_UID=pbuilder +GWH_GROUP=pbuilder +GWH_CHROOT=/home/vinaigrette/webhooks + +LOGDIR=/var/log/$NAME +LOGFILE=$NAME.log + +test -x $DAEMON || exit 0 + +# Include github-webhook defaults if available +if [ -f /etc/default/github-webhook ] ; then + . /etc/default/github-webhook +fi + +. /lib/lsb/init-functions +set -e + +function do_start { + mkdir -p $LOGDIR + chown -R $GWH_UID:$GWH_GROUP $LOGDIR + + start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \ + --chuid $GWH_UID:$GWH_GROUP --chdir $GWH_CHROOT \ + --background --make-pidfile --startas /bin/bash -- -c "exec python $DAEMON >> $LOGDIR/$LOGFILE 2>&1" +} + +function do_stop { + start-stop-daemon --stop --quiet --oknodo --retry 120 --pidfile /var/run/$NAME.pid +} + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + log_end_msg $? + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + log_end_msg $? + ;; + restart) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + sleep 1 + do_start + log_end_msg $? + ;; + *) + N=/etc/init.d/$NAME + echo "Usage: $N {start|stop|restart}" >&2 + exit 1 + ;; +esac + +exit 0