Adding github webhooks stuff

init.sh

@@ -1,5 +1,6 @@
 
 apt-get install nginx pbuilder reprepro rebuildd gawk sendxmpp -y
+apt-get install python-virtualenv python3-pip -y
 
 VINAIGRETTE_HOME="/home/vinaigrette"
 
@@ -16,10 +17,10 @@ git clone https://github.com/yunohost/ssowat
 git clone https://github.com/yunohost/moulinette
 
 cd yunohost
-git symbolic-ref refs/heads/jessie-stable refs/heads/stable
-git symbolic-ref refs/heads/jessie-testing refs/heads/testing
-git symbolic-ref refs/heads/jessie-unstable refs/heads/unstable
-git symbolic-ref refs/heads/stretch-unstable refs/heads/stretch
+git checkout stable   && git symbolic-ref refs/heads/jessie-stable refs/heads/stable
+git checkout testing  && git symbolic-ref refs/heads/jessie-testing refs/heads/testing
+git checkout unstable && git symbolic-ref refs/heads/jessie-unstable refs/heads/unstable
+git checkout stretch  && git symbolic-ref refs/heads/stretch-unstable refs/heads/stretch
 cd ..
 
 mkdir -p /var/www/repo/debian/conf/
@@ -42,3 +43,14 @@ echo "127.0.0.1 $REPO_URL" >> /etc/hosts
 service nginx reload
 
 rebuildd init
+
+
+cd $VINAIGRETTE_HOME/webhooks
+virtualenv venv
+source venv/bin/activate
+pip install -r requirements.txt
+
+cp $VINAIGRETTE_HOME/webhooks/service /etc/init.d/github-webhook
+systemctl daemon-reload
+updated-rc.d github-webhook defaults
+github-webhook

webhooks/requirements.txt

@@ -0,0 +1,5 @@
+bottle==0.12.9
+distribute>=0.6.24
+flup==1.0.3.dev20161029
+wsgiref==0.1.2
+urllib>=1.21.1

webhooks/server.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+import os
+import glob
+import json
+import time
+import shutil
+import tarfile
+import tempfile
+import threading
+import subprocess
+import collections
+import urllib.request
+from queue import Queue
+
+from bottle import route, request, run
+
+DISTRIBUTION = "jessie"
+BRANCHES=['stable', 'testing', 'unstable']
+
+# -- Variables
+
+# The backend to use for running the server
+# See: http://bottlepy.org/docs/0.12/deployment.html#switching-the-server-backend
+# Note: flup allows to run as FastCGI process
+SERVER_BACKEND = 'flup'
+
+# Host and port on which the server will listen
+HOST = '127.0.0.1'
+PORT = 9908
+
+# Path to the script to build a package
+BUILD_CMD = '/home/vinaigrette/scripts/build_deb'
+
+# Directory where build logs are stored
+BUILD_LOG_DIR = '/home/vinaigrette/webhooks/logs'
+
+# Number of worker threads
+WORKERS = 1
+
+# -- Types and methods
+
+# Represent a new release of a package
+PackageRelease = collections.namedtuple('PackageRelease', [
+    'name', 'tarball', 'tag', 'branch', 'pid',
+])
+
+
+def package_files(tar, pkgname, callback=None):
+    """Return the members of a package tarball safely."""
+    if not callable(callback):
+        callback = lambda m: None
+    for member in tar:
+        if member.name.startswith(pkgname):
+            callback(member.name)
+            yield member
+        else:
+            callback("nope: " + member.name)
+
+
+def build(pkg):
+    """Build the given PackageRelease."""
+    with open(os.path.join(
+            BUILD_LOG_DIR, "{0}.log".format(pkg.pid)), 'w') as logfile:
+        log = lambda m: print(m, file=logfile)
+        log("== Building {0} ==".format(pkg))
+
+        # create a temporary directory for the package
+        pkg_dir = tempfile.mkdtemp()
+
+        # download and extract tarball
+        log(":: Downloading and extract tarball to {0}...".format(pkg_dir))
+        tarball_stream = urllib.request.urlopen(pkg.tarball)
+        tarball = tarfile.open(fileobj=tarball_stream, mode="r|gz")
+        for member in tarball:
+            if member.name[0] not in ['/', '.']:
+                try:
+                    _, member.name = member.name.split('/', 1)
+                except ValueError:
+                    continue
+                log(member.name)
+                tarball.extract(member, pkg_dir)
+            else:
+                log("ignoring: {0}".format(member.name))
+        tarball.close()
+
+        # build the package
+        log("\n:: Building the Debian package...")
+        logfile.flush()
+        retcode = subprocess.call([
+                BUILD_CMD, '-d', pkg.branch,
+                '-c', DISTRIBUTION, '.'
+            ], cwd=pkg_dir, stdout=logfile, stderr=subprocess.STDOUT,
+        )
+
+        # cleaning...
+        shutil.rmtree(pkg_dir)
+        return retcode
+
+
+def worker():
+    while True:
+        item = queue.get()
+        if item is None:
+            break
+        build(item)
+        queue.task_done()
+
+
+# -- Web app and routes
+
+@route("/", method=['GET', 'POST'])
+def index():
+    if request.method == 'GET':
+        return ' Yolo '
+
+    if request.method == 'POST':
+
+        # Answer to GitHub events
+        if request.headers.get('X-GitHub-Event') == "ping":
+            return json.dumps({'msg': 'Hi!'})
+
+        # Only accept create event:
+        # https://developer.github.com/v3/activity/events/types/#releaseevent
+        if request.headers.get('X-GitHub-Event') != "release":
+            return json.dumps({'msg': "wrong event type"})
+
+        payload = request.json
+        if not payload:
+            return json.dumps({'msg': "invalid payload"})
+
+        # Only handle tag creation (create event also triggers for branch creation)
+        if payload['action'] != "published":
+            return json.dumps({'msg': "uninteresting action"})
+
+        # Validate tag and branch
+        tag = payload['release']['tag_name']
+        if not tag:
+            return json.dumps({'msg': "invalid tag"})
+        branch = payload['release']['target_commitish']
+        if branch not in BRANCHES:
+            return json.dumps({'msg': "uninteresting branch"})
+
+        # Init new package release and put it in the queue
+        package = PackageRelease(
+            name=payload['repository']['name'].lower(),
+            tarball=payload['release']['tarball_url'],
+            tag=tag, branch=branch, pid=int(time.time()),
+        )
+        queue.put(package, False)
+
+        # Return the pid to be able to track building
+        return "Queue id: {0}".format(package.pid)
+
+
+# -- Main program
+
+if __name__ == '__main__':
+    # Create the queue and the workers
+    queue = Queue()
+    threads = []
+    for i in range(WORKERS):
+        t = threading.Thread(target=worker)
+        t.start()
+        threads.append(t)
+
+    # Run the Web server
+    run(server=SERVER_BACKEND, host=HOST, port=PORT)
+
+    # Block until all tasks are done
+    queue.join()
+
+    # Stop workers
+    for i in range(WORKERS):
+        queue.put(None)
+        for t in threads:
+            t.join()

webhooks/service

@@ -0,0 +1,77 @@
+#! /bin/bash
+# github-webhook init script
+#
+### BEGIN INIT INFO
+# Provides:          github-webhook
+# Required-Start:    $remote_fs
+# Required-Stop:     $remote_fs
+# Should-Start:      $network
+# Should-Stop:       $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: rebuild daemon
+# Description:       daemon providing rebuild system
+#                    for Debian packages
+### END INIT INFO
+
+PATH=/home/vinaigrette/scripts/webhooks/venv/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON=/home/vinaitrette/webhooks/server.py
+NAME=github-webhook
+DESC="GitHub WebHook daemon"
+
+GWH_UID=pbuilder
+GWH_GROUP=pbuilder
+GWH_CHROOT=/home/vinaigrette/webhooks
+
+LOGDIR=/var/log/$NAME
+LOGFILE=$NAME.log
+
+test -x $DAEMON || exit 0
+
+# Include github-webhook defaults if available
+if [ -f /etc/default/github-webhook ] ; then
+	. /etc/default/github-webhook
+fi
+
+. /lib/lsb/init-functions
+set -e
+
+function do_start {
+  mkdir -p $LOGDIR
+  chown -R $GWH_UID:$GWH_GROUP $LOGDIR
+
+  start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \
+    --chuid $GWH_UID:$GWH_GROUP --chdir $GWH_CHROOT \
+    --background --make-pidfile --startas /bin/bash -- -c "exec python $DAEMON >> $LOGDIR/$LOGFILE 2>&1"
+}
+
+function do_stop {
+  start-stop-daemon --stop --quiet --oknodo --retry 120 --pidfile /var/run/$NAME.pid
+}
+
+case "$1" in
+  start)
+	log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        log_end_msg $?
+	;;
+  stop)
+	log_daemon_msg "Stopping $DESC" "$NAME"
+	do_stop
+	log_end_msg $?
+	;;
+  restart)
+	log_daemon_msg "Restarting $DESC" "$NAME"
+	do_stop
+        sleep 1
+        do_start
+	log_end_msg $?
+	;;
+  *)
+	N=/etc/init.d/$NAME
+	echo "Usage: $N {start|stop|restart}" >&2
+	exit 1
+	;;
+esac
+
+exit 0