vinaigrette/webhooks/server.py

#!/usr/bin/env python3
import os
import glob
import json
import time
import shutil
import tarfile
import tempfile
import threading
import subprocess
import collections
import urllib.request
from queue import Queue

from bottle import route, request, run

DISTRIBUTION = "jessie"
BRANCHES=['stable', 'testing', 'unstable']

# -- Variables

# The backend to use for running the server
# See: http://bottlepy.org/docs/0.12/deployment.html#switching-the-server-backend
# Note: flup allows to run as FastCGI process
SERVER_BACKEND = 'flup'

# Host and port on which the server will listen
HOST = '127.0.0.1'
PORT = 9908

# Path to the script to build a package
BUILD_CMD = '/home/vinaigrette/scripts/build_deb'

# Directory where build logs are stored
BUILD_LOG_DIR = '/home/vinaigrette/webhooks/logs'

# Number of worker threads
WORKERS = 1

# -- Types and methods

# Represent a new release of a package
PackageRelease = collections.namedtuple('PackageRelease', [
    'name', 'tarball', 'tag', 'branch', 'pid',
])


def package_files(tar, pkgname, callback=None):
    """Return the members of a package tarball safely."""
    if not callable(callback):
        callback = lambda m: None
    for member in tar:
        if member.name.startswith(pkgname):
            callback(member.name)
            yield member
        else:
            callback("nope: " + member.name)


def build(pkg):
    """Build the given PackageRelease."""
    with open(os.path.join(
            BUILD_LOG_DIR, "{0}.log".format(pkg.pid)), 'w') as logfile:
        log = lambda m: print(m, file=logfile)
        log("== Building {0} ==".format(pkg))

        # create a temporary directory for the package
        pkg_dir = tempfile.mkdtemp()

        # download and extract tarball
        log(":: Downloading and extract tarball to {0}...".format(pkg_dir))
        tarball_stream = urllib.request.urlopen(pkg.tarball)
        tarball = tarfile.open(fileobj=tarball_stream, mode="r|gz")
        for member in tarball:
            if member.name[0] not in ['/', '.']:
                try:
                    _, member.name = member.name.split('/', 1)
                except ValueError:
                    continue
                log(member.name)
                tarball.extract(member, pkg_dir)
            else:
                log("ignoring: {0}".format(member.name))
        tarball.close()

        # build the package
        log("\n:: Building the Debian package...")
        logfile.flush()
        retcode = subprocess.call([
                BUILD_CMD, '-d', pkg.branch,
                '-c', DISTRIBUTION, '.'
            ], cwd=pkg_dir, stdout=logfile, stderr=subprocess.STDOUT,
        )

        # cleaning...
        shutil.rmtree(pkg_dir)
        return retcode


def worker():
    while True:
        item = queue.get()
        if item is None:
            break
        build(item)
        queue.task_done()


# -- Web app and routes

@route("/", method=['GET', 'POST'])
def index():
    if request.method == 'GET':
        return ' Yolo '

    if request.method == 'POST':

        # Answer to GitHub events
        if request.headers.get('X-GitHub-Event') == "ping":
            return json.dumps({'msg': 'Hi!'})

        # Only accept create event:
        # https://developer.github.com/v3/activity/events/types/#releaseevent
        if request.headers.get('X-GitHub-Event') != "release":
            return json.dumps({'msg': "wrong event type"})

        payload = request.json
        if not payload:
            return json.dumps({'msg': "invalid payload"})

        # Only handle tag creation (create event also triggers for branch creation)
        if payload['action'] != "published":
            return json.dumps({'msg': "uninteresting action"})

        # Validate tag and branch
        tag = payload['release']['tag_name']
        if not tag:
            return json.dumps({'msg': "invalid tag"})
        branch = payload['release']['target_commitish']
        if branch not in BRANCHES:
            return json.dumps({'msg': "uninteresting branch"})

        # Init new package release and put it in the queue
        package = PackageRelease(
            name=payload['repository']['name'].lower(),
            tarball=payload['release']['tarball_url'],
            tag=tag, branch=branch, pid=int(time.time()),
        )
        queue.put(package, False)

        # Return the pid to be able to track building
        return "Queue id: {0}".format(package.pid)


# -- Main program

if __name__ == '__main__':
    # Create the queue and the workers
    queue = Queue()
    threads = []
    for i in range(WORKERS):
        t = threading.Thread(target=worker)
        t.start()
        threads.append(t)

    # Run the Web server
    run(server=SERVER_BACKEND, host=HOST, port=PORT)

    # Block until all tasks are done
    queue.join()

    # Stop workers
    for i in range(WORKERS):
        queue.put(None)
        for t in threads:
            t.join()