deis-build-hook

#!/usr/bin/env python
import argparse
import getpass
import json
import os
import shutil
import subprocess
import sys
import tempfile
import yaml

def parse_args():
    desc = """
Process a push by running it through the buildpack process

Note this script must be run as the `git` user.
"""
    parser = argparse.ArgumentParser(description=desc,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('src', action='store',
                        help="path to source repository")
    parser.add_argument('user', action='store',
                        help="name of user who started build",
                        default='system')
    parser.add_argument('--buildpack_root', action='store',
                        help="path to buildpack root directory",
                        default="/opt/deis/build/buildpacks")
    parser.add_argument('--image', action='store',
                        help="docker image to build from",
                        default="lucid64-build")
    # check execution environment
    if getpass.getuser() != 'git':
        parser.print_help()
        sys.exit(1)
    args = parser.parse_args()
    args.src = os.path.abspath(args.src)
    # store formation ID
    args.formation = '/'.join(args.src.split(os.path.sep)[-1:]).replace('.git', '')
    return args

def puts_step(s):
    sys.stdout.write("-----> %s\n" % s)
    sys.stdout.flush()

def puts_line():
    sys.stdout.write("\n")
    sys.stdout.flush()

def puts(s):
    sys.stdout.write("       " + s)
    sys.stdout.flush()

def get_buildpack_url(args):
    # TODO: replace with `deis config` lookup
    config = {}
    buildpack = config.get('BUILDPACK_URL')
    return buildpack
    
def detect(args, detect_dir):
    """
    Detect the repository type using the local buildpack library

    Default library location: /opt/deis/build/buildpacks

    Languages detected (in order):
     * Ruby
     * Node.js
     * Python
     * Java
     * Clojure
     * Go
    """
    for lang in ('ruby', 'nodejs', 'python', 'java', 'clojure', 'go'):
        buildpack_path = os.path.join(args.buildpack_root, lang)
        detect_cmd = os.path.join(buildpack_path, "bin", "detect")
        p = subprocess.Popen([detect_cmd, detect_dir], stdout=subprocess.PIPE)
        rc = p.wait()
        detected = p.stdout.read().replace('\n', '').capitalize()
        if rc == 0:
            if detected == 'Nodejs':
                detected = 'Node.js'
            puts_step("%(detected)s app detected" % locals())
            return buildpack_path
    sys.stderr.write('Could not detect buildpack, exiting...\n')
    sys.stderr.flush()
    sys.exit(1)

def read_procfile(build_dir):
    puts_step('Discovering process types')
    procfile = os.path.join(build_dir, 'Procfile')
    if not os.path.exists(procfile):
       puts("Error: Procfile missing from repository root")
       sys.exit(1)
    with open(procfile) as f:
        data = f.read()
    y = yaml.safe_load(data)
    p_types = y.keys()
    puts("Procfile declares types -> %s\n" % (' '.join(p_types)))
    return y

def _sizeof_fmt(num):
    for x in ['bytes','KB','MB','GB','TB']:
        if num < 1024.0:
            return "%3.1f %s" % (num, x)
        num /= 1024.0

def compile(args, buildpack):
    """
    Return a Docker container ready for compilation
    """
    images = []
    # create required directories
    deis_dir = os.path.join(args.src, 'deis')
    pack_dir = os.path.join(deis_dir, 'pack')
    build_dir = os.path.join(deis_dir, 'build')
    cache_dir = os.path.join(deis_dir, 'cache')
    slug_dir = os.path.join('/opt/deis/build/slugs') # make this dynamic
    for d in (deis_dir, build_dir, cache_dir, slug_dir):
        if not os.path.exists(d):
            os.mkdir(d)
    # buildpacks _should_ work with a read-only bind mount
    # but the python pack requires write access during
    # `python setup.py install` of vendored libs
    # .. so let's clone a repo-specific buildpack on every run
    if os.path.exists(pack_dir):
        shutil.rmtree(pack_dir)
    subprocess.check_call(['git', 'clone', buildpack, pack_dir], cwd=args.src,
                          stdout=args.devnull)
    # run the buildpack "compile" step
    # /build contains fresh repo to be built in place
    # /cache contains artifacts persisted across compiles
    cmd = ("docker run -d "
          "-b {buildpack}:/pack:rw "
          "-b {build_dir}:/build:rw "
          "-b {cache_dir}:/cache:rw "
          "{args.image} "
          "/pack/bin/compile /build /cache").format(**locals())
    container = subprocess.check_output(cmd, shell=True).strip('\n')
    # attach to the container to stream build output
    cmd = "docker attach {container}".format(**locals())
    subprocess.check_call(cmd, shell=True)
    # wait for the container to finish and check its return code
    cmd = "docker wait {container}".format(**locals())
    rc = subprocess.check_call(cmd, shell=True, stdout=args.devnull)
    if rc != 0:
        raise RuntimeError("Compile returned %s" % rc)
    # chown directories back to git.git after build/cache updates
    cmd = ("docker run "
          "-b {build_dir}:/build:rw "
          "-b {cache_dir}:/cache:rw "
          "{args.image} "
          "chown -R 325:325 /build /cache").format(**locals()) # maps to git.git
    subprocess.check_call(cmd, shell=True)
    # tar up a slug from the build directory
    sha = args.sha
    build_path = os.path.join(slug_dir, "%(sha)s.tar.gz" % locals())
    cmd = "tar -C {build_dir} --exclude=.git* -cz . -f {build_path}".format(**locals())
    subprocess.check_call(cmd, shell=True)
    return build_path, sha

def release(args, buildpack, slug_path):
    deis_dir = os.path.join(args.src, 'deis')
    build_dir = os.path.join(deis_dir, 'build')
    # call release and deserialize the result
    cmd = ("docker run "
          "-b {buildpack}:/pack:rw "
          "-b {build_dir}:/build:rw "
          "{args.image} "
          "/pack/bin/release /build").format(**locals())
    stdout = subprocess.check_output(cmd % locals(), shell=True)
    release_out = yaml.safe_load(stdout) # sets release['config_vars']
    build = {'formation': args.formation}
    build['config'] = release_out['config_vars']
    # update with slug url
    sha = args.sha
    build['ssh_key'] = args.user
    build['sha'] = args.sha
    build['url'] = "/%(sha)s.tar.gz" % locals()
    build['procfile'] = read_procfile(build_dir)
    # calculate slug size
    build['size'] = os.stat(slug_path).st_size
    build['checksum'] = subprocess.check_output(['sha256sum', slug_path]).split(' ')[0]
    puts_line()
    puts_step("Compiled slug size: %s" % _sizeof_fmt(build['size']))
    return build

def run(args, build):
    sys.stdout.write("       " + "Launching... ")
    sys.stdout.flush()
    p = subprocess.Popen(['sudo', '-u', 'deis', '/opt/deis/controller/bin/build-release-run'], 
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate(json.dumps(build))
    rc = p.wait()
    if rc != 0:
        raise RuntimeError('Build creation error {0}'.format(stderr))
    databag = json.loads(stdout)
    sys.stdout.write("done\n")
    sys.stdout.flush()
    puts_line()
    puts_step("{args.formation} deployed to Deis".format(**locals()))
    for proxy_fqdn in databag['nodes']['proxies'].values():
        puts("http://{proxy_fqdn}\n".format(**locals()))
    
if __name__ == '__main__':
    args = parse_args()
    args.devnull = open('/dev/null', 'w')
    puts_line()
    buildpack = get_buildpack_url(args)
    # completely wipe and re-clone the build directory
    deis_dir = os.path.join(args.src, 'deis')
    if not os.path.exists(deis_dir):
        os.mkdir(deis_dir)
    build_dir = os.path.join(deis_dir, 'build')
    if os.path.exists(build_dir):
        shutil.rmtree(build_dir)
    subprocess.check_call(['git', 'clone', args.src, build_dir], stdout=args.devnull)
    # get the sha of the repository we're building
    args.sha = subprocess.check_output(['cat', '.git/refs/heads/master'], cwd=build_dir).strip('\n')
    subprocess.check_call(['rm', '-rf', os.path.join(build_dir, '.git')]) # remove .git
    # look for custom buildpack
    if not buildpack:
        buildpack = detect(args, build_dir)
    # TODO: capture build output
    slug_path, sha = compile(args, buildpack)
    release = release(args, buildpack, slug_path)
    run(args, release)
    puts_line()