#!/usr/bin/env python import argparse import getpass import json import os import shutil import subprocess import sys import tempfile import yaml def parse_args(): desc = """ Process a push by running it through the buildpack process Note this script must be run as the `git` user. """ parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('src', action='store', help="path to source repository") parser.add_argument('user', action='store', help="name of user who started build", default='system') parser.add_argument('--buildpack_root', action='store', help="path to buildpack root directory", default="/opt/deis/build/buildpacks") parser.add_argument('--image', action='store', help="docker image to build from", default="lucid64-build") # check execution environment if getpass.getuser() != 'git': parser.print_help() sys.exit(1) args = parser.parse_args() args.src = os.path.abspath(args.src) # store formation ID args.formation = '/'.join(args.src.split(os.path.sep)[-1:]).replace('.git', '') return args def puts_step(s): sys.stdout.write("-----> %s\n" % s) sys.stdout.flush() def puts_line(): sys.stdout.write("\n") sys.stdout.flush() def puts(s): sys.stdout.write(" " + s) sys.stdout.flush() def get_buildpack_url(args): # TODO: replace with `deis config` lookup config = {} buildpack = config.get('BUILDPACK_URL') return buildpack def detect(args, detect_dir): """ Detect the repository type using the local buildpack library Default library location: /opt/deis/build/buildpacks Languages detected (in order): * Ruby * Node.js * Python * Java * Clojure * Go """ for lang in ('ruby', 'nodejs', 'python', 'java', 'clojure', 'go'): buildpack_path = os.path.join(args.buildpack_root, lang) detect_cmd = os.path.join(buildpack_path, "bin", "detect") p = subprocess.Popen([detect_cmd, detect_dir], stdout=subprocess.PIPE) rc = p.wait() detected = p.stdout.read().replace('\n', '').capitalize() if rc == 0: if detected == 'Nodejs': detected = 'Node.js' puts_step("%(detected)s app detected" % locals()) return buildpack_path sys.stderr.write('Could not detect buildpack, exiting...\n') sys.stderr.flush() sys.exit(1) def read_procfile(build_dir): puts_step('Discovering process types') procfile = os.path.join(build_dir, 'Procfile') if not os.path.exists(procfile): puts("Error: Procfile missing from repository root") sys.exit(1) with open(procfile) as f: data = f.read() y = yaml.safe_load(data) p_types = y.keys() puts("Procfile declares types -> %s\n" % (' '.join(p_types))) return y def _sizeof_fmt(num): for x in ['bytes','KB','MB','GB','TB']: if num < 1024.0: return "%3.1f %s" % (num, x) num /= 1024.0 def compile(args, buildpack): """ Return a Docker container ready for compilation """ images = [] # create required directories deis_dir = os.path.join(args.src, 'deis') pack_dir = os.path.join(deis_dir, 'pack') build_dir = os.path.join(deis_dir, 'build') cache_dir = os.path.join(deis_dir, 'cache') slug_dir = os.path.join('/opt/deis/build/slugs') # make this dynamic for d in (deis_dir, build_dir, cache_dir, slug_dir): if not os.path.exists(d): os.mkdir(d) # buildpacks _should_ work with a read-only bind mount # but the python pack requires write access during # `python setup.py install` of vendored libs # .. so let's clone a repo-specific buildpack on every run if os.path.exists(pack_dir): shutil.rmtree(pack_dir) subprocess.check_call(['git', 'clone', buildpack, pack_dir], cwd=args.src, stdout=args.devnull) # run the buildpack "compile" step # /build contains fresh repo to be built in place # /cache contains artifacts persisted across compiles cmd = ("docker run -d " "-b {buildpack}:/pack:rw " "-b {build_dir}:/build:rw " "-b {cache_dir}:/cache:rw " "{args.image} " "/pack/bin/compile /build /cache").format(**locals()) container = subprocess.check_output(cmd, shell=True).strip('\n') # attach to the container to stream build output cmd = "docker attach {container}".format(**locals()) subprocess.check_call(cmd, shell=True) # wait for the container to finish and check its return code cmd = "docker wait {container}".format(**locals()) rc = subprocess.check_call(cmd, shell=True, stdout=args.devnull) if rc != 0: raise RuntimeError("Compile returned %s" % rc) # chown directories back to git.git after build/cache updates cmd = ("docker run " "-b {build_dir}:/build:rw " "-b {cache_dir}:/cache:rw " "{args.image} " "chown -R 325:325 /build /cache").format(**locals()) # maps to git.git subprocess.check_call(cmd, shell=True) # tar up a slug from the build directory sha = args.sha build_path = os.path.join(slug_dir, "%(sha)s.tar.gz" % locals()) cmd = "tar -C {build_dir} --exclude=.git* -cz . -f {build_path}".format(**locals()) subprocess.check_call(cmd, shell=True) return build_path, sha def release(args, buildpack, slug_path): deis_dir = os.path.join(args.src, 'deis') build_dir = os.path.join(deis_dir, 'build') # call release and deserialize the result cmd = ("docker run " "-b {buildpack}:/pack:rw " "-b {build_dir}:/build:rw " "{args.image} " "/pack/bin/release /build").format(**locals()) stdout = subprocess.check_output(cmd % locals(), shell=True) release_out = yaml.safe_load(stdout) # sets release['config_vars'] build = {'formation': args.formation} build['config'] = release_out['config_vars'] # update with slug url sha = args.sha build['ssh_key'] = args.user build['sha'] = args.sha build['url'] = "/%(sha)s.tar.gz" % locals() build['procfile'] = read_procfile(build_dir) # calculate slug size build['size'] = os.stat(slug_path).st_size build['checksum'] = subprocess.check_output(['sha256sum', slug_path]).split(' ')[0] puts_line() puts_step("Compiled slug size: %s" % _sizeof_fmt(build['size'])) return build def run(args, build): sys.stdout.write(" " + "Launching... ") sys.stdout.flush() p = subprocess.Popen(['sudo', '-u', 'deis', '/opt/deis/controller/bin/build-release-run'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate(json.dumps(build)) rc = p.wait() if rc != 0: raise RuntimeError('Build creation error {0}'.format(stderr)) databag = json.loads(stdout) sys.stdout.write("done\n") sys.stdout.flush() puts_line() puts_step("{args.formation} deployed to Deis".format(**locals())) for proxy_fqdn in databag['nodes']['proxies'].values(): puts("http://{proxy_fqdn}\n".format(**locals())) if __name__ == '__main__': args = parse_args() args.devnull = open('/dev/null', 'w') puts_line() buildpack = get_buildpack_url(args) # completely wipe and re-clone the build directory deis_dir = os.path.join(args.src, 'deis') if not os.path.exists(deis_dir): os.mkdir(deis_dir) build_dir = os.path.join(deis_dir, 'build') if os.path.exists(build_dir): shutil.rmtree(build_dir) subprocess.check_call(['git', 'clone', args.src, build_dir], stdout=args.devnull) # get the sha of the repository we're building args.sha = subprocess.check_output(['cat', '.git/refs/heads/master'], cwd=build_dir).strip('\n') subprocess.check_call(['rm', '-rf', os.path.join(build_dir, '.git')]) # remove .git # look for custom buildpack if not buildpack: buildpack = detect(args, build_dir) # TODO: capture build output slug_path, sha = compile(args, buildpack) release = release(args, buildpack, slug_path) run(args, release) puts_line()