Merge pull request #1190 from deis/deis-build

`deis pull` as alternative to `git push` workflow

Author: Gabriel Monroy

builder/templates/builder

@@ -34,14 +34,17 @@ ENTRYPOINT ["/runner/init"]
 
 if __name__ == '__main__':
     user, repo, branch, app = parse_args()
-    # define image names
-    _id = uuid.uuid4().hex[:8]
-    tmp_image = "{app}:temp_{_id}".format(**locals())
-    target_image = "{{ .deis_registry_host }}:{{ .deis_registry_port }}/{app}".format(**locals())
     # create required directories
     repo_dir = os.path.join(os.getcwd(), repo)
     build_dir = os.path.join(repo_dir, 'build')
     cache_dir = os.path.join(repo_dir, 'cache')
+    # get sha of branch
+    with open(os.path.join(repo_dir, branch)) as f:
+        sha = f.read().strip('\n')
+    short_sha = sha[:8]
+    # define image names
+    tmp_image = "{app}:git-{short_sha}".format(**locals())
+    target_image = "{{ .deis_registry_host }}:{{ .deis_registry_port }}/{app}".format(**locals())
     for d in (cache_dir, build_dir):
         if not os.path.exists(d):
             os.mkdir(d)
@@ -111,6 +114,7 @@ if __name__ == '__main__':
             raise Exception('Could not push Docker image')
         # construct json body for posting to the build hook
         body = {}
+        body['sha'] = sha
         body['receive_user'] = user
         body['receive_repo'] = app
         body['image'] = target_image
@@ -162,8 +166,8 @@ if __name__ == '__main__':
         l = locals()
         shutil.rmtree(temp_dir)
         if 'container' in l:
-            subprocess.Popen('docker rm {container}'.format(**l), shell=True,
+            subprocess.Popen('docker rm -f {container}'.format(**l), shell=True,
                              stdout=open(os.devnull, 'w'), stderr=open(os.devnull, 'w'))
-        if 'tmp_image' in l:
-            subprocess.Popen('docker rmi {tmp_image}'.format(**l), shell=True,
+        if 'tmp_image' in l and 'target_image' in l:
+            subprocess.Popen('docker rmi -f {tmp_image} {target_image}'.format(**l), shell=True,
                              stdout=open(os.devnull, 'w'), stderr=open(os.devnull, 'w'))

client/deis.py

@@ -92,11 +92,30 @@ def __init__(self):
             self.cookies.clear_expired_cookies()
             self.cookies.save()
 
+    @property
+    def app(self):
+        """Retrieve the application's name."""
+        try:
+            return self._get_name_from_git_remote(self.git_root())
+        except EnvironmentError:
+            return os.path.basename(os.getcwd())
+
+    def is_git_app(self):
+        """Determines if this app is a git repository. This is important in special cases
+        where we need to know whether or not we should use Deis' automatic app name
+        generator, for example.
+        """
+        try:
+            self.git_root()
+            return True
+        except EnvironmentError:
+            return False
+
     def git_root(self):
         """
-        Return the absolute path from the git repository root
+        Returns the absolute path from the git repository root.
 
-        If no git repository exists, raise an EnvironmentError
+        If no git repository exists, raises an EnvironmentError.
         """
         try:
             git_root = subprocess.check_output(
@@ -106,15 +125,13 @@ def git_root(self):
             raise EnvironmentError('Current directory is not a git repository')
         return git_root
 
-    def get_app(self):
+    def _get_name_from_git_remote(self, git_root):
         """
-        Return the application name for the current directory
+        Retrieves the application name from a git repository root.
 
         The application is determined by parsing `git remote -v` output.
-        If no application is found, raise an EnvironmentError.
+        If no application is found, raises an EnvironmentError.
         """
-        git_root = self.git_root()
-        # try to match a deis remote
         remotes = subprocess.check_output(['git', 'remote', '-v'],
                                           cwd=git_root)
         m = re.search(r'^deis\W+(?P<url>\S+)\W+\(', remotes, re.MULTILINE)
@@ -127,8 +144,6 @@ def get_app(self):
             raise EnvironmentError("Could not parse: {url}".format(**locals()))
         return m.groupdict()['app']
 
-    app = property(get_app)
-
     def request(self, *args, **kwargs):
         """
         Issue an HTTP request with proper cookie handling including
@@ -391,19 +406,13 @@ def apps_create(self, args):
         --cluster=CLUSTER      target cluster to host application [default: dev]
         --no-remote            do not create a 'deis' git remote
         """
-        try:
-            self._session.git_root()  # check for a git repository
-        except EnvironmentError:
-            print('No git repository found, use `git init` to create one')
-            sys.exit(1)
-        try:
-            self._session.get_app()
-            print('Deis remote already exists')
-            sys.exit(1)
-        except EnvironmentError:
-            pass
         body = {}
-        app_name = args.get('<id>')
+        app_name = None
+        if not self._session.is_git_app():
+            app_name = self._session.app
+        # prevent app name from being reset to None
+        if args.get('<id>'):
+            app_name = args.get('<id>')
         if app_name:
             body.update({'id': app_name})
         cluster = args.get('--cluster')
@@ -423,8 +432,11 @@ def apps_create(self, args):
             data = response.json()
             app_id = data['id']
             print("done, created {}".format(app_id))
-            # add a git remote
-            # TODO: retrieve the hostname from service discovery
+            # set a git remote if necessary
+            try:
+                self._session.git_root()
+            except EnvironmentError:
+                return
             hostname = urlparse.urlparse(self._settings['controller']).netloc.split(':')[0]
             git_remote = "ssh://git@{hostname}:2222/{app_id}.git".format(**locals())
             if args.get('--no-remote'):
@@ -476,9 +488,9 @@ def apps_destroy(self, args):
         if response.status_code in (requests.codes.no_content,  # @UndefinedVariable
                                     requests.codes.not_found):  # @UndefinedVariable
             print('done in {}s'.format(int(time.time() - before)))
-            # If the requested app is in the current dir, delete the git remote
             try:
-                if app == self._session.app:
+                # If the requested app is a heroku app, delete the git remote
+                if self._session.is_git_app():
                     subprocess.check_call(
                         ['git', 'remote', 'rm', 'deis'],
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -729,9 +741,19 @@ def builds(self, args):
 
     def builds_create(self, args):
         """
-        Create a new build of an application
+        Creates a new build of an application. Imports an <image> and deploys it to Deis
+        as a new release.
 
         Usage: deis builds:create <image> [--app=<app>]
+
+        Arguments:
+          <image>
+            A fully-qualified docker image, either from DockerHub (e.g. deis/example-go)
+            or from an in-house registry (e.g. myregistry.example.com:5000/example-go).
+
+        Options:
+          --app=<app>
+            The uniquely identifiable name for the application.
         """
         app = args.get('--app')
         if not app:
@@ -1178,12 +1200,13 @@ def ps_scale(self, args):
         """
         app = args.get('--app')
         if not app:
-            app = self._session.get_app()
+            app = self._session.app
         body = {}
         for type_num in args.get('<type=num>'):
             typ, count = type_num.split('=')
             body.update({typ: int(count)})
-        print('Scaling processes... but first, coffee!')
+        sys.stdout.write('Scaling processes... but first, coffee!\n')
+        sys.stdout.flush()
         try:
             progress = TextProgress()
             progress.start()
@@ -1195,7 +1218,7 @@ def ps_scale(self, args):
             progress.cancel()
             progress.join()
         if response.status_code == requests.codes.no_content:  # @UndefinedVariable
-            print('done in {}s\n'.format(int(time.time() - before)))
+            print('done in {}s'.format(int(time.time() - before)))
             self.ps_list({}, app)
         else:
             raise ResponseError(response)
@@ -1449,7 +1472,7 @@ def releases_list(self, args):
             data = response.json()
             for item in data['results']:
                 item['created'] = readable_datetime(item['created'])
-                print("v{version:<6} {created:<33} {summary}".format(**item))
+                print("v{version:<6} {created:<24} {summary}".format(**item))
         else:
             raise ResponseError(response)
 
@@ -1470,9 +1493,18 @@ def releases_rollback(self, args):
         else:
             body = {}
         url = "/api/apps/{app}/releases/rollback".format(**locals())
-        response = self._dispatch('post', url, json.dumps(body))
-        if response.status_code == requests.codes.created:
-            print(response.json())
+        sys.stdout.write('Rolling back to v{version}... '.format(**locals()))
+        sys.stdout.flush()
+        try:
+            progress = TextProgress()
+            progress.start()
+            response = self._dispatch('post', url, json.dumps(body))
+        finally:
+            progress.cancel()
+            progress.join()
+        if response.status_code == requests.codes.created:  # @UndefinedVariable
+            new_version = response.json()['version']
+            print("done, v{}".format(new_version))
         else:
             raise ResponseError(response)
 
@@ -1499,6 +1531,7 @@ def shortcuts(self, args):
     ('register', 'auth:register'),
     ('login', 'auth:login'),
     ('logout', 'auth:logout'),
+    ('pull', 'builds:create'),
     ('scale', 'ps:scale'),
     ('rollback', 'releases:rollback'),
     ('sharing', 'perms:list'),

controller/Makefile

@@ -48,7 +48,7 @@ test: test-unit test-functional
 
 test-unit:
 	@if [ ! -d venv ]; then virtualenv venv; fi
-	venv/bin/pip install -q -r requirements.txt
+	venv/bin/pip install -q -r requirements.txt -r dev_requirements.txt
 	venv/bin/python manage.py test --noinput api
 
 test-functional:

controller/api/models.py

@@ -462,7 +462,7 @@ class Meta:
     def __str__(self):
         return "{0}-v{1}".format(self.app.id, self.version)
 
-    def new(self, user, config=None, build=None, summary=None, source_version=None):
+    def new(self, user, config=None, build=None, summary=None, source_version='latest'):
         """
         Create a new application release using the provided Build and Config
         on behalf of a user.
@@ -473,24 +473,28 @@ def new(self, user, config=None, build=None, summary=None, source_version=None):
             config = self.config
         if not build:
             build = self.build
-        if not source_version:
-            source_version = 'latest'
-        else:
-            source_version = 'v{}'.format(source_version)
-        # prepare release tag
+        # always create a release off the latest image
+        source_image = '{}:{}'.format(build.image, source_version)
+        # construct fully-qualified target image
         new_version = self.version + 1
         tag = 'v{}'.format(new_version)
-        image = build.image + ':{tag}'.format(**locals())
+        release_image = '{}:{}'.format(self.app.id, tag)
+        target_image = '{}:{}/{}'.format(
+            settings.REGISTRY_HOST, settings.REGISTRY_PORT, self.app.id)
         # create new release and auto-increment version
         release = Release.objects.create(
             owner=user, app=self.app, config=config,
-            build=build, version=new_version, image=image, summary=summary)
-        # publish release to registry as new docker image
-        repository_path = self.app.id
-        publish_release(repository_path,
+            build=build, version=new_version, image=target_image, summary=summary)
+        # IOW, this image did not come from the builder
+        if not build.sha:
+            # we assume that the image is not present on our registry,
+            # so shell out a task to pull in the repository
+            tasks.import_repository.delay(build.image, self.app.id).get()
+            # update the source image to the repository we just imported
+            source_image = self.app.id
+        publish_release(source_image,
                         config.values,
-                        tag,
-                        source_tag=source_version)
+                        release_image,)
         return release
 
     def previous(self):
@@ -516,7 +520,9 @@ def save(self, *args, **kwargs):
             # compare this build to the previous build
             old_build = prev_release.build if prev_release else None
             # if the build changed, log it and who pushed it
-            if self.build != old_build:
+            if self.version == 1:
+                self.summary += "{} created initial release".format(self.app.owner)
+            elif self.build != old_build:
                 if self.build.sha:
                     self.summary += "{} deployed {}".format(self.build.owner, self.build.sha[:7])
                 else:

controller/api/tasks.py

@@ -7,9 +7,11 @@
 
 from __future__ import unicode_literals
 
+import requests
 import threading
 
 from celery import task
+from django.conf import settings
 
 
 @task
@@ -34,6 +36,19 @@ def deploy_release(app, release):
     [t.join() for t in threads]
 
 
+@task
+def import_repository(source, target_repository):
+    """Imports an image from a remote registry into our own private registry"""
+    data = {
+        'src': source,
+    }
+    requests.post(
+        '{}/v1/repositories/{}/tags'.format(settings.REGISTRY_URL,
+                                            target_repository),
+        data=data,
+    )
+
+
 @task
 def start_containers(containers):
     create_threads = []
@@ -71,10 +86,10 @@ def run_command(c, command):
         if rc != 0:
             raise EnvironmentError('Could not pull image: {pull_image}'.format(**locals()))
         # run the command
-        docker_args = ' '.join(['-a', 'stdout', '-a', 'stderr', '--rm', image])
-        env_args = ' '.join(["-e '{k}={v}'".format(**locals())
-                             for k, v in release.config.values.items()])
-        command = "docker run {env_args} {docker_args} {command}".format(**locals())
+        docker_args = ' '.join(['--entrypoint=/bin/sh',
+                                '-a', 'stdout', '-a', 'stderr', '--rm', image])
+        escaped_command = command.replace("'", "'\\''")
+        command = r"docker run {docker_args} -c \'{escaped_command}\'".format(**locals())
         return c.run(command)
     finally:
         c.delete()