ref(scheduler): introduce deis/publisher

In Deis's scheduler abstraction, we run a "sidecar" systemd unit which
publishes the running host and port of the application to etcd for
service discovery.

The issues with this sidecar container are as follows:

- it must live on the same host as the container
- it must wait for the container to come up before publishing
- it relies on using the docker client directly

The last point is crucial. This means that we are effectively
monitoring the docker client for container uptime, which is not the
same as monitoring the container directly.

This has been refactored into a micro-service image called
deis/publisher. Publisher listens directly to a docker socket
bind-mounted into the container and listens to the docker events API
for running containers. Running containers that follow the same format
as our current naming scheme (e.g. hardy-woodsman_v2.web.1) are
published to etcd for service discovery. This allows us to remove the
hard dependency on running a sidecar container to publish the app
container to etcd. Instead, one container per host publishes all
apps running on that host.

MIGRATING: to migrate to this change from existing hosts, you can run
the following:

    ><> # rsync publisher/systemd/deis-publisher.service to every node
    ><> for i in deis-1.example.com deis-2.example.com; do ssh core@$i -c "sudo systemctl enable /home/core/deis-publisher.service && sudo systemctl start /home/core/deis-publisher"

Then, you'll want to remove each announce sidecar attached to the applications:

    ><> fleetctl destroy <appname>-_<version>.<proctype>.<conatiner ID>-announce

Author: Matthew Fisher

Makefile

@@ -4,8 +4,8 @@
 
 include includes.mk
 
-COMPONENTS=builder cache controller database logger registry router
-START_ORDER=logger database cache registry controller builder router
+COMPONENTS=builder cache controller database logger publisher registry router
+START_ORDER=publisher logger database cache registry controller builder router
 CLIENTS=client deisctl
 
 all: build run

controller/api/models.py

@@ -436,9 +436,6 @@ def _get_command(self):
 
     _command = property(_get_command)
 
-    def _command_announceable(self):
-        return self._command.lower() in ['start web', '']
-
     def clone(self, release):
         c = Container.objects.create(owner=self.owner,
                                      app=self.app,
@@ -459,7 +456,7 @@ def create(self):
                 name=job_id,
                 image=image,
                 command=self._command,
-                use_announcer=self._command_announceable(), **kwargs)
+                **kwargs)
         except Exception as e:
             err = '{} (create): {}'.format(job_id, e)
             log_event(self.app, err, logging.ERROR)
@@ -469,7 +466,7 @@ def create(self):
     def start(self):
         job_id = self._job_id
         try:
-            self._scheduler.start(job_id, self._command_announceable())
+            self._scheduler.start(job_id)
         except Exception as e:
             err = '{} (start): {}'.format(job_id, e)
             log_event(self.app, err, logging.WARNING)
@@ -479,7 +476,7 @@ def start(self):
     def stop(self):
         job_id = self._job_id
         try:
-            self._scheduler.stop(job_id, self._command_announceable())
+            self._scheduler.stop(job_id)
         except Exception as e:
             err = '{} (stop): {}'.format(job_id, e)
             log_event(self.app, err, logging.ERROR)
@@ -489,7 +486,7 @@ def stop(self):
     def destroy(self):
         job_id = self._job_id
         try:
-            self._scheduler.destroy(job_id, self._command_announceable())
+            self._scheduler.destroy(job_id)
         except Exception as e:
             err = '{} (destroy): {}'.format(job_id, e)
             log_event(self.app, err, logging.ERROR)

controller/scheduler/chaos.py

@@ -25,28 +25,28 @@ def tearDown(self):
 
     # job api
 
-    def create(self, name, image, command, use_announcer, **kwargs):
+    def create(self, name, image, command, **kwargs):
         if random.random() < CREATE_ERROR_RATE:
             raise RuntimeError
         return True
 
-    def start(self, name, use_announcer):
+    def start(self, name):
         """
         Start an idle job
         """
         if random.random() < START_ERROR_RATE:
             raise RuntimeError
         return True
 
-    def stop(self, name, use_announcer):
+    def stop(self, name):
         """
         Stop a running job
         """
         if random.random() < STOP_ERROR_RATE:
             raise RuntimeError
         return True
 
-    def destroy(self, name, use_announcer):
+    def destroy(self, name):
         """
         Destroy an existing job
         """

controller/scheduler/coreos.py

@@ -101,15 +101,12 @@ def _get_machines(self):
 
     # container api
 
-    def create(self, name, image, command='', template=None, use_announcer=True, **kwargs):
+    def create(self, name, image, command='', template=None, **kwargs):
         """Create a container"""
         self._create_container(name, image, command,
                                template or copy.deepcopy(CONTAINER_TEMPLATE), **kwargs)
         self._create_log(name, image, command, copy.deepcopy(LOG_TEMPLATE))
 
-        if use_announcer:
-            self._create_announcer(name, image, command, copy.deepcopy(ANNOUNCE_TEMPLATE))
-
     def _create_container(self, name, image, command, unit, **kwargs):
         l = locals().copy()
         l.update(re.match(MATCH, name).groupdict())
@@ -146,22 +143,10 @@ def _create_log(self, name, image, command, unit):
         # post unit to fleet
         self._put_unit(name+'-log', {"desiredState": "launched", "options": unit})
 
-    def _create_announcer(self, name, image, command, unit):
-        l = locals().copy()
-        l.update(re.match(MATCH, name).groupdict())
-        # construct unit from template
-        for f in unit:
-            f['value'] = f['value'].format(**l)
-        # post unit to fleet
-        self._put_unit(name+'-announce', {"desiredState": "launched", "options": unit})
-
-    def start(self, name, use_announcer=True):
+    def start(self, name):
         """Start a container"""
         self._wait_for_container(name)
 
-        if use_announcer:
-            self._wait_for_announcer(name)
-
     def _wait_for_container(self, name):
         # we bump to 20 minutes here to match the timeout on the router and in the app unit files
         for _ in range(1200):
@@ -177,24 +162,6 @@ def _wait_for_container(self, name):
         else:
             raise RuntimeError('container failed to start')
 
-    def _wait_for_announcer(self, name):
-        # wait a bit for the announcer to come up, otherwise we may have hit
-        # https://github.com/docker/docker/issues/8022
-        for _ in range(30):
-            states = self._get_state(name)
-            if states and len(states.get('states', [])) == 1:
-                state = states.get('states')[0]
-                subState = state.get('systemdSubState')
-                if subState == 'running':
-                    # wait for the router to be reconfigured
-                    time.sleep(10)
-                    break
-                elif subState == 'failed':
-                    raise RuntimeError('announcer failed to start')
-            time.sleep(1)
-        else:
-            raise RuntimeError('announcer timeout on start')
-
     def _wait_for_destroy(self, name):
         for _ in range(30):
             states = self._get_state(name)
@@ -204,15 +171,13 @@ def _wait_for_destroy(self, name):
         else:
             raise RuntimeError('timeout on container destroy')
 
-    def stop(self, name, use_announcer=True):
+    def stop(self, name):
         """Stop a container"""
         raise NotImplementedError
 
-    def destroy(self, name, use_announcer=True):
+    def destroy(self, name):
         """Destroy a container"""
         funcs = []
-        if use_announcer:
-            funcs.append(functools.partial(self._destroy_announcer, name))
         funcs.append(functools.partial(self._destroy_container, name))
         funcs.append(functools.partial(self._destroy_log, name))
         # call all destroy functions, ignoring any errors
@@ -226,9 +191,6 @@ def destroy(self, name, use_announcer=True):
     def _destroy_container(self, name):
         return self._delete_unit(name)
 
-    def _destroy_announcer(self, name):
-        return self._delete_unit(name+'-announce')
-
     def _destroy_log(self, name):
         return self._delete_unit(name+'-log')
 
@@ -350,18 +312,6 @@ def attach(self, name):
 ]
 
 
-ANNOUNCE_TEMPLATE = [
-    {"section": "Unit", "name": "Description", "value": "{name} announce"},
-    {"section": "Unit", "name": "BindsTo", "value": "{name}.service"},
-    {"section": "Service", "name": "EnvironmentFile", "value": "/etc/environment"},
-    {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "until docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name} >/dev/null 2>&1; do sleep 2; done; port=$(docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name}); if [[ -z $port ]]; then echo We have no port...; exit 1; fi; echo Waiting for $port/tcp...; until netstat -lnt | grep :$port >/dev/null; do sleep 1; done"'''},  # noqa
-    {"section": "Service", "name": "ExecStart", "value": '''/bin/sh -c "port=$(docker inspect -f '{{{{range $i, $e := .NetworkSettings.Ports }}}}{{{{$p := index $e 0}}}}{{{{$p.HostPort}}}}{{{{end}}}}' {name}); echo Connected to $COREOS_PRIVATE_IPV4:$port/tcp, publishing to etcd...; while netstat -lnt | grep :$port >/dev/null; do etcdctl set /deis/services/{app}/{name} $COREOS_PRIVATE_IPV4:$port --ttl 60 >/dev/null; sleep 45; done"'''},  # noqa
-    {"section": "Service", "name": "ExecStop", "value": "/usr/bin/etcdctl rm --recursive /deis/services/{app}/{name}"},  # noqa
-    {"section": "Service", "name": "TimeoutStartSec", "value": "20m"},
-    {"section": "X-Fleet", "name": "MachineOf", "value": "{name}.service"},
-]
-
-
 RUN_TEMPLATE = [
     {"section": "Unit", "name": "Description", "value": "{name} admin command"},
     {"section": "Service", "name": "ExecStartPre", "value": '''/bin/sh -c "IMAGE=$(etcdctl get /deis/registry/host 2>&1):$(etcdctl get /deis/registry/port 2>&1)/{image}; docker pull $IMAGE"'''},  # noqa

controller/scheduler/mock.py

@@ -20,25 +20,25 @@ def tearDown(self):
 
     # container api
 
-    def create(self, name, image, command, use_announcer, **kwargs):
+    def create(self, name, image, command, **kwargs):
         """
         Create a new container
         """
         return
 
-    def start(self, name, use_announcer):
+    def start(self, name):
         """
         Start a container
         """
         return
 
-    def stop(self, name, use_announcer):
+    def stop(self, name):
         """
         Stop a container
         """
         return
 
-    def destroy(self, name, use_announcer):
+    def destroy(self, name):
         """
         Destroy a container
         """

docs/understanding_deis/concepts.rst

@@ -99,9 +99,9 @@ changed, making it easy to rollback code and configuration.
 Run Stage
 ^^^^^^^^^
 The run stage shells out jobs to the scheduler. The scheduler is in control of balancing the
-processes evenly across the cluster, as well as the announcers and the loggers for each
-application. The scheduler uses SSH to submit jobs to each node in the cluster and updates
-the proxy component between releases, making zero downtime deployments possible.
+processes evenly across the cluster, as well as the loggers for each application. The
+scheduler uses SSH to submit jobs to each node in the cluster and updates the proxy
+component between releases, making zero downtime deployments possible.
 
 .. _concepts_backing_services:
 

publisher/Dockerfile

@@ -0,0 +1,7 @@
+FROM deis/go:latest
+
+WORKDIR /go/src/github.com/deis/deis/publisher
+CMD /go/bin/publisher
+
+ADD . /go/src/github.com/deis/deis/publisher
+RUN CGO_ENABLED=0 go get -a -ldflags '-s' github.com/deis/deis/publisher

publisher/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2014 OpDemand LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

publisher/Makefile

@@ -0,0 +1,46 @@
+include ../includes.mk
+
+DOCKER_IMAGE := deis/publisher
+BUILD_IMAGE := $(DOCKER_IMAGE)-build
+RELEASE_IMAGE := $(DOCKER_IMAGE):$(BUILD_TAG)
+REMOTE_IMAGE := $(REGISTRY)/$(RELEASE_IMAGE)
+
+build: check-docker
+	docker build -t $(BUILD_IMAGE) .
+	docker cp `docker run -d $(BUILD_IMAGE)`:/go/bin/publisher image/
+	docker build -t $(DOCKER_IMAGE) image
+	rm -rf image/publisher
+
+clean: check-docker check-registry
+	docker rmi $(RELEASE_IMAGE) $(REMOTE_IMAGE)
+
+full-clean: check-docker check-registry
+	docker images -q $(DOCKER_IMAGE) | xargs docker rmi -f
+	docker images -q $(REGISTRY)/$(DOCKER_IMAGE) | xargs docker rmi -f
+
+install: check-deisctl
+	deisctl install publisher
+
+dev-release: check-docker check-registry check-deisctl
+	docker tag $(RELEASE_IMAGE) $(REMOTE_IMAGE)
+	docker push $(REMOTE_IMAGE)
+	deisctl config publisher set image=$(REMOTE_IMAGE)
+
+release: check-docker
+	docker push $(DOCKER_IMAGE)
+
+restart: stop start
+
+run: install start
+
+start: check-deisctl
+	deisctl start publisher
+
+stop: check-deisctl
+	deisctl stop publisher
+
+test:
+	@echo no unit tests
+
+uninstall: check-deisctl
+	deisctl uninstall publisher

publisher/README.md

@@ -0,0 +1,33 @@
+publisher
+=========
+
+Publisher listens directly to a docker socket bind-mounted into the container and listens to the
+docker events API for running containers on the host. Deis applications are published to etcd for
+service discovery.
+
+## Running this Container
+
+Set $ETCD_HOST to be the IP address/hostname of the etcd endpoint you wish to target, and
+$HOST to be the IP address of the host running this container:
+
+    ><> docker run -d -v /var/run/docker.sock:/tmp/docker.sock -e ETCD_HOST=192.168.0.1 -e HOST=192.168.0.1 deis/publisher
+
+## Building from Source
+
+To build the image, run `make build`.
+
+The build/runtime environment is split into two parts:
+
+### The build environment
+
+Based on deis/go, this image installs Go and compiles publisher into a binary.
+
+### The runtime environment
+
+Leveraging the build environment, this image pulls in the standalone binary compiled in
+the build environment and injects it into a minimal standalone container, minimizing the
+disk space footprint that this image takes up. In fact, this image is < 5MB:
+
+    ><> docker images | grep publisher
+    deis/publisher                           master              7974d140b07d        11 minutes ago      4.678 MB
+    deis/publisher-build                     master              75983660e714        11 minutes ago      1.091 GB