feat(mesos): change docker cmd format and make ExecstartPre statements onle liners

fix(style): fix go vet erorrs for mesos

Author: smothiki

.gitignore

@@ -44,6 +44,10 @@ publisher/Godeps/
 publisher/image/bin/publisher
 router/Godeps/
 swarm/image/bin/
+mesos/**/bin/
+mesos/**/bindata/
+mesos/Dockerfile
+mesos/marathon-assembly.jar
 
 # coverage reports
 .coverage

controller/scheduler/mesos_marathon.py

@@ -1,17 +1,21 @@
 import re
 import time
+
 from django.conf import settings
 from marathon import MarathonClient
 from marathon.models import MarathonApp
-from .states import JobState
 from docker import Client
+
+from .states import JobState
 from .fleet import FleetHTTPClient
 
 # turn down standard marathon logging
 
 MATCH = re.compile(
     '(?P<app>[a-z0-9-]+)_?(?P<version>v[0-9]+)?\.?(?P<c_type>[a-z-_]+)?.(?P<c_num>[0-9]+)')
 RETRIES = 3
+POLL_ATTEMPTS = 30
+POLL_WAIT = 100
 
 
 class MarathonHTTPClient(object):
@@ -47,24 +51,23 @@ def create(self, name, image, command='', **kwargs):
         cpu = kwargs.get('cpu', {}).get(l['c_type'])
         if cpu:
             c = cpu
-        cpu = kwargs.get('cpu', {}).get(l['c_type'])
-        self.client.create_app(app_id,
-                               MarathonApp(cmd="docker run --name "+name+" -P "+image+" "+command,
-                                           mem=m, cpus=c))
+        cmd = "docker run --name {name} -P {image} {command}".format(**locals())
+        self.client.create_app(app_id, MarathonApp(cmd=cmd, mem=m, cpus=c))
         self.client.scale_app(app_id, 0, force=True)
-        for _ in xrange(30):
+        for _ in xrange(POLL_ATTEMPTS):
             if self.client.get_app(self._app_id(name)).tasks_running == 0:
                 return
             time.sleep(1)
 
     def start(self, name):
         """Start a container"""
         self.client.scale_app(self._app_id(name), 1, force=True)
-        for _ in xrange(30):
+        for _ in xrange(POLL_ATTEMPTS):
             if self.client.get_app(self._app_id(name)).tasks_running == 1:
-                return
+                break
             time.sleep(1)
-        raise RuntimeError("App Not Started")
+        host = self.client.get_app(self._app_id(name)).tasks[0].host
+        self._waitforcontainer(host, name)
 
     def stop(self, name):
         """Stop a container"""
@@ -79,21 +82,33 @@ def destroy(self, name):
         except:
             self.client.delete_app(self._app_id(name), force=True)
 
-    def _delete_container(self, host, name):
+    def _get_container_state(self, host, name):
         docker_cli = Client("tcp://{}:2375".format(host), timeout=1200, version='1.17')
         try:
-            if docker_cli.inspect_container(name)['State']:
-                docker_cli.remove_container(name, force=True)
+            if docker_cli.inspect_container(name)['State']['Running']:
+                return JobState.up
         except:
-            pass
+            return JobState.destroyed
+
+    def _waitforcontainer(self, host, name):
+        for _ in xrange(POLL_WAIT):
+            if self._get_container_state(host, name) == JobState.up:
+                return
+            time.sleep(1)
+        raise RuntimeError("App container Not Started")
+
+    def _delete_container(self, host, name):
+        docker_cli = Client("tcp://{}:2375".format(host), timeout=1200, version='1.17')
+        if docker_cli.inspect_container(name)['State']:
+            docker_cli.remove_container(name, force=True)
 
     def run(self, name, image, entrypoint, command):  # noqa
         """Run a one-off command"""
         return self.fleet.run(name, image, entrypoint, command)
 
     def state(self, name):
         try:
-            for _ in xrange(30):
+            for _ in xrange(POLL_ATTEMPTS):
                 if self.client.get_app(self._app_id(name)).tasks_running == 1:
                     return JobState.up
                 elif self.client.get_app(self._app_id(name)).tasks_running == 0:

deisctl/cmd/mesos.go

@@ -34,7 +34,7 @@ func InstallMesos(b backend.Backend) error {
 func installMesosServices(b backend.Backend, wg *sync.WaitGroup, outchan chan string, errchan chan error) {
 
 	outchan <- fmt.Sprintf("Zookeeper...")
-	b.Create([]string{"mesos-zk"}, wg, outchan, errchan)
+	b.Create([]string{"zookeeper"}, wg, outchan, errchan)
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Mesos Master...")
@@ -85,7 +85,7 @@ func uninstallMesosServices(b backend.Backend, wg *sync.WaitGroup, outchan chan
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Zookeeper...")
-	b.Destroy([]string{"mesos-zk"}, wg, outchan, errchan)
+	b.Destroy([]string{"zookeeper"}, wg, outchan, errchan)
 	wg.Wait()
 
 	return nil
@@ -116,7 +116,7 @@ func StartMesos(b backend.Backend) error {
 func startMesosServices(b backend.Backend, wg *sync.WaitGroup, outchan chan string, errchan chan error) {
 
 	outchan <- fmt.Sprintf("Zookeeper...")
-	b.Start([]string{"mesos-zk"}, wg, outchan, errchan)
+	b.Start([]string{"zookeeper"}, wg, outchan, errchan)
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Mesos Master...")
@@ -169,6 +169,6 @@ func stopMesosServices(b backend.Backend, wg *sync.WaitGroup, outchan chan strin
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Zookeeper...")
-	b.Stop([]string{"mesos-zk"}, wg, outchan, errchan)
+	b.Stop([]string{"zookeeper"}, wg, outchan, errchan)
 	wg.Wait()
 }

deisctl/units/deis-mesos-marathon.service

@@ -13,11 +13,7 @@ ExecStartPre=/bin/sh -c "etcdctl set /deis/scheduler/mesos/marathon $COREOS_PRIV
 ExecStartPre=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-marathon` && docker history $IMAGE >/dev/null 2>&1 || docker pull $IMAGE"
 ExecStartPre=-/usr/bin/docker kill deis-mesos-marathon
 ExecStartPre=-/usr/bin/docker rm deis-mesos-marathon
-ExecStart=/usr/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-marathon` && docker run \
-  --name=coreos-mesos-marathon \
-  --net=host \
-  -e HOST=$COREOS_PRIVATE_IPV4 \
-  $IMAGE"
+ExecStart=/usr/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-marathon` && docker run --name=deis-mesos-marathon --net=host -e HOST=$COREOS_PRIVATE_IPV4 $IMAGE"
 ExecStop=-/usr/bin/docker stop deis-mesos-marathon
 
 [Install]

deisctl/units/deis-mesos-master.service

@@ -12,13 +12,7 @@ ExecStartPre=-/usr/bin/docker kill deis-mesos-master
 ExecStartPre=-/usr/bin/docker rm deis-mesos-master
 ExecStartPre=/bin/sh -c "docker inspect deis-mesos-master-data >/dev/null 2>&1 || docker run --name deis-mesos-master-data -v /tmp/mesos-master alpine:3.1 /bin/true"
 ExecStartPre=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-master` && docker history $IMAGE >/dev/null 2>&1 || docker pull $IMAGE"
-ExecStart=/usr/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-master` && docker run \
-  --volumes-from=deis-mesos-master-data \
-  --name=deis-mesos-master \
-  --privileged \
-  --net=host \
-  -e HOST=$COREOS_PRIVATE_IPV4 \
-  $IMAGE"
+ExecStart=/usr/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-master` && docker run --volumes-from=deis-mesos-master-data --name=deis-mesos-master --privileged --net=host -e HOST=$COREOS_PRIVATE_IPV4 $IMAGE"
 ExecStop=-/usr/bin/docker stop deis-mesos-master
 
 [Install]

deisctl/units/deis-mesos-slave.service

@@ -11,17 +11,8 @@ TimeoutStartSec=0
 ExecStartPre=-/usr/bin/docker kill deis-mesos-slave
 ExecStartPre=-/usr/bin/docker rm deis-mesos-slave
 ExecStartPre=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-slave` && docker history $IMAGE >/dev/null 2>&1 || docker pull $IMAGE"
-ExecStart=/usr/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-slave` && docker run \
-  --name=deis-mesos-slave \
-  --net=host \
-  --privileged \
-  -e HOST=$COREOS_PRIVATE_IPV4 \
-  -v /sys:/sys \
-  -v /usr/bin/docker:/usr/bin/docker:ro \
-  -v /var/run/docker.sock:/var/run/docker.sock \
-  -v /lib64/libdevmapper.so.1.02:/lib/libdevmapper.so.1.02:ro \
-  $IMAGE"
-ExecStop=/usr/bin/docker stop coreos-mesos-slave
+ExecStart=/usr/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/mesos-slave` && docker run --name=deis-mesos-slave --net=host --privileged -e HOST=$COREOS_PRIVATE_IPV4 -v /sys:/sys -v /usr/bin/docker:/usr/bin/docker:ro -v /var/run/docker.sock:/var/run/docker.sock -v /lib64/libdevmapper.so.1.02:/lib/libdevmapper.so.1.02:ro $IMAGE"
+ExecStop=/usr/bin/docker stop deis-mesos-slave
 
 [Install]
 WantedBy=multi-user.target

deisctl/units/deis-mesos-zk.service deisctl/units/deis-zookeeper.servicedeisctl/units/deis-mesos-zk.service renamed to deisctl/units/deis-zookeeper.service

@@ -103,7 +103,6 @@ to `enable the remote API`_.
     **Known Issues**
 
     - It is not yet possible to change the default affinity filter.
-    - If swarm can't create all the containers requested during scale, deis rolls back the scale operation.
 
 To test the Swarm Scheduler backend, first install and start the swarm components:
 

docs/customizing_deis/choosing-a-scheduler.rst

@@ -1,3 +1,5 @@
+include ../includes.mk
+
 REPO = deis
 MESOS = 0.22.1-1.0.ubuntu1404
 MESOS_VERSION = v1.9.0
@@ -13,56 +15,108 @@ GOLINT = golint
 GOTEST = $(GO) test -cover -race -v
 GOVET = $(GO) vet
 
-GO_PACKAGES = pkg/boot pkg/confd pkg/etcd pkg/fleet pkg/log pkg/net
+COMPONENT = $(notdir $(repo_path))
+GO_PACKAGES = pkg/boot pkg/confd pkg/etcd pkg/fleet pkg/log pkg/net pkg/os pkg/types
 GO_PACKAGES_REPO_PATH = $(addprefix $(repo_path)/,$(GO_PACKAGES))
 
-build: test-style zookeeper-go build-tools mesos-template mesos-master mesos-slave mesos-marathon zookeeper
+MASTER_IMAGE = $(IMAGE_PREFIX)mesos-master:$(BUILD_TAG)
+MASTER_DEV_IMAGE = $(REGISTRY)$(MASTER_IMAGE)
+MARATHON_IMAGE = $(IMAGE_PREFIX)mesos-marathon:$(BUILD_TAG)
+MARATHON_DEV_IMAGE = $(REGISTRY)$(MARATHON_IMAGE)
+SLAVE_IMAGE = $(IMAGE_PREFIX)mesos-slave:$(BUILD_TAG)
+SLAVE_DEV_IMAGE = $(REGISTRY)$(SLAVE_IMAGE)
+ZOOKEEPER_IMAGE = $(IMAGE_PREFIX)zookeeper:$(BUILD_TAG)
+ZOOKEEPER_DEV_IMAGE = $(REGISTRY)$(ZOOKEEPER_IMAGE)
+
+build: test-style zookeeper-go mesos-template mesos-master mesos-slave mesos-marathon zookeeper
+
+install: check-deisctl
+	deisctl install $(COMPONENT)
+
+uninstall: check-deisctl
+	deisctl uninstall $(COMPONENT)
+
+start: check-deisctl
+	deisctl start $(COMPONENT)
+
+stop: check-deisctl
+	deisctl stop $(COMPONENT)
+
+restart: stop start
+
+run: install start
 
-mesos-go:
+dev-release: push set-image
+
+push: check-registry
+	docker tag -f $(MASTER_IMAGE) $(MASTER_DEV_IMAGE)
+	docker push $(MASTER_DEV_IMAGE)
+	docker tag -f $(SLAVE_IMAGE) $(SLAVE_DEV_IMAGE)
+	docker push $(SLAVE_DEV_IMAGE)
+	docker tag -f $(MARATHON_IMAGE) $(MARATHON_DEV_IMAGE)
+	docker push $(MARATHON_DEV_IMAGE)
+	docker tag -f $(ZOOKEEPER_IMAGE) $(ZOOKEEPER_DEV_IMAGE)
+	docker push $(ZOOKEEPER_DEV_IMAGE)
+
+set-image: check-deisctl
+	deisctl config mesos-master set image=$(MASTER_DEV_IMAGE)
+	deisctl config mesos-slave set image=$(SLAVE_DEV_IMAGE)
+	deisctl config mesos-marathon set image=$(MARATHON_DEV_IMAGE)
+	deisctl config zookeeper set image=$(ZOOKEEPER_DEV_IMAGE)
+
+release:
+	docker push $(MASTER_IMAGE)
+	docker push $(SLAVE_IMAGE)
+	docker push $(MARATHON_IMAGE)
+	docker push $(ZOOKEEPER_IMAGE)
+
+deploy: build dev-release restart
+
+setup-gotools:
+	go get -u github.com/jteeuwen/go-bindata/...
+
+mesos-go: setup-gotools
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o bin/master-boot pkg/boot/mesos/master/main.go
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o bin/slave-boot pkg/boot/mesos/slave/main.go
-	go-bindata -pkg bindata -o pkg/boot/mesos/marathon/bindata/bindata.go pkg/boot/mesos/marathon/bash/; \
+	go-bindata -pkg bindata -o bindata/marathon/bindata.go pkg/boot/mesos/marathon/bash/; \
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o bin/marathon-boot pkg/boot/mesos/marathon/main.go
 
 mesos-template:
 	sed "s/#VERSION#/$(MESOS)/g" template > Dockerfile
-	docker build -t $(REPO)/$@:$(MESOS_VERSION) .
+	docker build -t $(IMAGE_PREFIX)$@:$(MESOS_VERSION) .
 	rm -f Dockerfile
 
 mesos-master: mesos-go mesos-template
 	sed "s/#VERSION#/$(MESOS_VERSION)/g" master > Dockerfile
-	docker build -t $(REPO)/$@:$(MESOS_VERSION) .
+	docker build -t $(IMAGE_PREFIX)$@:$(BUILD_TAG) .
 	rm -f Dockerfile
 
 mesos-slave: mesos-go mesos-template
 	sed "s/#VERSION#/$(MESOS_VERSION)/g" slave > Dockerfile
-	docker build -t $(REPO)/$@:$(MESOS_VERSION) .
+	docker build -t $(IMAGE_PREFIX)$@:$(BUILD_TAG) .
 	rm -f Dockerfile
 
 build-mesos-marathon: mesos-template
 	sed "s/#MARATHON_VERSION#/$(MARATHON_VERSION)/;s/#VERSION#/$(MESOS_VERSION)/" build-marathon > Dockerfile
-	docker build -t $(REPO)/$@:$(MESOS_VERSION) .
+	docker build -t $(IMAGE_PREFIX)$@:$(BUILD_TAG) .
 	rm -f Dockerfile
 
-mesos-marathon: build-mesos-marathon mesos-go
+mesos-marathon: mesos-go build-mesos-marathon
 	cp marathon Dockerfile
-	docker cp `docker create $(REPO)/build-mesos-marathon:$(MESOS_VERSION) /bin/bash`:/marathon/target/marathon-assembly-$(MARATHON_VERSION).jar .
+	docker cp `docker create $(IMAGE_PREFIX)build-mesos-marathon:$(BUILD_TAG) /bin/bash`:/marathon/target/marathon-assembly-$(MARATHON_VERSION).jar .
 	mv marathon-assembly-$(MARATHON_VERSION).jar marathon-assembly.jar
 	sed "s/#MARATHON_VERSION#/$(MARATHON_VERSION)/;s/#VERSION#/$(MESOS_VERSION)/" marathon > Dockerfile
-	docker build -t $(REPO)/$@:$(MESOS_VERSION) .
+	docker build -t $(IMAGE_PREFIX)$@:$(BUILD_TAG) .
 	rm -f Dockerfile
 
 zookeeper: zookeeper-go
-	docker build -t $(REPO)/$@:$(ZOOKEEPER_VERSION) zookeeper/.
+	docker build -t $(IMAGE_PREFIX)$@:$(BUILD_TAG) zookeeper/.
 
 zookeeper-go:
 	echo "Building..."
-	go-bindata -pkg bindata -o pkg/boot/zookeeper/bindata/bindata.go pkg/boot/zookeeper/bash/; \
+	go-bindata -pkg bindata -o bindata/zookeeper/bindata.go pkg/boot/zookeeper/bash/; \
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o zookeeper/bin/boot pkg/boot/zookeeper/main/boot.go
 
-build-tools:
-	echo "Building tools..."
-
 test: mesos-go zookeeper-go
 	@$(GOFMT) -timeout 10s $(GO_PACKAGES)
 

mesos/Makefile

@@ -0,0 +1,23 @@
+# Mesos with Marathon Framework
+
+[Marathon Framework](https://github.com/mesosphere/marathon) components for use as
+an optional scheduler in Deis, the open source PaaS.
+
+## Usage
+
+Please consult the [Mesos with Marathon framework](http://docs.deis.io/en/latest/customizing_deis/choosing-a-scheduler/#mesos-with-marathon-framework)
+for instructions on installing and activating Mesos with Marathon Framework in Deis.
+
+## License
+
+© 2015 Engine Yard, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0>
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.