feat(platform): support placement options for each plane and router mesh

This updates deisctl code that creates and loads system units such that
platform components belonging to each of the control plane, data plane, and
router mesh can OPTIONALLY be scheduled only where fleet machine metadata
indicates.

The means of achieving this are that a "decorator" is applied to any system
unit just before scheduling if and only if this option is enabled by means
of the etcd key /deis/platform/enablePlacementOptions having been set to the
value true.  The decorator adds the correct metadata.

Nodes in existing clusters will lack the machine metadata referenced by the
system unit decorators.  That would make this a breaking change except for
the fact that this is entirely OPT-IN.

Users should be encouraged to begin adopting metadata that will permit them
to isolate their planes more easily in the future.  As such, user-data.example
is modified such that machines configured from that cloud-config are eligible
to host both planes and the router mesh.  This permits NEW small clusters
without any isolation to be constructed as easily as ever before and permits
the platform to be installed to such a cluster without complication.

The main benefit of this change is that advanced users can use custom cloud
config to isolate their planes using ONLY metadata and without also needing
to also manually customize their system units, as would have been required
in the past.

Author: krancour

Makefile

@@ -27,6 +27,7 @@ dev-cluster: discovery-url
 	ssh-add ~/.vagrant.d/insecure_private_key
 	deisctl config platform set sshPrivateKey=$(HOME)/.vagrant.d/insecure_private_key
 	deisctl config platform set domain=local3.deisapp.com
+	deisctl config platform set enablePlacementOptions=true
 	deisctl install platform
 
 discovery-url:

contrib/coreos/user-data.example

@@ -7,6 +7,7 @@ coreos:
     public-ip: $private_ipv4
     # allow etcd to slow down at times
     etcd_request_timeout: 3.0
+    metadata: controlPlane=true,dataPlane=true,routerMesh=true
   units:
   - name: etcd.service
     command: start

controller/scheduler/fleet.py

@@ -150,10 +150,10 @@ def _create_container(self, name, image, command, unit, **kwargs):
             f['value'] = f['value'].format(**l)
         # prepare tags only if one was provided
         tags = kwargs.get('tags', {})
-        if tags:
-            tagset = ' '.join(['"{}={}"'.format(k, v) for k, v in tags.viewitems()])
+        tagset = ' '.join(['"{}={}"'.format(k, v) for k, v in tags.viewitems()])
+        if settings.ENABLE_PLACEMENT_OPTIONS in ['true', 'True', 'TRUE', '1']:
             unit.append({"section": "X-Fleet", "name": "MachineMetadata",
-                         "value": tagset})
+                         "value": tagset + ' "dataPlane=true"'})
         # post unit to fleet
         self._put_unit(name, {"desiredState": "loaded", "options": unit})
 

controller/templates/confd_settings.py

@@ -24,6 +24,8 @@
 # platform domain must be provided
 DEIS_DOMAIN = '{{ getv "/deis/platform/domain" }}'
 
+ENABLE_PLACEMENT_OPTIONS = """{{ if exists "/deis/platform/enablePlacementOptions" }}{{ getv "/deis/platform/enablePlacementOptions" }}{{ else }}false{{end}}"""
+
 # use the private registry module
 REGISTRY_MODULE = 'registry.private'
 REGISTRY_URL = '{{ getv "/deis/registry/protocol" }}://{{ getv "/deis/registry/host" }}:{{ getv "/deis/registry/port" }}'  # noqa

deisctl/backend/fleet/create.go

@@ -3,6 +3,7 @@ package fleet
 import (
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -96,7 +97,15 @@ func (c *FleetClient) createServiceUnit(component string, num int) (name string,
 	if err != nil {
 		return "", nil, err
 	}
-	uf, err = NewUnit(component, c.templatePaths)
+	decorateStr, err := c.configBackend.GetWithDefault("/deis/platform/enablePlacementOptions", "false")
+	if err != nil {
+		return "", nil, err
+	}
+	decorate, err := strconv.ParseBool(decorateStr)
+	if err != nil {
+		return "", nil, err
+	}
+	uf, err = NewUnit(component, c.templatePaths, decorate)
 	if err != nil {
 		return
 	}

deisctl/backend/fleet/create_test.go

@@ -6,6 +6,9 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/deis/deis/deisctl/config/model"
+	"github.com/deis/deis/deisctl/test/mock"
+
 	"github.com/coreos/fleet/schema"
 )
 
@@ -30,7 +33,9 @@ func TestCreate(t *testing.T) {
 	testFleetClient := stubFleetClient{testUnits: []*schema.Unit{}, unitsMutex: &sync.Mutex{},
 		unitStatesMutex: &sync.Mutex{}}
 
-	c := &FleetClient{templatePaths: []string{name}, Fleet: &testFleetClient}
+	testConfigBackend := mock.ConfigBackend{Expected: []*model.ConfigNode{{Key: "/deis/platform/enablePlacementOptions", Value: "true"}}}
+
+	c := &FleetClient{templatePaths: []string{name}, Fleet: &testFleetClient, configBackend: testConfigBackend}
 
 	var errOutput string
 	var wg sync.WaitGroup

deisctl/backend/fleet/fleet.go

@@ -6,13 +6,16 @@ import (
 	"path"
 	"text/tabwriter"
 
+	"github.com/deis/deis/deisctl/config"
+
 	"github.com/coreos/fleet/client"
 	"github.com/coreos/fleet/machine"
 )
 
 // FleetClient used to wrap Fleet API calls
 type FleetClient struct {
-	Fleet client.API
+	Fleet         client.API
+	configBackend config.Backend
 
 	// used to cache MachineStates
 	machineStates map[string]*machine.MachineState
@@ -25,7 +28,7 @@ type FleetClient struct {
 
 // NewClient returns a client used to communicate with Fleet
 // using the Registry API
-func NewClient() (*FleetClient, error) {
+func NewClient(cb config.Backend) (*FleetClient, error) {
 	client, err := getRegistryClient()
 	if err != nil {
 		return nil, err
@@ -41,6 +44,6 @@ func NewClient() (*FleetClient, error) {
 	out := new(tabwriter.Writer)
 	out.Init(os.Stdout, 0, 8, 1, '\t', 0)
 
-	return &FleetClient{Fleet: client, templatePaths: templatePaths, runner: sshCommandRunner{},
+	return &FleetClient{Fleet: client, configBackend: cb, templatePaths: templatePaths, runner: sshCommandRunner{},
 		out: out, errWriter: os.Stderr}, nil
 }

deisctl/backend/fleet/fleet_test.go

@@ -6,6 +6,9 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/deis/deis/deisctl/config/model"
+	"github.com/deis/deis/deisctl/test/mock"
+
 	"github.com/coreos/fleet/machine"
 	"github.com/coreos/fleet/schema"
 )
@@ -176,8 +179,10 @@ func TestNewClient(t *testing.T) {
 	// set required flags
 	Flags.Endpoint = "http://127.0.0.1:4001"
 
+	testConfigBackend := mock.ConfigBackend{Expected: []*model.ConfigNode{}}
+
 	// instantiate client
-	_, err := NewClient()
+	_, err := NewClient(testConfigBackend)
 	if err != nil {
 		t.Fatal(err)
 	}

deisctl/backend/fleet/scale_test.go

@@ -7,6 +7,9 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/deis/deis/deisctl/config/model"
+	"github.com/deis/deis/deisctl/test/mock"
+
 	"github.com/coreos/fleet/schema"
 )
 
@@ -30,7 +33,9 @@ func TestScaleUp(t *testing.T) {
 
 	testFleetClient := stubFleetClient{testUnits: testUnits, testUnitStates: []*schema.UnitState{}, unitsMutex: &sync.Mutex{}, unitStatesMutex: &sync.Mutex{}}
 
-	c := &FleetClient{templatePaths: []string{name}, Fleet: &testFleetClient}
+	testConfigBackend := mock.ConfigBackend{Expected: []*model.ConfigNode{{Key: "/deis/platform/enablePlacementOptions", Value: "true"}}}
+
+	c := &FleetClient{templatePaths: []string{name}, Fleet: &testFleetClient, configBackend: testConfigBackend}
 
 	var errOutput string
 	var wg sync.WaitGroup

deisctl/backend/fleet/unit.go

@@ -11,6 +11,21 @@ import (
 	"github.com/coreos/fleet/unit"
 )
 
+// path hierarchy for finding systemd service templates
+var templatePaths = []string{
+	os.Getenv("DEISCTL_UNITS"),
+	path.Join(os.Getenv("HOME"), ".deis", "units"),
+	"/var/lib/deis/units",
+}
+
+// and the same for systemd service "decorators" for optionally isolating
+// control plane, data plane, and router mesh
+var decoratorPaths = []string{
+	path.Join(os.Getenv("DEISCTL_UNITS"), "decorators"),
+	path.Join(os.Getenv("HOME"), ".deis", "units", "decorators"),
+	"/var/lib/deis/units/decorators",
+}
+
 // Units returns a list of units filtered by target
 func (c *FleetClient) Units(target string) (units []string, err error) {
 	allUnits, err := c.Fleet.Units()
@@ -56,14 +71,19 @@ func (c *FleetClient) lastUnit(component string) (num int, err error) {
 
 // NewUnit takes a component type and returns a Fleet unit
 // that includes the relevant systemd service template
-func NewUnit(component string, templatePaths []string) (uf *unit.UnitFile, err error) {
+func NewUnit(component string, templatePaths []string, decorate bool) (uf *unit.UnitFile, err error) {
 	template, err := readTemplate(component, templatePaths)
 	if err != nil {
 		return
 	}
-	uf, err = unit.NewUnitFile(string(template))
-	if err != nil {
-		return
+	if decorate {
+		decorator, err := readDecorator(component)
+		if err != nil {
+			return nil, err
+		}
+		uf, err = unit.NewUnitFile(string(template) + "\n" + string(decorator))
+	} else {
+		uf, err = unit.NewUnitFile(string(template))
 	}
 	return
 }
@@ -104,3 +124,26 @@ func readTemplate(component string, templatePaths []string) (out []byte, err err
 	}
 	return
 }
+
+// readDecorator returns the contents of a file containing a snippet that can
+// optionally be grafted on to the end of a corresponding systemd unit to
+// achieve isolation of the control plane, data plane, and router mesh
+func readDecorator(component string) (out []byte, err error) {
+	decoratorName := "deis-" + component + ".service.decorator"
+	var decoratorFile string
+
+	// look in $DEISCTL_UNITS env var, then the local and global root paths
+	for _, p := range decoratorPaths {
+		filename := path.Join(p, decoratorName)
+		if _, err := os.Stat(filename); err == nil {
+			decoratorFile = filename
+			break
+		}
+	}
+
+	if decoratorFile == "" {
+		return
+	}
+	out, err = ioutil.ReadFile(decoratorFile)
+	return
+}