Merge pull request #4175 from krancour/graceful-upgrade

feat(deisctl): add graceful upgrade

Author: krancour

deisctl/backend/backend.go

@@ -12,6 +12,7 @@ type Backend interface {
 	Start([]string, *sync.WaitGroup, io.Writer, io.Writer)
 	Stop([]string, *sync.WaitGroup, io.Writer, io.Writer)
 	Scale(string, int, *sync.WaitGroup, io.Writer, io.Writer)
+	RollingRestart(string, *sync.WaitGroup, io.Writer, io.Writer)
 	SSH(string) error
 	SSHExec(string, string) error
 	ListUnits() error

deisctl/backend/fleet/rolling_restart.go

@@ -0,0 +1,37 @@
+package fleet
+
+import (
+	"fmt"
+	"io"
+	"sync"
+)
+
+// RollingRestart for instance units
+func (c *FleetClient) RollingRestart(component string, wg *sync.WaitGroup, out, ew io.Writer) {
+	if component != "router" {
+		fmt.Fprint(ew, "invalid component. supported for: router")
+		return
+	}
+
+	components, err := c.Units(component)
+	if err != nil {
+		io.WriteString(ew, err.Error())
+		return
+	}
+	if len(components) < 1 {
+		fmt.Fprint(ew, "rolling restart requires at least 1 component")
+		return
+	}
+	for num := range components {
+		unitName := fmt.Sprintf("%s@%v", component, num+1)
+
+		c.Stop([]string{unitName}, wg, out, ew)
+		wg.Wait()
+		c.Destroy([]string{unitName}, wg, out, ew)
+		wg.Wait()
+		c.Create([]string{unitName}, wg, out, ew)
+		wg.Wait()
+		c.Start([]string{unitName}, wg, out, ew)
+		wg.Wait()
+	}
+}

deisctl/client/client.go

@@ -9,6 +9,8 @@ import (
 	"github.com/deis/deis/deisctl/backend"
 	"github.com/deis/deis/deisctl/backend/fleet"
 	"github.com/deis/deis/deisctl/cmd"
+	"github.com/deis/deis/deisctl/config"
+	"github.com/deis/deis/deisctl/config/etcd"
 	"github.com/deis/deis/deisctl/units"
 
 	docopt "github.com/docopt/docopt-go"
@@ -28,11 +30,15 @@ type DeisCtlClient interface {
 	Status(argv []string) error
 	Stop(argv []string) error
 	Uninstall(argv []string) error
+	UpgradePrep(argv []string) error
+	UpgradeTakeover(argv []string) error
+	RollingRestart(argv []string) error
 }
 
 // Client uses a backend to implement the DeisCtlClient interface.
 type Client struct {
-	Backend backend.Backend
+	Backend       backend.Backend
+	configBackend config.Backend
 }
 
 // NewClient returns a Client using the requested backend.
@@ -54,7 +60,56 @@ func NewClient(requestedBackend string) (*Client, error) {
 	default:
 		return nil, errors.New("invalid backend")
 	}
-	return &Client{Backend: backend}, nil
+
+	cb, err := etcd.NewConfigBackend()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Client{Backend: backend, configBackend: cb}, nil
+}
+
+// UpgradePrep prepares a running cluster to be upgraded
+func (c *Client) UpgradePrep(argv []string) error {
+	usage := `Prepare platform for graceful upgrade.
+
+Usage:
+  deisctl upgrade-prep [options]
+`
+	if _, err := docopt.Parse(usage, argv, true, "", false); err != nil {
+		return err
+	}
+
+	return cmd.UpgradePrep(c.Backend)
+}
+
+// UpgradeTakeover gracefully restarts a cluster prepared with upgrade-prep
+func (c *Client) UpgradeTakeover(argv []string) error {
+	usage := `Complete the upgrade of a prepped cluster.
+
+Usage:
+  deisctl upgrade-takeover [options]
+`
+	if _, err := docopt.Parse(usage, argv, true, "", false); err != nil {
+		return err
+	}
+
+	return cmd.UpgradeTakeover(c.Backend, c.configBackend)
+}
+
+// RollingRestart attempts a rolling restart of an instance unit
+func (c *Client) RollingRestart(argv []string) error {
+	usage := `Perform a rolling restart of an instance unit.
+
+Usage:
+  deisctl rolling-restart <target>
+`
+	args, err := docopt.Parse(usage, argv, true, "", false)
+	if err != nil {
+		return err
+	}
+
+	return cmd.RollingRestart(args["<target>"].(string), c.Backend)
 }
 
 // Config gets or sets a configuration value from the cluster.
@@ -105,7 +160,7 @@ Examples:
 		key = args["<key>"].([]string)
 	}
 
-	return cmd.Config(args["<target>"].(string), action, key)
+	return cmd.Config(args["<target>"].(string), action, key, c.configBackend)
 }
 
 // Install loads the definitions of components from local unit files.
@@ -140,7 +195,7 @@ Options:
 	}
 	cmd.RouterMeshSize = uint8(parsedValue)
 
-	return cmd.Install(args["<target>"].([]string), c.Backend, cmd.CheckRequiredKeys)
+	return cmd.Install(args["<target>"].([]string), c.Backend, c.configBackend, cmd.CheckRequiredKeys)
 }
 
 // Journal prints log output for the specified components.

deisctl/cmd/cmd.go

@@ -97,14 +97,24 @@ func Start(targets []string, b backend.Backend) error {
 	return nil
 }
 
-// CheckRequiredKeys exist in etcd
-func CheckRequiredKeys() error {
-	if err := config.CheckConfig("/deis/platform/", "domain"); err != nil {
+// RollingRestart restart instance unit in a rolling manner
+func RollingRestart(target string, b backend.Backend) error {
+	var wg sync.WaitGroup
+
+	b.RollingRestart(target, &wg, Stdout, Stderr)
+	wg.Wait()
+
+	return nil
+}
+
+// CheckRequiredKeys exist in config backend
+func CheckRequiredKeys(cb config.Backend) error {
+	if err := config.CheckConfig("/deis/platform/", "domain", cb); err != nil {
 		return fmt.Errorf(`Missing platform domain, use:
 deisctl config platform set domain=<your-domain>`)
 	}
 
-	if err := config.CheckConfig("/deis/platform/", "sshPrivateKey"); err != nil {
+	if err := config.CheckConfig("/deis/platform/", "sshPrivateKey", cb); err != nil {
 		fmt.Printf(`Warning: Missing sshPrivateKey, "deis run" will be unavailable. Use:
 deisctl config platform set sshPrivateKey=<path-to-key>
 `)
@@ -276,15 +286,15 @@ func Journal(targets []string, b backend.Backend) error {
 
 // Install loads the definitions of components from local unit files.
 // After Install, the components will be available to Start.
-func Install(targets []string, b backend.Backend, checkKeys func() error) error {
+func Install(targets []string, b backend.Backend, cb config.Backend, checkKeys func(config.Backend) error) error {
 
 	// if target is platform, install all services
 	if len(targets) == 1 {
 		switch targets[0] {
 		case PlatformCommand:
-			return InstallPlatform(b, checkKeys, false)
+			return InstallPlatform(b, cb, checkKeys, false)
 		case StatelessPlatformCommand:
-			return InstallPlatform(b, checkKeys, true)
+			return InstallPlatform(b, cb, checkKeys, true)
 		case mesos:
 			return InstallMesos(b)
 		case swarm:
@@ -429,11 +439,11 @@ func splitScaleTarget(target string) (c string, num int, err error) {
 
 // Config gets or sets a configuration value from the cluster.
 //
-// A configuration value is stored and retrieved from a key/value store (in this case, etcd)
+// A configuration value is stored and retrieved from a key/value store
 // at /deis/<component>/<config>. Configuration values are typically used for component-level
 // configuration, such as enabling TLS for the routers.
-func Config(target string, action string, key []string) error {
-	if err := config.Config(target, action, key); err != nil {
+func Config(target string, action string, key []string, cb config.Backend) error {
+	if err := config.Config(target, action, key, cb); err != nil {
 		return err
 	}
 	return nil

deisctl/cmd/cmd_test.go

@@ -13,6 +13,9 @@ import (
 	"testing"
 
 	"github.com/deis/deis/deisctl/backend"
+	"github.com/deis/deis/deisctl/config"
+	"github.com/deis/deis/deisctl/config/model"
+	"github.com/deis/deis/deisctl/test/mock"
 	"github.com/deis/deis/deisctl/units"
 )
 
@@ -21,6 +24,7 @@ type backendStub struct {
 	stoppedUnits     []string
 	installedUnits   []string
 	uninstalledUnits []string
+	restartedUnits   []string
 	expected         bool
 }
 
@@ -46,6 +50,10 @@ func (backend *backendStub) Scale(component string, num int, wg *sync.WaitGroup,
 		backend.expected = false
 	}
 }
+func (backend *backendStub) RollingRestart(target string, wg *sync.WaitGroup, out, ew io.Writer) {
+	backend.restartedUnits = append(backend.restartedUnits, target)
+}
+
 func (backend *backendStub) ListUnits() error {
 	return nil
 }
@@ -79,7 +87,7 @@ func (backend *backendStub) SSHExec(target, command string) error {
 
 var _ backend.Backend = &backendStub{}
 
-func fakeCheckKeys() error {
+func fakeCheckKeys(cb config.Backend) error {
 	return nil
 }
 
@@ -270,6 +278,57 @@ func TestStartSwarm(t *testing.T) {
 	}
 }
 
+func TestRollingRestart(t *testing.T) {
+	t.Parallel()
+
+	b := backendStub{}
+	expected := []string{"router"}
+
+	RollingRestart("router", &b)
+
+	if !reflect.DeepEqual(b.restartedUnits, expected) {
+		t.Error(fmt.Errorf("Expected %v, Got %v", expected, b.restartedUnits))
+	}
+}
+
+func TestUpgradePrep(t *testing.T) {
+	t.Parallel()
+
+	b := backendStub{}
+	expected := []string{"database", "registry@*", "controller", "builder", "logger", "logspout", "store-volume",
+		"store-gateway@*", "store-metadata", "store-daemon", "store-monitor"}
+
+	UpgradePrep(&b)
+
+	if !reflect.DeepEqual(b.stoppedUnits, expected) {
+		t.Error(fmt.Errorf("Expected %v, Got %v", expected, b.stoppedUnits))
+	}
+}
+
+func TestUpgradeTakeover(t *testing.T) {
+	t.Parallel()
+	testMock := mock.ConfigBackend{Expected: []*model.ConfigNode{{Key: "/deis/services/app1", Value: "foo", TTL: 10},
+		{Key: "/deis/services/app2", Value: "8000", TTL: 10}}}
+
+	b := backendStub{}
+	expectedRestarted := []string{"router"}
+	expectedStarted := []string{"publisher", "store-monitor", "store-daemon", "store-metadata",
+		"store-gateway@*", "store-volume", "logger", "logspout", "database", "registry@*",
+		"controller", "builder", "publisher", "router@*", "database", "registry@*",
+		"controller", "builder", "publisher", "router@*"}
+
+	if err := doUpgradeTakeOver(&b, testMock); err != nil {
+		t.Error(fmt.Errorf("Takeover failed: %v", err))
+	}
+
+	if !reflect.DeepEqual(b.restartedUnits, expectedRestarted) {
+		t.Error(fmt.Errorf("Expected %v, Got %v", expectedRestarted, b.restartedUnits))
+	}
+	if !reflect.DeepEqual(b.startedUnits, expectedStarted) {
+		t.Error(fmt.Errorf("Expected %v, Got %v", expectedStarted, b.startedUnits))
+	}
+}
+
 func TestStop(t *testing.T) {
 	t.Parallel()
 
@@ -419,9 +478,11 @@ func TestInstall(t *testing.T) {
 	t.Parallel()
 
 	b := backendStub{}
+	cb := mock.ConfigBackend{}
+
 	expected := []string{"router@1", "router@2"}
 
-	Install(expected, &b, fakeCheckKeys)
+	Install(expected, &b, &cb, fakeCheckKeys)
 
 	if !reflect.DeepEqual(b.installedUnits, expected) {
 		t.Error(fmt.Errorf("Expected %v, Got %v", expected, b.installedUnits))
@@ -432,11 +493,13 @@ func TestInstallPlatform(t *testing.T) {
 	t.Parallel()
 
 	b := backendStub{}
+	cb := mock.ConfigBackend{}
+
 	expected := []string{"store-daemon", "store-monitor", "store-metadata", "store-volume",
 		"store-gateway@1", "logger", "logspout", "database", "registry@1",
 		"controller", "builder", "publisher", "router@1", "router@2", "router@3"}
 
-	Install([]string{"platform"}, &b, fakeCheckKeys)
+	Install([]string{"platform"}, &b, &cb, fakeCheckKeys)
 
 	if !reflect.DeepEqual(b.installedUnits, expected) {
 		t.Error(fmt.Errorf("Expected %v, Got %v", expected, b.installedUnits))
@@ -447,12 +510,14 @@ func TestInstallPlatformWithCustomRouterMeshSize(t *testing.T) {
 	t.Parallel()
 
 	b := backendStub{}
+	cb := mock.ConfigBackend{}
+
 	expected := []string{"store-daemon", "store-monitor", "store-metadata", "store-volume",
 		"store-gateway@1", "logger", "logspout", "database", "registry@1",
 		"controller", "builder", "publisher", "router@1", "router@2", "router@3", "router@4", "router@5"}
 	RouterMeshSize = 5
 
-	Install([]string{"platform"}, &b, fakeCheckKeys)
+	Install([]string{"platform"}, &b, &cb, fakeCheckKeys)
 	RouterMeshSize = DefaultRouterMeshSize
 
 	if !reflect.DeepEqual(b.installedUnits, expected) {
@@ -464,10 +529,12 @@ func TestInstallStatelessPlatform(t *testing.T) {
 	t.Parallel()
 
 	b := backendStub{}
+	cb := mock.ConfigBackend{}
+
 	expected := []string{"logspout", "registry@1",
 		"controller", "builder", "publisher", "router@1", "router@2", "router@3"}
 
-	Install([]string{"stateless-platform"}, &b, fakeCheckKeys)
+	Install([]string{"stateless-platform"}, &b, &cb, fakeCheckKeys)
 
 	if !reflect.DeepEqual(b.installedUnits, expected) {
 		t.Error(fmt.Errorf("Expected %v, Got %v", expected, b.installedUnits))
@@ -478,9 +545,11 @@ func TestInstallSwarm(t *testing.T) {
 	t.Parallel()
 
 	b := backendStub{}
+	cb := mock.ConfigBackend{}
+
 	expected := []string{"swarm-node", "swarm-manager"}
 
-	Install([]string{"swarm"}, &b, fakeCheckKeys)
+	Install([]string{"swarm"}, &b, &cb, fakeCheckKeys)
 
 	if !reflect.DeepEqual(b.installedUnits, expected) {
 		t.Error(fmt.Errorf("Expected %v, Got %v", expected, b.installedUnits))

deisctl/cmd/platform.go

@@ -6,14 +6,15 @@ import (
 	"sync"
 
 	"github.com/deis/deis/deisctl/backend"
+	"github.com/deis/deis/deisctl/config"
 	"github.com/deis/deis/pkg/prettyprint"
 )
 
 // InstallPlatform loads all components' definitions from local unit files.
 // After InstallPlatform, all components will be available for StartPlatform.
-func InstallPlatform(b backend.Backend, checkKeys func() error, stateless bool) error {
+func InstallPlatform(b backend.Backend, cb config.Backend, checkKeys func(config.Backend) error, stateless bool) error {
 
-	if err := checkKeys(); err != nil {
+	if err := checkKeys(cb); err != nil {
 		return err
 	}