Merge pull request #2307 from carmstrong/deisctl_wait_on_stop

fix(deisctl): use wait groups for stopping and uninstalling store

Author: carmstrong

deisctl/cmd/cmd.go

@@ -131,7 +131,10 @@ func startDefaultServices(b backend.Backend, wg *sync.WaitGroup, outchan chan st
 	b.Start([]string{"store-metadata"}, wg, outchan, errchan)
 	wg.Wait()
 
-	b.Start([]string{"store-volume", "store-gateway"}, wg, outchan, errchan)
+	// we start gateway first to give metadata time to come up for volume
+	b.Start([]string{"store-gateway"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Start([]string{"store-volume"}, wg, outchan, errchan)
 	wg.Wait()
 
 	// start logging subsystem first to collect logs from other components
@@ -221,9 +224,14 @@ func stopDefaultServices(b backend.Backend, wg *sync.WaitGroup, outchan chan str
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Storage subsystem...")
-	b.Stop([]string{"store-gateway", "store-volume", "store-metadata", "store-monitor", "store-daemon"}, wg, outchan, errchan)
+	b.Stop([]string{"store-volume", "store-gateway"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Stop([]string{"store-metadata"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Stop([]string{"store-daemon"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Stop([]string{"store-monitor"}, wg, outchan, errchan)
 	wg.Wait()
-
 }
 
 func Restart(b backend.Backend, targets []string) error {
@@ -379,7 +387,13 @@ func uninstallAllServices(b backend.Backend, wg *sync.WaitGroup, outchan chan st
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Storage subsystem...")
-	b.Destroy([]string{"store-gateway", "store-volume", "store-metadata", "store-monitor", "store-daemon"}, wg, outchan, errchan)
+	b.Destroy([]string{"store-volume", "store-gateway"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Destroy([]string{"store-metadata"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Destroy([]string{"store-daemon"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Destroy([]string{"store-monitor"}, wg, outchan, errchan)
 	wg.Wait()
 
 	return nil

docs/managing_deis/upgrading-deis.rst

@@ -24,11 +24,17 @@ and then reinstall platform components.
 
     In-place upgrades are supported starting from Deis version 0.14.0
 
+.. note::
+
+    Occasionally, when upgrading Deis, we have seen deisctl stuck in a restart loop on ``deis-store`` components.
+    The workaround for this is to restart the store-daemons with ``deisctl restart store-daemon``. If run while
+    the original ``deisctl start platform`` command is still running, it should resume almost immediately.
+
 Use the following steps to perform an in-place upgrade of your Deis cluster.
 
 .. code-block:: console
 
-    $ deisctl uninstall platform
+    $ deisctl stop platform && deisctl uninstall platform
     $ deisctl config platform set version=v0.15.0
     $ deisctl install platform
     $ deisctl start platform