fix(store): fix shared etcd key defaults

In store-monitor's bin/boot script, we set default etcd keys that
are used to template ceph.conf for all store components. Previously,
this logic was inside the monitor setup lock logic, meaning it's only
executed once for the lifetime of a cluster. This breaks when adding new
keys to ceph.conf and this logic, since upgraded clusters will not
execute this logic a second time, thus skipping the creation of new
(and necessary) keys. The cluster will not come up on an upgrade
because confd cannot template ceph.conf in the store components.

Because *all* store monitors now set these defaults before checking
if they're the master, we have to ensure that all monitors are setting
the same default values and not writing over a different default set
by a competing monitor. We do this by removing the ability to set
environment variable overrides for these values. It never really made
sense anyway, since a user can always override these by manually setting
the etcd key before cluster start.

Author: carmstrong

store/monitor/bin/boot

@@ -8,13 +8,6 @@ ETCD="$HOST:$ETCD_PORT"
 ETCD_PATH=${ETCD_PATH:-/deis/store}
 HOSTNAME=`hostname`
 
-# These defaults are for 3 hosts
-NUM_STORES=${NUM_STORES:-3}
-PG_NUM=${PG_NUM:-64}
-## We set this to the number of PGs before re-evaluating the PG count so users upgrading don't see the warning
-## Now, 12 pools * 64 pgs per pool = 768 PGs per OSD
-PGS_PER_OSD_WARNING=${PGS_PER_OSD_WARNING:-1536}
-
 function etcd_set_default {
   set +e
   etcdctl --no-sync -C $ETCD mk $ETCD_PATH/$1 $2 >/dev/null 2>&1
@@ -25,21 +18,25 @@ function etcd_set_default {
   set -e
 }
 
+# set some defaults in etcd - these are templated in ceph.conf
+# These defaults are sane for 3 hosts, and may need to be tweaked for larger clusters.
+etcd_set_default delayStart 15
+etcd_set_default size 3 # maintain 3 copies of all data
+etcd_set_default minSize 1 # since we have 3 copies of data, the cluster can operate with just one host up
+etcd_set_default pgNum 64 # this gives us a reasonable number of placement groups per host, assuming 3 hosts and 12 pools
+
+# New clusters use 768 PGs per host (12 pools * 64 PGs per pool = 768 PGs per OSD)
+# However, upgraded clusters may still use 128 PGs per pool, so we set this to 1536 PGs per host to suppress the
+# "too many placement groups per host" warning
+etcd_set_default maxPGsPerOSDWarning 1536
+
 if ! etcdctl --no-sync -C $ETCD get ${ETCD_PATH}/monSetupComplete >/dev/null 2>&1 ; then
   echo "store-monitor: Ceph hasn't yet been deployed. Trying to deploy..."
   # let's rock and roll. we need to obtain a lock so we can ensure only one machine is trying to deploy the cluster
   if etcdctl --no-sync -C $ETCD mk ${ETCD_PATH}/monSetupLock $HOSTNAME >/dev/null 2>&1 \
   || [[ `etcdctl --no-sync -C $ETCD get ${ETCD_PATH}/monSetupLock` == "$HOSTNAME" ]] ; then
     echo "store-monitor: obtained the lock to proceed with setting up."
 
-    # set some defaults in etcd if they're not passed in as environment variables
-    # these are templated in ceph.conf
-    etcd_set_default delayStart 15
-    etcd_set_default maxPGsPerOSDWarning ${PGS_PER_OSD_WARNING}
-    etcd_set_default minSize 1
-    etcd_set_default pgNum ${PG_NUM}
-    etcd_set_default size ${NUM_STORES}
-
     # Generate administrator key
     ceph-authtool /etc/ceph/ceph.client.admin.keyring --create-keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'