Revert "fix(store): remove pg_num reconfiguration"

This reverts commit 39a5e7f.

Author: carmstrong

store/gateway/bin/boot

@@ -30,6 +30,38 @@ until confd -onetime -node $ETCD -config-file /app/confd.toml >/dev/null 2>/dev/
     sleep $(($ETCD_TTL/2))  # sleep for half the TTL
 done
 
+# set the number of placement groups for the default pools - they come up with defaults that are too low
+if ! etcdctl --no-sync -C $ETCD get /deis/store/defaultPoolsConfigured >/dev/null 2>&1 ; then
+  echo "store-gateway: setting pg_num values for default pools..."
+  function set_until_success {
+    set +e
+    ceph osd pool set $1 pg_num $2 2>/dev/null
+    PG_SET=$?
+    until [[ $PG_SET -eq 0 ]]; do
+      sleep 5
+      ceph osd pool set $1 pg_num $2 2>/dev/null
+      PG_SET=$?
+    done
+
+    ceph osd pool set $1 pgp_num $2 2>/dev/null
+    PGP_SET=$?
+    until [[ $PGP_SET -eq 0 ]]; do
+      sleep 5
+      ceph osd pool set $1 pgp_num $2 2>/dev/null
+      PGP_SET=$?
+    done
+    set -e
+  }
+
+  PG_NUM=`etcdctl --no-sync -C $ETCD get /deis/store/pgNum`
+
+  set_until_success data ${PG_NUM}
+  set_until_success rbd ${PG_NUM}
+  set_until_success metadata ${PG_NUM}
+
+  etcdctl --no-sync -C $ETCD set /deis/store/defaultPoolsConfigured youBetcha >/dev/null
+fi
+
 # we generate a key for the gateway. we can do this because we have the client key templated out
 if ! etcdctl --no-sync -C $ETCD get /deis/store/gatewayKeyring >/dev/null 2>&1 ; then
   ceph-authtool --create-keyring /etc/ceph/ceph.client.radosgw.keyring
@@ -97,4 +129,4 @@ if [[ ! -z $EXTERNAL_PORT ]]; then
 
 fi
 
-wait
+wait

store/tests/store_test.go

@@ -25,6 +25,9 @@ func TestStore(t *testing.T) {
 	// prep etcd with the monitor hostname -- this is done in an ExecStartPre in the monitor unit
 	etcdutils.SetSingle(t, "/deis/store/hosts/"+host, hostname, etcdPort)
 
+	// since we're only running one OSD, our default of 128 placement groups is too large
+	etcdutils.SetSingle(t, "/deis/store/pgNum", "64", etcdPort)
+
 	// test deis-store-monitor
 	fmt.Printf("--- Run deis/store-monitor:%s at %s\n", tag, host)
 	name := "deis-store-monitor-" + tag

tests/mock/mock.go

@@ -58,6 +58,9 @@ func RunMockCeph(t *testing.T, name string, cli *client.DockerCli, etcdPort stri
 
 	etcdutils.SetSingle(t, "/deis/store/hosts/"+utils.HostAddress(), utils.HostAddress(), etcdPort)
 
+	// since we're only running one OSD, our default of 128 placement groups is too large
+	etcdutils.SetSingle(t, "/deis/store/pgNum", "64", etcdPort)
+
 	monitorName := name + "-monitor"
 	RunMockCephMonitor(t, monitorName, etcdPort)