Skip to content

Commit 575f68d

Browse files
carmstrongcarmstrong
committed
Merge pull request #3645 from carmstrong/store-fix_pg_count
fix(store): lower number of placement groups
2 parents c0e662e + 6c9765b commit 575f68d

7 files changed

Lines changed: 18 additions & 47 deletions

File tree

‎docs/customizing_deis/store_daemon_settings.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ setting description
3535
/deis/store/adminKeyring keyring for an admin user to access the Ceph cluster (set by store-monitor)
3636
/deis/store/fsid Ceph filesystem ID (set by store-monitor)
3737
/deis/store/hosts/* deis-monitor hosts (set by store-monitor)
38+
/deis/store/maxPGsPerOSDWarning threshold for warning on number of placement groups per OSD (set by store-monitor)
3839
/deis/store/monKeyring keyring for the monitor to access the Ceph cluster (set by store-monitor)
3940
/deis/store/monSetupComplete set when the Ceph cluster setup is complete (set by store-monitor)
4041
/deis/store/monSetupLock host of store-monitor that completed setup (set by store-monitor)

‎docs/customizing_deis/store_gateway_settings.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ The following etcd keys are set by the store-gateway component, typically in its
2222
================================== =================================================================================
2323
setting description
2424
================================== =================================================================================
25-
/deis/store/defaultPoolsConfigured set once the gateway configures the default storage pools on first boot
2625
/deis/store/gatewayKeyring keyring for the gateway user to access the Ceph cluster (generated by Ceph)
2726
/deis/store/gateway/accessKey S3 API access used to access store-gateway (generated by Ceph)
2827
/deis/store/gateway/host IP address of the host running store-gateway
@@ -40,6 +39,7 @@ setting description
4039
/deis/store/adminKeyring keyring for an admin user to access the Ceph cluster (set by store-monitor)
4140
/deis/store/fsid Ceph filesystem ID (set by store-monitor)
4241
/deis/store/hosts/* deis-monitor hosts (set by store-monitor)
42+
/deis/store/maxPGsPerOSDWarning threshold for warning on number of placement groups per OSD (set by store-monitor)
4343
/deis/store/monKeyring keyring for the monitor to access the Ceph cluster (set by store-monitor)
4444
/deis/store/monSetupComplete set when the Ceph cluster setup is complete (set by store-monitor)
4545
/deis/store/monSetupLock host of store-monitor that completed setup (set by store-monitor)

‎docs/customizing_deis/store_metadata_settings.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ setting description
3535
/deis/store/adminKeyring keyring for an admin user to access the Ceph cluster (set by store-monitor)
3636
/deis/store/fsid Ceph filesystem ID (set by store-monitor)
3737
/deis/store/hosts/* deis-monitor hosts (set by store-monitor)
38+
/deis/store/maxPGsPerOSDWarning threshold for warning on number of placement groups per OSD (set by store-monitor)
3839
/deis/store/monKeyring keyring for the monitor to access the Ceph cluster (set by store-monitor)
3940
/deis/store/monSetupComplete set when the Ceph cluster setup is complete (set by store-monitor)
4041
/deis/store/monSetupLock host of store-monitor that completed setup (set by store-monitor)

‎docs/customizing_deis/store_monitor_settings.rst‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,20 @@ Settings set by store-monitor
1919
-----------------------------
2020
The following etcd keys are set by the store-monitor component, typically in its /bin/boot script.
2121

22-
============================= =================================================================================
22+
=============================== ==================================================================================
2323
setting description
24-
============================= =================================================================================
24+
=============================== ==================================================================================
2525
/deis/store/adminKeyring keyring for an admin user to access the Ceph cluster
2626
/deis/store/fsid Ceph filesystem ID
2727
/deis/store/hosts/$HOST hostname (not IP) of the host running this store-monitor instance
28+
/deis/store/maxPGsPerOSDWarning threshold for warning on number of placement groups per OSD (set by store-monitor)
2829
/deis/store/monKeyring keyring for the monitor to access the Ceph cluster
2930
/deis/store/monSetupComplete set when the Ceph cluster setup is complete
3031
/deis/store/monSetupLock IP address of the monitor instance that is or has set up the Ceph cluster
3132
/deis/store/minSize minimum number of store-daemons necessary for the cluster to accept writes
3233
/deis/store/pgNum number of Ceph placement groups for the storage pools
3334
/deis/store/size number of replicas for data stored in Ceph
34-
============================= =================================================================================
35+
=============================== ==================================================================================
3536

3637
Settings used by store-monitor
3738
------------------------------

‎store/base/templates/ceph.conf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ fsid = {{ getv "/deis/store/fsid" }}
33
mon initial members = {{ getv "/deis/store/monSetupLock" }}
44
mon host = {{ join (getvs "/deis/store/hosts/*") "," }}
55
mon addr = {{ range $index, $element := (gets "/deis/store/hosts/*") }}{{if $index}},{{end}}{{ base $element.Key }}:6789{{ end }}
6+
mon pg warn max per osd = {{ getv "/deis/store/maxPGsPerOSDWarning" }}
67
auth cluster required = cephx
78
auth service required = cephx
89
auth client required = cephx

‎store/gateway/bin/boot‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -31,45 +31,6 @@ until confd -onetime -node $ETCD --confdir /app --log-level error; do
3131
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
3232
done
3333

34-
# set the number of placement groups for the default pools - they come up with defaults that are too low
35-
if ! etcdctl --no-sync -C $ETCD get /deis/store/defaultPoolsConfigured >/dev/null 2>&1 ; then
36-
echo "store-gateway: setting pg_num values for default pools..."
37-
function set_until_success {
38-
set +e
39-
40-
echo "store-gateway: checking pool $1..."
41-
if ! ceph osd pool get $1 pg_num | grep "pg_num: $2" ; then
42-
ceph osd pool set $1 pg_num $2 2>/dev/null
43-
PG_SET=$?
44-
until [[ $PG_SET -eq 0 ]]; do
45-
sleep 5
46-
ceph osd pool set $1 pg_num $2 2>/dev/null
47-
PG_SET=$?
48-
done
49-
fi
50-
51-
if ! ceph osd pool get $1 pgp_num | grep "pgp_num: $2" ; then
52-
ceph osd pool set $1 pgp_num $2 2>/dev/null
53-
PGP_SET=$?
54-
until [[ $PGP_SET -eq 0 ]]; do
55-
sleep 5
56-
ceph osd pool set $1 pgp_num $2 2>/dev/null
57-
PGP_SET=$?
58-
done
59-
fi
60-
61-
set -e
62-
}
63-
64-
PG_NUM=`etcdctl --no-sync -C $ETCD get /deis/store/pgNum`
65-
66-
set_until_success data ${PG_NUM}
67-
set_until_success rbd ${PG_NUM}
68-
set_until_success metadata ${PG_NUM}
69-
70-
etcdctl --no-sync -C $ETCD set /deis/store/defaultPoolsConfigured youBetcha >/dev/null
71-
fi
72-
7334
# we generate a key for the gateway. we can do this because we have the client key templated out
7435
if ! etcdctl --no-sync -C $ETCD get /deis/store/gatewayKeyring >/dev/null 2>&1 ; then
7536
ceph-authtool --create-keyring /etc/ceph/ceph.client.radosgw.keyring

‎store/monitor/bin/boot‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,15 @@ set -e
66
ETCD_PORT=${ETCD_PORT:-4001}
77
ETCD="$HOST:$ETCD_PORT"
88
ETCD_PATH=${ETCD_PATH:-/deis/store}
9-
NUM_STORES=${NUM_STORES:-3}
10-
PG_NUM=${PG_NUM:-128} # default for 3 OSDs
119
HOSTNAME=`hostname`
1210

11+
# These defaults are for 3 hosts
12+
NUM_STORES=${NUM_STORES:-3}
13+
PG_NUM=${PG_NUM:-64}
14+
## We set this to the number of PGs before re-evaluating the PG count so users upgrading don't see the warning
15+
## Now, 12 pools * 64 pgs per pool = 768 PGs per OSD
16+
PGS_PER_OSD_WARNING=${PGS_PER_OSD_WARNING:-1536}
17+
1318
function etcd_set_default {
1419
set +e
1520
etcdctl --no-sync -C $ETCD mk $ETCD_PATH/$1 $2 >/dev/null 2>&1
@@ -29,10 +34,11 @@ if ! etcdctl --no-sync -C $ETCD get ${ETCD_PATH}/monSetupComplete >/dev/null 2>&
2934

3035
# set some defaults in etcd if they're not passed in as environment variables
3136
# these are templated in ceph.conf
32-
etcd_set_default size ${NUM_STORES}
37+
etcd_set_default delayStart 15
38+
etcd_set_default maxPGsPerOSDWarning ${PGS_PER_OSD_WARNING}
3339
etcd_set_default minSize 1
3440
etcd_set_default pgNum ${PG_NUM}
35-
etcd_set_default delayStart 15
41+
etcd_set_default size ${NUM_STORES}
3642

3743
# Generate administrator key
3844
ceph-authtool /etc/ceph/ceph.client.admin.keyring --create-keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'

0 commit comments

Comments
 (0)