chore(postgresql-cluster): optimization of database parameters

Author: EamonZhang

addons/postgresql-cluster/15/chart/postgresql-cluster/templates/networkpolicy.yaml

@@ -23,11 +23,16 @@ spec:
     - ports:
       - port: 5432
       - port: 9000
+      - port: 80
+      - port: 8008
     {{- if and .Values.metrics.enabled }}
       - port: {{ .Values.metrics.containerPort }}
-    {{ end }} 
+    {{ end }}
     {{- if or .Values.networkPolicy.allowCurrentNamespace .Values.networkPolicy.allowNamespaces }}
       from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: backup
       {{- if .Values.networkPolicy.allowCurrentNamespace }}
         - namespaceSelector:
             matchLabels:

addons/postgresql-cluster/15/chart/postgresql-cluster/values.yaml

@@ -54,9 +54,6 @@ preInitScript: |
   restapi:
     listen: 0.0.0.0:8008
     connect_address: 0.0.0.0:8008
-    authentication:
-      username: NzUwNjg3MTEtMDgzOS00YTNkLWEyNjAt
-      password: YjJjMDNjYjQtMDA0Ny00NTgwLTgwYjMt
   bootstrap:
     dcs:
       ttl: 30
@@ -75,16 +72,19 @@ preInitScript: |
         - host replication postgres 0.0.0.0/0 scram-sha-256
         custom_conf: '/opt/drycc/postgresql/config/custom_conf.conf'
         parameters:
-          wal_level: hot_standby
+          max_connections: {{ .Values.patroni.pgParameters.max_connections }}
+          max_worker_processes: {{ .Values.patroni.pgParameters.max_worker_processes }}
+          max_parallel_workers: {{ .Values.patroni.pgParameters.max_parallel_workers }}
+          wal_level: logical
           hot_standby: "on"
-          max_connections: 1005
-          max_worker_processes: 8
           max_wal_senders: 10
           max_replication_slots: 10
           hot_standby_feedback: on
           max_prepared_transactions: 0
           max_locks_per_transaction: 64
           wal_log_hints: "on"
+          wal_keep_size: "1 GB"
+          max_slot_wal_keep_size: {{ .Values.patroni.pgParameters.max_slot_wal_keep_size | quote }}
           track_commit_timestamp: "off"
           archive_mode: "on"
           archive_timeout: 300s
@@ -194,14 +194,19 @@ postgresql:
     log_min_duration_statement = 1000
     max_wal_size = 4GB
     min_wal_size = 4GB
-    max_connections = 1005
-    max_worker_processes = 8
     max_wal_senders = 10
     max_replication_slots = 10
     max_prepared_transactions = 0
     max_locks_per_transaction = 64
 
 patroni:
+  pgParameters:
+    max_worker_processes: 64
+    max_parallel_workers: 32
+    max_connections: 2000
+    max_slot_wal_keep_size: '2 GB'
+
+
   ## @param patroni.podAnnotations Additional pod annotations for Postgresql patroni pods
   ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
   ##

addons/postgresql-cluster/15/plans/standard-16c64g400/values.yaml

@@ -1,11 +1,16 @@
 ## @param fullnameOverride String to fully override common.names.fullname template
 ##
 fullnameOverride: hb-postgresql-cluster-standard-400
+patroni:
+  pgParameters:
+    max_worker_processes: 32
+    max_parallel_workers: 16
+    max_connections: 2000
+    max_slot_wal_keep_size: '10 GB'
 
 postgresql:
   config: |-
     # Connectivity
-    max_connections = 2000
     superuser_reserved_connections = 3
 
     # Memory Settings
@@ -28,15 +33,14 @@ postgresql:
     # Checkpointing: 
     checkpoint_timeout  = '30 min' 
     checkpoint_completion_target = 0.9
-    max_wal_size = '6 GB'
+    max_wal_size = '16 GB'
     min_wal_size = '2 GB'
 
     # WAL writing
     wal_compression = on
     wal_buffers = -1    # auto-tuned by Postgres till maximum of segment size (16MB by default)
     wal_writer_delay = 200ms
     wal_writer_flush_after = 1MB
-    wal_keep_size = '10 GB'
 
     # Background writer
     bgwriter_delay = 200ms
@@ -45,7 +49,6 @@ postgresql:
     bgwriter_flush_after = 0
 
     # Parallel queries: 
-    max_worker_processes = 16
     max_parallel_workers_per_gather = 8
     max_parallel_maintenance_workers = 8
     max_parallel_workers = 16

addons/postgresql-cluster/15/plans/standard-2c4g20/values.yaml

@@ -2,10 +2,16 @@
 ##
 fullnameOverride: hb-postgresql-cluster-standard-20
 
+patroni:
+  pgParameters:
+    max_worker_processes: 4
+    max_parallel_workers: 2
+    max_connections: 1000
+    max_slot_wal_keep_size: '2 GB'
+
 postgresql:
   config: |-
     # Connectivity
-    max_connections = 1000
     superuser_reserved_connections = 3
 
     # Memory Settings
@@ -29,35 +35,30 @@ postgresql:
     checkpoint_timeout  = '15 min' 
     checkpoint_completion_target = 0.9
     max_wal_size = '2 GB'
-    min_wal_size = '512 MB'
+    min_wal_size = '1 GB'
           
 
     # WAL writing
     wal_compression = on
     wal_buffers = -1    # auto-tuned by Postgres till maximum of segment size (16MB by default)
     wal_writer_delay = 200ms
     wal_writer_flush_after = 1MB
-    wal_keep_size = '1 GB'
-
 
     # Background writer
     bgwriter_delay = 200ms
     bgwriter_lru_maxpages = 100
     bgwriter_lru_multiplier = 2.0
     bgwriter_flush_after = 0
 
-    # Parallel queries: 
-    max_worker_processes = 2
+    # Parallel queries:
     max_parallel_workers_per_gather = 1
     max_parallel_maintenance_workers = 1
-    max_parallel_workers = 2
     parallel_leader_participation = on
 
     # Advanced features 
     enable_partitionwise_join = on 
     enable_partitionwise_aggregate = on
     jit = on
-    max_slot_wal_keep_size = '1000 MB'
     track_wal_io_timing = on
     maintenance_io_concurrency = 100
 

addons/postgresql-cluster/15/plans/standard-2c8g50/values.yaml

@@ -2,10 +2,16 @@
 ##
 fullnameOverride: hb-postgresql-cluster-standard-50
 
+patroni:
+  pgParameters:
+    max_worker_processes: 4
+    max_parallel_workers: 2
+    max_connections: 2000
+    max_slot_wal_keep_size: '5 GB'
+
 postgresql:
   config: |-
     # Connectivity
-    max_connections = 1000
     superuser_reserved_connections = 3
 
     # Memory Settings
@@ -28,15 +34,14 @@ postgresql:
     # Checkpointing: 
     checkpoint_timeout  = '15 min' 
     checkpoint_completion_target = 0.9
-    max_wal_size = '2 GB'
-    min_wal_size = '512 MB'
+    max_wal_size = '4 GB'
+    min_wal_size = '1 GB'
 
     # WAL writing
     wal_compression = on
     wal_buffers = -1    # auto-tuned by Postgres till maximum of segment size (16MB by default)
     wal_writer_delay = 200ms
     wal_writer_flush_after = 1MB
-    wal_keep_size = '3 GB'
 
 
     # Background writer
@@ -46,7 +51,6 @@ postgresql:
     bgwriter_flush_after = 0
 
     # Parallel queries: 
-    max_worker_processes = 2
     max_parallel_workers_per_gather = 1
     max_parallel_maintenance_workers = 1
     max_parallel_workers = 2

addons/postgresql-cluster/15/plans/standard-32c128g800/values.yaml

@@ -2,10 +2,16 @@
 ##
 fullnameOverride: hb-postgresql-cluster-standard-800
 
+patroni:
+  pgParameters:
+    max_worker_processes: 64
+    max_parallel_workers: 32
+    max_connections: 2000
+    max_slot_wal_keep_size: '100 GB'
+
 postgresql:
   config: |-
     # Connectivity
-    max_connections = 2000
     superuser_reserved_connections = 3
 
     # Memory Settings
@@ -28,15 +34,14 @@ postgresql:
     # Checkpointing: 
     checkpoint_timeout  = '30 min' 
     checkpoint_completion_target = 0.9
-    max_wal_size = '8 GB'
+    max_wal_size = '64 GB'
     min_wal_size = '4 GB'
 
     # WAL writing
     wal_compression = on
     wal_buffers = -1    # auto-tuned by Postgres till maximum of segment size (16MB by default)
     wal_writer_delay = 200ms
     wal_writer_flush_after = 1MB
-    wal_keep_size = '10 GB'
 
     # Background writer
     bgwriter_delay = 200ms
@@ -45,7 +50,6 @@ postgresql:
     bgwriter_flush_after = 0
 
     # Parallel queries: 
-    max_worker_processes = 32
     max_parallel_workers_per_gather = 16
     max_parallel_maintenance_workers = 16
     max_parallel_workers = 32

addons/postgresql-cluster/15/plans/standard-32c64g4000/values.yaml

@@ -2,10 +2,16 @@
 ##
 fullnameOverride: hb-postgresql-cluster-standard-800
 
+patroni:
+  pgParameters:
+    max_worker_processes: 64
+    max_parallel_workers: 32
+    max_connections: 2000
+    max_slot_wal_keep_size: '200 GB'
+
 postgresql:
   config: |-
     # Connectivity
-    max_connections = 2000
     superuser_reserved_connections = 3
 
     # Memory Settings
@@ -29,14 +35,13 @@ postgresql:
     checkpoint_timeout  = '30 min' 
     checkpoint_completion_target = 0.9
     max_wal_size = '32 GB'
-    min_wal_size = '16 GB'
+    min_wal_size = '4 GB'
 
     # WAL writing
     wal_compression = on
     wal_buffers = -1    # auto-tuned by Postgres till maximum of segment size (16MB by default)
     wal_writer_delay = 200ms
     wal_writer_flush_after = 1MB
-    wal_keep_size = '60 GB'
 
     # Background writer
     bgwriter_delay = 200ms
@@ -45,7 +50,6 @@ postgresql:
     bgwriter_flush_after = 0
 
     # Parallel queries: 
-    max_worker_processes = 32
     max_parallel_workers_per_gather = 16
     max_parallel_maintenance_workers = 16
     max_parallel_workers = 32

addons/postgresql-cluster/15/plans/standard-4c16g100/values.yaml

@@ -2,10 +2,16 @@
 ##
 fullnameOverride: hb-postgresql-cluster-standard-100
 
+patroni:
+  pgParameters:
+    max_worker_processes: 8
+    max_parallel_workers: 4
+    max_connections: 2000
+    max_slot_wal_keep_size: '10 GB'
+
 postgresql:
   config: |-
     # Connectivity
-    max_connections = 2000
     superuser_reserved_connections = 3
 
     # Memory Settings
@@ -28,15 +34,14 @@ postgresql:
     # Checkpointing: 
     checkpoint_timeout  = '25 min' 
     checkpoint_completion_target = 0.9
-    max_wal_size = '4 GB'
+    max_wal_size = '8 GB'
     min_wal_size = '2 GB'
 
     # WAL writing
     wal_compression = on
     wal_buffers = -1    # auto-tuned by Postgres till maximum of segment size (16MB by default)
     wal_writer_delay = 200ms
     wal_writer_flush_after = 1MB
-    wal_keep_size = '4 GB'
 
 
     # Background writer
@@ -46,7 +51,6 @@ postgresql:
     bgwriter_flush_after = 0
 
     # Parallel queries: 
-    max_worker_processes = 4
     max_parallel_workers_per_gather = 2
     max_parallel_maintenance_workers = 2
     max_parallel_workers = 4

addons/postgresql-cluster/15/plans/standard-8c32g200/values.yaml

@@ -2,10 +2,16 @@
 ##
 fullnameOverride: hb-postgresql-cluster-standard-200
 
+patroni:
+  pgParameters:
+    max_worker_processes: 16
+    max_parallel_workers: 8
+    max_connections: 2000
+    max_slot_wal_keep_size: '10 GB'
+
 postgresql:
   config: |-
     # Connectivity
-    max_connections = 2000
     superuser_reserved_connections = 3
 
     # Memory Settings
@@ -28,15 +34,14 @@ postgresql:
     # Checkpointing: 
     checkpoint_timeout  = '25 min' 
     checkpoint_completion_target = 0.9
-    max_wal_size = '5 GB'
+    max_wal_size = '16 GB'
     min_wal_size = '3 GB'
 
     # WAL writing
     wal_compression = on
     wal_buffers = -1    # auto-tuned by Postgres till maximum of segment size (16MB by default)
     wal_writer_delay = 200ms
     wal_writer_flush_after = 1MB
-    wal_keep_size = '5 GB'
 
     # Background writer
     bgwriter_delay = 200ms
@@ -45,7 +50,6 @@ postgresql:
     bgwriter_flush_after = 0
 
     # Parallel queries: 
-    max_worker_processes = 8
     max_parallel_workers_per_gather = 4
     max_parallel_maintenance_workers = 4
     max_parallel_workers = 8