feat(workflow): add redis and postgres persistence support

Author: duanhongyi

_scripts/install.sh

@@ -110,13 +110,42 @@ function configure_mirrors {
 function install_k3s_server {
   configure_os
   configure_mirrors
-  INSTALL_K3S_EXEC="server ${INSTALL_K3S_EXEC} --flannel-backend=none --disable=traefik --disable-kube-proxy --disable=local-storage --disable=servicelb --cluster-cidr=10.233.0.0/16"
+  INSTALL_K3S_EXEC="server ${INSTALL_K3S_EXEC} --flannel-backend=none --disable=traefik --disable-kube-proxy --disable=local-storage --cluster-cidr=10.233.0.0/16"
   if [[ -n "${K3S_DATA_DIR}" ]] ; then
     INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --data-dir=${K3S_DATA_DIR}/rancher/k3s"
   fi
   if [[ -z "${K3S_URL}" ]] ; then
     INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --cluster-init"
   fi
+  if [[ "${BGP_ENABLED:-false}" == "true" ]] ; then
+    if [[ -z "${BGP_CONFIG_FILE}" ]] ; then
+      echo -e "\\033[31m---> Please set the BGP_CONFIG_FILE variable.\\033[0m"
+      echo -e "\\033[31m---> For example:\\033[0m"
+      echo -e "\\033[31m---> export BGP_CONFIG_FILE=./bgp.yaml\\033[0m"
+      echo -e "\\033[31m---> For details, please check bgp.yaml in the current directory\\033[0m"
+      cat << EOF >  "./bgp.yaml"
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bgp-config
+  namespace: kube-system
+data:
+  config.yaml: |
+    peers:
+      - peer-address: 10.0.0.1
+        peer-asn: 64512
+        my-asn: 64512
+    address-pools:
+      - name: default
+        protocol: bgp
+        addresses:
+          - 192.0.2.0/24
+EOF
+      exit 1  
+    else
+      INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --disable=servicelb"
+    fi
+  fi
   curl -sfL "${k3s_install_url}" |INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC" sh -s -
 }
 
@@ -142,15 +171,11 @@ function install_components {
     --set k8sServiceHost=${api_server[1]} \
     --set k8sServicePort=${api_server[2]} \
     --set hostPort.enabled=true \
+    --set bgp.enabled=${BGP_ENABLED:-false} \
+    --set bgp.announce.loadbalancerIP=true \
+    --set bgp.announce.podCIDR=true \
     --namespace kube-system --wait
-  helm install metallb drycc/metallb --namespace metallb --create-namespace --wait -f - <<EOF
-configInline:
-  address-pools:
-   - name: default
-     protocol: layer2
-     addresses:
-     - ${METALLB_ADDRESS_POOLS:-172.16.0.0/12}
-EOF
+
   helm install traefik drycc/traefik \
     --namespace traefik \
     --create-namespace --wait -f - <<EOF
@@ -178,6 +203,7 @@ function install_openebs {
   helm install openebs drycc/openebs \
     --namespace openebs \
     --create-namespace \
+    --set localprovisioner.basePath=${LOCAL_PROVISIONER_PATH:-"/var/openebs/local"} \
     --set nfs-provisioner.enabled=true --wait
   kubectl patch storageclass ${DEFAULT_STORAGE_CLASS:-"openebs-hostpath"} \
     -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
@@ -240,6 +266,9 @@ EOF
     --set global.ingressClass=traefik \
     --set fluentd.daemonEnvironment.CONTAINER_TAIL_PARSER_TYPE="/^(?<time>.+) (?<stream>stdout|stderr)( (?<tags>.))? (?<log>.*)$/" \
     --set controller.appStorageClass=${CONTROLLER_APP_STORAGE_CLASS:-"openebs-kernel-nfs"} \
+    --set redis.persistence.enabled=true \
+    --set redis.persistence.size=${REDIS_PERSISTENCE_SIZE:-5Gi} \
+    --set redis.persistence.storageClass=${REDIS_PERSISTENCE_STORAGE_CLASS:-""} \
     --set minio.persistence.enabled=true \
     --set minio.persistence.size=${MINIO_PERSISTENCE_SIZE:-20Gi} \
     --set minio.persistence.storageClass=${MINIO_PERSISTENCE_STORAGE_CLASS:-""} \
@@ -258,6 +287,9 @@ EOF
     --set passport.adminPassword=${DRYCC_ADMIN_PASSWORD} \
     --set database.limitsMemory="256Mi" \
     --set database.limitsHugepages2Mi="256Mi" \
+    --set database.persistence.enabled=true \
+    --set database.persistence.size=${DATABASE_PERSISTENCE_SIZE:-5Gi} \
+    --set database.persistence.storageClass=${DATABASE_PERSISTENCE_STORAGE_CLASS:-""} \
     --namespace drycc \
     --values /tmp/drycc-values.yaml \
     --create-namespace --wait --timeout 30m0s
@@ -318,53 +350,6 @@ EOF
   echo -e "\\033[32m---> Helmbroker password: $HELMBROKER_PASSWORD\\033[0m"
 }
 
-function configure_haproxy {
-  BUILDER_IP=$(kubectl get svc drycc-builder -n drycc -o="jsonpath={.status.loadBalancer.ingress[0].ip}")
-  INGRESS_IP=$(kubectl get svc traefik -n traefik -o="jsonpath={.status.loadBalancer.ingress[0].ip}")
-
-  if [[ "${USE_HAPROXY:-true}" == "true" ]] ; then
-    cat << EOF > "/etc/haproxy/haproxy.cfg"
-global
-   log /dev/log    local0
-   log /dev/log    local1 notice
-   chroot /var/lib/haproxy
-   stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
-   stats timeout 30s
-   user haproxy
-   group haproxy
-   daemon
-listen http
-   bind *:${HAPROXY_HTTP_PORT:-80}
-   mode tcp
-   maxconn 100000
-   timeout connect 60s
-   timeout client  30000
-   timeout server  30000
-   server ingress ${INGRESS_IP}:80 check
-listen https
-   bind *:${HAPROXY_HTTPS_PORT:-443}
-   mode tcp
-   maxconn 100000
-   timeout connect 60s
-   timeout client  30000
-   timeout server  30000
-   server ingress ${INGRESS_IP}:443 check
-listen builder
-   bind *:${HAPROXY_BUILDER_PORT:-2222}
-   mode tcp
-   maxconn 100000
-   timeout connect 60s
-   timeout client  30000
-   timeout server  30000
-   server builder ${BUILDER_IP}:2222 check
-EOF
-  fi
-
-  mkdir -p /run/haproxy
-  systemctl enable haproxy
-  systemctl restart haproxy
-}
-
 export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
 
 if [[ -z "$@" ]] ; then
@@ -373,7 +358,6 @@ if [[ -z "$@" ]] ; then
   install_components
   install_openebs
   install_drycc
-  configure_haproxy
   install_helmbroker
   echo -e "\\033[32m---> Installation complete, enjoy life...\\033[0m"
 else

charts/workflow/values.yaml

@@ -204,13 +204,32 @@ database:
   # If left empty they will be generated using randAlphaNum
   username: ""
   password: ""
+  persistence:
+    enabled: false # Set to true to enable persistence
+    accessMode: ReadWriteOnce
+    size: 5Gi # PVC size
+    ## database data Persistent Volume Storage Class
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    ## Storage class of PV to bind. By default it looks for standard storage class.
+    ## If the PV uses a different storage class, specify that here.
+    storageClass: ""
+    volumeName: ""
 
 redis:
   # The following parameters are configured only when using an on-cluster Redis instance
   replicas: 1
   # The following parameters are configured only when using an off-cluster Redis instance
   addrs: "" # A list of clusters: "127.0.0.1:7001/1,127.0.0.2:7002/1"
   password: "redis password" # "" == no password
+  persistence:
+    enabled: false # Set to true to enable persistence
+    size: 5Gi
+    storageClass: ""
 
 rabbitmq:
   # Configure the following ONLY if using an off-cluster rabbitmq
@@ -222,7 +241,6 @@ rabbitmq:
   # GCP PDs and EBS volumes are supported only
   persistence:
     enabled: false # Set to true to enable persistence
-    accessMode: ReadWriteOnce
     size: 5Gi # PVC size
     ## rabbitmq data Persistent Volume Storage Class
     ## If defined, storageClassName: <storageClass>
@@ -234,7 +252,6 @@ rabbitmq:
     ## Storage class of PV to bind. By default it looks for standard storage class.
     ## If the PV uses a different storage class, specify that here.
     storageClass: ""
-    volumeName: ""
 
 influxdb:
   # URL configuration is only available in off-cluster Influx database

src/quickstart/install-workflow.md

@@ -60,24 +60,6 @@ For RHEL, CentOS, and EKS with EKS Kubernetes Worker AMI with AmazonLinux2 image
 $ yum install nfs-utils
 ```
 
-### Installing haproxy
-We use metallb as the loadblance component.
-If the host managed by metallb has a public IP, we don't need to install haproxy.
-Otherwise, we need to use haproxy to expose ports such as `80` and `443 `.
-Ensure that ports `80`, `443` and `2222` cannot be occupied by other applications on the host, and focus on services such as httpd and nginx.
-
-For Debian and Ubuntu, use this command:
-
-```
-$ apt-get install haproxy
-```
-
-For RHEL, CentOS, and EKS with EKS Kubernetes Worker AMI with AmazonLinux2 image, use this command:
-
-```
-$ yum install haproxy
-```
-
 ### Installing curl
 
 For Debian and Ubuntu, use this command:
@@ -189,17 +171,19 @@ $ curl -sfL https://drycc.cc/install.sh | bash -s - install_k3s_agent
 When using this method to install drycc, the following environment variables can be used to configure the installation:
 
 ENVIRONMENT VARIABLE                       | DESCRIPTION
--------------------------------------------|---------------------------------------------------------------------------------------------------------
+-------------------------------------------|---------------------------------------------------------------------------------------------
 PLATFORM_DOMAIN                            | Required item, specify drycc's domain name
 DRYCC_ADMIN_USERNAME                       | Required item, specify drycc's admin username
 DRYCC_ADMIN_PASSWORD                       | Required item, specify drycc's admin password
 CERT_MANAGER_ENABLED                       | Whether to use automatic certificate. It is `true` by default
 CHANNEL                                    | By default, `stable` channel will be installed. You can also specify `testing`
 REGISTRIES_FILE                            | The `registers.yaml` file path used by k3s.
-USE_HAPROXY                                | Haproxy is enabled by default. If you want to turn it off, this value is false
-METALLB_ADDRESS_POOLS                      | IP pool for LoadBalancer. The default is `172.16.0.0/12`
+BGP_ENABLED                                | Whether BGP is enabled or not. It is false by default.
+BGP_CONFIG_FILE                            | The bgp config file path used by k3s, after BGP is enabled, the env is required
 INSTALL_DRYCC_MIRROR                       | Specify the accelerated mirror location. Currently, only `cn` is supported
 CONTROLLER_APP_STORAGE_CLASS               | StorageClass allocated by `drycc volumes`; default storageClass is used by default
+REDIS_PERSISTENCE_SIZE                     | The size of the persistence space allocated to `redis`, which is `5Gi` by default
+REDIS_PERSISTENCE_STORAGE_CLASS            | StorangeClass of `redis`; default storangeclass is used by default
 MINIO_PERSISTENCE_SIZE                     | The size of the persistence space allocated to `minio`, which is `20Gi` by default
 MINIO_PERSISTENCE_STORAGE_CLASS            | StorangeClass of `minio`; default storangeclass is used by default
 MONITOR_GRAFANA_PERSISTENCE_SIZE           | The size of the persistence space allocated to `monitor.grafana`, which is `5Gi` by default
@@ -208,13 +192,13 @@ INFLUXDB_PERSISTENCE_SIZE                  | The size of the persistence space a
 INFLUXDB_PERSISTENCE_STORAGE_CLASS         | StorangeClass of `influxdb`; default storangeclass is used by default
 RABBITMQ_PERSISTENCE_SIZE                  | The size of the persistence space allocated to `rabbitmq`, which is `5Gi` by default
 RABBITMQ_PERSISTENCE_STORAGE_CLASS         | StorangeClass of `rabbitmq`; default storangeclass is used by default
+DATABASE_PERSISTENCE_SIZE                  | The size of the persistence space allocated to `database`, which is `5Gi` by default
+DATABASE_PERSISTENCE_STORAGE_CLASS         | StorangeClass of `database`; default storangeclass is used by default
 HELMBROKER_PERSISTENCE_SIZE                | The size of the persistence space allocated to `helmbroker`, which is `5Gi` by default
 HELMBROKER_PERSISTENCE_STORAGE_CLASS       | StorangeClass of `helmbroker`; default storangeclass is used by default
 K3S_DATA_DIR                               | The config of k3s data dir; If not set, the default path is used
-LONGHORN_DATA_PATH                         | The defaultDataPath of longhorn; If not set, the default path is used
-HAPROXY_HTTP_PORT                    | The drycc ingress http port of haproxy for external exposure, default `80`
-HAPROXY_HTTPS_PORT                   | The drycc ingress https port of haproxy for external exposure, default `443`
-HAPROXY_BUILDER_PORT                 | The drycc builder port of haproxy for external exposure, default `2222`
+DEFAULT_STORAGE_CLASS                      | K3s default stroageclass, If not set, the `openebs-hostpath` stroageclass is used
+LOCAL_PROVISIONER_PATH                     | Local path storage path, If not set, the `/var/openebs/local` path is used
 
 Since the installation script will install k3s, other environment variables can refer to k3s installation [environment variables](https://rancher.com/docs/k3s/latest/en/installation/install-options/).