docs(workflow): add fluentbit component

duanhongyi · duanhongyi · commit 84ae1a8cab0e · 2023-08-17T11:53:07.000+08:00
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ Please see below for links and descriptions of each component:
 - [controller](https://github.com/drycc/controller) - Workflow API server
 - [builder](https://github.com/drycc/builder) - Git server and source-to-image component
 - [imagebuilder](https://github.com/drycc/imagebuilder) - The builder for [Docker](https://www.docker.com/) and [CNCF Buildpacks](https://buildpacks.io/) based applications
-- [fluentd](https://github.com/drycc/fluentd) - Backend log shipping mechanism for `drycc logs`
+- [fluentbit](https://github.com/drycc/fluentbit) - Backend log shipping mechanism for `drycc logs`
 - [postgres](https://github.com/drycc/postgres) - The central database
 - [registry](https://github.com/drycc/registry) - The Docker registry
 - [logger](https://github.com/drycc/logger) - The (in-memory) log buffer for `drycc logs`
diff --git a/_scripts/install.sh b/_scripts/install.sh
@@ -439,10 +439,8 @@ timeseries:
     size: ${TIMESERIES_PERSISTENCE_SIZE:-5Gi}
     storageClass: ${TIMESERIES_PERSISTENCE_STORAGE_CLASS:-""}
 
-fluentd:
-  imageRegistry: ${DRYCC_REGISTRY} 
-  daemonEnvironment:
-    CONTAINER_TAIL_PARSER_TYPE: "/^(?<time>.+) (?<stream>stdout|stderr)( (?<tags>.))? (?<log>.*)$/"
+fluentbit:
+  imageRegistry: ${DRYCC_REGISTRY}
 
 controller:
   apiReplicas: ${CONTROLLER_API_REPLICAS}
diff --git a/charts/workflow/Chart.yaml b/charts/workflow/Chart.yaml
@@ -16,7 +16,7 @@ dependencies:
 - name: timeseries
   repository: oci://registry.drycc.cc/charts-testing
   version: x.x.x
-- name: fluentd
+- name: fluentbit
   repository: oci://registry.drycc.cc/charts-testing
   version: x.x.x
 - name: gateway
diff --git a/charts/workflow/values.yaml b/charts/workflow/values.yaml
@@ -237,14 +237,9 @@ rabbitmq:
     ## If the PV uses a different storage class, specify that here.
     storageClass: ""
 
-fluentd:
-  syslog:
-    # Configure the following ONLY if using Fluentd to send log messages to both
-    # the Logger component and external syslog endpoint
-    # external syslog endpoint url
-    host: ""
-    # external syslog endpoint port
-    port: ""
+fluentbit:
+  # Interval to flush output (seconds)
+  flush: 5
 
 monitor:
   grafana:
diff --git a/src/_includes/install-workflow.md b/src/_includes/install-workflow.md
@@ -44,10 +44,10 @@ drycc-builder-hy3xv            1/1       Running   5          5m
 drycc-controller-g3cu8         1/1       Running   5          5m
 drycc-controller-celery-cmxxn  3/3       Running   0          5m
 drycc-database-rad1o           1/1       Running   0          5m
-drycc-logger-fluentd-1v8uk     1/1       Running   0          5m
-drycc-logger-fluentd-esm60     1/1       Running   0          5m
+drycc-logger-fluentbit-1v8uk   1/1       Running   0          5m
+drycc-logger-fluentbit-esm60   1/1       Running   0          5m
 drycc-logger-sm8b3             1/1       Running   0          5m
-drycc-storage-4ww3t              1/1       Running   0          5m
+drycc-storage-4ww3t            1/1       Running   0          5m
 drycc-registry-asozo           1/1       Running   1          5m
 drycc-rabbitmq-0               1/1       Running   0          5m
 ```
diff --git a/src/installing-workflow/gateway.md b/src/installing-workflow/gateway.md
@@ -52,8 +52,8 @@ drycc-builder-hy3xv            1/1       Running   5          5m
 drycc-controller-g3cu8         1/1       Running   5          5m
 drycc-controller-celery-cmxxn  3/3       Running   0          5m
 drycc-database-rad1o           1/1       Running   0          5m
-drycc-logger-fluentd-1v8uk     1/1       Running   0          5m
-drycc-logger-fluentd-esm60     1/1       Running   0          5m
+drycc-logger-fluentbit-1v8uk   1/1       Running   0          5m
+drycc-logger-fluentbit-esm60   1/1       Running   0          5m
 drycc-logger-sm8b3             1/1       Running   0          5m
 drycc-storage-4ww3t            1/1       Running   0          5m
 drycc-registry-asozo           1/1       Running   1          5m
diff --git a/src/installing-workflow/index.md b/src/installing-workflow/index.md
@@ -68,7 +68,7 @@ $ helm install drycc oci://registry.drycc.cc/charts/workflow \
     --set imagebuilder.imageRegistry=quay.io \
     --set controller.imageRegistry=quay.io \
     --set database.imageRegistry=quay.io \
-    --set fluentd.imageRegistry=quay.io \
+    --set fluentbit.imageRegistry=quay.io \
     --set redis.imageRegistry=quay.io \
     --set rabbitmq.imageRegistry=quay.io \
     --set logger.imageRegistry=quay.io \
@@ -106,9 +106,9 @@ drycc-controller-3953262871-pncgq         1/1       Running   2          4m
 drycc-controller-celery-cmxxn             3/3       Running   0          4m
 drycc-database-83844344-47ld6             1/1       Running   0          4m
 drycc-logger-176328999-wjckx              1/1       Running   4          4m
-drycc-logger-fluentd-zxnqb                1/1       Running   0          4m
+drycc-logger-fluentbit-zxnqb              1/1       Running   0          4m
 drycc-redis-304849759-1f35p               1/1       Running   0          4m
-drycc-storage-676004970-nxqgt               1/1       Running   0          4m
+drycc-storage-676004970-nxqgt             1/1       Running   0          4m
 drycc-monitor-grafana-432627134-lnl2h     1/1       Running   0          4m
 drycc-monitor-telegraf-wmcmn              1/1       Running   1          4m
 drycc-registry-756475849-lwc6b            1/1       Running   1          4m
diff --git a/src/managing-workflow/platform-logging.md b/src/managing-workflow/platform-logging.md
@@ -1,8 +1,8 @@
 # Platform Logging
 
-The logging platform is made up of 2 components - [Fluentd](https://github.com/drycc/fluentd) and [Logger](https://github.com/drycc/logger).
+The logging platform is made up of 2 components - [Fluentbit](https://github.com/drycc/fluentbit) and [Logger](https://github.com/drycc/logger).
 
-[Fluentd](https://github.com/drycc/fluentd) runs on every worker node of the cluster and is deployed as a [Daemon Set](http://kubernetes.io/v1.1/docs/admin/daemons.html). The Fluentd pods capture all of the stderr and stdout streams of every container running on the host (even those not hosted directly by kubernetes). Once the log message arrives in our [custom fluentd plugin](https://github.com/drycc/fluentd/tree/main/rootfs/opt/fluentd/drycc-output) we determine where the message originated.
+[Fluentbit](https://github.com/drycc/fluentbit) runs on every worker node of the cluster and is deployed as a [Daemon Set](http://kubernetes.io/v1.1/docs/admin/daemons.html). The Fluentbit pods capture all of the stderr and stdout streams of every container running on the host (even those not hosted directly by kubernetes). Once the log message arrives in our [custom fluentbit plugin](https://github.com/drycc/fluentbit/tree/main/plugin) we determine where the message originated.
 
 If the message was from the [Workflow Controller](https://github.com/drycc/controller) or from an application deployed via workflow we send it to the logs topic on the local [Redis Stream](http://redis.io) instance.
 
@@ -27,9 +27,9 @@ If the `drycc logs` command encounters an error it will return the following mes
 
 ```
 Error: There are currently no log messages. Please check the following things:
-1) Logger and fluentd pods are running.
+1) Logger and fluentbit pods are running.
 2) The application is writing logs to the logger component by checking that an entry in the ring buffer was created: kubectl  --namespace=drycc logs <logger pod>
-3) Making sure that the container logs were mounted properly into the fluentd pod: kubectl --namespace=drycc exec <fluentd pod> ls /var/log/containers
+3) Making sure that the container logs were mounted properly into the fluentbit pod: kubectl --namespace=drycc exec <fluentbit pod> ls /var/log/containers
 ```
 
 ## Architecture Diagram
@@ -43,78 +43,12 @@ Error: There are currently no log messages. Please check the following things:
                             │                           │                
                             ▼                           │                
 ┌────────┐             ┌─────────┐    logs/metrics   ┌──────────────┐     
-│App Logs│──Log File──▶│ fluentd │───────topics─────▶│ Redis Stream │     
+│App Logs│──Log File──▶│Fluentbit│───────topics─────▶│ Redis Stream │     
 └────────┘             └─────────┘                   └──────────────┘     
                                                                           
 ```
 
 ## Default Configuration
 
-By default the Fluentd pod can be configured to talk to numerous syslog endpoints. So for example it is possible to have Fluentd send log messages to both the Logger component and [Papertrail](https://papertrailapp.com/). This allows production deployments of Drycc to satisfy stringent logging requirements such as offsite backups of log data.
-
-Configuring Fluentd to talk to multiple syslog endpoints means modifying the Fluentd daemonset
-manifest.
-This means you will need to fetch the chart with `helm fetch oci://registry.drycc.cc/charts/workflow --untar`, then
-modify `workflow/charts/fluentd/templates/logger-fluentd-daemon.yaml` with the following:
-
-```
-env:
-- name: "SYSLOG_HOST_1"
-  value: "my.syslog.host"
-- name: "SYSLOG_PORT_1"
-  value: "5144"
-  ....
-- name: "SYSLOG_HOST_N"
-  value: "my.syslog.host.n"
-- name: "SYSLOG_PORT_N"
-  value: "51333"
-```
-
-If you only need to talk to 1 Syslog endpoint you can use the following configuration within your chart:
-
-```
-env:
-- name: "SYSLOG_HOST"
-  value: "my.syslog.host"
-- name: "SYSLOG_PORT"
-  value: "5144"
-```
-
-Then run `helm install ./workflow --namespace drycc` to install the modified chart.
-
-### Customizing:
-
-We currently support logging information to Syslog, Elastic Search, and Sumo Logic. However, we will gladly accept pull requests that add support to other locations. For more information please visit the [fluentd repository](https://github.com/drycc/fluentd).
-
-
-### Custom Fluentd Plugins
-
-That are many output plugins available for [Fluentd](https://github.com/search?q=fluentd+output&ref=opensearch). But, we purposefully do not ship our Fluentd image with these installed. Instead, we provide a mechanism that allows users to install a plugin at startup time of the container and configure it. 
-
-If you would like to install a plugin you can set an environment variable such as the following: `FLUENTD_PLUGIN_N=some-fluentd-plugin` where N is a positive integer that is incremented for every plugin you wish to install. After you set this value you must then set the configuration text for the `FILTER` or `STORE` plugin you are installing. You can do that by setting `CUSTOM_STORE_N=configuration text` where N is the corresponding index value of the plugin you just installed.
-
-Here is an example of setting the values directly in the manifest of the daemonset. 
-
-```
-env:
-  - name: "FLUENTD_PLUGIN_1"
-    value: "fluent-plugin-kafka"
-  - name: "CUSTOM_STORE_1"
-    value: |
-      <store>
-        @type kafka \
-        default_topic some_topic
-      </store>
-```
-
-Or you could configure it using the `daemonEnvironment` key in the `values.yaml`:
-
-```
-fluentd:
-  daemonEnvironment:
-    FLUENTD_PLUGIN_1: "fluent-plugin-kafka"
-    CUSTOM_STORE_1: "|\n              <store>\n                @type kafka\n                        default_topic some_topic\n                        </store>"
-    INSTALL_BUILD_TOOLS: "|\n              true"
-```
-
-For more information please see the [Custom Plugins](https://github.com/drycc/fluentd#custom-plugins) section of the README.
+Fluent Bit is based in a pluggable architecture where different plugins plays a major role in the data pipeline, more than 70 built-in plugins available.
+Please refer to charts [values.yaml](https://github.com/drycc/fluentbit/blob/main/charts/fluentbit/values.yaml) for specific configurations.
diff --git a/src/managing-workflow/tuning-component-settings.md b/src/managing-workflow/tuning-component-settings.md
@@ -115,26 +115,18 @@ Setting           | Description
 BACKUP_FREQUENCY  | how often the database should perform a base backup (default: "12h")
 BACKUPS_TO_RETAIN | number of base backups the backing store should retain (default: 5)
 
-## Customizing Fluentd
-
-The following values can be changed in the `values.yaml` file or by using the `--set` flag with the Helm CLI.
-
-Key               | Default | Description
-------------------| --------| ---------------------------------
-syslog.host | "" | Host value of a syslog endpoint
-syslog.port | "" | Port value of a syslog endpoint
-sources.startScript | false | Capture kubernetes start script logs
-sources.docker | false | Capture docker daemon logs
-sources.etcd | false | Capture etcd logs
-sources.kubelet | false | Capture kubelet logs
-sources.kubeApi | false | Capture Kubernetes API logs
-sources.controller | false | Capture Kubernetes Controller logs
-sources.scheduler | false | Capture Kubernetes Scheduler logs
-output.disableDrycc | false | Disable the Drycc output plugin
-boot.installBuildTools | false | Install the build tools package. This is useful when using custom plugins
-daemonEnvironment | | Takes key-value pairs and turns them into environment variables.
-
-For more information about the various environment variables that can be set please see the [README](https://github.com/drycc/fluentd/blob/main/README.md)
+## Customizing Fluentbit
+
+The following values can be changed in the `values.yaml` file or by using the `--values` flag with the Helm CLI.
+
+Key               | Description
+------------------| ---------------------------------
+config.service    | The service section defines the global properties of the service.
+config.inputs     | An input section defines a source (related to an input plugin).
+config.filters    | A filter section defines a filter (related to a filter plugin)
+config.outputs    | The outputs section specify a destination that certain records should follow after a Tag match.
+
+For more information about the various variables that can be set please see the [fluentbit](https://github.com/drycc/fluentbit).
 
 ## Customizing the Logger
 
@@ -203,7 +195,7 @@ API_VERSION       | The version number Workflow Manager sends to the versions AP
 [downward-api]: http://kubernetes.io/docs/user-guide/downward-api/
 [gunicorn]: http://gunicorn.org/
 [kubernetes-deployment-revision]: http://kubernetes.io/docs/user-guide/deployments/#revision-history-limit
-[logger]: ../understanding-workflow/components.md#logger-fluentd-logger
+[logger]: ../understanding-workflow/components.md#logger-fluentbit-logger
 [monitor]: ../understanding-workflow/components.md#monitor
 [pull-policy]: http://kubernetes.io/docs/user-guide/images/
 [registry]: ../understanding-workflow/components.md#registry
diff --git a/src/managing-workflow/upgrading-workflow.md b/src/managing-workflow/upgrading-workflow.md
@@ -48,10 +48,10 @@ drycc-controller-1410285775-ipc34         1/1       Running   3          5m
 drycc-controller-celery-694f75749b-cmxxn  3/3       Running   0          5m
 drycc-database-e7c5z                      1/1       Running   0          5m
 drycc-logger-cgjup                        1/1       Running   3          5m
-drycc-logger-fluentd-45h7j                1/1       Running   0          5m
-drycc-logger-fluentd-4z7lw                1/1       Running   0          5m
-drycc-logger-fluentd-k2wsw                1/1       Running   0          5m
-drycc-logger-fluentd-skdw4                1/1       Running   0          5m
+drycc-logger-fluentbit-45h7j              1/1       Running   0          5m
+drycc-logger-fluentbit-4z7lw              1/1       Running   0          5m
+drycc-logger-fluentbit-k2wsw              1/1       Running   0          5m
+drycc-logger-fluentbit-skdw4              1/1       Running   0          5m
 drycc-redis-8nazu                         1/1       Running   0          5m
 drycc-monitor-grafana-tm266               1/1       Running   0          5m
 drycc-monitor-telegraf-51zel              1/1       Running   1          5m
diff --git a/src/troubleshooting/kubectl.md b/src/troubleshooting/kubectl.md
@@ -14,9 +14,9 @@ running, run:
 	drycc-controller-h6lk6         0/1       ContainerCreating   0          4s
 	drycc-controller-celery-cmxxn  0/3       ContainerCreating   0          4s
 	drycc-database-56v39           0/1       ContainerCreating   0          4s
-	drycc-logger-fluentd-xihr1     0/1       Pending             0          2s
+	drycc-logger-fluentbit-xihr1   0/1       Pending             0          2s
 	drycc-logger-grupg             0/1       ContainerCreating   0          3s
-	drycc-storage-c2exb              0/1       Pending             0          3s
+	drycc-storage-c2exb            0/1       Pending             0          3s
 	drycc-monitor-grafana-9ccur    0/1       Pending             0          3s
 	drycc-monitor-telegraf-dc3y3   0/1       Pending             0          2s
 	drycc-registry-5bor6           0/1       Pending             0          3s
diff --git a/src/understanding-workflow/components.md b/src/understanding-workflow/components.md
@@ -88,22 +88,22 @@ images generated from the builder component. Registry persists the Docker image
 images to either local storage (in development mode) or to object storage
 configured for the cluster.
 
-## Logger: fluentd, logger
+## Logger: fluentbit, logger
 
-The logging subsystem consists of two components. Fluentd handles log shipping
+The logging subsystem consists of two components. Fluentbit handles log shipping
 and logger maintains a ring-buffer of application logs.
 
 
-**Project Location:** [drycc/fluentd](https://github.com/drycc/fluentd)
+**Project Location:** [drycc/fluentbit](https://github.com/drycc/fluentbit)
 
-Fluentd is deployed to your Kubernetes cluster via Daemon Sets. Fluentd
+Fluentbit is deployed to your Kubernetes cluster via Daemon Sets. Fluentbit
 subscribes to all container logs, decorates the output with Kubernetes metadata
 and can be configured to drain logs to multiple destinations. By default,
-fluentd ships logs to the logger component, which powers `drycc logs`.
+Fluentbit ships logs to the logger component, which powers `drycc logs`.
 
 **Project Location:** [drycc/logger](https://github.com/drycc/logger)
 
-The `logger` component receives log streams from `fluentd`, collating by
+The `logger` component receives log streams from `fluentbit`, collating by
 Application name. Logger does not persist logs to disk, instead maintaining an
 in-memory ring buffer. For more information on logger see the [project
 documentation][logger-documentation].
