V3.0.2 (#46)

* Enable log collection for PersistentVolume and PersistentVolumeClaim

* Multi process worker support - final set of changes

* Updates to custom docker image instruction

* V3.0.2 changelog, charts version changes

* minor change for logs configmap template

* add oci_images.tf; update dashboard import instructions

* review comments, pull in change from PR#39

---------

Co-authored-by: Naga Raju Barri <naga.barri@oracle.com>
Co-authored-by: paliwalparitosh <paritoshedx@gmail.com>

Author: 3 people

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Change Log
 
+## 2023-07-24
+### Added
+- Support Fluentd's [Multi Process Workers](https://docs.fluentd.org/deployment/multi-process-workers). 
+- Custom Container Image for Fluentd using OL8-Slim as base Image.
+- PV, PVC Objects Collection 
+### Changed
+- Instructions and dependency versions updates to custom container image for Fluentd using OL8 as base image.
+- ClusterRole updates to add read permission for `storage.k8s.io` api group to support PV, PVC Objects collection. 
+### Deprecating
+- Custom Container Image for Fluentd using Debian and OL8 as base Image. 
+
+## 2023-07-19
+### Added
+- Helm repo throguh Github pages.
+
 ## 2023-06-14
 ### Added
 - Kubernetes Metrics Collection to OCI Monitoring using OCI Management Agent.

README.md

@@ -138,7 +138,7 @@ Dashboards needs to be imported manually. Below is an example for importing Dash
 
 1. Download and configure [OCI CLI](https://docs.oracle.com/en-us/iaas/Content/API/SDKDocs/cliinstall.htm) or open cloud-shell where OCI CLI is pre-installed. Alternative methods like REST API, SDK, Terraform etc can also be used.
 2. Find the **OCID** of the compartment, where the dashboards need to be imported.
-3. Download the dashboard JSONs from [here](logan/terraform/oke/modules/dashboards/dashboards_json/) (TBD).
+3. Download the dashboard JSONs from [here](terraform/modules/dashboards/dashboards_json/).
 4. **Replace** all the instances of the keyword - "`${compartment_ocid}`" in the JSONs with the **Compartment OCID** identified in previous step.
     * Following command is for quick reference that can be used in a linux/cloud-shell envirnment :
 
@@ -153,6 +153,7 @@ Dashboards needs to be imported manually. Below is an example for importing Dash
     oci management-dashboard dashboard import --from-json file://node.json
     oci management-dashboard dashboard import --from-json file://workload.json
     oci management-dashboard dashboard import --from-json file://pod.json
+    oci management-dashboard dashboard import --from-json file://service-type-lb.json
     ```
 
 ##### 4 Uninstall

charts/common/templates/clusterrole.yaml

@@ -23,6 +23,7 @@ rules:
       - batch
       - discovery.k8s.io
       - metrics.k8s.io
+      - storage.k8s.io
     resources:
       - '*'
     verbs:

charts/logan/Chart.yaml

@@ -5,11 +5,11 @@ apiVersion: v2
 name: oci-onm-logan
 description: Charts for sending Kubernetes platform logs, compute logs, and Kubernetes Objects information to OCI Logging Analytics.
 type: application
-version: 3.0.1
+version: 3.0.2
 appVersion: "3.0.0"
 
 dependencies:
 - name: oci-onm-common
-  version: "3.0.1"
+  version: "3.0.2"
   repository: "file://../common"
   condition: oci-onm-common.enabled

charts/logan/templates/logs-configmap.yaml

@@ -151,6 +151,10 @@ fluentd:
    file: fluent.conf
    # -- Base directory on the node (with read write permission) for storing fluentd plugins related data.
    baseDir: /var/log
+   # -- To enable MultiProcessWorkers, set this to (> 0) the number of required workers. Defaults to 0.
+   # -- Set corresponding workerId using worker field against each source/log. When enabled MultiProcessWorkers, the default worker would be 0 until unless set for each source/log.
+   multiProcessWorkers: 0
+
    # Configuration for oci-logging-analytics fluentd output plugin
    ociLoggingAnalyticsOutputPlugin:
       # -- OCI API Key profile to use, if multiple profiles are found in the OCI API config file.
@@ -229,6 +233,9 @@ fluentd:
          #"Third Key": "Third Value"
       #ociLAEntityID:
       #encoding:
+      # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+      #worker:
+
       logs:
          # -- Kube Proxy logs collection configuration
          kube-proxy:
@@ -245,6 +252,8 @@ fluentd:
             #ociLAEntityID:
             #ociLALogGroupID:
             #encoding:
+            # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+           #worker:
 
          # -- Kube Flannel logs collection configuration
          kube-flannel:
@@ -296,6 +305,16 @@ fluentd:
             multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
    # Configuration for Linux System specific logs like CronLogs and SecureLogs
    linuxSystem:
+      # Setting the following properties will override the default/generic configuration and applies to all Kubernetes system logs
+      #ociLALogGroupID:
+      #metadata:
+         #"Client Host Region": "America"
+         #"Environment": "Production"
+         #"Third Key": "Third Value"
+      #ociLAEntityID:
+      #encoding:
+      # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+      #worker:
       logs:
          # -- Linux CRON logs collection configuration
          cronlog:
@@ -375,19 +394,13 @@ fluentd:
          - '"/var/log/containers/csi-oci-node-*.log"'
          - '"/var/log/containers/proxymux-client-*.log"'
          - '"/var/log/containers/cluster-autoscaler-*.log"'
+      # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+      #worker: 1
 
    # -- Configuration for any custom logs which are not part of the default configuration defined in this file.
    # All the pod/container logs will be collected as per "genericContainerLogs" section.
    # Use this section to create a custom configuration for any of the container logs.
    # Also, you can use this section to define configuration for any other log path existing on a Kubernetes worker node
-   #custom-id1:
-         #path: /var/log/containers/custom*.log
-         # Logging Analytics log source to use for parsing and processing the logs:
-         #ociLALogSourceName: "Custom1 Logs"
-         # The regular expression pattern for the starting line in case of multi-line logs.
-         #multilineStartRegExp:
-         # Set isContainerLog to false if the log is not a container log (/var/log/containers/*.log). Default value is true.
-         #isContainerLog: true
    customLogs:
       #  A unique identifier to represent the configuration for a single log path
       #custom-id1:
@@ -398,6 +411,16 @@ fluentd:
          #multilineStartRegExp:
          # Set isContainerLog to false if the log is not a container log (/var/log/containers/*.log). Default value is true.
          #isContainerLog: true
+         # Setting the following properties will override the default/generic configuration and applies to all Kubernetes system logs
+         #ociLALogGroupID:
+         #metadata:
+            #"Client Host Region": "America"
+            #"Environment": "Production"
+            #"Third Key": "Third Value"
+         #ociLAEntityID:
+         #encoding:
+         # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+         #worker:
       #custom-id2:
          #path: /var/log/custom/*.log
          # Logging Analytics log source to use for parsing and processing the logs:
@@ -436,6 +459,10 @@ fluentd:
             api_endpoint: ""
          events:
             api_endpoint: ""
+         persistent_volumes:
+            api_endpoint: ""
+         persistent_volume_claims:
+            api_endpoint: ""
          daemon_sets:
             api_endpoint: apis/apps
          replica_sets:

charts/logan/values.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.0.1
+version: 3.0.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -28,11 +28,11 @@ appVersion: "3.0.0"
 
 dependencies:
 - name: oci-onm-common
-  version: "3.0.1"
+  version: "3.0.2"
   repository: "file://../common"
   condition: oci-onm-common.enabled
 - name: oci-onm-logan
-  version: "3.0.1"
+  version: "3.0.2"
   repository: "file://../logan"
   condition: oci-onm-logan.enabled
 - name: oci-onm-mgmt-agent

charts/oci-onm/Chart.yaml

@@ -195,4 +195,83 @@ oci-onm-logan:
       #   -----BEGIN RSA PRIVATE KEY-----
       #   XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
       #   -----END RSA PRIVATE KEY-----
+``` 
+
+### Enable Multi Process Workers
+
+**Note**: This feature will not work with custom fluentd configuration and if you set custom fluentd configuration, it would be placed under worker 0.
+
+To enable multi-process workers feature of Fluentd, set value of `multiProcessWorkers` under oci-onm-logan to the desired number of workers. By default it is set 0, indicating that the feature is disabled. To assign different workers for different logs, you may set `worker` field to the desired worker id against each of the individual logs or group of logs as supported in the confguration. The default worker id is set to 0 for all the logs when enabling `multiProcessWorkers` feature. The following are few examples,
+ 
+#### Example1  
+
+Enable multi-process worker feature with 2 workers and set the worker id 1 to all container logs (except custom logs) and keeping the default worker id (0) for the remaining all logs. 
+
 ```      
+..       
+..       
+oci-onm-logan:
+  ..  
+  ..  
+  fluentd:
+    ...   
+    ...
+    multiProcessWorkers: 2
+    ...
+    ...
+    genericContainerLogs:
+      ...
+      ... 
+      worker: 1
+```       
+
+#### Example2                 
+
+Enable multi-process worker feature with 3 workers and set the worker id 1 to all container logs (except custom logs), worker id 2 to Kube Proxy, Linux Syslog and Kubelet logs, and keeping the default worker id (0) for the remaining all logs.    
+
+```
+..
+..
+oci-onm-logan:
+  ..
+  ..
+  fluentd:
+    ...
+    ...
+    multiProcessWorkers: 3
+    ...
+    ...
+    kubernetesSystem:
+    ...
+    ...
+      logs:
+        ...
+        ...
+        kube-proxy:
+        ...
+        ...
+        worker: 2
+        ...
+        ...
+    ...
+    ...
+    linuxSystem:
+      ...
+      ...
+      logs:
+        ...
+        ...
+        syslog:
+          ...
+          ...
+          worker: 2
+          ...
+          ...
+    ...
+    ...
+    genericContainerLogs:
+      ...
+      ...
+      worker: 1
+```       
+

docs/FAQ.md

@@ -12,7 +12,7 @@ By default, pre-built images by Oracle are used.
 ##### Fluentd Container Image
 
 - Download all the files from the below mentioned dir into a local machine having access to internet and docker installed.
-  - [OL8](logan/docker-images/v1.0/oraclelinux/8/)
+  - [OL8-Slim](logan/docker-images/v1.0/oraclelinux/8-slim/)
 - Run the following command to build the image.
     - `docker build -t oci-la-fluentd-collector-custom -f Dockerfile .`
 - The docker image built from the above step, can either be pushed to Docker Hub or OCI Container Registry (OCIR) or to a Local Docker Registry depending on the requirements.

docs/custom-images.md

@@ -0,0 +1,88 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+### Build the docker image using multi-stage build
+
+## To build/install all the dependencies
+
+FROM container-registry.oracle.com/os/oraclelinux:8-slim AS builder
+
+USER root
+WORKDIR /fluentd
+
+# Environment variables
+ENV PATH /fluentd/vendor/bundle/ruby/2.7.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.7.0:$GEM_PATH
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.7.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+COPY Gemfile* /fluentd/
+
+# Install ruby, ruby-libs along with rubygems and bundler.
+RUN microdnf -y module enable ruby:2.7 \
+# Install ruby (it's dependencies gdbm-libs) and ruby-libs, disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs ruby-2.7.6 ruby-libs-2.7.6 gdbm-libs \
+# Install rubygems (it's dependencies rubygem-openssl rubygem-psych), disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs rubygems-3.1.6 \
+ && gem install bundler -v 2.3.25 \
+# Install development dependent packages for gems native installation
+ && microdnf -y install --nodocs gcc make redhat-rpm-config openssl ruby-devel gcc-c++ libtool libffi-devel bzip2 git \
+# Install Fluentd, it's dependencies along with other run time dependencies for OCI Logging Analytics Solution
+ && bundle config silence_root_warning true \ 
+ && bundle config --local path /fluentd/vendor/bundle \
+ && bundle config --global jobs 9 \
+ && bundle install --gemfile=/fluentd/Gemfile \
+# Install tini, init for containers (from EPEL repo)
+ && microdnf -y install oracle-epel-release-el8 \
+ && microdnf -y install tini-0.19.0 \
+# Install jemalloc (custom make with no docs)
+ && cd /tmp && ls /tmp \
+ && git clone -b 5.3.0 https://github.com/jemalloc/jemalloc.git && cd jemalloc/ \  
+ && ./autogen.sh && make && make install_bin install_include install_lib \
+ && mv lib/libjemalloc.so.2 /usr/lib
+
+## To build the final docker image
+
+FROM container-registry.oracle.com/os/oraclelinux:8-slim
+
+USER root
+WORKDIR /fluentd
+
+# Environment variables
+ENV PATH /fluentd/vendor/bundle/ruby/2.7.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.7.0:$GEM_PATH
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.7.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+# Install ruby, ruby-libs along with rubygems and bundler.
+RUN microdnf -y module enable ruby:2.7 \
+# Install ruby (it's dependencies gdbm-libs) and ruby-libs, disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs ruby-2.7.6 ruby-libs-2.7.6 gdbm-libs \ 
+# Install rubygems (it's dependencies rubygem-openssl rubygem-psych), disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs rubygems-3.1.6 \ 
+ && gem install bundler -v 2.3.25 \
+ && bundle config --local path /fluentd/vendor/bundle \
+# clear caches
+ && microdnf clean all \
+ && rm -rf /var/cache/dnf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem     
+
+# Copy binaries (tini & jemallco) and rubygems bundler environment from build stage
+COPY --from=builder /fluentd /fluentd
+COPY --from=builder /usr/bin/tini /usr/bin/tini
+COPY --from=builder /usr/lib/libjemalloc.so.2 /usr/lib/libjemalloc.so.2
+
+RUN mkdir -p /fluentd/etc /fluentd/plugins \
+ && touch /fluentd/etc/disable.conf
+
+# Environment variables
+ENV FLUENTD_CONF="/fluentd/etc/fluent.conf"
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+
+COPY entrypoint.sh /fluentd/entrypoint.sh
+# Give execution permission to entrypoint.sh
+RUN ["chmod", "+x", "/fluentd/entrypoint.sh"]
+
+# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib
+ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"]