Skip to content

Commit 9359c85

Browse files
choudharypoojachoudharypooja
authored
feat(lmlogs): priorityClassName systemd support for lmlogs (#493)
* fix(logs): logs version upgrade in lm-container * feat(lmlogs): support for priorityClassName and systemd logs in lm-logs chart * feat(lmlogs): adding boolean flag for enabling/disabling systemd conf * feat(lmlogs): bumping up the logs chart version * feat(lmlogs): bumping up the logs chart app version
1 parent 34da2ba commit 9359c85

File tree

7 files changed

+185
-37
lines changed

7 files changed

+185
-37
lines changed

‎charts/lm-logs/Chart.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
apiVersion: v2
22
description: A Helm chart for sending k8s logs to Logic Monitor
33
name: lm-logs
4-
version: 0.7.0-rc01
4+
version: 0.8.0-rc01
55
maintainers:
66
- email: dev@logicmonitor.com
77
name: LogicMonitor
88
icon: https://logicmonitor.github.io/helm-charts-qa/lm_logo.png
9-
appVersion: 1.3.0
9+
appVersion: 1.4.0
1010
home: https://logicmonitor.github.io/helm-charts-qa

‎charts/lm-logs/Dockerfile

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,21 @@
11
FROM fluent/fluentd-kubernetes-daemonset:v1.16-debian-forward-1
2+
23
USER root
3-
RUN gem install fluent-plugin-lm-logs -v 1.2.8
4-
RUN gem install fluent-plugin-multi-format-parser -v 1.0.0
4+
5+
RUN apt-get update && \
6+
apt-get install -y --no-install-recommends \
7+
build-essential \
8+
libsystemd-dev \
9+
pkg-config \
10+
ruby-dev && \
11+
gem install fluent-plugin-systemd -v 1.1.1 && \
12+
gem install fluent-plugin-lm-logs -v 1.2.8 && \
13+
gem install fluent-plugin-multi-format-parser -v 1.0.0 && \
14+
apt-get purge -y \
15+
build-essential \
16+
libsystemd-dev \
17+
pkg-config \
18+
ruby-dev && \
19+
apt-get autoremove -y && \
20+
apt-get clean && \
21+
rm -rf /var/lib/apt/lists/*

‎charts/lm-logs/templates/_helpers.tpl

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,3 +130,66 @@ Return the appropriate apiVersion for rbac.
130130
{{- define "fluentd-image" -}}
131131
"{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
132132
{{- end -}}
133+
134+
{{- define "fluentd.lmMatch" -}}
135+
<match {{ .tag }}>
136+
@type lm
137+
company_name {{ if .context.Values.lm_company_name }}{{ .context.Values.lm_company_name }}{{ else }}{{ required "A valid .Values.lm_company_name or .Values.global.account entry is required!" .context.Values.global.account }}{{ end }}
138+
company_domain {{ .context.Values.lm_company_domain | default .context.Values.global.companyDomain | default "logicmonitor.com" }}
139+
resource_mapping '{{ .resource_mapping }}'
140+
resource_type {{ .context.Values.fluent.resource_type | default "" }}
141+
{{- if and (or .context.Values.lm_access_id .context.Values.global.accessID) (or .context.Values.lm_access_key .context.Values.global.accessKey) }}
142+
access_id {{ .context.Values.lm_access_id | default .context.Values.global.accessID }}
143+
access_key {{ .context.Values.lm_access_key | default .context.Values.global.accessKey }}
144+
{{- else if .context.Values.lm_bearer_token }}
145+
bearer_token {{ .context.Values.lm_bearer_token }}
146+
{{- else }}
147+
{{ required "Either specify valid lm_access_id and lm_access_key both or lm_bearer_token for authentication with LogicMonitor." .context.Values.lm_bearer_token }}
148+
{{- end }}
149+
debug false
150+
compression gzip
151+
{{ include "logsource.userAgent" .context | nindent 8 }}
152+
include_metadata {{ hasKey .context.Values.fluent "include_metadata" | ternary .context.Values.fluent.include_metadata true }}
153+
device_less_logs {{ .context.Values.fluent.device_less_logs | default false }}
154+
<buffer>
155+
@type memory
156+
flush_interval {{ .context.Values.fluent.buffer.memory.flush_interval | default "1s" }}
157+
chunk_limit_size {{ .context.Values.fluent.buffer.memory.chunk_limit_size | default "8m" }}
158+
flush_thread_count {{ .context.Values.fluent.buffer.memory.flush_thread_count | default "8" }}
159+
</buffer>
160+
</match>
161+
{{- end }}
162+
163+
{{/*
164+
Optional systemd.conf block for ConfigMap rendering (with context passing)
165+
*/}}
166+
{{- define "fluentd.systemdConfBlock" -}}
167+
{{- $ctx := .context -}}
168+
{{- if $ctx.Values.useSystemdConf }}
169+
{{- if not $ctx.Values.systemd.conf }}
170+
systemd.conf: |
171+
<source>
172+
@type systemd
173+
path /run/log/journal
174+
tag systemd.al2023
175+
read_from_head false
176+
<storage>
177+
@type local
178+
persistent false
179+
</storage>
180+
</source>
181+
182+
<match systemd.al2023>
183+
@type relabel
184+
@label @PROCESS_AFTER_CONCAT
185+
</match>
186+
{{- else }}
187+
systemd.conf: |
188+
{{ $ctx.Values.systemd.conf | indent 2 }}
189+
{{- end }}
190+
{{- else }}
191+
systemd.conf: ""
192+
{{- end }}
193+
{{- end }}
194+
195+

‎charts/lm-logs/templates/configmap.yaml

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,34 @@ metadata:
99
data:
1010
fluent.conf: |
1111
@include kubernetes.conf
12+
{{- if .Values.useSystemdConf }}
13+
@include systemd.conf
14+
{{- end }}
1215
1316
<system>
1417
log_level "#{ENV['FLUENT_LOG_LEVEL'] || 'warn'}"
1518
</system>
1619
1720
<label @PROCESS_AFTER_CONCAT>
21+
{{- if .Values.useSystemdConf }}
22+
<filter {{ .Values.systemd.tag | default "systemd.al2023" }}>
23+
@type record_transformer
24+
enable_ruby
25+
<record>
26+
message ${record["MESSAGE"]}
27+
timestamp ${record["__REALTIME_TIMESTAMP"]}
28+
source_tag {{ .Values.systemd.tag | default "systemd.al2023" }}
29+
{{- if .Values.fluent.device_less_logs }}
30+
resource.service.name ${ENV['K8S_NODE_NAME']}
31+
{{- end }}
32+
{{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
33+
{{ include "kubernetes.cluster_name" . | nindent 10 }}
34+
{{- end }}
35+
</record>
36+
remove_keys MESSAGE
37+
</filter>
38+
{{- end }}
39+
1840
<filter kubernetes.**>
1941
@type kubernetes_metadata
2042
@id filter_kube_metadata
@@ -35,43 +57,21 @@ data:
3557
timestamp ${record["time"]}
3658
{{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
3759
{{ include "kubernetes.cluster_name" . | nindent 8 }}
38-
{{- end}}
39-
{{- if .Values.fluent.device_less_logs }}
40-
resource.service.name ${record.dig("kubernetes","labels","app") != nil ? record.dig("kubernetes","labels","app") : record.dig("kubernetes","labels","app_kubernetes_io/name") != nil ? record.dig("kubernetes","labels","app_kubernetes_io/name") : record.dig("kubernetes","container_name") != nil ? record.dig("kubernetes","container_name") : record.dig("kubernetes","pod_name") != nil ? record.dig("kubernetes","pod_name") : "unknown" }
60+
{{- end }}
61+
{{- if .Values.fluent.device_less_logs }}
62+
resource.service.name '${record.dig("kubernetes","labels","app") != nil ? record.dig("kubernetes","labels","app") : record.dig("kubernetes","labels","app_kubernetes_io/name") != nil ? record.dig("kubernetes","labels","app_kubernetes_io/name") : record.dig("kubernetes","container_name") != nil ? record.dig("kubernetes","container_name") : record.dig("kubernetes","pod_name") != nil ? record.dig("kubernetes","pod_name") : "unknown"}'
4163
resource.service.namespace ${record["kubernetes"]["namespace_name"]}
42-
{{- end}}
64+
{{- end }}
4365
</record>
4466
remove_keys log
4567
</filter>
4668
47-
<match kubernetes.**>
48-
@type lm
49-
company_name {{ if .Values.lm_company_name }} {{ .Values.lm_company_name }} {{ else }} {{ required "A valid .Values.lm_company_name or .Values.global.account entry is required!" .Values.global.account }} {{ end }}
50-
company_domain {{ .Values.lm_company_domain | default .Values.global.companyDomain | default "logicmonitor.com" }}
51-
resource_mapping {"kubernetes.pod_name": "auto.name"}
52-
resource_type {{ .Values.fluent.resource_type | default "" }}
53-
{{- if and ( or .Values.lm_access_id .Values.global.accessID ) ( or .Values.lm_access_key .Values.global.accessKey) }}
54-
access_id {{ .Values.lm_access_id | default .Values.global.accessID }}
55-
access_key {{ .Values.lm_access_key | default .Values.global.accessKey }}
56-
{{- else if .Values.lm_bearer_token }}
57-
bearer_token {{ .Values.lm_bearer_token }}
58-
{{- else }} {{ required "Either specify valid lm_access_id and lm_access_key both or lm_bearer_token for authentication with LogicMonitor." .Values.lm_bearer_token }}
59-
{{- end}}
60-
debug false
61-
compression gzip
62-
{{ include "logsource.userAgent" . | nindent 8 }}
63-
include_metadata {{ hasKey .Values.fluent "include_metadata" | ternary .Values.fluent.include_metadata true }}
64-
device_less_logs {{ .Values.fluent.device_less_logs | default false }}
65-
<buffer>
66-
@type memory
67-
flush_interval {{ .Values.fluent.buffer.memory.flush_interval | default "1s" }}
68-
chunk_limit_size {{ .Values.fluent.buffer.memory.chunk_limit_size | default "8m" }}
69-
flush_thread_count {{ .Values.fluent.buffer.memory.flush_thread_count | default "8"}}
70-
</buffer>
71-
</match>
69+
{{ include "fluentd.lmMatch" (dict "tag" "kubernetes.**" "resource_mapping" "{\"kubernetes.pod_name\": \"auto.name\"}" "context" .) | nindent 6 }}
70+
{{- if .Values.useSystemdConf }}
71+
{{ include "fluentd.lmMatch" (dict "tag" (.Values.systemd.tag | default "systemd.al2023") "resource_mapping" "{\"_HOSTNAME\": \"system.hostname\"}" "context" .) | nindent 6 }}
72+
{{- end }}
7273
</label>
7374
74-
7575
kubernetes.conf: |
7676
<source>
7777
@type tail
@@ -99,7 +99,7 @@ data:
9999
<filter kubernetes.**>
100100
@type concat
101101
key {{ .Values.kubernetes.multiline_concat_key }}
102-
seperator ""
102+
separator ""
103103
multiline_start_regexp {{ .Values.kubernetes.multiline_start_regexp }}
104104
timeout_label @PROCESS_AFTER_CONCAT
105105
</filter>
@@ -109,4 +109,4 @@ data:
109109
@label @PROCESS_AFTER_CONCAT
110110
</match>
111111
112-
112+
{{ include "fluentd.systemdConfBlock" (dict "context" .) | indent 2 }}

‎charts/lm-logs/templates/deamonset.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,15 @@ spec:
1616
{{- include "fluentd.selectorLabels" . | nindent 8 }}
1717
spec:
1818
serviceAccountName: {{ include "fluentd.serviceAccountName" . }}
19+
{{- if .Values.priorityClassName }}
20+
priorityClassName: {{ .Values.priorityClassName | quote }}
21+
{{- end }}
1922
containers:
2023
- name: {{ .Chart.Name }}
24+
{{- if .Values.securityContext }}
2125
securityContext:
2226
{{- toYaml .Values.securityContext | nindent 12 }}
27+
{{- end }}
2328
image: {{ include "fluentd-image" . }}
2429
env:
2530
- name: K8S_NODE_NAME

‎charts/lm-logs/values.schema.json

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,52 @@
381381
"$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount"
382382
},
383383
"uniqueItems": true
384+
},
385+
"systemd": {
386+
"type": ["object", "null"],
387+
"default": null,
388+
"description": "Optional systemd log source configuration for AL2023 or similar systems",
389+
"properties": {
390+
"tag": {
391+
"type": "string",
392+
"description": "The Fluentd tag used for systemd logs",
393+
"examples": ["systemd.al2023"]
394+
},
395+
"conf": {
396+
"type": "string",
397+
"description": "Custom Fluentd systemd.conf content to be included and processed if defined"
398+
}
399+
},
400+
"additionalProperties": false
401+
},
402+
"securityContext": {
403+
"type": ["object", "null"],
404+
"default": null,
405+
"properties": {
406+
"privileged": {
407+
"type": "boolean"
408+
},
409+
"runAsUser": {
410+
"type": "integer"
411+
}
412+
},
413+
"additionalProperties": false
414+
},
415+
"priorityClassName": {
416+
"$id": "#/properties/priorityClassName",
417+
"type": ["string", "null"],
418+
"title": "Priority Class Name",
419+
"description": "The Kubernetes PriorityClass to assign to Fluentd pods. If set, this determines pod preemption and scheduling priority.",
420+
"default": null,
421+
"examples": [
422+
""
423+
],
424+
"$comment": "tf:optional"
425+
},
426+
"useSystemdConf": {
427+
"type": "boolean",
428+
"default": true,
429+
"description": "If true and no custom systemd.conf is provided, a default configuration for systemd will be injected. If false, systemd will be disabled regardless of conf."
384430
}
385431
},
386432
"additionalProperties": false,

‎charts/lm-logs/values.yaml

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
image:
22
repository: logicmonitor/lm-logs-k8s-fluentd
33
pullPolicy: Always
4-
tag: "1.3.0"
4+
tag: "1.4.0"
55

66
lm_access_id: ""
77
lm_access_key: ""
@@ -41,6 +41,16 @@ fluent:
4141
chunk_limit_size: 8m
4242
flush_thread_count: 8
4343

44+
systemd:
45+
tag: ""
46+
conf: ""
47+
48+
useSystemdConf: false
49+
50+
securityContext: null
51+
52+
priorityClassName: null
53+
4454
kubernetes:
4555
multiline_start_regexp: /^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/
4656
cluster_name: ""
@@ -58,6 +68,10 @@ volumes:
5868
- name: varlibdockercontainers
5969
hostPath:
6070
path: /var/lib/docker/containers
71+
- name: journal
72+
hostPath:
73+
path: /run/log/journal
74+
6175

6276
volumeMounts:
6377
- name: fluentconf
@@ -67,3 +81,6 @@ volumeMounts:
6781
- name: varlibdockercontainers
6882
mountPath: /var/lib/docker/containers
6983
readOnly: true
84+
- name: journal
85+
mountPath: /run/log/journal
86+
readOnly: true

0 commit comments

Comments
 (0)