Skip to content

Commit b0b5e93

Browse files
siddharthcksiddharthck
authored
ci(lm-logs): add multiline_concat_key param for lm-logs (#314)
* ci(lm-logs): add multiline_concat_key param for lm-logs Add multiline_concat_key to determine which key to be used while concatenating multi line logs with Fluentd. Update Readme for lm-logs. * fix(lm-logs): fix concat plugin log-loss Use labels to route concat-timeout logs to LM Reuse label to route normal logs to LM * fix(lm-logs): assign version for lm-logs 0.5.1-rc01 Rectify versioning for lm-logs
1 parent b598c37 commit b0b5e93

File tree

5 files changed

+84
-57
lines changed

5 files changed

+84
-57
lines changed

‎charts/lm-logs/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: v2
22
description: A Helm chart for sending k8s logs to Logic Monitor
33
name: lm-logs
4-
version: 0.5.0-rc01
4+
version: 0.5.1-rc01
55
maintainers:
66
- email: [email protected]
77
name: LogicMonitor

‎charts/lm-logs/README.md‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ The following tables lists the configurable parameters of the lm-logs chart and
4040
| `affinity` | Affinity for pod assignment | `{}` (evaluated as a template) |
4141
| `env` | Map to add extra environment variables | `{}` |
4242
| `kubernetes.multiline_start_regexp` | Regexp to match beginning of multiline | `/^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/` |
43-
| `kubernetes.cluster_name` | ClusterName given while adding k8s cluster | `""` |
43+
| `kubernetes.cluster_name` | ClusterName given while adding k8s cluster | `""` |
44+
| `kubernetes.multiline_concat_key` | Key to look for fluentD to concatenate multiline logs | `"log"` |
45+
4446

4547
### Avaialble Environment variables
4648
For descriptions see: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
@@ -76,3 +78,19 @@ Anomaly detection will be done on `namespace` and `service`
7678
#### Multiline log support for k8s lm logs
7779
To use regexp to match beginning of multiline set `kubernetes.multiline_start_regexp=<some-regex-pattern>`
7880
by default the regex is set to `/^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/`
81+
82+
### Logs appearing in cri format
83+
If conatiner runtime is containerD or cri-o, on lm-logs ui you might see logs with prefix eg.
84+
```
85+
2016-10-06T00:17:09.669794202Z stdout F The content of the log entry 1
86+
```
87+
To solve this we need to install lm-logs with following command :
88+
```
89+
helm upgrade --install -n <namespace> \
90+
--set lm_company_name="<comapny>" \
91+
--set lm_access_id="<access_id>" \
92+
--set lm_access_key="<access_key"> \
93+
--set env.FLUENT_CONTAINER_TAIL_PARSER_TYPE="cri" \
94+
--set kubernetes.multiline_concat_key="message" \
95+
lm-logs logicmonitor/lm-logs
96+
```

‎charts/lm-logs/templates/configmap.yaml‎

Lines changed: 60 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -14,45 +14,61 @@ data:
1414
log_level "#{ENV['FLUENT_LOG_LEVEL'] || 'warn'}"
1515
</system>
1616
17-
<filter kubernetes.**>
18-
@type record_transformer
19-
enable_ruby
20-
<record>
21-
message ${record["log"]} ${record["message"]}
22-
timestamp ${record["time"]}
23-
{{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
24-
{{ include "kubernetes.cluster_name" . | nindent 8 }}
25-
{{- end}}
26-
{{- if .Values.fluent.device_less_logs }}
27-
resource.service.name ${record.dig("kubernetes","labels","app") != nil ? record.dig("kubernetes","labels","app") : record.dig("kubernetes","labels","app_kubernetes_io/name") != nil ? record.dig("kubernetes","labels","app_kubernetes_io/name") : record.dig("kubernetes","container_name") != nil ? record.dig("kubernetes","container_name") : record.dig("kubernetes","pod_name") != nil ? record.dig("kubernetes","pod_name") : "unknown" }
28-
resource.service.namespace ${record["kubernetes"]["namespace_name"]}
17+
<label @PROCESS_AFTER_CONCAT>
18+
<filter kubernetes.**>
19+
@type kubernetes_metadata
20+
@id filter_kube_metadata
21+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
22+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
23+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
24+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
25+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
26+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
27+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
28+
</filter>
29+
30+
<filter kubernetes.**>
31+
@type record_transformer
32+
enable_ruby
33+
<record>
34+
message ${record["log"]} ${record["message"]}
35+
timestamp ${record["time"]}
36+
{{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
37+
{{ include "kubernetes.cluster_name" . | nindent 8 }}
38+
{{- end}}
39+
{{- if .Values.fluent.device_less_logs }}
40+
resource.service.name ${record.dig("kubernetes","labels","app") != nil ? record.dig("kubernetes","labels","app") : record.dig("kubernetes","labels","app_kubernetes_io/name") != nil ? record.dig("kubernetes","labels","app_kubernetes_io/name") : record.dig("kubernetes","container_name") != nil ? record.dig("kubernetes","container_name") : record.dig("kubernetes","pod_name") != nil ? record.dig("kubernetes","pod_name") : "unknown" }
41+
resource.service.namespace ${record["kubernetes"]["namespace_name"]}
42+
{{- end}}
43+
</record>
44+
remove_keys log
45+
</filter>
46+
47+
<match kubernetes.**>
48+
@type lm
49+
company_name {{ if .Values.lm_company_name }} {{ .Values.lm_company_name }} {{ else }} {{ required "A valid .Values.lm_company_name or .Values.global.account entry is required!" .Values.global.account }} {{ end }}
50+
resource_mapping {"kubernetes.pod_name": "auto.name"}
51+
{{- if and ( or .Values.lm_access_id .Values.global.accessID ) ( or .Values.lm_access_key .Values.global.accessKey) }}
52+
access_id {{ .Values.lm_access_id | default .Values.global.accessID }}
53+
access_key {{ .Values.lm_access_key | default .Values.global.accessKey }}
54+
{{- else if .Values.lm_bearer_token }}
55+
bearer_token {{ .Values.lm_bearer_token }}
56+
{{- else }} {{ required "Either specify valid lm_access_id and lm_access_key both or lm_bearer_token for authentication with LogicMonitor." .Values.lm_bearer_token }}
2957
{{- end}}
30-
</record>
31-
remove_keys log
32-
</filter>
58+
debug false
59+
compression gzip
60+
include_metadata {{ hasKey .Values.fluent "include_metadata" | ternary .Values.fluent.include_metadata true }}
61+
device_less_logs {{ .Values.fluent.device_less_logs | default false }}
62+
<buffer>
63+
@type memory
64+
flush_interval {{ .Values.fluent.buffer.memory.flush_interval | default "1s" }}
65+
chunk_limit_size {{ .Values.fluent.buffer.memory.chunk_limit_size | default "8m" }}
66+
flush_thread_count {{ .Values.fluent.buffer.memory.flush_thread_count | default "8"}}
67+
</buffer>
68+
</match>
69+
</label>
70+
3371
34-
<match kubernetes.**>
35-
@type lm
36-
company_name {{ if .Values.lm_company_name }} {{ .Values.lm_company_name }} {{ else }} {{ required "A valid .Values.lm_company_name or .Values.global.account entry is required!" .Values.global.account }} {{ end }}
37-
resource_mapping {"kubernetes.pod_name": "auto.name"}
38-
{{- if and ( or .Values.lm_access_id .Values.global.accessID ) ( or .Values.lm_access_key .Values.global.accessKey) }}
39-
access_id {{ .Values.lm_access_id | default .Values.global.accessID }}
40-
access_key {{ .Values.lm_access_key | default .Values.global.accessKey }}
41-
{{- else if .Values.lm_bearer_token }}
42-
bearer_token {{ .Values.lm_bearer_token }}
43-
{{- else }} {{ required "Either specify valid lm_access_id and lm_access_key both or lm_bearer_token for authentication with LogicMonitor." .Values.lm_bearer_token }}
44-
{{- end}}
45-
debug false
46-
compression gzip
47-
include_metadata {{ hasKey .Values.fluent "include_metadata" | ternary .Values.fluent.include_metadata true }}
48-
device_less_logs {{ .Values.fluent.device_less_logs | default false }}
49-
<buffer>
50-
@type memory
51-
flush_interval {{ .Values.fluent.buffer.memory.flush_interval | default "1s" }}
52-
chunk_limit_size {{ .Values.fluent.buffer.memory.chunk_limit_size | default "8m" }}
53-
flush_thread_count {{ .Values.fluent.buffer.memory.flush_thread_count | default "8"}}
54-
</buffer>
55-
</match>
5672
kubernetes.conf: |
5773
<source>
5874
@type tail
@@ -79,26 +95,15 @@ data:
7995
8096
<filter kubernetes.**>
8197
@type concat
82-
key log
98+
key {{ .Values.kubernetes.multiline_concat_key }}
8399
seperator ""
84100
multiline_start_regexp {{ .Values.kubernetes.multiline_start_regexp }}
85-
timeout_label @NORMAL
101+
timeout_label @PROCESS_AFTER_CONCAT
86102
</filter>
87103
88-
<label @NORMAL>
89-
<match kubernetes.**>
90-
@type stdout
91-
</match>
92-
</label>
104+
<match kubernetes.**>
105+
@type relabel
106+
@label @PROCESS_AFTER_CONCAT
107+
</match>
108+
93109
94-
<filter kubernetes.**>
95-
@type kubernetes_metadata
96-
@id filter_kube_metadata
97-
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
98-
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
99-
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
100-
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
101-
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
102-
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
103-
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
104-
</filter>

‎charts/lm-logs/values.schema.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,9 @@
313313
},
314314
"cluster_name" : {
315315
"type": "string"
316+
},
317+
"multiline_concat_key" : {
318+
"type": "string"
316319
}
317320
}
318321
},

‎charts/lm-logs/values.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ fluent:
3939

4040
kubernetes:
4141
multiline_start_regexp: /^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/
42+
multiline_concat_key: log
4243

4344
nodeSelector: {}
4445
affinity: {}

0 commit comments

Comments
 (0)