Add monitoring to Trino (opendatahub-io#475)

Signed-off-by: Ricardo Martinelli de Oliveira <[email protected]>
eb-oss · Sep 7, 2021 · 9f38fb6 · 9f38fb6
1 parent d1a352a
commit 9f38fb6
Show file tree

Hide file tree

Showing 8 changed files with 124 additions and 20 deletions.
diff --git a/trino/base/kustomization.yaml b/trino/base/kustomization.yaml
@@ -16,6 +16,7 @@ resources:
 - hive-config.yaml
 - hive-jmx-config.yaml
 - trino-config-secret.yaml
+- trino-config-monitoring-secret.yaml
 - trino-catalog-secret.yaml
 - trino-coordinator-dc.yaml
 - trino-service.yaml
@@ -126,13 +127,6 @@ vars:
       kind: ConfigMap
     fieldref:
       fieldpath: data.trino_memory_limit
-  - name: trino_image
-    objref:
-      name: trino-config
-      apiVersion: v1
-      kind: ConfigMap
-    fieldref:
-      fieldpath: data.trino_image
 
 configurations:
   - params.yaml
@@ -146,4 +140,4 @@ images:
     newTag: 2.3.3-002
   - name: trino
     newName: quay.io/opendatahub/trino
-    newTag: latest
+    newTag: '361'
diff --git a/trino/base/params.env b/trino/base/params.env
@@ -11,4 +11,3 @@ trino_cpu_request=300m
 trino_cpu_limit=1
 trino_memory_request=4Gi
 trino_memory_limit=4Gi
-trino_image=quay.io/opendatahub/trino:355
diff --git a/trino/base/params.yaml b/trino/base/params.yaml
@@ -18,7 +18,5 @@ varReference:
     kind: Deployment
   - path: spec/template/spec/containers[]/resources/limits/memory
     kind: Deployment
-  - path: spec/template/spec/containers[]/image
-    kind: Deployment
   - path: spec/template/spec/containers[]/env[]/valueFrom/secretKeyRef/name
     kind: StatefulSet
diff --git a/trino/base/trino-config-monitoring-secret.yaml b/trino/base/trino-config-monitoring-secret.yaml
@@ -0,0 +1,80 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trino-config-monitoring
+stringData:
+  config.yaml: |-
+    ---
+    lowercaseOutputName: true
+    lowercaseOutputLabelNames: true
+    attrNameSnakeCase: false
+    rules:
+      # capture percentile and set quantile label
+      - pattern: 'trino.plugin.hive<type=(.+), name=hive><>(.+AllTime).P(\\d+): (.*)'
+        name: 'trino_hive_$1_$2_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+        labels:
+          quantile: '0.$3'
+      # match non-percentiles
+      - pattern: 'trino.plugin.hive<type=(.+), name=hive><>(.+AllTime.+): (.*)'
+        name: 'trino_hive_$1_$2_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+      # counts
+      - pattern: 'trino.plugin.hive<type=(.+), name=hive><>(.+TotalCount.*): (.*)'
+        name: 'trino_hive_$1_$2_total'
+        type: COUNTER
+      # capture percentile and set quantile label
+      - pattern: 'trino.plugin.hive.s3<type=(.+), name=hive><>(.+AllTime).P(\\d+): (.*)'
+        name: 'trino_hive_s3_$1_$2_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+        labels:
+          quantile: '0.$3'
+      # match non-percentiles
+      - pattern: 'trino.plugin.hive.s3<type=(.+), name=hive><>(.+AllTime.+): (.*)'
+        name: 'trino_hive_s3_$1_$2_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+      # counts
+      - pattern: 'trino.plugin.hive.s3<type=(.+), name=hive><>(.+TotalCount.*): (.*)'
+        name: 'trino_hive_s3_$1_$2_total'
+        type: COUNTER
+      # capture percentile and set quantile label
+      - pattern: 'trino.plugin.hive.metastore.thrift<type=(.+), name=hive><>(.+AllTime).P(\\d+): (.*)'
+        name: 'trino_hive_metastore_thrift_$1_$2_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+        labels:
+          quantile: '0.$3'
+      # match non-percentiles
+      - pattern: 'trino.plugin.hive.metastore.thrift<type=(.+), name=hive><>(.+AllTime.+): (.*)'
+        name: 'trino_hive_metastore_thrift_$1_$2_count_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+      # counts
+      - pattern: 'trino.plugin.hive.metastore.thrift<type=(.+), name=hive><>(.+TotalCount.*): (.*)'
+        name: 'trino_hive_metastore_thrift_$1_$2_total'
+        type: COUNTER
+      - pattern: 'trino.execution<name=(.+)><>(.+AllTime).P(\d+): (.*)'
+        name: 'trino_execution_$1_$2_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+        labels:
+          quantile: '0.$3'
+      - pattern: 'trino.execution<name=(.+)><>(.+AllTime.+): (.*)'
+        name: 'trino_execution_$1_$2_seconds'
+        type: GAUGE
+        valueFactor: 0.001
+      # counts
+      - pattern: 'trino.execution<name=(.+)><>(.+TotalCount.*): (.*)'
+        name: 'trino_execution_$1_$2_total'
+        type: COUNTER
+      - pattern: 'trino.memory<type=(.*), name=(.*)><>(.+): (.*)'
+        name: 'trino_memory_$1_$2_$3'
+        type: GAUGE
+      - pattern: 'trino.failuredetector<name=HeartbeatFailureDetector><>ActiveCount: (.*)'
+        name: 'trino_heartbeatdetector_activecount'
+        type: GAUGE
diff --git a/trino/base/trino-config-secret.yaml b/trino/base/trino-config-secret.yaml
@@ -14,9 +14,26 @@ stringData:
     coordinator=false
     http-server.http.port=8080
     discovery.uri=http://trino-service:8080
-  jvm.config: |-
+  jvm-coordinator.config: |-
     -server
-    -Xmx16G
+    -XX:InitialRAMPercentage=50.0
+    -XX:MaxRAMPercentage=75.0
+    -XX:-UseBiasedLocking
+    -XX:+UseG1GC
+    -XX:G1HeapRegionSize=32M
+    -XX:+ExplicitGCInvokesConcurrent
+    -XX:+ExitOnOutOfMemoryError
+    -XX:+HeapDumpOnOutOfMemoryError
+    -XX:ReservedCodeCacheSize=512M
+    -XX:PerMethodRecompilationCutoff=10000
+    -XX:PerBytecodeRecompilationCutoff=10000
+    -Djdk.attach.allowAttachSelf=true
+    -Djdk.nio.maxCachedBufferSize=2000000
+    -javaagent:/usr/lib/trino/lib/jmx_exporter.jar=8082:/etc/trino/prometheus/config.yaml
+  jvm-worker.config: |-
+    -server
+    -XX:InitialRAMPercentage=50.0
+    -XX:MaxRAMPercentage=75.0
     -XX:-UseBiasedLocking
     -XX:+UseG1GC
     -XX:G1HeapRegionSize=32M

diff --git a/trino/base/trino-coordinator-dc.yaml b/trino/base/trino-coordinator-dc.yaml
@@ -26,12 +26,19 @@ spec:
             items:
               - key: config-coordinator.properties
                 path: config-coordinator.properties
-              - key: jvm.config
-                path: jvm.config
+              - key: jvm-coordinator.config
+                path: jvm-coordinator.config
               - key: log.properties
                 path: log.properties
               - key: node.properties
                 path: node.properties
+        - name: prometheus-exporter
+          secret:
+            secretName: trino-config-monitoring
+            defaultMode: 420
+            items:
+              - key: config.yaml
+                path: config.yaml
         - name: trino-catalogs-volume
           secret:
             secretName: trino-catalog
@@ -72,8 +79,11 @@ spec:
           volumeMounts:
             - name: trino-config-volume
               mountPath: /etc/trino
+            - name: prometheus-exporter
+              mountPath: /etc/trino/prometheus
             - name: trino-catalogs-volume
               mountPath: /etc/trino/catalog
-          image: $(trino_image)
+          image: trino
           args:
             - --config=/etc/trino/config-coordinator.properties
+            - --jvm-config=/etc/trino/jvm-coordinator.config
diff --git a/trino/base/trino-service.yaml b/trino/base/trino-service.yaml
@@ -7,9 +7,14 @@ metadata:
     role: trino-coordinator
 spec:
   ports:
-    - protocol: TCP
+    - name: trino-endpoint
+      protocol: TCP
       port: 8080
       targetPort: 8080
+    - name: metrics
+      protocol: TCP
+      port: 8082
+      targetPort: 8082
   selector:
     instance: trino
     role: trino-coordinator
diff --git a/trino/base/trino-worker-dc.yaml b/trino/base/trino-worker-dc.yaml
@@ -27,8 +27,8 @@ spec:
             items:
               - key: config-worker.properties
                 path: config-worker.properties
-              - key: jvm.config
-                path: jvm.config
+              - key: jvm-worker.config
+                path: jvm-worker.config
               - key: log.properties
                 path: log.properties
               - key: node.properties
@@ -75,6 +75,7 @@ spec:
               mountPath: /etc/trino
             - name: trino-catalogs-volume
               mountPath: /etc/trino/catalog
-          image: $(trino_image)
+          image: trino
           args:
             - --config=/etc/trino/config-worker.properties
+            - --jvm-config=/etc/trino/jvm-worker.config