Add Exemplars support to Prometheus 1.x (#4867)

Issues:

1. prometheus-metrics-tracer-initializer is needed:
We would only need SpanContext from prometheus-metrics-tracer-common
but ExemplarSampler imports SpanContextSupplier so that is also needed
from prometheus-metrics-tracer-initializer.

2. Testing is not easy because of the rate limiter since the Prometheus
Java Client does not have a clock abstraction that we could mock/fake.

Author: jonatan-ivanov

gradle/libs.versions.toml

@@ -69,7 +69,7 @@ newrelic-api = "5.14.0"
 # Kotlin 1.7 sample will fail from OkHttp 4.12.0 due to okio dependency being a Kotlin 1.9 module
 okhttp = "4.11.0"
 postgre = "42.7.2"
-prometheus = "1.1.0"
+prometheus = "1.2.0"
 prometheusSimpleClient = "0.16.0"
 reactor = "2022.0.16"
 rest-assured = "5.4.0"

implementations/micrometer-registry-prometheus/build.gradle

@@ -4,10 +4,11 @@ dependencies {
     api project(':micrometer-core')
 
     api('io.prometheus:prometheus-metrics-core') {
-        // We don't need this nor some of its dependencies:
-        // io.prometheus:prometheus-metrics-tracer-otel
-        // io.prometheus:prometheus-metrics-tracer-otel-agent
-        exclude(group: 'io.prometheus', module: 'prometheus-metrics-tracer-initializer')
+        // We only need SpanContext from prometheus-metrics-tracer-common so we should
+        // exclude(group: 'io.prometheus', module: 'prometheus-metrics-tracer-initializer')
+        // But right now we cannot since ExemplarSampler imports SpanContextSupplier
+        exclude(group: 'io.prometheus', module: 'prometheus-metrics-tracer-otel')
+        exclude(group: 'io.prometheus', module: 'prometheus-metrics-tracer-otel-agent')
     }
     implementation 'io.prometheus:prometheus-metrics-exposition-formats'
 

implementations/micrometer-registry-prometheus/src/main/java/io/micrometer/prometheusmetrics/DefaultExemplarSamplerFactory.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2024 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.micrometer.prometheusmetrics;
+
+import io.prometheus.metrics.config.ExemplarsProperties;
+import io.prometheus.metrics.config.PrometheusProperties;
+import io.prometheus.metrics.core.exemplars.ExemplarSampler;
+import io.prometheus.metrics.core.exemplars.ExemplarSamplerConfig;
+import io.prometheus.metrics.tracer.common.SpanContext;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * Default implementation of {@link ExemplarSamplerFactory}.
+ *
+ * @author Jonatan Ivanov
+ * @since 1.13.0
+ */
+class DefaultExemplarSamplerFactory implements ExemplarSamplerFactory {
+
+    private final ExemplarsProperties exemplarProperties = PrometheusProperties.get().getExemplarProperties();
+
+    private final ConcurrentMap<Integer, ExemplarSamplerConfig> exemplarSamplerConfigsByNumberOfExemplars = new ConcurrentHashMap<>();
+
+    private final ConcurrentMap<double[], ExemplarSamplerConfig> exemplarSamplerConfigsByHistogramUpperBounds = new ConcurrentHashMap<>();
+
+    private final SpanContext spanContext;
+
+    public DefaultExemplarSamplerFactory(SpanContext spanContext) {
+        this.spanContext = spanContext;
+    }
+
+    @Override
+    public ExemplarSampler createExemplarSampler(int numberOfExemplars) {
+        ExemplarSamplerConfig config = exemplarSamplerConfigsByNumberOfExemplars.computeIfAbsent(numberOfExemplars,
+                key -> new ExemplarSamplerConfig(exemplarProperties, numberOfExemplars));
+        return new ExemplarSampler(config, spanContext);
+    }
+
+    @Override
+    public ExemplarSampler createExemplarSampler(double[] histogramClassicUpperBounds) {
+        ExemplarSamplerConfig config = exemplarSamplerConfigsByHistogramUpperBounds.computeIfAbsent(
+                histogramClassicUpperBounds,
+                key -> new ExemplarSamplerConfig(exemplarProperties, histogramClassicUpperBounds));
+        return new ExemplarSampler(config, spanContext);
+    }
+
+}

implementations/micrometer-registry-prometheus/src/main/java/io/micrometer/prometheusmetrics/ExemplarSamplerFactory.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2024 VMware, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.micrometer.prometheusmetrics;
+
+import io.prometheus.metrics.core.exemplars.ExemplarSampler;
+
+/**
+ * A factory that creates {@link ExemplarSampler} instances with the desired properties.
+ *
+ * @author Jonatan Ivanov
+ * @since 1.13.0
+ */
+interface ExemplarSamplerFactory {
+
+    /**
+     * Creates an {@link ExemplarSampler} that stores the defined amount of exemplars.
+     * @param numberOfExemplars the amount of exemplars stored by the sampler.
+     * @return a new {@link ExemplarSampler} instance.
+     */
+    ExemplarSampler createExemplarSampler(int numberOfExemplars);
+
+    /**
+     * Creates an {@link ExemplarSampler} that stores exemplars for the defined histogram
+     * buckets. This means as many exemplars as many buckets are defined.
+     * @param histogramUpperBounds histogram buckets to store exemplars for.
+     * @return a new {@link ExemplarSampler} instance.
+     */
+    ExemplarSampler createExemplarSampler(double[] histogramUpperBounds);
+
+}

implementations/micrometer-registry-prometheus/src/main/java/io/micrometer/prometheusmetrics/PrometheusCounter.java

@@ -15,15 +15,13 @@
  */
 package io.micrometer.prometheusmetrics;
 
-import io.micrometer.common.lang.NonNull;
 import io.micrometer.common.lang.Nullable;
 import io.micrometer.core.instrument.AbstractMeter;
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Meter;
 import io.prometheus.metrics.core.exemplars.ExemplarSampler;
 import io.prometheus.metrics.model.snapshots.Exemplar;
 
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.DoubleAdder;
 
 /**
@@ -36,26 +34,24 @@ public class PrometheusCounter extends AbstractMeter implements Counter {
 
     private final DoubleAdder count = new DoubleAdder();
 
-    private final AtomicReference<Exemplar> exemplar = new AtomicReference<>();
-
     @Nullable
     private final ExemplarSampler exemplarSampler;
 
     PrometheusCounter(Meter.Id id) {
         this(id, null);
     }
 
-    PrometheusCounter(Meter.Id id, @Nullable ExemplarSampler exemplarSampler) {
+    PrometheusCounter(Meter.Id id, @Nullable ExemplarSamplerFactory exemplarSamplerFactory) {
         super(id);
-        this.exemplarSampler = exemplarSampler;
+        this.exemplarSampler = exemplarSamplerFactory != null ? exemplarSamplerFactory.createExemplarSampler(1) : null;
     }
 
     @Override
     public void increment(double amount) {
         if (amount > 0) {
             count.add(amount);
             if (exemplarSampler != null) {
-                updateExemplar(amount, exemplarSampler);
+                exemplarSampler.observe(amount);
             }
         }
     }
@@ -67,18 +63,7 @@ public double count() {
 
     @Nullable
     Exemplar exemplar() {
-        return exemplar.get();
-    }
-
-    // Similar to exemplar.updateAndGet(...) but it does nothing if the next value is null
-    private void updateExemplar(double amount, @NonNull ExemplarSampler exemplarSampler) {
-        // Exemplar prev;
-        // Exemplar next;
-        // do {
-        // prev = exemplar.get();
-        // next = exemplarSampler.sample(amount, prev);
-        // }
-        // while (next != null && next != prev && !exemplar.compareAndSet(prev, next));
+        return exemplarSampler != null ? exemplarSampler.collect().getLatest() : null;
     }
 
 }

implementations/micrometer-registry-prometheus/src/main/java/io/micrometer/prometheusmetrics/PrometheusDistributionSummary.java

@@ -15,16 +15,14 @@
  */
 package io.micrometer.prometheusmetrics;
 
-import io.micrometer.common.lang.NonNull;
 import io.micrometer.common.lang.Nullable;
 import io.micrometer.core.instrument.AbstractDistributionSummary;
 import io.micrometer.core.instrument.Clock;
 import io.micrometer.core.instrument.DistributionSummary;
 import io.micrometer.core.instrument.distribution.*;
 import io.prometheus.metrics.core.exemplars.ExemplarSampler;
-import io.prometheus.metrics.model.snapshots.Exemplar;
+import io.prometheus.metrics.model.snapshots.Exemplars;
 
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.DoubleAdder;
 import java.util.concurrent.atomic.LongAdder;
 
@@ -50,13 +48,8 @@ public class PrometheusDistributionSummary extends AbstractDistributionSummary {
     @Nullable
     private final ExemplarSampler exemplarSampler;
 
-    @Nullable
-    private final AtomicReference<Exemplar> lastExemplar;
-
-    private boolean histogramExemplarsEnabled = false;
-
     PrometheusDistributionSummary(Id id, Clock clock, DistributionStatisticConfig distributionStatisticConfig,
-            double scale, @Nullable ExemplarSampler exemplarSampler) {
+            double scale, @Nullable ExemplarSamplerFactory exemplarSamplerFactory) {
         super(id, clock,
                 DistributionStatisticConfig.builder()
                     .percentilesHistogram(false)
@@ -68,22 +61,13 @@ public class PrometheusDistributionSummary extends AbstractDistributionSummary {
         this.max = new TimeWindowMax(clock, distributionStatisticConfig);
 
         if (distributionStatisticConfig.isPublishingHistogram()) {
-            PrometheusHistogram prometheusHistogram = new PrometheusHistogram(clock, distributionStatisticConfig,
-                    exemplarSampler);
-            this.histogram = prometheusHistogram;
-            this.histogramExemplarsEnabled = prometheusHistogram.isExemplarsEnabled();
+            this.histogram = new PrometheusHistogram(clock, distributionStatisticConfig, exemplarSamplerFactory);
+            this.exemplarSampler = null;
         }
         else {
             this.histogram = null;
-        }
-
-        if (!this.histogramExemplarsEnabled && exemplarSampler != null) {
-            this.exemplarSampler = exemplarSampler;
-            this.lastExemplar = new AtomicReference<>();
-        }
-        else {
-            this.exemplarSampler = null;
-            this.lastExemplar = null;
+            this.exemplarSampler = exemplarSamplerFactory != null ? exemplarSamplerFactory.createExemplarSampler(1)
+                    : null;
         }
     }
 
@@ -96,40 +80,17 @@ protected void recordNonNegative(double amount) {
         if (histogram != null) {
             histogram.recordDouble(amount);
         }
-        if (!histogramExemplarsEnabled && exemplarSampler != null) {
-            updateLastExemplar(amount, exemplarSampler);
+        else if (exemplarSampler != null) {
+            exemplarSampler.observe(amount);
         }
     }
 
-    // Similar to exemplar.updateAndGet(...) but it does nothing if the next value is null
-    private void updateLastExemplar(double amount, @NonNull ExemplarSampler exemplarSampler) {
-        // Exemplar prev;
-        // Exemplar next;
-        // do {
-        // prev = lastExemplar.get();
-        // next = exemplarSampler.sample(amount, prev);
-        // }
-        // while (next != null && next != prev && !lastExemplar.compareAndSet(prev,
-        // next));
-    }
-
-    @Nullable
-    Exemplar[] histogramExemplars() {
-        if (histogramExemplarsEnabled) {
+    Exemplars exemplars() {
+        if (histogram != null) {
             return ((PrometheusHistogram) histogram).exemplars();
         }
         else {
-            return null;
-        }
-    }
-
-    @Nullable
-    Exemplar lastExemplar() {
-        if (histogramExemplarsEnabled) {
-            return ((PrometheusHistogram) histogram).lastExemplar();
-        }
-        else {
-            return lastExemplar != null ? lastExemplar.get() : null;
+            return exemplarSampler != null ? exemplarSampler.collect() : Exemplars.EMPTY;
         }
     }