diff --git a/benchmarks/benchmarks-core/build.gradle b/benchmarks/benchmarks-core/build.gradle index ffa6f509c8..0af6b50c59 100644 --- a/benchmarks/benchmarks-core/build.gradle +++ b/benchmarks/benchmarks-core/build.gradle @@ -5,6 +5,7 @@ plugins { dependencies { jmh project(':micrometer-core') jmh project(':micrometer-registry-prometheus') + jmh project(':micrometer-registry-otlp') jmh 'io.dropwizard.metrics5:metrics-core:latest.release' jmh 'io.prometheus:simpleclient_common:latest.release' diff --git a/benchmarks/benchmarks-core/src/jmh/java/io/micrometer/benchmark/compare/CompareOTLPHistograms.java b/benchmarks/benchmarks-core/src/jmh/java/io/micrometer/benchmark/compare/CompareOTLPHistograms.java new file mode 100644 index 0000000000..b43143a415 --- /dev/null +++ b/benchmarks/benchmarks-core/src/jmh/java/io/micrometer/benchmark/compare/CompareOTLPHistograms.java @@ -0,0 +1,337 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.benchmark.compare; + +import java.util.Iterator; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +//CHECKSTYLE:OFF +import com.google.common.collect.Iterators; +////CHECKSTYLE:ON +import io.micrometer.common.lang.Nullable; +import io.micrometer.core.instrument.Clock; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.registry.otlp.AggregationTemporality; +import io.micrometer.registry.otlp.HistogramFlavour; +import io.micrometer.registry.otlp.OtlpConfig; +import io.micrometer.registry.otlp.OtlpMeterRegistry; + +/** + * @author Lenin Jaganathan + */ +@Fork(1) +@Measurement(iterations = 2) +@Warmup(iterations = 2) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Threads(16) +public class CompareOTLPHistograms { + + @State(Scope.Thread) + public static class Data { + + Iterator dataIterator; + + @Setup(Level.Iteration) + public void setup() { + final Random r = new Random(1234567891L); + dataIterator = Iterators.cycle(Stream.generate(() -> { + long randomNumber; + do { + randomNumber = Math.round(Math.exp(2.0 + r.nextGaussian())); + } + while (randomNumber < 1 || randomNumber > 60000); + return randomNumber; + }).limit(1048576).collect(Collectors.toList())); + } + + } + + @State(Scope.Benchmark) + public static class DistributionsWithoutHistogramCumulative { + + MeterRegistry registry; + + Timer timer; + + DistributionSummary distributionSummary; + + @Setup(Level.Iteration) + public void setup() { + registry = new OtlpMeterRegistry(); + distributionSummary = DistributionSummary.builder("ds").register(registry); + timer = Timer.builder("timer").register(registry); + } + + @TearDown(Level.Iteration) + public void tearDown(Blackhole hole) { + hole.consume(distributionSummary.takeSnapshot()); + } + + } + + @State(Scope.Benchmark) + public static class DistributionsWithoutHistogramDelta { + + OtlpConfig otlpConfig = new OtlpConfig() { + + @Override + public AggregationTemporality aggregationTemporality() { + return AggregationTemporality.DELTA; + } + + @Nullable + @Override + public String get(final String key) { + return null; + } + }; + + MeterRegistry registry; + + Timer timer; + + DistributionSummary distributionSummary; + + @Setup(Level.Iteration) + public void setup() { + registry = new OtlpMeterRegistry(otlpConfig, Clock.SYSTEM); + distributionSummary = DistributionSummary.builder("ds").register(registry); + timer = Timer.builder("timer").register(registry); + } + + @TearDown(Level.Iteration) + public void tearDown(Blackhole hole) { + hole.consume(distributionSummary.takeSnapshot()); + } + + } + + @State(Scope.Benchmark) + public static class ExplicitBucketHistogramCumulative { + + MeterRegistry registry; + + Timer timer; + + DistributionSummary distributionSummary; + + @Setup(Level.Iteration) + public void setup() { + registry = new OtlpMeterRegistry(); + distributionSummary = DistributionSummary.builder("ds").publishPercentileHistogram().register(registry); + timer = Timer.builder("timer").publishPercentileHistogram().register(registry); + } + + @TearDown(Level.Iteration) + public void tearDown(Blackhole hole) { + hole.consume(distributionSummary.takeSnapshot()); + } + + } + + @State(Scope.Benchmark) + public static class ExplicitBucketHistogramDelta { + + OtlpConfig otlpConfig = new OtlpConfig() { + + @Override + public AggregationTemporality aggregationTemporality() { + return AggregationTemporality.DELTA; + } + + @Nullable + @Override + public String get(final String key) { + return null; + } + }; + + MeterRegistry registry; + + Timer timer; + + DistributionSummary distributionSummary; + + @Setup(Level.Iteration) + public void setup() { + registry = new OtlpMeterRegistry(otlpConfig, Clock.SYSTEM); + distributionSummary = DistributionSummary.builder("ds").publishPercentileHistogram().register(registry); + timer = Timer.builder("timer").publishPercentileHistogram().register(registry); + } + + @TearDown(Level.Iteration) + public void tearDown(Blackhole hole) { + hole.consume(distributionSummary.takeSnapshot()); + } + + } + + @State(Scope.Benchmark) + public static class ExponentialHistogramCumulative { + + MeterRegistry registry; + + OtlpConfig otlpConfig = new OtlpConfig() { + @Override + public HistogramFlavour histogramFlavour() { + return HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM; + } + + @Nullable + @Override + public String get(final String key) { + return null; + } + }; + + Timer timer; + + DistributionSummary distributionSummary; + + @Setup(Level.Iteration) + public void setup() { + registry = new OtlpMeterRegistry(otlpConfig, Clock.SYSTEM); + distributionSummary = DistributionSummary.builder("ds").publishPercentileHistogram().register(registry); + timer = Timer.builder("timer").publishPercentileHistogram().register(registry); + } + + @TearDown(Level.Iteration) + public void tearDown(Blackhole hole) { + hole.consume(distributionSummary.takeSnapshot()); + } + + } + + @State(Scope.Benchmark) + public static class ExponentialHistogramDelta { + + MeterRegistry registry; + + OtlpConfig otlpConfig = new OtlpConfig() { + + @Override + public AggregationTemporality aggregationTemporality() { + return AggregationTemporality.DELTA; + } + + @Override + public HistogramFlavour histogramFlavour() { + return HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM; + } + + @Nullable + @Override + public String get(final String key) { + return null; + } + }; + + Timer timer; + + DistributionSummary distributionSummary; + + @Setup(Level.Iteration) + public void setup() { + registry = new OtlpMeterRegistry(otlpConfig, Clock.SYSTEM); + distributionSummary = DistributionSummary.builder("ds").publishPercentileHistogram().register(registry); + timer = Timer.builder("timer").publishPercentileHistogram().register(registry); + } + + @TearDown(Level.Iteration) + public void tearDown(Blackhole hole) { + hole.consume(distributionSummary.takeSnapshot()); + } + + } + + @Benchmark + public void otlpCumulativeDs(DistributionsWithoutHistogramCumulative state, Data data) { + state.distributionSummary.record(data.dataIterator.next()); + } + + @Benchmark + public void otlpDeltaDs(DistributionsWithoutHistogramDelta state, Data data) { + state.distributionSummary.record(data.dataIterator.next()); + } + + @Benchmark + public void otlpCumulativeExplicitBucketHistogramDs(ExplicitBucketHistogramCumulative state, Data data) { + state.distributionSummary.record(data.dataIterator.next()); + } + + @Benchmark + public void otlpDeltaExplicitBucketHistogramDs(ExplicitBucketHistogramDelta state, Data data) { + state.distributionSummary.record(data.dataIterator.next()); + } + + @Benchmark + public void oltpCumulativeExponentialHistogramDs(ExponentialHistogramCumulative state, Data data) { + state.distributionSummary.record(data.dataIterator.next()); + } + + @Benchmark + public void oltpDeltaExponentialHistogramDs(ExponentialHistogramDelta state, Data data) { + state.distributionSummary.record(data.dataIterator.next()); + } + + @Benchmark + public void otlpCumulativeTimer(DistributionsWithoutHistogramCumulative state, Data data) { + state.timer.record(data.dataIterator.next(), TimeUnit.MILLISECONDS); + } + + @Benchmark + public void otlpDeltaTimer(DistributionsWithoutHistogramDelta state, Data data) { + state.timer.record(data.dataIterator.next(), TimeUnit.MILLISECONDS); + } + + @Benchmark + public void otlpCumulativeExplicitBucketHistogramTimer(ExplicitBucketHistogramCumulative state, Data data) { + state.timer.record(data.dataIterator.next(), TimeUnit.MILLISECONDS); + } + + @Benchmark + public void otlpDeltaExplicitBucketHistogramTimer(ExplicitBucketHistogramDelta state, Data data) { + state.timer.record(data.dataIterator.next(), TimeUnit.MILLISECONDS); + } + + @Benchmark + public void oltpCumulativeExponentialHistogramTimer(ExponentialHistogramCumulative state, Data data) { + state.timer.record(data.dataIterator.next(), TimeUnit.MILLISECONDS); + } + + @Benchmark + public void oltpDeltaExponentialHistogramTimer(ExponentialHistogramDelta state, Data data) { + state.timer.record(data.dataIterator.next(), TimeUnit.MILLISECONDS); + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder().include(CompareOTLPHistograms.class.getSimpleName()).build(); + new Runner(opt).run(); + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/HistogramFlavour.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/HistogramFlavour.java new file mode 100644 index 0000000000..8b25681e86 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/HistogramFlavour.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp; + +/** + * Histogram Flavour to be used while recording distributions, + * + * @see OTLP + * Configuration + * @author Lenin Jaganathan + * @since 1.12.0 + */ +public enum HistogramFlavour { + + EXPLICIT_BUCKET_HISTOGRAM, BASE2_EXPONENTIAL_BUCKET_HISTOGRAM; + + /** + * Converts a string to {@link HistogramFlavour} by using a case-insensitive matching. + */ + public static HistogramFlavour fromString(final String histogramPreference) { + if (BASE2_EXPONENTIAL_BUCKET_HISTOGRAM.name().equalsIgnoreCase(histogramPreference)) { + return BASE2_EXPONENTIAL_BUCKET_HISTOGRAM; + } + return EXPLICIT_BUCKET_HISTOGRAM; + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpConfig.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpConfig.java index 0a93d1e4bf..87cf01b9d5 100644 --- a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpConfig.java +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpConfig.java @@ -141,6 +141,52 @@ default Map headers() { keyValue -> keyValue.substring(keyValue.indexOf('=') + 1).trim(), (l, r) -> r)); } + /** + * Histogram type to be preferred when histogram publishing is enabled. By default + * {@link HistogramFlavour#EXPLICIT_BUCKET_HISTOGRAM} is used for the supported + * meters. When this is set to + * {@link HistogramFlavour#BASE2_EXPONENTIAL_BUCKET_HISTOGRAM} and publishPercentiles + * are enabled {@link io.micrometer.registry.otlp.internal.Base2ExponentialHistogram} + * is used for recording distributions. + * + *

+ * Note: If specific SLO's are added as part of meters, this property is not honored + * and {@link HistogramFlavour#EXPLICIT_BUCKET_HISTOGRAM} is used for those meters. + * @return - histogram flavour to be used + * + * @since 1.12.0 + */ + default HistogramFlavour histogramFlavour() { + String histogramPreference = System.getenv().get("OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION"); + if (histogramPreference == null) { + return getEnum(this, HistogramFlavour.class, "histogramFlavour") + .orElse(HistogramFlavour.EXPLICIT_BUCKET_HISTOGRAM); + } + return HistogramFlavour.fromString(histogramPreference); + } + + /** + * Max scale to use for + * {@link io.micrometer.registry.otlp.internal.Base2ExponentialHistogram} + * @return maxScale + * + * @since 1.12.0 + */ + default int maxScale() { + return getInteger(this, "maxScale").orElse(20); + } + + /** + * Maximum number of buckets to be used for + * {@link io.micrometer.registry.otlp.internal.Base2ExponentialHistogram} + * @return - maxBuckets + * + * @since 1.12.0 + */ + default int maxBucketCount() { + return getInteger(this, "maxBucketCount").orElse(160); + } + @Override default Validated validate() { return checkAll(this, c -> PushRegistryConfig.validate(c), checkRequired("url", OtlpConfig::url), diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeDistributionSummary.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeDistributionSummary.java index 2ecd5acc97..7392b722fa 100644 --- a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeDistributionSummary.java +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeDistributionSummary.java @@ -15,21 +15,29 @@ */ package io.micrometer.registry.otlp; +import io.micrometer.common.lang.Nullable; import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; import io.micrometer.core.instrument.distribution.*; +import io.micrometer.registry.otlp.internal.Base2ExponentialHistogram; +import io.micrometer.registry.otlp.internal.ExponentialHistogramSnapShot; import java.util.concurrent.TimeUnit; -class OtlpCumulativeDistributionSummary extends CumulativeDistributionSummary implements StartTimeAwareMeter { +class OtlpCumulativeDistributionSummary extends CumulativeDistributionSummary + implements StartTimeAwareMeter, OtlpHistogramSupport { + + private final HistogramFlavour histogramFlavour; private final long startTimeNanos; OtlpCumulativeDistributionSummary(Id id, Clock clock, DistributionStatisticConfig distributionStatisticConfig, - double scale, boolean supportsAggregablePercentiles) { + double scale, OtlpConfig otlpConfig) { super(id, clock, distributionStatisticConfig, scale, - OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, AggregationTemporality.CUMULATIVE)); + OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, otlpConfig)); this.startTimeNanos = TimeUnit.MILLISECONDS.toNanos(clock.wallTime()); + this.histogramFlavour = OtlpMeterRegistry.histogramFlavour(otlpConfig.histogramFlavour(), + distributionStatisticConfig); } @Override @@ -37,4 +45,13 @@ public long getStartTimeNanos() { return this.startTimeNanos; } + @Override + @Nullable + public ExponentialHistogramSnapShot getExponentialHistogramSnapShot() { + if (histogramFlavour == HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM) { + return ((Base2ExponentialHistogram) histogram).getLatestExponentialHistogramSnapshot(); + } + return null; + } + } diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeTimer.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeTimer.java index e2519f1ab3..c0ad223005 100644 --- a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeTimer.java +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpCumulativeTimer.java @@ -15,21 +15,28 @@ */ package io.micrometer.registry.otlp; +import io.micrometer.common.lang.Nullable; import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.cumulative.CumulativeTimer; import io.micrometer.core.instrument.distribution.*; import io.micrometer.core.instrument.distribution.pause.PauseDetector; +import io.micrometer.registry.otlp.internal.Base2ExponentialHistogram; +import io.micrometer.registry.otlp.internal.ExponentialHistogramSnapShot; import java.util.concurrent.TimeUnit; -class OtlpCumulativeTimer extends CumulativeTimer implements StartTimeAwareMeter { +class OtlpCumulativeTimer extends CumulativeTimer implements StartTimeAwareMeter, OtlpHistogramSupport { + + private final HistogramFlavour histogramFlavour; private final long startTimeNanos; OtlpCumulativeTimer(Id id, Clock clock, DistributionStatisticConfig distributionStatisticConfig, - PauseDetector pauseDetector, TimeUnit baseTimeUnit) { + PauseDetector pauseDetector, TimeUnit baseTimeUnit, OtlpConfig otlpConfig) { super(id, clock, distributionStatisticConfig, pauseDetector, baseTimeUnit, - OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, AggregationTemporality.CUMULATIVE)); + OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, otlpConfig, baseTimeUnit)); + this.histogramFlavour = OtlpMeterRegistry.histogramFlavour(otlpConfig.histogramFlavour(), + distributionStatisticConfig); this.startTimeNanos = TimeUnit.MILLISECONDS.toNanos(clock.wallTime()); } @@ -38,4 +45,13 @@ public long getStartTimeNanos() { return this.startTimeNanos; } + @Override + @Nullable + public ExponentialHistogramSnapShot getExponentialHistogramSnapShot() { + if (histogramFlavour == HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM) { + return ((Base2ExponentialHistogram) histogram).getLatestExponentialHistogramSnapshot(); + } + return null; + } + } diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpHistogramSupport.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpHistogramSupport.java new file mode 100644 index 0000000000..abe54cb5ca --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpHistogramSupport.java @@ -0,0 +1,24 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp; + +import io.micrometer.registry.otlp.internal.ExponentialHistogramSnapShot; + +interface OtlpHistogramSupport { + + ExponentialHistogramSnapShot getExponentialHistogramSnapShot(); + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpMeterRegistry.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpMeterRegistry.java index b1510e77c3..ab6b6c9bae 100644 --- a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpMeterRegistry.java +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpMeterRegistry.java @@ -37,6 +37,9 @@ import io.micrometer.core.instrument.util.TimeUtils; import io.micrometer.core.ipc.http.HttpSender; import io.micrometer.core.ipc.http.HttpUrlConnectionSender; +import io.micrometer.registry.otlp.internal.CumulativeBase2ExponentialHistogram; +import io.micrometer.registry.otlp.internal.DeltaBase2ExponentialHistogram; +import io.micrometer.registry.otlp.internal.ExponentialHistogramSnapShot; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; @@ -45,6 +48,7 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -73,6 +77,11 @@ public class OtlpMeterRegistry extends PushMeterRegistry { private static final ThreadFactory DEFAULT_THREAD_FACTORY = new NamedThreadFactory("otlp-metrics-publisher"); + private static final ExponentialHistogramDataPoint.Buckets EMPTY_EXPONETIAL_HISTOGRAM_BUCKETS = ExponentialHistogramDataPoint.Buckets + .newBuilder() + .addAllBucketCounts(Collections.emptyList()) + .build(); + private static final double[] EMPTY_SLO_WITH_POSITIVE_INF = new double[] { Double.POSITIVE_INFINITY }; private final InternalLogger logger = InternalLoggerFactory.getInstance(OtlpMeterRegistry.class); @@ -184,18 +193,17 @@ protected Counter newCounter(Meter.Id id) { protected Timer newTimer(Meter.Id id, DistributionStatisticConfig distributionStatisticConfig, PauseDetector pauseDetector) { return isCumulative() - ? new OtlpCumulativeTimer(id, this.clock, distributionStatisticConfig, pauseDetector, getBaseTimeUnit()) - : new OtlpStepTimer(id, clock, distributionStatisticConfig, pauseDetector, getBaseTimeUnit(), - config.step().toMillis()); + ? new OtlpCumulativeTimer(id, this.clock, distributionStatisticConfig, pauseDetector, getBaseTimeUnit(), + config) + : new OtlpStepTimer(id, clock, distributionStatisticConfig, pauseDetector, getBaseTimeUnit(), config); } @Override protected DistributionSummary newDistributionSummary(Meter.Id id, DistributionStatisticConfig distributionStatisticConfig, double scale) { return isCumulative() - ? new OtlpCumulativeDistributionSummary(id, this.clock, distributionStatisticConfig, scale, true) - : new OtlpStepDistributionSummary(id, clock, distributionStatisticConfig, scale, - config.step().toMillis()); + ? new OtlpCumulativeDistributionSummary(id, this.clock, distributionStatisticConfig, scale, config) + : new OtlpStepDistributionSummary(id, clock, distributionStatisticConfig, scale, config); } @Override @@ -379,6 +387,37 @@ Metric writeHistogramSupport(HistogramSupport histogramSupport) { return metricBuilder.build(); } + ExponentialHistogramSnapShot exponentialHistogramSnapShot = getExponentialHistogramSnapShot(histogramSupport); + if (exponentialHistogramSnapShot != null) { + ExponentialHistogramDataPoint.Builder exponentialDataPoint = ExponentialHistogramDataPoint.newBuilder() + .addAllAttributes(tags) + .setStartTimeUnixNano(startTimeNanos) + .setTimeUnixNano(getTimeUnixNano()) + .setCount(count) + .setSum(total) + .setScale(exponentialHistogramSnapShot.scale()) + .setZeroCount(exponentialHistogramSnapShot.zeroCount()) + .setZeroThreshold(exponentialHistogramSnapShot.zeroThreshold()) + .setPositive(ExponentialHistogramDataPoint.Buckets.newBuilder() + .addAllBucketCounts(exponentialHistogramSnapShot.bucketsCount()) + .setOffset(exponentialHistogramSnapShot.offset()) + .build()) + // Micrometer doesn't support negative recordings. + .setNegative(EMPTY_EXPONETIAL_HISTOGRAM_BUCKETS); + + if (isDelta()) { + exponentialDataPoint + .setMax(isTimeBased ? histogramSnapshot.max(getBaseTimeUnit()) : histogramSnapshot.max()); + } + + return metricBuilder + .setExponentialHistogram(ExponentialHistogram.newBuilder() + .setAggregationTemporality(otlpAggregationTemporality) + .addDataPoints(exponentialDataPoint) + .build()) + .build(); + } + HistogramDataPoint.Builder histogramDataPoint = HistogramDataPoint.newBuilder() .addAllAttributes(tags) .setStartTimeUnixNano(startTimeNanos) @@ -415,6 +454,16 @@ Metric writeHistogramSupport(HistogramSupport histogramSupport) { .build(); } + @Nullable + private static ExponentialHistogramSnapShot getExponentialHistogramSnapShot( + final HistogramSupport histogramSupport) { + if (histogramSupport instanceof OtlpHistogramSupport) { + return ((OtlpHistogramSupport) histogramSupport).getExponentialHistogramSnapShot(); + } + + return null; + } + // VisibleForTesting Metric writeFunctionTimer(FunctionTimer functionTimer) { return getMetricBuilder(functionTimer.getId()) @@ -497,35 +546,36 @@ Iterable getResourceAttributes() { } static Histogram getHistogram(Clock clock, DistributionStatisticConfig distributionStatisticConfig, - AggregationTemporality aggregationTemporality) { - return getHistogram(clock, distributionStatisticConfig, aggregationTemporality, 0); + OtlpConfig otlpConfig) { + return getHistogram(clock, distributionStatisticConfig, otlpConfig, null); } - static Histogram getHistogram(Clock clock, DistributionStatisticConfig distributionStatisticConfig, - AggregationTemporality aggregationTemporality, long stepMillis) { - // While publishing to OTLP, we export either Histogram datapoint / Summary + static Histogram getHistogram(final Clock clock, final DistributionStatisticConfig distributionStatisticConfig, + final OtlpConfig otlpConfig, @Nullable final TimeUnit baseTimeUnit) { + // While publishing to OTLP, we export either Histogram datapoint (Explicit Bucket + // or Exponential) / Summary // datapoint. So, we will make the histogram either of them and not both. // Though AbstractTimer/Distribution Summary prefers publishing percentiles, // exporting of histograms over percentiles is preferred in OTLP. if (distributionStatisticConfig.isPublishingHistogram()) { - double[] sloWithPositiveInf = getSloWithPositiveInf(distributionStatisticConfig); - if (AggregationTemporality.isCumulative(aggregationTemporality)) { - return new TimeWindowFixedBoundaryHistogram(clock, DistributionStatisticConfig.builder() - // effectively never roll over - .expiry(Duration.ofDays(1825)) - .serviceLevelObjectives(sloWithPositiveInf) - .percentiles() - .bufferLength(1) - .build() - .merge(distributionStatisticConfig), true, false); + if (histogramFlavour(otlpConfig.histogramFlavour(), + distributionStatisticConfig) == HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM) { + Double minimumExpectedValue = distributionStatisticConfig.getMinimumExpectedValueAsDouble(); + if (minimumExpectedValue == null) { + minimumExpectedValue = 0.0; + } + + return otlpConfig.aggregationTemporality() == AggregationTemporality.DELTA + ? new DeltaBase2ExponentialHistogram(otlpConfig.maxScale(), otlpConfig.maxBucketCount(), + minimumExpectedValue, baseTimeUnit, clock, otlpConfig.step().toMillis()) + : new CumulativeBase2ExponentialHistogram(otlpConfig.maxScale(), otlpConfig.maxBucketCount(), + minimumExpectedValue, baseTimeUnit); } - if (AggregationTemporality.isDelta(aggregationTemporality) && stepMillis > 0) { - return new OtlpStepBucketHistogram(clock, stepMillis, - DistributionStatisticConfig.builder() - .serviceLevelObjectives(sloWithPositiveInf) - .build() - .merge(distributionStatisticConfig), - true, false); + + Histogram explicitBucketHistogram = getExplicitBucketHistogram(clock, distributionStatisticConfig, + otlpConfig.aggregationTemporality(), otlpConfig.step().toMillis()); + if (explicitBucketHistogram != null) { + return explicitBucketHistogram; } } @@ -535,6 +585,47 @@ static Histogram getHistogram(Clock clock, DistributionStatisticConfig distribut return NoopHistogram.INSTANCE; } + static HistogramFlavour histogramFlavour(HistogramFlavour preferredHistogramFlavour, + DistributionStatisticConfig distributionStatisticConfig) { + + final double[] serviceLevelObjectiveBoundaries = distributionStatisticConfig + .getServiceLevelObjectiveBoundaries(); + if (distributionStatisticConfig.isPublishingHistogram() + && preferredHistogramFlavour == HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM + && (serviceLevelObjectiveBoundaries == null || serviceLevelObjectiveBoundaries.length == 0)) { + return HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM; + } + return HistogramFlavour.EXPLICIT_BUCKET_HISTOGRAM; + } + + @Nullable + private static Histogram getExplicitBucketHistogram(final Clock clock, + final DistributionStatisticConfig distributionStatisticConfig, + final AggregationTemporality aggregationTemporality, final long stepMillis) { + + double[] sloWithPositiveInf = getSloWithPositiveInf(distributionStatisticConfig); + if (AggregationTemporality.isCumulative(aggregationTemporality)) { + return new TimeWindowFixedBoundaryHistogram(clock, DistributionStatisticConfig.builder() + // effectively never roll over + .expiry(Duration.ofDays(1825)) + .serviceLevelObjectives(sloWithPositiveInf) + .percentiles() + .bufferLength(1) + .build() + .merge(distributionStatisticConfig), true, false); + } + if (AggregationTemporality.isDelta(aggregationTemporality) && stepMillis > 0) { + return new OtlpStepBucketHistogram(clock, stepMillis, + DistributionStatisticConfig.builder() + .serviceLevelObjectives(sloWithPositiveInf) + .build() + .merge(distributionStatisticConfig), + true, false); + } + + return null; + } + // VisibleForTesting static double[] getSloWithPositiveInf(DistributionStatisticConfig distributionStatisticConfig) { double[] sloBoundaries = distributionStatisticConfig.getServiceLevelObjectiveBoundaries(); diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepDistributionSummary.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepDistributionSummary.java index ef9897e5bc..0ba3da603c 100644 --- a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepDistributionSummary.java +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepDistributionSummary.java @@ -15,14 +15,19 @@ */ package io.micrometer.registry.otlp; +import io.micrometer.common.lang.Nullable; import io.micrometer.core.instrument.AbstractDistributionSummary; import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import io.micrometer.registry.otlp.internal.Base2ExponentialHistogram; +import io.micrometer.registry.otlp.internal.ExponentialHistogramSnapShot; import java.util.concurrent.atomic.DoubleAdder; import java.util.concurrent.atomic.LongAdder; -class OtlpStepDistributionSummary extends AbstractDistributionSummary { +class OtlpStepDistributionSummary extends AbstractDistributionSummary implements OtlpHistogramSupport { + + private final HistogramFlavour histogramFlavour; private final LongAdder count = new LongAdder(); @@ -38,14 +43,16 @@ class OtlpStepDistributionSummary extends AbstractDistributionSummary { * @param clock clock * @param distributionStatisticConfig distribution statistic configuration * @param scale scale - * @param stepMillis step in milliseconds + * @param otlpConfig config for registry */ OtlpStepDistributionSummary(Id id, Clock clock, DistributionStatisticConfig distributionStatisticConfig, - double scale, long stepMillis) { - super(id, scale, OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, - AggregationTemporality.DELTA, stepMillis)); - this.countTotal = new OtlpStepTuple2<>(clock, stepMillis, 0L, 0.0, count::sumThenReset, total::sumThenReset); - this.max = new StepMax(clock, stepMillis); + double scale, OtlpConfig otlpConfig) { + super(id, scale, OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, otlpConfig)); + this.countTotal = new OtlpStepTuple2<>(clock, otlpConfig.step().toMillis(), 0L, 0.0, count::sumThenReset, + total::sumThenReset); + this.max = new StepMax(clock, otlpConfig.step().toMillis()); + this.histogramFlavour = OtlpMeterRegistry.histogramFlavour(otlpConfig.histogramFlavour(), + distributionStatisticConfig); } @Override @@ -70,6 +77,15 @@ public double max() { return max.poll(); } + @Override + @Nullable + public ExponentialHistogramSnapShot getExponentialHistogramSnapShot() { + if (histogramFlavour == HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM) { + return ((Base2ExponentialHistogram) histogram).getLatestExponentialHistogramSnapshot(); + } + return null; + } + /** * This is an internal method not meant for general use. *

diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepTimer.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepTimer.java index 49d644aaf5..19f2ea005f 100644 --- a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepTimer.java +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/OtlpStepTimer.java @@ -15,16 +15,21 @@ */ package io.micrometer.registry.otlp; +import io.micrometer.common.lang.Nullable; import io.micrometer.core.instrument.AbstractTimer; import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import io.micrometer.core.instrument.distribution.pause.PauseDetector; import io.micrometer.core.instrument.util.TimeUtils; +import io.micrometer.registry.otlp.internal.Base2ExponentialHistogram; +import io.micrometer.registry.otlp.internal.ExponentialHistogramSnapShot; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; -class OtlpStepTimer extends AbstractTimer { +class OtlpStepTimer extends AbstractTimer implements OtlpHistogramSupport { + + private final HistogramFlavour histogramFlavour; private final LongAdder count = new LongAdder(); @@ -41,14 +46,17 @@ class OtlpStepTimer extends AbstractTimer { * @param distributionStatisticConfig distribution statistic configuration * @param pauseDetector pause detector * @param baseTimeUnit base time unit - * @param stepDurationMillis step in milliseconds + * @param otlpConfig config of the registry */ OtlpStepTimer(Id id, Clock clock, DistributionStatisticConfig distributionStatisticConfig, - PauseDetector pauseDetector, TimeUnit baseTimeUnit, long stepDurationMillis) { - super(id, clock, pauseDetector, baseTimeUnit, OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, - AggregationTemporality.DELTA, stepDurationMillis)); - countTotal = new OtlpStepTuple2<>(clock, stepDurationMillis, 0L, 0L, count::sumThenReset, total::sumThenReset); - max = new StepMax(clock, stepDurationMillis); + PauseDetector pauseDetector, TimeUnit baseTimeUnit, OtlpConfig otlpConfig) { + super(id, clock, pauseDetector, otlpConfig.baseTimeUnit(), + OtlpMeterRegistry.getHistogram(clock, distributionStatisticConfig, otlpConfig, baseTimeUnit)); + countTotal = new OtlpStepTuple2<>(clock, otlpConfig.step().toMillis(), 0L, 0L, count::sumThenReset, + total::sumThenReset); + max = new StepMax(clock, otlpConfig.step().toMillis()); + this.histogramFlavour = OtlpMeterRegistry.histogramFlavour(otlpConfig.histogramFlavour(), + distributionStatisticConfig); } @Override @@ -88,4 +96,13 @@ void _closingRollover() { } } + @Override + @Nullable + public ExponentialHistogramSnapShot getExponentialHistogramSnapShot() { + if (histogramFlavour == HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM) { + return ((Base2ExponentialHistogram) histogram).getLatestExponentialHistogramSnapshot(); + } + return null; + } + } diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/Base2ExponentialHistogram.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/Base2ExponentialHistogram.java new file mode 100644 index 0000000000..5c07d5751a --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/Base2ExponentialHistogram.java @@ -0,0 +1,275 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; + +import io.micrometer.common.lang.Nullable; +import io.micrometer.core.instrument.distribution.Histogram; +import io.micrometer.core.instrument.distribution.HistogramSnapshot; +import io.micrometer.core.instrument.util.TimeUtils; + +/** + * A ExponentialHistogram implementation that compresses bucket boundaries using an + * exponential formula (Base2 exponent), making it suitable for conveying high dynamic + * range data with small relative error. This is an implementation of the Exponential + * Histogram as per the OTLP specification. The internal implementations uses the + * techniques outlined in the OTLP specification mentioned above. + * + * @author Lenin Jaganathen + * @since 1.12.0 + */ +public abstract class Base2ExponentialHistogram implements Histogram { + + private final int maxScale; + + private final int maxBucketsCount; + + private final double zeroThreshold; + + @Nullable + private final TimeUnit baseUnit; + + private final LongAdder zeroCount = new LongAdder(); + + private CircularCountHolder circularCountHolder; + + private IndexProvider base2IndexProvider; + + private int scale; + + /** + * Creates an Base2ExponentialHistogram that records positive values. + * @param maxScale - maximum scale that can be used. The recordings start with this + * scale and gets downscaled to the scale that supports recording data within + * maxBucketsCount. + * @param maxBucketsCount - maximum number of buckets that can be used for + * distribution. + * @param zeroThreshold - values less than or equal to this are considered in zero + * count and recorded in the histogram. If less than 0, this is rounded to zero. In + * case of recording time, this should be in nanoseconds. + * @param baseUnit - an Optional TimeUnit. If set to a non-null unit, the recorded + * values are converted to this unit. + */ + Base2ExponentialHistogram(int maxScale, int maxBucketsCount, double zeroThreshold, @Nullable TimeUnit baseUnit) { + this.maxScale = maxScale; + this.scale = maxScale; + this.maxBucketsCount = maxBucketsCount; + this.baseUnit = baseUnit; + // Convert the zeroThreshold to baseUnit. + this.zeroThreshold = Math.max(baseUnit != null ? TimeUtils.nanosToUnit(zeroThreshold, baseUnit) : zeroThreshold, + 0.0); + + this.circularCountHolder = new CircularCountHolder(maxBucketsCount); + this.base2IndexProvider = IndexProviderFactory.getIndexProviderForScale(scale); + } + + /** + * Returns the latest snapshot of recordings from + * {@link Base2ExponentialHistogram#takeExponentialHistogramSnapShot()} and not the + * current set of values. It is recommended to use this method to consume values + * recorded in this Histogram as this will provide consistency in recorded values. + */ + public abstract ExponentialHistogramSnapShot getLatestExponentialHistogramSnapshot(); + + /** + * Takes a snapshot of the values that are recorded. + */ + abstract void takeExponentialHistogramSnapShot(); + + int getScale() { + return scale; + } + + /** + * Provides a bridge to Micrometer {@link HistogramSnapshot}. + */ + @Override + public synchronized HistogramSnapshot takeSnapshot(final long count, final double total, final double max) { + this.takeExponentialHistogramSnapShot(); + return new HistogramSnapshot(count, total, max, null, null, null); + } + + /** + * Returns the snapshot of current recorded values.. + */ + ExponentialHistogramSnapShot getCurrentValuesSnapshot() { + return (circularCountHolder.isEmpty() && zeroCount.longValue() == 0) + ? DefaultExponentialHistogramSnapShot.getEmptySnapshotForScale(scale) + : new DefaultExponentialHistogramSnapShot(scale, getOffset(), zeroCount.longValue(), zeroThreshold, + getBucketCounts()); + } + + /** + * Records the value to the Histogram. While measuring time, this value will be + * converted to {@link Base2ExponentialHistogram#baseUnit}. + * @param value - value to be recorded in the Histogram. (in + * {@link TimeUnit#NANOSECONDS} if recording time.) + */ + @Override + public void recordLong(final long value) { + recordDouble(value); + } + + /** + * Records the value to the Histogram. While measuring time, this value will be + * converted {@link Base2ExponentialHistogram#baseUnit}. + * @param value - value to be recorded in the Histogram. (in + * {@link TimeUnit#NANOSECONDS} if recording time.) + */ + @Override + public void recordDouble(double value) { + if (baseUnit != null) { + value = TimeUtils.nanosToUnit(value, baseUnit); + } + + if (value <= zeroThreshold) { + zeroCount.increment(); + return; + } + + int index = base2IndexProvider.getIndexForValue(value); + if (!circularCountHolder.increment(index, 1)) { + downScale(getDownScaleFactor(index)); + index = base2IndexProvider.getIndexForValue(value); + circularCountHolder.increment(index, 1); + } + } + + /** + * Reduces the scale of the histogram by downScaleFactor. The buckets are merged to + * align with the exponential scale. + * @param downScaleFactor - the factor to downscale this histogram. + */ + private synchronized void downScale(int downScaleFactor) { + if (downScaleFactor == 0) { + return; + } + + if (!circularCountHolder.isEmpty()) { + CircularCountHolder newCounts = new CircularCountHolder(maxBucketsCount); + + for (int i = circularCountHolder.getStartIndex(); i <= circularCountHolder.getEndIndex(); i++) { + long count = circularCountHolder.getValueAtIndex(i); + if (count > 0) { + newCounts.increment(i >> downScaleFactor, count); + } + } + this.circularCountHolder = newCounts; + } + + this.updateScale(this.scale - downScaleFactor); + } + + private void updateScale(int newScale) { + if (newScale > maxScale) { + newScale = maxScale; + } + this.scale = newScale; + this.base2IndexProvider = IndexProviderFactory.getIndexProviderForScale(scale); + } + + /** + * Provide a downscale factor for the {@link Base2ExponentialHistogram} so that the + * value can be recorded within {@link Base2ExponentialHistogram#maxBucketsCount}. + * @param index - the index to which current value belongs to. + * @return a factor by which {@link Base2ExponentialHistogram#scale} should be + * decreased. + */ + private synchronized int getDownScaleFactor(final long index) { + long newStart = Math.min(index, circularCountHolder.getStartIndex()); + long newEnd = Math.max(index, circularCountHolder.getEndIndex()); + + int scaleDownFactor = 0; + while (newEnd - newStart + 1 > maxBucketsCount) { + newStart >>= 1; + newEnd >>= 1; + scaleDownFactor++; + } + return scaleDownFactor; + } + + /** + * Provides a factor by which {@link Base2ExponentialHistogram#scale} can be increased + * so that the values can still be represented using + * {@link Base2ExponentialHistogram#maxBucketsCount}. This does not reset the last + * used scale but makes the best attempt based on data recorded for last interval. In + * most cases the range of values recorded within an {@link Base2ExponentialHistogram} + * instance stays same, and we should avoid re-scaling to minimize garbage creation. + * This applies only for + * {@link io.micrometer.registry.otlp.AggregationTemporality#DELTA} where values are + * reset for every interval. + * @return - a factor by which the {@link Base2ExponentialHistogram#scale} should be + * increased. + */ + private int getUpscaleFactor() { + if (!circularCountHolder.isEmpty()) { + int indexDelta = circularCountHolder.getEndIndex() - circularCountHolder.getStartIndex() + 1; + if (indexDelta == 1) { + return maxScale - scale; + } + return (int) Math.floor(Math.log(maxBucketsCount / (double) indexDelta) / Math.log(2)); + } + // When there are no recordings we will fall back to max scale. + return maxScale - scale; + } + + private int getOffset() { + if (circularCountHolder.isEmpty()) { + return 0; + } + return circularCountHolder.getStartIndex(); + } + + /** + * Returns the list of buckets representing the values recorded. This is always less + * than or equal to {@link Base2ExponentialHistogram#maxBucketsCount}. + */ + private List getBucketCounts() { + if (circularCountHolder.isEmpty()) { + return Collections.emptyList(); + } + + int length = circularCountHolder.getEndIndex() - circularCountHolder.getStartIndex() + 1; + + long[] countsArr = new long[length]; + for (int i = 0; i < length; i++) { + countsArr[i] = circularCountHolder.getValueAtIndex(i + circularCountHolder.getStartIndex()); + } + return Arrays.stream(countsArr).boxed().collect(Collectors.toList()); + } + + /** + * Reset the current values and possibly increase the scale based on current recorded + * values; + */ + synchronized void reset() { + int upscaleFactor = getUpscaleFactor(); + if (upscaleFactor > 0) { + this.updateScale(this.scale + upscaleFactor); + } + + this.circularCountHolder.reset(); + this.zeroCount.reset(); + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/CircularCountHolder.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/CircularCountHolder.java new file mode 100644 index 0000000000..42fa3d0e87 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/CircularCountHolder.java @@ -0,0 +1,102 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.util.concurrent.atomic.AtomicLongArray; + +class CircularCountHolder { + + private final AtomicLongArray counts; + + private final int length; + + private int startIndex; + + private int endIndex; + + private int baseIndex; + + CircularCountHolder(int size) { + this.length = size; + this.counts = new AtomicLongArray(size); + this.baseIndex = Integer.MIN_VALUE; + this.startIndex = Integer.MIN_VALUE; + this.endIndex = Integer.MIN_VALUE; + } + + int getStartIndex() { + return startIndex; + } + + int getEndIndex() { + return endIndex; + } + + long getValueAtIndex(int index) { + return counts.get(getRelativeIndex(index)); + } + + boolean isEmpty() { + return baseIndex == Integer.MIN_VALUE; + } + + boolean increment(int index, long incrementBy) { + if (baseIndex == Integer.MIN_VALUE) { + this.baseIndex = index; + this.startIndex = index; + this.endIndex = index; + this.counts.addAndGet(0, incrementBy); + return true; + } + + if (index > endIndex) { + if ((long) index - startIndex + 1 > length) { + return false; + } + endIndex = index; + } + else if (index < startIndex) { + if ((long) endIndex - index + 1 > length) { + return false; + } + startIndex = index; + } + + counts.addAndGet(getRelativeIndex(index), incrementBy); + return true; + } + + private int getRelativeIndex(int index) { + int result = index - baseIndex; + if (result >= length) { + result -= length; + } + else if (result < 0) { + result += length; + } + return result; + } + + void reset() { + for (int i = 0; i < counts.length(); i++) { + counts.set(i, 0); + } + this.baseIndex = Integer.MIN_VALUE; + this.endIndex = Integer.MIN_VALUE; + this.startIndex = Integer.MIN_VALUE; + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/CumulativeBase2ExponentialHistogram.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/CumulativeBase2ExponentialHistogram.java new file mode 100644 index 0000000000..bad18f732b --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/CumulativeBase2ExponentialHistogram.java @@ -0,0 +1,59 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.util.concurrent.TimeUnit; + +import io.micrometer.common.lang.Nullable; + +/** + * A {@link Base2ExponentialHistogram} that tracks values cumulatively from first recorded + * value. + */ +public class CumulativeBase2ExponentialHistogram extends Base2ExponentialHistogram { + + private ExponentialHistogramSnapShot exponentialHistogramSnapShot; + + /** + * Creates an Base2ExponentialHistogram that record positive values cumulatively. + * @param maxScale - maximum scale that can be used. The recordings start with this + * scale and gets downscaled to the scale that supports recording data within + * maxBucketsCount. + * @param maxBucketsCount - maximum number of buckets that can be used for + * distribution. + * @param zeroThreshold - values less than or equal to this are considered in zero + * count and recorded in the histogram. If less than 0, this is rounded to zero. In + * case of recording time, this should be in nanoseconds. + * @param baseUnit - an Optional TimeUnit. If set to a non-null unit, the recorded + * values are converted to this unit. + */ + public CumulativeBase2ExponentialHistogram(final int maxScale, final int maxBucketsCount, + final double zeroThreshold, @Nullable final TimeUnit baseUnit) { + super(maxScale, maxBucketsCount, zeroThreshold, baseUnit); + this.exponentialHistogramSnapShot = DefaultExponentialHistogramSnapShot.getEmptySnapshotForScale(maxScale); + } + + @Override + public ExponentialHistogramSnapShot getLatestExponentialHistogramSnapshot() { + return this.exponentialHistogramSnapShot; + } + + @Override + synchronized void takeExponentialHistogramSnapShot() { + this.exponentialHistogramSnapShot = getCurrentValuesSnapshot(); + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/DefaultExponentialHistogramSnapShot.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/DefaultExponentialHistogramSnapShot.java new file mode 100644 index 0000000000..82afa73300 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/DefaultExponentialHistogramSnapShot.java @@ -0,0 +1,88 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class DefaultExponentialHistogramSnapShot implements ExponentialHistogramSnapShot { + + private static final Map emptySnapshotCache = new HashMap<>(); + + private final int scale; + + private final int offset; + + private final long zeroCount; + + private final double zeroThreshold; + + private final List bucketsCount; + + DefaultExponentialHistogramSnapShot(int scale, int offset, long zeroCount, double zeroThreshold, + List bucketsCount) { + this.scale = scale; + this.offset = offset; + this.zeroCount = zeroCount; + this.zeroThreshold = zeroThreshold; + this.bucketsCount = Collections.unmodifiableList(bucketsCount); + } + + @Override + public int scale() { + return scale; + } + + @Override + public long zeroCount() { + return zeroCount; + } + + @Override + public int offset() { + return offset; + } + + @Override + public List bucketsCount() { + return bucketsCount; + } + + @Override + public double zeroThreshold() { + return zeroThreshold; + } + + @Override + public boolean isEmpty() { + return bucketsCount.isEmpty() && zeroCount == 0; + } + + static ExponentialHistogramSnapShot getEmptySnapshotForScale(int scale) { + return emptySnapshotCache.computeIfAbsent(scale, + key -> new DefaultExponentialHistogramSnapShot(key, 0, 0, 0.0, Collections.emptyList())); + } + + @Override + public String toString() { + return "DefaultExponentialHistogramSnapShot{" + "scale=" + scale() + ", zeroCount=" + zeroCount() + + ", zeroThreshold=" + zeroThreshold() + ", {bucketsCountLength=" + bucketsCount().size() + ", offset=" + + offset() + ", " + "bucketsCount=" + bucketsCount() + '}'; + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/DeltaBase2ExponentialHistogram.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/DeltaBase2ExponentialHistogram.java new file mode 100644 index 0000000000..d43d8e88b4 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/DeltaBase2ExponentialHistogram.java @@ -0,0 +1,88 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import io.micrometer.common.lang.Nullable; +import io.micrometer.core.instrument.Clock; +import io.micrometer.core.instrument.step.StepValue; + +/** + * A {@link Base2ExponentialHistogram} where values are reset after every Step. + * Internally, this uses {@link StepValue} to roll the HistogramSnapshot for every step. + */ +public class DeltaBase2ExponentialHistogram extends Base2ExponentialHistogram { + + private final StepValue stepExponentialHistogramSnapShot; + + /** + * Creates an Base2ExponentialHistogram that record positive values and resets for + * every step. This doesn't move the step window during recording but this does so on + * calling {@link Base2ExponentialHistogram#takeSnapshot(long, double, double)} ()}. + * @param maxScale - maximum scale that can be used. The recordings start with this + * scale and gets downscaled to the scale that supports recording data within + * maxBucketsCount. + * @param maxBucketsCount - maximum number of buckets that can be used for + * distribution. + * @param zeroThreshold - values less than or equal to this are considered in zero + * count and recorded in the histogram. If less than 0, this is rounded to zero. In + * case of recording time, this should be in nanoseconds. + * @param baseUnit - an Optional TimeUnit. If set to a non-null unit, the recorded + * values are converted to this unit. + * @param clock - clock to be used. + * @param stepMillis - window for delta aggregation + */ + public DeltaBase2ExponentialHistogram(final int maxScale, final int maxBucketsCount, final double zeroThreshold, + @Nullable final TimeUnit baseUnit, final Clock clock, final long stepMillis) { + super(maxScale, maxBucketsCount, zeroThreshold, baseUnit); + this.stepExponentialHistogramSnapShot = new StepExponentialHistogramSnapShot(clock, stepMillis, maxScale); + } + + @Override + public ExponentialHistogramSnapShot getLatestExponentialHistogramSnapshot() { + return stepExponentialHistogramSnapShot.poll(); + } + + @Override + synchronized void takeExponentialHistogramSnapShot() { + stepExponentialHistogramSnapShot.poll(); + } + + private class StepExponentialHistogramSnapShot extends StepValue { + + public StepExponentialHistogramSnapShot(final Clock clock, final long stepMillis, final int maxScale) { + super(clock, stepMillis, DefaultExponentialHistogramSnapShot.getEmptySnapshotForScale(maxScale)); + } + + @Override + protected Supplier valueSupplier() { + return () -> { + ExponentialHistogramSnapShot latestSnapShot = getCurrentValuesSnapshot(); + reset(); + return latestSnapShot; + }; + } + + @Override + protected ExponentialHistogramSnapShot noValue() { + return DefaultExponentialHistogramSnapShot.getEmptySnapshotForScale(getScale()); + } + + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/ExponentialHistogramSnapShot.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/ExponentialHistogramSnapShot.java new file mode 100644 index 0000000000..47a9a5f6d5 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/ExponentialHistogramSnapShot.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.util.List; + +public interface ExponentialHistogramSnapShot { + + /** + * Returns the scale of the ExponentialHistogram. + */ + int scale(); + + /** + * Returns the count of values that are less than or equal to + * {@link ExponentialHistogramSnapShot#zeroThreshold()}. + */ + long zeroCount(); + + /** + * Returns the index of the first entry in the positive Bucket counts list. + */ + int offset(); + + /** + * Returns the count of positive range of exponential buckets. + */ + List bucketsCount(); + + /** + * Returns the threshold below which (inclusive) the values are counted in + * {@link ExponentialHistogramSnapShot#zeroCount()}. + */ + double zeroThreshold(); + + boolean isEmpty(); + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/IndexProvider.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/IndexProvider.java new file mode 100644 index 0000000000..7d64f9e18e --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/IndexProvider.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +interface IndexProvider { + + int getIndexForValue(final double value); + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/IndexProviderFactory.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/IndexProviderFactory.java new file mode 100644 index 0000000000..0292f1ef04 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/IndexProviderFactory.java @@ -0,0 +1,127 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.util.HashMap; +import java.util.Map; + +/** + * A factory that provides the {@link IndexProvider} for a given scale. + * + * @see Exponentioal + * Buckets + */ +class IndexProviderFactory { + + private static final Map indexProviderCache = new HashMap<>(); + + private static final IndexProvider ZERO_SCALE_INDEX_PROVIDER = new ZeroScaleIndexProvider(); + + private IndexProviderFactory() { + } + + static IndexProvider getIndexProviderForScale(int scale) { + if (scale > 0) { + return indexProviderCache.computeIfAbsent(scale, PositiveScaleIndexProvider::new); + } + else if (scale < 0) { + return indexProviderCache.computeIfAbsent(scale, NegativeScaleIndexProvider::new); + } + return ZERO_SCALE_INDEX_PROVIDER; + } + + /** + * Use Use + * the Logarithm Function to calculate index for positive scale. + */ + private static class PositiveScaleIndexProvider implements IndexProvider { + + private final double scaleFactor; + + PositiveScaleIndexProvider(int scale) { + this.scaleFactor = Math.scalb(Math.log(Math.E) / Math.log(2), scale); + } + + @Override + public int getIndexForValue(final double value) { + // NOTE: Is it worth handling the mapping of exact powers of 2 as mentioned in + // the spec? + return (int) Math.ceil(Math.log(value) * scaleFactor) - 1; + } + + } + + /** + * Use Extract the Exponent to calculate index + * for zero scale. + */ + private static class ZeroScaleIndexProvider implements IndexProvider { + + // IEEE 754 double-precision constants + private static final long SIGNIFICAND_MASK = 0x000FFFFFFFFFFFFFL; + + private static final long EXPONENT_MASK = 0x7FF0000000000000L; + + private static final int SIGNIFICAND_WIDTH = 52; + + private static final int EXPONENT_BIAS = 1023; + + ZeroScaleIndexProvider() { + } + + @Override + public int getIndexForValue(final double value) { + long rawBits = Double.doubleToLongBits(value); + long rawExponent = (rawBits & EXPONENT_MASK) >> SIGNIFICAND_WIDTH; + long rawFragment = rawBits & SIGNIFICAND_MASK; + if (rawExponent == 0) { + rawExponent -= Long.numberOfLeadingZeros(rawFragment - 1) - 12; + } + int ieeeExponent = (int) (rawExponent - EXPONENT_BIAS); + + if (rawFragment == 0) { + return ieeeExponent - 1; + } + return ieeeExponent; + } + + } + + /** + * Use Index + * computation for negative scale to calculate index for negative scale. + */ + private static class NegativeScaleIndexProvider implements IndexProvider { + + private final int scale; + + private NegativeScaleIndexProvider(final int scale) { + this.scale = scale; + } + + @Override + public int getIndexForValue(final double value) { + return ZERO_SCALE_INDEX_PROVIDER.getIndexForValue(value) >> -scale; + } + + } + +} diff --git a/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/package-info.java b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/package-info.java new file mode 100644 index 0000000000..cbfb161025 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/main/java/io/micrometer/registry/otlp/internal/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullFields +@NonNullApi +package io.micrometer.registry.otlp.internal; + +import io.micrometer.common.lang.NonNullApi; +import io.micrometer.common.lang.NonNullFields; diff --git a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpCumulativeMeterRegistryTest.java b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpCumulativeMeterRegistryTest.java index c8d1e5d2e1..e61b55c3df 100644 --- a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpCumulativeMeterRegistryTest.java +++ b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpCumulativeMeterRegistryTest.java @@ -17,12 +17,15 @@ import io.micrometer.core.instrument.*; import io.micrometer.core.instrument.binder.BaseUnits; +import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint; +import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import org.junit.jupiter.api.Test; import java.lang.management.CompilationMXBean; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; +import java.time.Duration; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -35,6 +38,22 @@ protected OtlpConfig otlpConfig() { return OtlpConfig.DEFAULT; } + @Override + OtlpConfig exponentialHistogramOtlpConfig() { + return new OtlpConfig() { + + @Override + public HistogramFlavour histogramFlavour() { + return HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM; + } + + @Override + public String get(final String key) { + return null; + } + }; + } + @Test void gauge() { Gauge cpus = Gauge @@ -517,4 +536,92 @@ void testMetricsStartAndEndTime() { assertThat(getDataPoint.apply(counter).getTimeUnixNano()).isEqualTo(60001000000L); } + @Test + void testExponentialHistogramWithTimer() { + Timer timer = Timer.builder(METER_NAME) + .description(METER_DESCRIPTION) + .tags(Tags.of(meterTag)) + .publishPercentileHistogram() + .register(registryWithExponentialHistogram); + timer.record(Duration.ofMillis(1)); + timer.record(Duration.ofMillis(100)); + timer.record(Duration.ofMillis(1000)); + + Metric metric = writeToMetric(timer); + assertThat(metric.getExponentialHistogram().getDataPointsCount()).isPositive(); + + ExponentialHistogramDataPoint exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + assertExponentialHistogram(metric, 3, 1101, 0.0, 1, 5); + ExponentialHistogramDataPoint.Buckets buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(212); + assertThat(buckets.getBucketCountsCount()).isEqualTo(107); + assertThat(buckets.getBucketCountsList().get(0)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(106)).isEqualTo(1); + assertThat(buckets.getBucketCountsList()).filteredOn(v -> v == 0).hasSize(105); + + long previousEndTime = exponentialHistogramDataPoint.getTimeUnixNano(); + + clock.add(exponentialHistogramOtlpConfig().step()); + timer.record(Duration.ofMillis(10000)); + + metric = writeToMetric(timer); + exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + assertThat(exponentialHistogramDataPoint.getTimeUnixNano() - previousEndTime) + .isEqualTo(otlpConfig().step().toNanos()); + + assertExponentialHistogram(metric, 4, 11101, 0.0, 1, 4); + + buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(106); + assertThat(buckets.getBucketCountsCount()).isEqualTo(107); + assertThat(buckets.getBucketCountsList().get(0)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(53)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(106)).isEqualTo(1); + assertThat(buckets.getBucketCountsList()).filteredOn(v -> v == 0).hasSize(104); + } + + @Test + void testExponentialHistogramDs() { + DistributionSummary ds = DistributionSummary.builder(METER_NAME) + .description(METER_DESCRIPTION) + .tags(Tags.of(meterTag)) + .publishPercentileHistogram() + .register(registryWithExponentialHistogram); + ds.record(1); + ds.record(100); + ds.record(1000); + + Metric metric = writeToMetric(ds); + assertThat(metric.getExponentialHistogram().getDataPointsCount()).isPositive(); + + ExponentialHistogramDataPoint exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + assertExponentialHistogram(metric, 3, 1101, 0.0, 1, 5); + ExponentialHistogramDataPoint.Buckets buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(212); + assertThat(buckets.getBucketCountsCount()).isEqualTo(107); + assertThat(buckets.getBucketCountsList().get(0)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(106)).isEqualTo(1); + assertThat(buckets.getBucketCountsList()).filteredOn(v -> v == 0).hasSize(105); + + long previousEndTime = exponentialHistogramDataPoint.getTimeUnixNano(); + + clock.add(exponentialHistogramOtlpConfig().step()); + ds.record(10000); + + metric = writeToMetric(ds); + exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + assertThat(exponentialHistogramDataPoint.getTimeUnixNano() - previousEndTime) + .isEqualTo(otlpConfig().step().toNanos()); + + assertExponentialHistogram(metric, 4, 11101, 0.0, 1, 4); + + buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(106); + assertThat(buckets.getBucketCountsCount()).isEqualTo(107); + assertThat(buckets.getBucketCountsList().get(0)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(53)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(106)).isEqualTo(1); + assertThat(buckets.getBucketCountsList()).filteredOn(v -> v == 0).hasSize(104); + } + } diff --git a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpDeltaMeterRegistryTest.java b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpDeltaMeterRegistryTest.java index 0513ffe50a..9e52466df2 100644 --- a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpDeltaMeterRegistryTest.java +++ b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpDeltaMeterRegistryTest.java @@ -21,6 +21,7 @@ import io.micrometer.core.instrument.distribution.CountAtBucket; import io.micrometer.core.instrument.distribution.HistogramSnapshot; import io.micrometer.core.instrument.util.TimeUtils; +import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint; import io.opentelemetry.proto.metrics.v1.HistogramDataPoint; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; @@ -67,6 +68,26 @@ public String get(String key) { }; } + @Override + OtlpConfig exponentialHistogramOtlpConfig() { + return new OtlpConfig() { + @Override + public AggregationTemporality aggregationTemporality() { + return DELTA; + } + + @Override + public HistogramFlavour histogramFlavour() { + return HistogramFlavour.BASE2_EXPONENTIAL_BUCKET_HISTOGRAM; + } + + @Override + public String get(String key) { + return null; + } + }; + } + @Test void gauge() { Gauge gauge = Gauge.builder(METER_NAME, new AtomicInteger(5), AtomicInteger::doubleValue).register(registry); @@ -443,6 +464,114 @@ void scheduledRolloverDistributionSummary() { 170, 150); } + @Test + void testExponentialHistogramWithTimer() { + Timer timer = Timer.builder(METER_NAME) + .description(METER_DESCRIPTION) + .tags(Tags.of(meterTag)) + .publishPercentileHistogram() + .register(registryWithExponentialHistogram); + timer.record(Duration.ofMillis(1)); + timer.record(Duration.ofMillis(100)); + timer.record(Duration.ofMillis(1000)); + + clock.add(exponentialHistogramOtlpConfig().step()); + registryWithExponentialHistogram.publish(); + timer.record(Duration.ofMillis(10000)); + + Metric metric = writeToMetric(timer); + assertThat(metric.getExponentialHistogram().getDataPointsCount()).isPositive(); + ExponentialHistogramDataPoint exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + assertExponentialHistogram(metric, 3, 1101, 1000.0, 1, 5); + ExponentialHistogramDataPoint.Buckets buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(212); + assertThat(buckets.getBucketCountsCount()).isEqualTo(107); + assertThat(buckets.getBucketCountsList().get(0)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(106)).isEqualTo(1); + assertThat(buckets.getBucketCountsList()).filteredOn(v -> v == 0).hasSize(105); + + clock.add(exponentialHistogramOtlpConfig().step()); + metric = writeToMetric(timer); + exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + + // Note the difference here, if it cumulative we had gone to a lower scale to + // accommodate 1, 100, 1000, + // 10000 but since the first 3 values are reset after the step. We will still be + // able to record 10000 in the + // same scale. + assertExponentialHistogram(metric, 1, 10000, 10000.0, 0, 5); + buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(425); + assertThat(buckets.getBucketCountsCount()).isEqualTo(1); + + timer.record(Duration.ofMillis(10001)); + clock.add(exponentialHistogramOtlpConfig().step()); + metric = writeToMetric(timer); + exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + + // Since, the range of recorded values in the last step is low, the histogram + // would have been rescaled to Max + // scale. + assertExponentialHistogram(metric, 1, 10001, 10001.0, 0, 20); + buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(13933327); + assertThat(buckets.getBucketCountsCount()).isEqualTo(1); + } + + @Test + void testExponentialHistogramDs() { + DistributionSummary ds = DistributionSummary.builder(METER_NAME) + .description(METER_DESCRIPTION) + .tags(Tags.of(meterTag)) + .publishPercentileHistogram() + .register(registryWithExponentialHistogram); + ds.record(1); + ds.record(100); + ds.record(1000); + + clock.add(exponentialHistogramOtlpConfig().step()); + registryWithExponentialHistogram.publish(); + ds.record(10000); + + Metric metric = writeToMetric(ds); + assertThat(metric.getExponentialHistogram().getDataPointsCount()).isPositive(); + ExponentialHistogramDataPoint exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + assertExponentialHistogram(metric, 3, 1101, 1000.0, 1, 5); + ExponentialHistogramDataPoint.Buckets buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(212); + assertThat(buckets.getBucketCountsCount()).isEqualTo(107); + assertThat(buckets.getBucketCountsList().get(0)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(106)).isEqualTo(1); + assertThat(buckets.getBucketCountsList()).filteredOn(v -> v == 0).hasSize(105); + + clock.add(exponentialHistogramOtlpConfig().step()); + metric = writeToMetric(ds); + exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + + // Mote the difference here, if it cumulative we had gone to a lower scale to + // accommodate 1, 100, 1000, + // 10000 but since the first 3 values are reset after the step. We will still be + // able to record 10000 in the + // same scale. + assertExponentialHistogram(metric, 1, 10000, 10000.0, 0, 5); + buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(425); + assertThat(buckets.getBucketCountsCount()).isEqualTo(1); + + ds.record(10001); + clock.add(exponentialHistogramOtlpConfig().step()); + metric = writeToMetric(ds); + exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + + // Since, the range of recorded values in the last step is low, the histogram + // would have been rescaled to Max + // scale. + assertExponentialHistogram(metric, 1, 10001, 10001.0, 0, 20); + buckets = exponentialHistogramDataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(13933327); + assertThat(buckets.getBucketCountsCount()).isEqualTo(1); + } + @Issue("#3773") @Test void shortLivedPublish() { diff --git a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpMeterRegistryTest.java b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpMeterRegistryTest.java index f529596921..e8f59bf678 100644 --- a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpMeterRegistryTest.java +++ b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/OtlpMeterRegistryTest.java @@ -17,6 +17,8 @@ import io.micrometer.core.instrument.*; import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import io.micrometer.core.instrument.Timer; +import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint; import io.opentelemetry.proto.metrics.v1.HistogramDataPoint; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; @@ -24,9 +26,7 @@ import java.io.IOException; import java.time.Duration; -import java.util.Collections; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -48,10 +48,14 @@ abstract class OtlpMeterRegistryTest { protected MockClock clock = new MockClock(); - protected OtlpMeterRegistry registry = new OtlpMeterRegistry(otlpConfig(), clock); + OtlpMeterRegistry registry = new OtlpMeterRegistry(otlpConfig(), clock); + + OtlpMeterRegistry registryWithExponentialHistogram = new OtlpMeterRegistry(exponentialHistogramOtlpConfig(), clock); abstract OtlpConfig otlpConfig(); + abstract OtlpConfig exponentialHistogramOtlpConfig(); + // If the service.name was not specified, SDKs MUST fallback to 'unknown_service' @Test void unknownServiceByDefault() { @@ -107,59 +111,149 @@ void timeGauge() { @Test void distributionWithPercentileShouldWriteSummary() { - Timer timer = Timer.builder("timer") + Timer.Builder timer = Timer.builder("timer") .description(METER_DESCRIPTION) .tags(Tags.of(meterTag)) - .publishPercentiles(0.5, 0.9) - .register(registry); + .publishPercentiles(0.5, 0.9); - DistributionSummary ds = DistributionSummary.builder("ds") + DistributionSummary.Builder ds = DistributionSummary.builder("ds") .description(METER_DESCRIPTION) .tags(Tags.of(meterTag)) - .publishPercentiles(0.5, 0.9) - .register(registry); + .publishPercentiles(0.5, 0.9); + + assertThat(writeToMetric(timer.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.SUMMARY.getNumber()); + assertThat(writeToMetric(ds.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.SUMMARY.getNumber()); + assertThat(writeToMetric(timer.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.SUMMARY.getNumber()); + assertThat(writeToMetric(ds.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.SUMMARY.getNumber()); + } - assertThat(writeToMetric(timer).getDataCase().getNumber()).isEqualTo(Metric.DataCase.SUMMARY.getNumber()); - assertThat(writeToMetric(ds).getDataCase().getNumber()).isEqualTo(Metric.DataCase.SUMMARY.getNumber()); + @Test + void distributionWithPercentileHistogramShouldWriteHistogramOrExponentialHistogram() { + Timer.Builder timer = Timer.builder("timer") + .description(METER_DESCRIPTION) + .tags(Tags.of(meterTag)) + .publishPercentileHistogram(); + + DistributionSummary.Builder ds = DistributionSummary.builder("ds") + .description(METER_DESCRIPTION) + .tags(Tags.of(meterTag)) + .publishPercentileHistogram(); + + assertThat(writeToMetric(timer.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + assertThat(writeToMetric(ds.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + assertThat(writeToMetric(timer.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.EXPONENTIAL_HISTOGRAM.getNumber()); + assertThat(writeToMetric(ds.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.EXPONENTIAL_HISTOGRAM.getNumber()); } @Test - void distributionWithPercentileAndHistogramShouldWriteHistogramDataPoint() { - Timer timer = Timer.builder("timer") + void distributionWithPercentileAndHistogramShouldWriteHistogramOrExponentialHistogram() { + Timer.Builder timer = Timer.builder("timer") .description(METER_DESCRIPTION) .tags(Tags.of(meterTag)) .publishPercentiles(0.5, 0.9) - .publishPercentileHistogram() - .serviceLevelObjectives(Duration.ofMillis(1)) - .register(registry); + .publishPercentileHistogram(); - DistributionSummary ds = DistributionSummary.builder("ds") + DistributionSummary.Builder ds = DistributionSummary.builder("ds") .description(METER_DESCRIPTION) .tags(Tags.of(meterTag)) .publishPercentiles(0.5, 0.9) - .publishPercentileHistogram() - .serviceLevelObjectives(1.0) - .register(registry); - - assertThat(writeToMetric(timer).getDataCase().getNumber()).isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); - assertThat(writeToMetric(ds).getDataCase().getNumber()).isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + .publishPercentileHistogram(); + + assertThat(writeToMetric(timer.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + assertThat(writeToMetric(ds.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + assertThat(writeToMetric(timer.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.EXPONENTIAL_HISTOGRAM.getNumber()); + assertThat(writeToMetric(ds.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.EXPONENTIAL_HISTOGRAM.getNumber()); } @Test - void distributionWithHistogramShouldWriteHistogramDataPoint() { - Timer timer = Timer.builder("timer") + void distributionWithSLOShouldWriteHistogramDataPoint() { + Timer.Builder timer = Timer.builder("timer") .description(METER_DESCRIPTION) .tags(Tags.of(meterTag)) - .serviceLevelObjectives(Duration.ofMillis(1)) - .register(registry); - DistributionSummary ds = DistributionSummary.builder("ds") + .serviceLevelObjectives(Duration.ofMillis(1)); + DistributionSummary.Builder ds = DistributionSummary.builder("ds") .description(METER_DESCRIPTION) .tags(Tags.of(meterTag)) - .serviceLevelObjectives(1.0) - .register(registry); + .serviceLevelObjectives(1.0); + + assertThat(writeToMetric(timer.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + assertThat(writeToMetric(ds.register(registry)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + assertThat(writeToMetric(timer.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + assertThat(writeToMetric(ds.register(registryWithExponentialHistogram)).getDataCase().getNumber()) + .isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + } + + @Test + void testZeroCountForExponentialHistogram() { + Timer timerWithZero1ms = Timer.builder("zero_count_1ms") + .publishPercentileHistogram() + .register(registryWithExponentialHistogram); + Timer timerWithZero1ns = Timer.builder("zero_count_1ns") + .publishPercentileHistogram() + .minimumExpectedValue(Duration.ofNanos(1)) + .register(registryWithExponentialHistogram); + + timerWithZero1ms.record(Duration.ofNanos(1)); + timerWithZero1ms.record(Duration.ofMillis(1)); + timerWithZero1ns.record(Duration.ofNanos(1)); + timerWithZero1ns.record(Duration.ofMillis(1)); + + clock.add(exponentialHistogramOtlpConfig().step()); + + ExponentialHistogramDataPoint dataPoint = writeToMetric(timerWithZero1ms).getExponentialHistogram() + .getDataPoints(0); + assertThat(dataPoint.getZeroCount()).isEqualTo(2); + assertThat(dataPoint.getCount()).isEqualTo(2); + assertThat(dataPoint.getPositive().getBucketCountsCount()).isZero(); + + dataPoint = writeToMetric(timerWithZero1ns).getExponentialHistogram().getDataPoints(0); + assertThat(dataPoint.getZeroCount()).isEqualTo(1); + assertThat(dataPoint.getCount()).isEqualTo(2); + assertThat(dataPoint.getPositive().getBucketCountsCount()).isEqualTo(1); + assertThat(dataPoint.getPositive().getBucketCountsList()).isEqualTo(List.of(1L)); + } + + @Test + void timerShouldRecordInBaseUnitForExponentialHistogram() { + Timer timer = Timer.builder("timer_with_different_units") + .minimumExpectedValue(Duration.ofNanos(1)) + .publishPercentileHistogram() + .register(registryWithExponentialHistogram); + + timer.record(Duration.ofNanos(1000)); // 0.001 Milliseconds + timer.record(Duration.ofMillis(1)); + timer.record(Duration.ofSeconds(1)); // 1000 Milliseconds - assertThat(writeToMetric(timer).getDataCase().getNumber()).isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); - assertThat(writeToMetric(ds).getDataCase().getNumber()).isEqualTo(Metric.DataCase.HISTOGRAM.getNumber()); + clock.add(exponentialHistogramOtlpConfig().step()); + + Metric metric = writeToMetric(timer); + ExponentialHistogramDataPoint dataPoint = metric.getExponentialHistogram().getDataPoints(0); + + assertThat(dataPoint.getCount()).isEqualTo(3); + assertThat(dataPoint.getSum()).isEqualTo(1001.001); + + ExponentialHistogramDataPoint.Buckets buckets = dataPoint.getPositive(); + assertThat(buckets.getOffset()).isEqualTo(-80); + assertThat(buckets.getBucketCountsCount()).isEqualTo(160); + assertThat(buckets.getBucketCountsList().get(0)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(79)).isEqualTo(1); + assertThat(buckets.getBucketCountsList().get(159)).isEqualTo(1); + assertThat(buckets.getBucketCountsList()).filteredOn(v -> v == 0).hasSize(157); } @Test @@ -206,14 +300,11 @@ protected void assertHistogram(Metric metric, long startTime, long endTime, Stri .isEqualTo(AggregationTemporality.toOtlpAggregationTemporality(otlpConfig().aggregationTemporality())); HistogramDataPoint histogram = metric.getHistogram().getDataPoints(0); - assertThat(metric.getName()).isEqualTo(METER_NAME); - assertThat(metric.getDescription()).isEqualTo(METER_DESCRIPTION); - assertThat(metric.getUnit()).isEqualTo(unit); + assertMetricMetadata(metric, Optional.of(unit)); assertThat(histogram.getStartTimeUnixNano()).isEqualTo(startTime); assertThat(histogram.getTimeUnixNano()).isEqualTo(endTime); assertThat(histogram.getCount()).isEqualTo(count); assertThat(histogram.getSum()).isEqualTo(sum); - assertThat(histogram.getAttributesCount()).isEqualTo(1); assertThat(histogram.getAttributes(0).getKey()).isEqualTo(meterTag.getKey()); assertThat(histogram.getAttributes(0).getValue().getStringValue()).isEqualTo(meterTag.getValue()); @@ -230,8 +321,7 @@ protected void assertHistogram(Metric metric, long startTime, long endTime, Stri protected void assertSum(Metric metric, long startTime, long endTime, double expectedValue) { NumberDataPoint sumDataPoint = metric.getSum().getDataPoints(0); - assertThat(metric.getName()).isEqualTo(METER_NAME); - assertThat(metric.getDescription()).isEqualTo(METER_DESCRIPTION); + assertMetricMetadata(metric, Optional.empty()); assertThat(sumDataPoint.getStartTimeUnixNano()).isEqualTo(startTime); assertThat(sumDataPoint.getTimeUnixNano()).isEqualTo(endTime); assertThat(sumDataPoint.getAsDouble()).isEqualTo(expectedValue); @@ -242,4 +332,23 @@ protected void assertSum(Metric metric, long startTime, long endTime, double exp .isEqualTo(AggregationTemporality.toOtlpAggregationTemporality(otlpConfig().aggregationTemporality())); } + protected void assertExponentialHistogram(Metric metric, long count, double sum, double max, long zeroCount, + long scale) { + assertThat(metric.getExponentialHistogram().getDataPointsCount()).isPositive(); + ExponentialHistogramDataPoint exponentialHistogramDataPoint = metric.getExponentialHistogram().getDataPoints(0); + assertThat(exponentialHistogramDataPoint.getCount()).isEqualTo(count); + assertThat(exponentialHistogramDataPoint.getSum()).isEqualTo(sum); + assertThat(exponentialHistogramDataPoint.getMax()).isEqualTo(max); + + assertThat(exponentialHistogramDataPoint.getScale()).isEqualTo(scale); + assertThat(exponentialHistogramDataPoint.getZeroCount()).isEqualTo(zeroCount); + assertThat(exponentialHistogramDataPoint.getNegative().getBucketCountsCount()).isZero(); + } + + private void assertMetricMetadata(final Metric metric, Optional unitOptional) { + assertThat(metric.getName()).isEqualTo(METER_NAME); + assertThat(metric.getDescription()).isEqualTo(METER_DESCRIPTION); + unitOptional.ifPresent(unit -> assertThat(metric.getUnit()).isEqualTo(unit)); + } + } diff --git a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/Base2ExponentialHistogramTest.java b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/Base2ExponentialHistogramTest.java new file mode 100644 index 0000000000..7d71eac0cf --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/Base2ExponentialHistogramTest.java @@ -0,0 +1,253 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class Base2ExponentialHistogramTest { + + private static final long MILLI_SCALE = 1000L * 1000L; + + private static final int MAX_SCALE = 10; + + private static final int MAX_BUCKETS_COUNT = 16; + + private Base2ExponentialHistogram base2ExponentialHistogram; + + @BeforeEach + void setUp() { + /* + * By default, we are using 16 bucket counts since it is easy to manipulate these + * buckets for upScaling and downscaling. Some of the facts/number to be used in + * this test, For scale 10, base is 1.0006771306930664 index 0-15 corresponds to + * bounds of (1.0, 1.010889286052] and Scale 0, is easier to assert things as + * values are more human-readable. + */ + + base2ExponentialHistogram = new CumulativeBase2ExponentialHistogram(MAX_SCALE, MAX_BUCKETS_COUNT, 1.0, null); + } + + @Test + void testRecordDouble() { + // 1 Always belongs to index 0. + base2ExponentialHistogram.recordDouble(1.000000000001); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE); + assertThat(base2ExponentialHistogram.getCurrentValuesSnapshot().zeroCount()).isZero(); + assertThat(getAllBucketsCountSum(base2ExponentialHistogram.getCurrentValuesSnapshot())).isEqualTo(1); + } + + @Test + void testRecordTimeBased() { + base2ExponentialHistogram = new CumulativeBase2ExponentialHistogram(MAX_SCALE, MAX_BUCKETS_COUNT, MILLI_SCALE, + TimeUnit.MILLISECONDS); + base2ExponentialHistogram.recordLong(Duration.ofMillis(1).toNanos()); + base2ExponentialHistogram.recordLong(Duration.ofMillis(2).toNanos()); // This + // should be + // same as + // calling + // recordDouble(2). + + ExponentialHistogramSnapShot currentSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentSnapshot.zeroCount()).isEqualTo(1); + assertThat(currentSnapshot.scale()).isEqualTo(MAX_SCALE); + assertThat(getAllBucketsCountSum(currentSnapshot)).isEqualTo(1); + assertThat(base2ExponentialHistogram.getCurrentValuesSnapshot().offset()).isEqualTo(1023); + } + + @Test + void testRecordTimeBasedInSeconds() { + base2ExponentialHistogram = new CumulativeBase2ExponentialHistogram(MAX_SCALE, MAX_BUCKETS_COUNT, MILLI_SCALE, + TimeUnit.MILLISECONDS); + base2ExponentialHistogram = new CumulativeBase2ExponentialHistogram(MAX_SCALE, MAX_BUCKETS_COUNT, MILLI_SCALE, + TimeUnit.SECONDS); + + base2ExponentialHistogram.recordLong(Duration.ofMillis(1).toNanos()); + + // This should be same as calling recordDouble(0.05). + base2ExponentialHistogram.recordLong(Duration.ofMillis(50).toNanos()); + + ExponentialHistogramSnapShot currentSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentSnapshot.zeroCount()).isEqualTo(1); + assertThat(currentSnapshot.scale()).isEqualTo(MAX_SCALE); + assertThat(getAllBucketsCountSum(currentSnapshot)).isEqualTo(1); + assertThat(base2ExponentialHistogram.getCurrentValuesSnapshot().offset()).isEqualTo(-4426); + + base2ExponentialHistogram.recordLong(Duration.ofMillis(90).toNanos()); + currentSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentSnapshot.zeroCount()).isEqualTo(1); + assertThat(currentSnapshot.scale()).isEqualTo(4); + assertThat(getAllBucketsCountSum(currentSnapshot)).isEqualTo(2); + assertThat(base2ExponentialHistogram.getCurrentValuesSnapshot().offset()).isEqualTo(-70); + } + + @Test + void testZeroThreshHold() { + base2ExponentialHistogram.recordDouble(1.0); + base2ExponentialHistogram.recordDouble(0.0); + base2ExponentialHistogram.recordDouble(0.5); + + ExponentialHistogramSnapShot currentSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentSnapshot.zeroCount()).isEqualTo(3); + assertThat(currentSnapshot.scale()).isEqualTo(MAX_SCALE); + assertThat(getAllBucketsCountSum(currentSnapshot)).isZero(); + } + + @Test + void testDownScale() { + base2ExponentialHistogram.recordDouble(1.0001); + + ExponentialHistogramSnapShot currentSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentSnapshot.zeroCount()).isZero(); + assertThat(currentSnapshot.scale()).isEqualTo(MAX_SCALE); + assertThat(getAllBucketsCountSum(currentSnapshot)).isEqualTo(1); + + base2ExponentialHistogram.recordDouble(1.011); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE - 1); + + base2ExponentialHistogram.recordDouble(512); + assertThat(base2ExponentialHistogram.getScale()).isZero(); + + base2ExponentialHistogram.recordDouble(65537); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(-1); + } + + @Test + void testUpscale() { + base2ExponentialHistogram.recordDouble(1.0001); + base2ExponentialHistogram.recordDouble(512); // Scale is 0 now. + + base2ExponentialHistogram.reset(); + assertThat(base2ExponentialHistogram.getScale()).isZero(); + + base2ExponentialHistogram.recordDouble(1.0001); + base2ExponentialHistogram.reset(); + // When there is only one recording we expect the scale to be reset to maxScale. + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE); + + base2ExponentialHistogram.recordDouble(1.0001); + base2ExponentialHistogram.recordDouble(512); + base2ExponentialHistogram.reset(); + + // We will still be recording in higher scale, i.e 0. + base2ExponentialHistogram.recordDouble(1.0001); + base2ExponentialHistogram.recordDouble(4); + assertThat(base2ExponentialHistogram.getScale()).isZero(); + + // Now 1-4 uses only 3 buckets in scale 0 and the best scale to record values + // under 4 with 16 buckets will be 3. + base2ExponentialHistogram.reset(); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(3); + + base2ExponentialHistogram.recordDouble(1.0001); + base2ExponentialHistogram.recordDouble(2); + + // Now (1-2] uses only 8 buckets in scale 3 and the best scale to record values + // between (1,2] with 16 buckets + // will be 4. + base2ExponentialHistogram.reset(); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(4); + + base2ExponentialHistogram.reset(); + // When no values are recorded, we MUST fall back to maximum scale. + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE); + } + + @Test + void testValuesAtIndices() { + ExponentialHistogramSnapShot currentValueSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentValueSnapshot.bucketsCount()).isEmpty(); + + base2ExponentialHistogram.recordDouble(1.0001); + currentValueSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentValueSnapshot.offset()).isZero(); + assertThat(currentValueSnapshot.bucketsCount().get(0)).isEqualTo(1); + assertThat(currentValueSnapshot.bucketsCount()).filteredOn(value -> value == 0).isEmpty(); + + base2ExponentialHistogram.recordDouble(1.0008); + + base2ExponentialHistogram.recordDouble(1.0076); + base2ExponentialHistogram.recordDouble(1.008); + currentValueSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentValueSnapshot.offset()).isZero(); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE); + assertThat(currentValueSnapshot.bucketsCount().get(0)).isEqualTo(1); + assertThat(currentValueSnapshot.bucketsCount().get(1)).isEqualTo(1); + assertThat(currentValueSnapshot.bucketsCount().get(11)).isEqualTo(2); + assertThat(currentValueSnapshot.bucketsCount()).filteredOn(value -> value == 0).hasSize(9); + + // We will record a value that will downscale by 1 and this should merge 2 + // consecutive buckets into one. + base2ExponentialHistogram.recordDouble(1.012); + currentValueSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentValueSnapshot.offset()).isZero(); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE - 1); + assertThat(currentValueSnapshot.bucketsCount().get(0)).isEqualTo(2); + assertThat(currentValueSnapshot.bucketsCount().get(5)).isEqualTo(2); + assertThat(currentValueSnapshot.bucketsCount().get(8)).isEqualTo(1); + assertThat(currentValueSnapshot.bucketsCount()).filteredOn(value -> value == 0).hasSize(6); + + // The base will reduced by a factor of more than one, + base2ExponentialHistogram.recordDouble(4); + currentValueSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentValueSnapshot.offset()).isZero(); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(3); + assertThat(currentValueSnapshot.bucketsCount().get(0)).isEqualTo(5); + assertThat(currentValueSnapshot.bucketsCount().get(15)).isEqualTo(1); + assertThat(currentValueSnapshot.bucketsCount()).filteredOn(value -> value == 0).hasSize(14); + } + + @Test + void testUpscaleForNegativeScale() { + base2ExponentialHistogram.recordDouble(2); + base2ExponentialHistogram.recordDouble(65537); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(-1); + base2ExponentialHistogram.reset(); + + base2ExponentialHistogram.recordDouble(2); + base2ExponentialHistogram.reset(); + assertThat(base2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE); + } + + @Test + void reset() { + base2ExponentialHistogram.recordDouble(1); + base2ExponentialHistogram.recordDouble(2); + + ExponentialHistogramSnapShot currentSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentSnapshot.zeroCount()).isEqualTo(1); + assertThat(currentSnapshot.scale()).isEqualTo(MAX_SCALE); + assertThat(currentSnapshot.offset()).isEqualTo(1023); + assertThat(getAllBucketsCountSum(currentSnapshot)).isEqualTo(1); + + base2ExponentialHistogram.reset(); + currentSnapshot = base2ExponentialHistogram.getCurrentValuesSnapshot(); + assertThat(currentSnapshot.zeroCount()).isZero(); + assertThat(currentSnapshot.scale()).isEqualTo(MAX_SCALE); + assertThat(currentSnapshot.offset()).isZero(); + assertThat(getAllBucketsCountSum(currentSnapshot)).isZero(); + } + + static long getAllBucketsCountSum(ExponentialHistogramSnapShot snapShot) { + return snapShot.bucketsCount().stream().mapToLong(Long::longValue).sum(); + } + +} diff --git a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/CumulativeBase2ExponentialHistogramTest.java b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/CumulativeBase2ExponentialHistogramTest.java new file mode 100644 index 0000000000..e1dee80d7c --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/CumulativeBase2ExponentialHistogramTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class CumulativeBase2ExponentialHistogramTest { + + private static final int MAX_SCALE = 10; + + private CumulativeBase2ExponentialHistogram cumulativeBase2ExponentialHistogram; + + @BeforeEach + void setUp() { + cumulativeBase2ExponentialHistogram = new CumulativeBase2ExponentialHistogram(MAX_SCALE, 16, 1.0, null); + } + + @Test + void testDataIsAccumulatedCumulatively() { + cumulativeBase2ExponentialHistogram.recordDouble(2.0); + cumulativeBase2ExponentialHistogram.recordDouble(2.1); + + cumulativeBase2ExponentialHistogram.takeSnapshot(0, 0, 0); + ExponentialHistogramSnapShot exponentialHistogramSnapShot = cumulativeBase2ExponentialHistogram + .getLatestExponentialHistogramSnapshot(); + + assertThat(Base2ExponentialHistogramTest.getAllBucketsCountSum(exponentialHistogramSnapShot)).isEqualTo(2); + assertThat(exponentialHistogramSnapShot.scale()).isEqualTo(7); + + cumulativeBase2ExponentialHistogram.recordDouble(4); + cumulativeBase2ExponentialHistogram.takeSnapshot(0, 0, 0); + exponentialHistogramSnapShot = cumulativeBase2ExponentialHistogram.getLatestExponentialHistogramSnapshot(); + assertThat(Base2ExponentialHistogramTest.getAllBucketsCountSum(exponentialHistogramSnapShot)).isEqualTo(3); + assertThat(exponentialHistogramSnapShot.scale()).isEqualTo(3); + } + +} diff --git a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/DeltaBase2ExponentialHistogramTest.java b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/DeltaBase2ExponentialHistogramTest.java new file mode 100644 index 0000000000..c51ee26532 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/DeltaBase2ExponentialHistogramTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import java.time.Duration; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import io.micrometer.core.instrument.MockClock; + +import static org.assertj.core.api.Assertions.assertThat; + +class DeltaBase2ExponentialHistogramTest { + + private static final int MAX_SCALE = 10; + + private MockClock clock; + + private final Duration step = Duration.ofMillis(10); + + private DeltaBase2ExponentialHistogram deltaBase2ExponentialHistogram; + + @BeforeEach + void setUp() { + clock = new MockClock(); + deltaBase2ExponentialHistogram = new DeltaBase2ExponentialHistogram(MAX_SCALE, 16, 1.0, null, clock, + step.toMillis()); + } + + @Test + void snapshotShouldBeSameForOneStep() { + deltaBase2ExponentialHistogram.recordDouble(1.0); + deltaBase2ExponentialHistogram.recordDouble(2.0); + + ExponentialHistogramSnapShot exponentialHistogramSnapShot = deltaBase2ExponentialHistogram + .getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.zeroCount()).isZero(); + assertThat(Base2ExponentialHistogramTest.getAllBucketsCountSum(exponentialHistogramSnapShot)).isZero(); + + clock.add(step); + exponentialHistogramSnapShot = deltaBase2ExponentialHistogram.getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.zeroCount()).isEqualTo(1); + assertThat(exponentialHistogramSnapShot.scale()).isEqualTo(MAX_SCALE); + assertThat(Base2ExponentialHistogramTest.getAllBucketsCountSum(exponentialHistogramSnapShot)).isEqualTo(1); + + clock.add(step.dividedBy(2)); + deltaBase2ExponentialHistogram.recordDouble(4.0); + deltaBase2ExponentialHistogram.recordDouble(1024.0); + exponentialHistogramSnapShot = deltaBase2ExponentialHistogram.getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.zeroCount()).isEqualTo(1); + assertThat(exponentialHistogramSnapShot.scale()).isEqualTo(MAX_SCALE); + assertThat(exponentialHistogramSnapShot.offset()).isEqualTo(1023); + assertThat(Base2ExponentialHistogramTest.getAllBucketsCountSum(exponentialHistogramSnapShot)).isEqualTo(1); + + clock.add(step.dividedBy(2)); + exponentialHistogramSnapShot = deltaBase2ExponentialHistogram.getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.zeroCount()).isZero(); + assertThat(exponentialHistogramSnapShot.scale()).isZero(); + assertThat(exponentialHistogramSnapShot.offset()).isEqualTo(1); + assertThat(Base2ExponentialHistogramTest.getAllBucketsCountSum(exponentialHistogramSnapShot)).isEqualTo(2); + + clock.add(step); + exponentialHistogramSnapShot = deltaBase2ExponentialHistogram.getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.zeroCount()).isZero(); + assertThat(exponentialHistogramSnapShot.scale()).isZero(); + assertThat(exponentialHistogramSnapShot.offset()).isZero(); + assertThat(Base2ExponentialHistogramTest.getAllBucketsCountSum(exponentialHistogramSnapShot)).isZero(); + + // By this time, the histogram should be rescaled. + assertThat(deltaBase2ExponentialHistogram.getScale()).isEqualTo(MAX_SCALE); + } + + @Test + void testRescalingAfterSnapshot() { + deltaBase2ExponentialHistogram.recordDouble(1.0); + deltaBase2ExponentialHistogram.recordDouble(2.0); + deltaBase2ExponentialHistogram.recordDouble(1024.0); + + clock.add(step); + ExponentialHistogramSnapShot exponentialHistogramSnapShot = deltaBase2ExponentialHistogram + .getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.scale()).isZero(); + + deltaBase2ExponentialHistogram.recordDouble(2.0); + deltaBase2ExponentialHistogram.recordDouble(4.0); + clock.add(step); + exponentialHistogramSnapShot = deltaBase2ExponentialHistogram.getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.scale()).isZero(); + + deltaBase2ExponentialHistogram.recordDouble(2.0); + deltaBase2ExponentialHistogram.recordDouble(4.0); + clock.add(step); + exponentialHistogramSnapShot = deltaBase2ExponentialHistogram.getLatestExponentialHistogramSnapshot(); + assertThat(exponentialHistogramSnapShot.scale()).isEqualTo(3); + } + +} diff --git a/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/IndexProviderFactoryTest.java b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/IndexProviderFactoryTest.java new file mode 100644 index 0000000000..6f638babd6 --- /dev/null +++ b/implementations/micrometer-registry-otlp/src/test/java/io/micrometer/registry/otlp/internal/IndexProviderFactoryTest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.registry.otlp.internal; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class IndexProviderFactoryTest { + + @Test + void testIndexProviderCache() { + assertThat(IndexProviderFactory.getIndexProviderForScale(0)) + .isEqualTo(IndexProviderFactory.getIndexProviderForScale(0)); + assertThat(IndexProviderFactory.getIndexProviderForScale(1)) + .isEqualTo(IndexProviderFactory.getIndexProviderForScale(1)); + assertThat(IndexProviderFactory.getIndexProviderForScale(-1)) + .isEqualTo(IndexProviderFactory.getIndexProviderForScale(-1)); + } + + @Test + void testGetIndexForValueForZeroScale() { + IndexProvider indexProvider = IndexProviderFactory.getIndexProviderForScale(0); + assertThat(indexProvider.getIndexForValue(1)).isEqualTo(-1); + assertThat(indexProvider.getIndexForValue(1.5)).isZero(); + assertThat(indexProvider.getIndexForValue(2)).isZero(); + + assertThat(indexProvider.getIndexForValue(Math.pow(2, 1023))).isEqualTo(1022); + assertThat(indexProvider.getIndexForValue(Double.MAX_VALUE)).isEqualTo(1023); + + assertThat(indexProvider.getIndexForValue(Math.pow(2, -1021))).isEqualTo(-1022); + assertThat(indexProvider.getIndexForValue(Double.MIN_VALUE)).isEqualTo(-1075); + } + + @Test + void testGetIndexForValueForPositiveScale() { + IndexProvider indexProvider = IndexProviderFactory.getIndexProviderForScale(1); + assertThat(indexProvider.getIndexForValue(1)).isEqualTo(-1); + assertThat(indexProvider.getIndexForValue(1.4)).isZero(); + assertThat(indexProvider.getIndexForValue(2)).isEqualTo(1); + + double tmp = (Math.pow(2, 1023) - Math.pow(2, 1022)) / 1.99; + assertThat(indexProvider.getIndexForValue(Math.pow(2, 1023) + tmp)).isEqualTo(2046); + assertThat(indexProvider.getIndexForValue(Double.MAX_VALUE)).isEqualTo(2047); + + assertThat(indexProvider.getIndexForValue(Double.MIN_VALUE)).isEqualTo(-2149); + } + + @Test + void testGetIndexForNegativeScale() { + IndexProvider indexProvider = IndexProviderFactory.getIndexProviderForScale(-1); + assertThat(indexProvider.getIndexForValue(1)).isEqualTo(-1); + assertThat(indexProvider.getIndexForValue(4)).isZero(); + assertThat(indexProvider.getIndexForValue(4.1)).isEqualTo(1); + + assertThat(indexProvider.getIndexForValue(Math.pow(2, 1021))).isEqualTo(510); + assertThat(indexProvider.getIndexForValue(Double.MAX_VALUE)).isEqualTo(511); + + assertThat(indexProvider.getIndexForValue(Double.MIN_NORMAL)).isEqualTo(-512); + assertThat(indexProvider.getIndexForValue(Double.MIN_VALUE)).isEqualTo(-538); + } + +}