Record Metrics for Reminder #4831
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| // SPDX-FileCopyrightText: Copyright 2025 The Minder Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // Package metrics provides metrics for the reminder service | ||
| package metrics | ||
|
|
||
| import ( | ||
| "go.opentelemetry.io/otel/metric" | ||
| ) | ||
|
|
||
| // Default bucket boundaries in seconds for the delay histograms | ||
| var delayBuckets = []float64{ | ||
| 60, // 1 minute | ||
| 300, // 5 minutes | ||
| 600, // 10 minutes | ||
| 1800, // 30 minutes | ||
| 3600, // 1 hour | ||
| 7200, // 2 hours | ||
| 10800, // 3 hours | ||
| 18000, // 5 hours | ||
| 25200, // 7 hours | ||
| 36000, // 10 hours | ||
| } | ||
|
|
||
| // Metrics contains all the metrics for the reminder service | ||
| type Metrics struct { | ||
| // Time between when a reminder became eligible and when it was sent | ||
| SendDelay metric.Float64Histogram | ||
|
|
||
| // Time between when a reminder became eligible and when it was sent for the first time | ||
| NewSendDelay metric.Float64Histogram | ||
|
|
||
| // Current number of reminders in the batch | ||
| BatchSize metric.Int64Histogram | ||
| } | ||
|
|
||
| // NewMetrics creates a new metrics instance | ||
| func NewMetrics(meter metric.Meter) (*Metrics, error) { | ||
| sendDelay, err := meter.Float64Histogram( | ||
| "send_delay", | ||
| metric.WithDescription("Time between reminder becoming eligible and actual send (seconds)"), | ||
| metric.WithUnit("s"), | ||
| metric.WithExplicitBucketBoundaries(delayBuckets...), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| newSendDelay, err := meter.Float64Histogram( | ||
| "new_send_delay", | ||
| metric.WithDescription("Time between reminder becoming eligible and actual send (seconds) for first time reminders"), | ||
| metric.WithUnit("s"), | ||
| metric.WithExplicitBucketBoundaries(delayBuckets...), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| batchSize, err := meter.Int64Histogram( | ||
| "batch_size", | ||
| metric.WithDescription("Current number of reminders in the batch"), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &Metrics{ | ||
| SendDelay: sendDelay, | ||
| NewSendDelay: newSendDelay, | ||
| BatchSize: batchSize, | ||
| }, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| // SPDX-FileCopyrightText: Copyright 2025 The Minder Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package reminder | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus/promhttp" | ||
| "github.com/rs/zerolog" | ||
| "go.opentelemetry.io/otel" | ||
| "go.opentelemetry.io/otel/exporters/prometheus" | ||
| sdkmetric "go.opentelemetry.io/otel/sdk/metric" | ||
| "go.opentelemetry.io/otel/sdk/resource" | ||
| semconv "go.opentelemetry.io/otel/semconv/v1.17.0" | ||
| ) | ||
|
|
||
| const ( | ||
| metricsPath = "/metrics" | ||
| readHeaderTimeout = 2 * time.Second | ||
| ) | ||
|
|
||
| func (r *reminder) startMetricServer(ctx context.Context) error { | ||
| logger := zerolog.Ctx(ctx) | ||
|
|
||
| prometheusExporter, err := prometheus.New( | ||
| prometheus.WithNamespace("reminder"), | ||
| ) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create Prometheus exporter: %w", err) | ||
| } | ||
|
|
||
| res := resource.NewWithAttributes( | ||
| semconv.SchemaURL, | ||
| semconv.ServiceName("reminder"), | ||
| // TODO: Make this auto-generated | ||
| semconv.ServiceVersion("v0.1.0"), | ||
| ) | ||
|
|
||
| mp := sdkmetric.NewMeterProvider( | ||
| sdkmetric.WithReader(prometheusExporter), | ||
| sdkmetric.WithResource(res), | ||
| ) | ||
|
|
||
| otel.SetMeterProvider(mp) | ||
|
|
||
| mux := http.NewServeMux() | ||
| mux.Handle(metricsPath, promhttp.Handler()) | ||
|
|
||
| server := &http.Server{ | ||
| Addr: r.cfg.MetricServer.GetAddress(), | ||
| Handler: mux, | ||
| ReadHeaderTimeout: readHeaderTimeout, | ||
| } | ||
| logger.Info().Msgf("starting metrics server on %s", server.Addr) | ||
|
|
||
| // Start the metrics server | ||
| go func() { | ||
| err = server.ListenAndServe() | ||
| if err != nil && !errors.Is(err, http.ErrServerClosed) { | ||
| logger.Err(err).Msg("error starting metrics server") | ||
| } | ||
| }() | ||
|
|
||
| // Watch for context cancellation or stop signal to shutdown the metrics server | ||
| go func() { | ||
| select { | ||
| case <-ctx.Done(): | ||
| case <-r.stop: | ||
| } | ||
|
|
||
| // shutdown the metrics server when either the context is done or when reminder is stopped | ||
| shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), 5*time.Second) | ||
| defer shutdownRelease() | ||
|
|
||
| logger.Info().Msg("shutting down metrics server") | ||
|
|
||
| if err = server.Shutdown(shutdownCtx); err != nil { | ||
| logger.Err(err).Msg("error shutting down metrics server") | ||
| } | ||
|
|
||
| if err = mp.Shutdown(shutdownCtx); err != nil { | ||
| logger.Err(err).Msg("error shutting down metrics provider") | ||
| } | ||
|
|
||
| close(r.metricsServerDone) | ||
| }() | ||
|
|
||
| return nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why do you need an
http.ServeMuxhere, rather than simply passingpromhttp.Handlerto thehttp.Serverbelow?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we want to serve the metrics on the root path? I don't see any problem with that, but it's just that I've seen
/metricsbeing commonly used.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If we have a separate port for the metrics, I don't see a reason not to serve it on all the paths off the root. Fetching
/metricsshould still work.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
(On the other hand, I don't have a strong complaint about putting this at
/metrics)