Skip to content

Add custom label for per-RPC metrics #12552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ejona86 wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Add custom label for per-RPC metrics #12552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
248b6be
Add custom label for per-RPC metrics
ejona86 Dec 5, 2025
451b38c
Add tests, and fix a case
ejona86 Dec 23, 2025
aa30a7f
Add a test for RLS, too
ejona86 Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions api/src/main/java/io/grpc/Grpc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,13 @@ private Grpc() {
public static final Attributes.Key<SSLSession> TRANSPORT_ATTR_SSL_SESSION =
Attributes.Key.create("io.grpc.Grpc.TRANSPORT_ATTR_SSL_SESSION");

/**
* The value for the custom label of per-RPC metrics. Defaults to empty string when unset. Must
* not be set to {@code null}.
*/
public static final CallOptions.Key<String> CALL_OPTION_CUSTOM_LABEL =
CallOptions.Key.createWithDefault("io.grpc.Grpc.CALL_OPTION_CUSTOM_LABEL", "");

/**
* Annotation for transport attributes. It follows the annotation semantics defined
* by {@link Attributes}.
Expand Down
60 changes: 41 additions & 19 deletions opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.BACKEND_SERVICE_KEY;
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.BAGGAGE_KEY;
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.CUSTOM_LABEL_KEY;
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.LOCALITY_KEY;
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.METHOD_KEY;
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.STATUS_KEY;
Expand All @@ -39,6 +40,7 @@
import io.grpc.Deadline;
import io.grpc.ForwardingClientCall.SimpleForwardingClientCall;
import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener;
import io.grpc.Grpc;
import io.grpc.Metadata;
import io.grpc.MethodDescriptor;
import io.grpc.ServerStreamTracer;
Expand Down Expand Up @@ -94,6 +96,7 @@ final class OpenTelemetryMetricsModule {
private final Supplier<Stopwatch> stopwatchSupplier;
private final boolean localityEnabled;
private final boolean backendServiceEnabled;
private final boolean customLabelEnabled;
private final ImmutableList<OpenTelemetryPlugin> plugins;

OpenTelemetryMetricsModule(Supplier<Stopwatch> stopwatchSupplier,
Expand All @@ -103,6 +106,7 @@ final class OpenTelemetryMetricsModule {
this.stopwatchSupplier = checkNotNull(stopwatchSupplier, "stopwatchSupplier");
this.localityEnabled = optionalLabels.contains(LOCALITY_KEY.getKey());
this.backendServiceEnabled = optionalLabels.contains(BACKEND_SERVICE_KEY.getKey());
this.customLabelEnabled = optionalLabels.contains(CUSTOM_LABEL_KEY.getKey());
this.plugins = ImmutableList.copyOf(plugins);
}

Expand Down Expand Up @@ -249,7 +253,7 @@ public void streamClosed(Status status) {
statusCode = Code.DEADLINE_EXCEEDED;
}
}
attemptsState.attemptEnded();
attemptsState.attemptEnded(info.getCallOptions());
recordFinishedAttempt();
}

Expand All @@ -273,6 +277,10 @@ void recordFinishedAttempt() {
}
builder.put(BACKEND_SERVICE_KEY, savedBackendService);
}
if (module.customLabelEnabled) {
builder.put(
CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
}
for (OpenTelemetryPlugin.ClientStreamPlugin plugin : streamPlugins) {
plugin.addLabels(builder);
}
Expand Down Expand Up @@ -318,6 +326,7 @@ static final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory
CallAttemptsTracerFactory(
OpenTelemetryMetricsModule module,
String target,
CallOptions callOptions,
String fullMethodName,
List<OpenTelemetryPlugin.ClientCallPlugin> callPlugins) {
this.module = checkNotNull(module, "module");
Expand All @@ -327,9 +336,14 @@ static final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory
this.attemptDelayStopwatch = module.stopwatchSupplier.get();
this.callStopWatch = module.stopwatchSupplier.get().start();

io.opentelemetry.api.common.Attributes attribute = io.opentelemetry.api.common.Attributes.of(
METHOD_KEY, fullMethodName,
TARGET_KEY, target);
AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder()
.put(METHOD_KEY, fullMethodName)
.put(TARGET_KEY, target);
if (module.customLabelEnabled) {
builder.put(
CUSTOM_LABEL_KEY, callOptions.getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
}
io.opentelemetry.api.common.Attributes attribute = builder.build();

// Record here in case mewClientStreamTracer() would never be called.
if (module.resource.clientAttemptCountCounter() != null) {
Expand All @@ -353,9 +367,14 @@ public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata metada
// CallAttemptsTracerFactory constructor. attemptsPerCall will be non-zero after the first
// attempt, as first attempt cannot be a transparent retry.
if (attemptsPerCall.get() > 0) {
io.opentelemetry.api.common.Attributes attribute =
io.opentelemetry.api.common.Attributes.of(METHOD_KEY, fullMethodName,
TARGET_KEY, target);
AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder()
.put(METHOD_KEY, fullMethodName)
.put(TARGET_KEY, target);
if (module.customLabelEnabled) {
builder.put(
CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
}
io.opentelemetry.api.common.Attributes attribute = builder.build();
if (module.resource.clientAttemptCountCounter() != null) {
module.resource.clientAttemptCountCounter().add(1, attribute);
}
Expand Down Expand Up @@ -383,7 +402,7 @@ private ClientTracer newClientTracer(StreamInfo info) {
}

// Called whenever each attempt is ended.
void attemptEnded() {
void attemptEnded(CallOptions callOptions) {
boolean shouldRecordFinishedCall = false;
synchronized (lock) {
if (--activeStreams == 0) {
Expand All @@ -395,11 +414,11 @@ void attemptEnded() {
}
}
if (shouldRecordFinishedCall) {
recordFinishedCall();
recordFinishedCall(callOptions);
}
}

void callEnded(Status status) {
void callEnded(Status status, CallOptions callOptions) {
callStopWatch.stop();
this.status = status;
boolean shouldRecordFinishedCall = false;
Expand All @@ -415,11 +434,11 @@ void callEnded(Status status) {
}
}
if (shouldRecordFinishedCall) {
recordFinishedCall();
recordFinishedCall(callOptions);
}
}

void recordFinishedCall() {
void recordFinishedCall(CallOptions callOptions) {
Context otelContext = otelContextWithBaggage();
if (attemptsPerCall.get() == 0) {
ClientTracer tracer = newClientTracer(null);
Expand All @@ -430,11 +449,13 @@ void recordFinishedCall() {
callLatencyNanos = callStopWatch.elapsed(TimeUnit.NANOSECONDS);

// Base attributes
io.opentelemetry.api.common.Attributes baseAttributes =
io.opentelemetry.api.common.Attributes.of(
METHOD_KEY, fullMethodName,
TARGET_KEY, target
);
AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder()
.put(METHOD_KEY, fullMethodName)
.put(TARGET_KEY, target);
if (module.customLabelEnabled) {
builder.put(CUSTOM_LABEL_KEY, callOptions.getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
}
io.opentelemetry.api.common.Attributes baseAttributes = builder.build();

// Duration
if (module.resource.clientCallDurationCounter() != null) {
Expand Down Expand Up @@ -660,11 +681,12 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
callOptions = plugin.filterCallOptions(callOptions);
}
}
final CallOptions finalCallOptions = callOptions;
// Only record method name as an attribute if isSampledToLocalTracing is set to true,
// which is true for all generated methods. Otherwise, programatically
// created methods result in high cardinality metrics.
final CallAttemptsTracerFactory tracerFactory = new CallAttemptsTracerFactory(
OpenTelemetryMetricsModule.this, target,
OpenTelemetryMetricsModule.this, target, callOptions,
recordMethodName(method.getFullMethodName(), method.isSampledToLocalTracing()),
callPlugins);
ClientCall<ReqT, RespT> call =
Expand All @@ -679,7 +701,7 @@ public void start(Listener<RespT> responseListener, Metadata headers) {
new SimpleForwardingClientCallListener<RespT>(responseListener) {
@Override
public void onClose(Status status, Metadata trailers) {
tracerFactory.callEnded(status);
tracerFactory.callEnded(status, finalCallOptions);
super.onClose(status, trailers);
}
},
Expand Down
3 changes: 3 additions & 0 deletions opentelemetry/src/main/java/io/grpc/opentelemetry/internal/OpenTelemetryConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ public final class OpenTelemetryConstants {
public static final AttributeKey<String> BACKEND_SERVICE_KEY =
AttributeKey.stringKey("grpc.lb.backend_service");

public static final AttributeKey<String> CUSTOM_LABEL_KEY =
AttributeKey.stringKey("grpc.client.call.custom");

public static final AttributeKey<String> DISCONNECT_ERROR_KEY =
AttributeKey.stringKey("grpc.disconnect_error");

Expand Down
Loading