feat: propagate trace context in mtls clients

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go and run tests
         uses: ./.github/actions/setup-go-service
         with:
-          go-version: "1.26.2"
+          go-version: "1.26.3"
           cache: false
           run-go-test: "true"
           go-test-race: "true"
@@ -40,7 +40,7 @@ jobs:
       - name: Set up Go and run golangci-lint
         uses: ./.github/actions/setup-go-service
         with:
-          go-version: "1.26.2"
+          go-version: "1.26.3"
           cache: false
           run-golangci-lint: "true"
           golangci-lint-args: "--timeout=5m ./..."
@@ -56,7 +56,7 @@ jobs:
       - name: Set up Go and run security scans
         uses: ./.github/actions/setup-go-service
         with:
-          go-version: "1.26.2"
+          go-version: "1.26.3"
           cache: false
           run-gosec: "true"
           run-govulncheck: "true"

go.mod

@@ -7,6 +7,7 @@ require (
 	connectrpc.com/otelconnect v0.7.2
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/alicebob/miniredis/v2 v2.37.0
+	github.com/bufbuild/httplb v0.4.1
 	github.com/evalops/proto v0.0.0-20260414193513-3db7075bd55b
 	github.com/go-chi/chi/v5 v5.2.5
 	github.com/golang-migrate/migrate/v4 v4.19.1
@@ -28,7 +29,6 @@ require (
 require (
 	github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bufbuild/httplb v0.4.1 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -56,11 +56,11 @@ require (
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/crypto v0.49.0 // indirect
-	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/crypto v0.50.0 // indirect
+	golang.org/x/net v0.53.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/sys v0.43.0 // indirect
+	golang.org/x/text v0.36.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect

go.sum

@@ -76,6 +76,8 @@ github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc=
 github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
+github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
 github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
 github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
 github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
@@ -165,17 +167,17 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
-golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
-golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
-golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
+golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
-golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=

identityclient/client.go

@@ -18,7 +18,6 @@ import (
 
 	identityv1 "github.com/evalops/proto/gen/go/identity/v1"
 	"github.com/evalops/service-runtime/mtls"
-	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -159,7 +158,7 @@ func New(config Config) *Client {
 		httpClient = http.DefaultClient
 	}
 	usesMTLSCert := httpClientUsesMTLSCertificate(httpClient)
-	httpClient = tracedHTTPClient(httpClient)
+	httpClient = mtls.TraceHTTPClient(httpClient)
 	maxSize := config.MaxCacheSize
 	if maxSize <= 0 {
 		maxSize = defaultMaxCacheSize
@@ -177,19 +176,6 @@ func New(config Config) *Client {
 	}
 }
 
-func tracedHTTPClient(client *http.Client) *http.Client {
-	if client == nil {
-		client = http.DefaultClient
-	}
-	cloned := *client
-	baseTransport := cloned.Transport
-	if baseTransport == nil {
-		baseTransport = http.DefaultTransport
-	}
-	cloned.Transport = otelhttp.NewTransport(baseTransport)
-	return &cloned
-}
-
 // NewClient creates a Client that introspects tokens at the given URL.
 func NewClient(introspectURL string, requestTimeout time.Duration, httpClient *http.Client) *Client {
 	return New(Config{
@@ -422,11 +408,22 @@ func httpClientUsesMTLSCertificate(client *http.Client) bool {
 	if client == nil {
 		return false
 	}
-	transport, ok := client.Transport.(*http.Transport)
-	if !ok || transport == nil {
-		return false
+	transport := client.Transport
+	if transport == nil {
+		transport = http.DefaultTransport
+	}
+	for transport != nil {
+		httpTransport, ok := transport.(*http.Transport)
+		if ok {
+			return tlsConfigHasClientCertificate(httpTransport.TLSClientConfig)
+		}
+		wrapped, ok := transport.(interface{ Unwrap() http.RoundTripper })
+		if !ok {
+			return false
+		}
+		transport = wrapped.Unwrap()
 	}
-	return tlsConfigHasClientCertificate(transport.TLSClientConfig)
+	return false
 }
 
 func tlsConfigHasClientCertificate(cfg *tls.Config) bool {

identityclient/client_test.go

@@ -18,6 +18,7 @@ import (
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/propagation"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
 )
 
 type roundTripFunc func(*http.Request) (*http.Response, error)
@@ -67,6 +68,65 @@ func TestConfigured(t *testing.T) {
 	}
 }
 
+func TestConfiguredDetectsMTLSCertificatesThroughTracedTransport(t *testing.T) {
+	if !New(Config{
+		ServiceTokensURL: "https://identity.internal/v1/service-tokens",
+		HTTPClient: mtls.TraceHTTPClient(&http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{Certificates: []tls.Certificate{{}}},
+			},
+		}),
+	}).ServiceTokensConfigured() {
+		t.Fatal("expected traced mtls-authenticated service tokens to be configured")
+	}
+}
+
+func TestNewDoesNotDoubleWrapTracedHTTPClient(t *testing.T) {
+	originalProvider := otel.GetTracerProvider()
+	originalPropagator := otel.GetTextMapPropagator()
+	recorder := tracetest.NewSpanRecorder()
+	tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
+	otel.SetTracerProvider(tracerProvider)
+	otel.SetTextMapPropagator(propagation.TraceContext{})
+	t.Cleanup(func() {
+		otel.SetTracerProvider(originalProvider)
+		otel.SetTextMapPropagator(originalPropagator)
+		_ = tracerProvider.Shutdown(context.Background())
+	})
+
+	httpClient := mtls.TraceHTTPClient(&http.Client{
+		Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+			return &http.Response{
+				StatusCode: http.StatusOK,
+				Body:       io.NopCloser(strings.NewReader(`{"active":true,"organization_id":"org-123"}`)),
+				Header:     make(http.Header),
+			}, nil
+		}),
+	})
+	client := New(Config{
+		IntrospectURL:  "https://identity.test/v1/tokens/introspect",
+		RequestTimeout: time.Second,
+		HTTPClient:     httpClient,
+	})
+
+	ctx, span := tracerProvider.Tracer("identityclient-test").Start(context.Background(), "root")
+	defer span.End()
+
+	if _, err := client.Introspect(ctx, "write-token"); err != nil {
+		t.Fatalf("introspect: %v", err)
+	}
+
+	httpSpanCount := 0
+	for _, ended := range recorder.Ended() {
+		if ended.Name() == "HTTP POST" {
+			httpSpanCount++
+		}
+	}
+	if httpSpanCount != 1 {
+		t.Fatalf("expected one HTTP client span, got %d", httpSpanCount)
+	}
+}
+
 func TestIntrospectSuccess(t *testing.T) {
 	server := testutil.NewTestServer(t, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 		if got := request.Header.Get("Authorization"); got != "Bearer write-token" {

mtls/mtls.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 )
 
 // ClientConfig holds TLS settings for an outbound mTLS client.
@@ -59,15 +61,15 @@ func BuildServerTLSConfig(cfg ServerConfig) (*tls.Config, error) {
 	return tlsConfig, nil
 }
 
-// BuildHTTPClient returns an *http.Client configured with the given mTLS settings,
-// or http.DefaultClient when the config is empty.
+// BuildHTTPClient returns an *http.Client configured with the given mTLS
+// settings and OpenTelemetry HTTP propagation.
 func BuildHTTPClient(cfg ClientConfig) (*http.Client, error) {
 	tlsConfig, err := BuildClientTLSConfig(cfg)
 	if err != nil {
 		return nil, err
 	}
 	if tlsConfig == nil {
-		return http.DefaultClient, nil
+		return TraceHTTPClient(http.DefaultClient), nil
 	}
 
 	transport, ok := http.DefaultTransport.(*http.Transport)
@@ -76,7 +78,50 @@ func BuildHTTPClient(cfg ClientConfig) (*http.Client, error) {
 	}
 	clone := transport.Clone()
 	clone.TLSClientConfig = tlsConfig
-	return &http.Client{Transport: clone}, nil
+	return TraceHTTPClient(&http.Client{Transport: clone}), nil
+}
+
+// TraceHTTPClient clones client and wraps its transport with OTel propagation.
+func TraceHTTPClient(client *http.Client) *http.Client {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	cloned := *client
+	transport := cloned.Transport
+	if transport == nil {
+		transport = http.DefaultTransport
+	}
+	cloned.Transport = traceRoundTripper(transport)
+	return &cloned
+}
+
+func traceRoundTripper(transport http.RoundTripper) http.RoundTripper {
+	if transport == nil {
+		transport = http.DefaultTransport
+	}
+	if _, ok := transport.(*tracedTransport); ok {
+		return transport
+	}
+	if _, ok := transport.(*otelhttp.Transport); ok {
+		return transport
+	}
+	return &tracedTransport{
+		base:   transport,
+		traced: otelhttp.NewTransport(transport),
+	}
+}
+
+type tracedTransport struct {
+	base   http.RoundTripper
+	traced http.RoundTripper
+}
+
+func (t *tracedTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+	return t.traced.RoundTrip(request)
+}
+
+func (t *tracedTransport) Unwrap() http.RoundTripper {
+	return t.base
 }
 
 // BuildClientTLSConfig returns a *tls.Config for an outbound mTLS client, or nil when all fields are empty.

mtls/mtls_test.go

@@ -1,13 +1,18 @@
 package mtls
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 )
 
 func TestBuildClientTLSConfigReturnsNilWhenUnset(t *testing.T) {
@@ -39,13 +44,48 @@ func TestBuildServerTLSConfigRequiresCertAndKeyTogether(t *testing.T) {
 	}
 }
 
-func TestBuildHTTPClientReturnsDefaultClientWhenUnset(t *testing.T) {
+func TestBuildHTTPClientPropagatesTraceContextWhenUnset(t *testing.T) {
+	originalProvider := otel.GetTracerProvider()
+	originalPropagator := otel.GetTextMapPropagator()
+	tracerProvider := sdktrace.NewTracerProvider()
+	otel.SetTracerProvider(tracerProvider)
+	otel.SetTextMapPropagator(propagation.TraceContext{})
+	t.Cleanup(func() {
+		otel.SetTracerProvider(originalProvider)
+		otel.SetTextMapPropagator(originalPropagator)
+		_ = tracerProvider.Shutdown(context.Background())
+	})
+
+	var traceParent string
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		traceParent = request.Header.Get("traceparent")
+		writer.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
 	client, err := BuildHTTPClient(ClientConfig{})
 	if err != nil {
 		t.Fatalf("build http client: %v", err)
 	}
-	if client != http.DefaultClient {
-		t.Fatal("expected default client")
+	if client == http.DefaultClient {
+		t.Fatal("expected traced client, got http.DefaultClient")
+	}
+
+	ctx, span := tracerProvider.Tracer("mtls-test").Start(context.Background(), "root")
+	defer span.End()
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
+	if err != nil {
+		t.Fatalf("build request: %v", err)
+	}
+	response, err := client.Do(request)
+	if err != nil {
+		t.Fatalf("client.Do: %v", err)
+	}
+	response.Body.Close()
+
+	if traceParent == "" {
+		t.Fatal("expected traceparent header")
 	}
 }