Switched tracing from Sentry to OpenTelemetry

- despite Sentry using OTel under the hood, I'm finding it way to hard
  to get everything working in GCP as expected because we're being
  forced to do things in the Sentry way so we should just go all-in on
  doing things with raw OTel and GCP

Author: daniellockyer

package.json

@@ -44,12 +44,17 @@
   "dependencies": {
     "@fedify/fedify": "1.2.3",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "2.4.1",
+    "@google-cloud/opentelemetry-cloud-trace-propagator": "0.20.0",
     "@google-cloud/pubsub": "4.8.0",
     "@hono/node-server": "1.13.5",
     "@js-temporal/polyfill": "0.4.4",
     "@logtape/logtape": "0.7.1",
+    "@opentelemetry/api": "1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "0.52.1",
     "@opentelemetry/exporter-trace-otlp-proto": "0.54.2",
+    "@opentelemetry/sdk-node": "0.54.2",
     "@opentelemetry/sdk-trace-base": "1.27.0",
+    "@opentelemetry/sdk-trace-node": "1.27.0",
     "@sentry/node": "8.37.1",
     "hono": "4.6.9",
     "jsonwebtoken": "9.0.2",

src/app.ts

@@ -29,7 +29,6 @@ import {
     isLogLevel,
     withContext,
 } from '@logtape/logtape';
-import { SEMANTIC_ATTRIBUTE_HTTP_REQUEST_METHOD } from '@sentry/core';
 import * as Sentry from '@sentry/node';
 import { Hono, type Context as HonoContext, type Next } from 'hono';
 import { cors } from 'hono/cors';
@@ -350,34 +349,9 @@ app.use(async (ctx, next) => {
             return event;
         });
 
-        return Sentry.continueTrace(
-            {
-                sentryTrace:
-                    ctx.req.header('sentry-trace') ||
-                    (traceId && spanId ? `${traceId}-${spanId}-1` : undefined),
-                baggage: ctx.req.header('baggage'),
-            },
-            () => {
-                return Sentry.startSpan(
-                    {
-                        op: 'http.server',
-                        name: `${ctx.req.method} ${ctx.req.path}`,
-
-                        attributes: {
-                            ...extra,
-                            [SEMANTIC_ATTRIBUTE_HTTP_REQUEST_METHOD]:
-                                ctx.req.method,
-                            'service.name': 'activitypub',
-                        },
-                    },
-                    () => {
-                        return withContext(extra, () => {
-                            return next();
-                        });
-                    },
-                );
-            },
-        );
+        return withContext(extra, () => {
+            return next();
+        });
     });
 });
 

src/instrumentation.ts

@@ -1,52 +1,84 @@
+import { trace } from '@opentelemetry/api';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { NodeSDK } from '@opentelemetry/sdk-node';
 import {
     BatchSpanProcessor,
     SimpleSpanProcessor,
 } from '@opentelemetry/sdk-trace-base';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import * as Sentry from '@sentry/node';
 
 if (process.env.SENTRY_DSN) {
-    const client = Sentry.init({
+    Sentry.init({
         dsn: process.env.SENTRY_DSN,
         environment: process.env.NODE_ENV || 'unknown',
         release: process.env.K_REVISION,
-        tracesSampleRate: 1.0,
     });
+}
+
+const sdk = new NodeSDK({
+    instrumentations: getNodeAutoInstrumentations({
+        '@opentelemetry/instrumentation-mysql2': {
+            addSqlCommenterCommentToQueries: true,
+        },
+    }),
+});
+
+const provider = new NodeTracerProvider();
+
+if (process.env.K_SERVICE) {
+    const { TraceExporter } = await import(
+        '@google-cloud/opentelemetry-cloud-trace-exporter'
+    );
+    provider.addSpanProcessor(new BatchSpanProcessor(new TraceExporter({})));
+
+    const { CloudPropagator } = await import(
+        '@google-cloud/opentelemetry-cloud-trace-propagator'
+    );
+    provider.register({
+        propagator: new CloudPropagator(),
+    });
+}
+
+if (process.env.NODE_ENV === 'testing') {
+    const { OTLPTraceExporter } = await import(
+        '@opentelemetry/exporter-trace-otlp-proto'
+    );
+
+    provider.addSpanProcessor(
+        new SimpleSpanProcessor(
+            new OTLPTraceExporter({
+                url: 'http://jaeger:4318/v1/traces',
+            }),
+        ),
+    );
+
+    provider.register();
+}
+
+export const tracer = trace.getTracer('activitypub');
 
-    if (process.env.K_SERVICE) {
-        const { TraceExporter } = await import(
-            '@google-cloud/opentelemetry-cloud-trace-exporter'
-        );
-
-        client?.traceProvider?.addSpanProcessor(
-            new BatchSpanProcessor(new TraceExporter({})),
-        );
-    }
-
-    if (process.env.NODE_ENV === 'testing') {
-        const { OTLPTraceExporter } = await import(
-            '@opentelemetry/exporter-trace-otlp-proto'
-        );
-
-        client?.traceProvider?.addSpanProcessor(
-            new SimpleSpanProcessor(
-                new OTLPTraceExporter({
-                    url: 'http://jaeger:4318/v1/traces',
-                }),
-            ),
-        );
-    }
+try {
+    sdk.start();
+} catch (e) {
+    console.error(e);
 }
 
 export function spanWrapper<TArgs extends unknown[], TReturn>(
-    fn: (...args: TArgs) => TReturn,
+    fn: (...args: TArgs) => TReturn | Promise<TReturn>,
 ) {
-    return (...args: TArgs) => {
-        return Sentry.startSpan(
-            {
-                op: 'fn',
-                name: fn.name,
-            },
-            () => fn(...args),
-        );
+    return async (...args: TArgs) => {
+        return tracer.startActiveSpan(fn.name || 'anonymous', async (span) => {
+            try {
+                const result = await Promise.resolve(fn(...args));
+                span.end();
+                return result;
+            } catch (error) {
+                span.recordException(error as Error);
+                span.setStatus({ code: 2 }); // OpenTelemetry ERROR status
+                span.end();
+                throw error;
+            }
+        });
     };
 }