Switched tracing from Sentry to OpenTelemetry

- despite Sentry using OTel under the hood, I'm finding it way to hard
  to get everything working in GCP as expected because we're being
  forced to do things in the Sentry way so we should just go all-in on
  doing things with raw OTel and GCP
- also introduces `development` NODE_ENV so we can disable traces in
  tests

Author: daniellockyer

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - MYSQL_HOST=mysql
       - MYSQL_PORT=3306
       - MYSQL_DATABASE=activitypub
-      - NODE_ENV=testing
+      - NODE_ENV=development
       - ALLOW_PRIVATE_ADDRESS=true
       - SKIP_SIGNATURE_VERIFICATION=true
     command: yarn build:watch

package.json

@@ -45,12 +45,17 @@
   "dependencies": {
     "@fedify/fedify": "1.2.4",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "2.4.1",
+    "@google-cloud/opentelemetry-cloud-trace-propagator": "0.20.0",
     "@google-cloud/pubsub": "4.8.0",
     "@hono/node-server": "1.13.5",
     "@js-temporal/polyfill": "0.4.4",
     "@logtape/logtape": "0.7.1",
+    "@opentelemetry/api": "1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "0.52.1",
     "@opentelemetry/exporter-trace-otlp-proto": "0.54.2",
+    "@opentelemetry/sdk-node": "0.54.2",
     "@opentelemetry/sdk-trace-base": "1.27.0",
+    "@opentelemetry/sdk-trace-node": "1.27.0",
     "@sentry/node": "8.37.1",
     "hono": "4.6.9",
     "jsonwebtoken": "9.0.2",

src/app.ts

@@ -29,7 +29,6 @@ import {
     isLogLevel,
     withContext,
 } from '@logtape/logtape';
-import { SEMANTIC_ATTRIBUTE_HTTP_REQUEST_METHOD } from '@sentry/core';
 import * as Sentry from '@sentry/node';
 import { Hono, type Context as HonoContext, type Next } from 'hono';
 import { cors } from 'hono/cors';
@@ -158,10 +157,10 @@ export const fedify = createFederation<ContextData>({
     kv: fedifyKv,
     skipSignatureVerification:
         process.env.SKIP_SIGNATURE_VERIFICATION === 'true' &&
-        process.env.NODE_ENV === 'testing',
+        ['development', 'testing'].includes(process.env.NODE_ENV || ''),
     allowPrivateAddress:
         process.env.ALLOW_PRIVATE_ADDRESS === 'true' &&
-        process.env.NODE_ENV === 'testing',
+        ['development', 'testing'].includes(process.env.NODE_ENV || ''),
 });
 
 export const db = await KnexKvStore.create(client, 'key_value');
@@ -350,34 +349,9 @@ app.use(async (ctx, next) => {
             return event;
         });
 
-        return Sentry.continueTrace(
-            {
-                sentryTrace:
-                    ctx.req.header('sentry-trace') ||
-                    (traceId && spanId ? `${traceId}-${spanId}-1` : undefined),
-                baggage: ctx.req.header('baggage'),
-            },
-            () => {
-                return Sentry.startSpan(
-                    {
-                        op: 'http.server',
-                        name: `${ctx.req.method} ${ctx.req.path}`,
-
-                        attributes: {
-                            ...extra,
-                            [SEMANTIC_ATTRIBUTE_HTTP_REQUEST_METHOD]:
-                                ctx.req.method,
-                            'service.name': 'activitypub',
-                        },
-                    },
-                    () => {
-                        return withContext(extra, () => {
-                            return next();
-                        });
-                    },
-                );
-            },
-        );
+        return withContext(extra, () => {
+            return next();
+        });
     });
 });
 

src/instrumentation.ts

@@ -1,52 +1,84 @@
+import { trace } from '@opentelemetry/api';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { NodeSDK } from '@opentelemetry/sdk-node';
 import {
     BatchSpanProcessor,
     SimpleSpanProcessor,
 } from '@opentelemetry/sdk-trace-base';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import * as Sentry from '@sentry/node';
 
 if (process.env.SENTRY_DSN) {
-    const client = Sentry.init({
+    Sentry.init({
         dsn: process.env.SENTRY_DSN,
         environment: process.env.NODE_ENV || 'unknown',
         release: process.env.K_REVISION,
-        tracesSampleRate: 1.0,
     });
+}
+
+const sdk = new NodeSDK({
+    instrumentations: getNodeAutoInstrumentations({
+        '@opentelemetry/instrumentation-mysql2': {
+            addSqlCommenterCommentToQueries: true,
+        },
+    }),
+});
+
+const provider = new NodeTracerProvider();
+
+if (process.env.K_SERVICE) {
+    const { TraceExporter } = await import(
+        '@google-cloud/opentelemetry-cloud-trace-exporter'
+    );
+    provider.addSpanProcessor(new BatchSpanProcessor(new TraceExporter({})));
+
+    const { CloudPropagator } = await import(
+        '@google-cloud/opentelemetry-cloud-trace-propagator'
+    );
+    provider.register({
+        propagator: new CloudPropagator(),
+    });
+}
+
+if (process.env.NODE_ENV === 'development') {
+    const { OTLPTraceExporter } = await import(
+        '@opentelemetry/exporter-trace-otlp-proto'
+    );
+
+    provider.addSpanProcessor(
+        new SimpleSpanProcessor(
+            new OTLPTraceExporter({
+                url: 'http://jaeger:4318/v1/traces',
+            }),
+        ),
+    );
+
+    provider.register();
+}
+
+export const tracer = trace.getTracer('activitypub');
 
-    if (process.env.K_SERVICE) {
-        const { TraceExporter } = await import(
-            '@google-cloud/opentelemetry-cloud-trace-exporter'
-        );
-
-        client?.traceProvider?.addSpanProcessor(
-            new BatchSpanProcessor(new TraceExporter({})),
-        );
-    }
-
-    if (process.env.NODE_ENV === 'testing') {
-        const { OTLPTraceExporter } = await import(
-            '@opentelemetry/exporter-trace-otlp-proto'
-        );
-
-        client?.traceProvider?.addSpanProcessor(
-            new SimpleSpanProcessor(
-                new OTLPTraceExporter({
-                    url: 'http://jaeger:4318/v1/traces',
-                }),
-            ),
-        );
-    }
+try {
+    sdk.start();
+} catch (e) {
+    console.error(e);
 }
 
 export function spanWrapper<TArgs extends unknown[], TReturn>(
-    fn: (...args: TArgs) => TReturn,
+    fn: (...args: TArgs) => TReturn | Promise<TReturn>,
 ) {
     return (...args: TArgs) => {
-        return Sentry.startSpan(
-            {
-                op: 'fn',
-                name: fn.name,
-            },
-            () => fn(...args),
-        );
+        return tracer.startActiveSpan(fn.name || 'anonymous', async (span) => {
+            try {
+                const result = await Promise.resolve(fn(...args));
+                span.end();
+                return result;
+            } catch (error) {
+                span.recordException(error as Error);
+                span.setStatus({ code: 2 }); // OpenTelemetry ERROR status
+                span.end();
+                throw error;
+            }
+        });
     };
 }