feat(active_job): support for tracing

.devcontainer/.env.example

@@ -12,5 +12,11 @@ SENTRY_E2E_SVELTE_APP_PORT=4001
 SENTRY_E2E_RAILS_APP_URL="http://localhost:4000"
 SENTRY_E2E_SVELTE_APP_URL="http://localhost:4001"
 
+# ActiveJob queue adapter under test: async | inline | sidekiq | resque | delayed_job
+SENTRY_E2E_ACTIVE_JOB_ADAPTER="async"
+
+# Redis for the sidekiq/resque adapters (the Compose service is named "redis")
+REDIS_URL="redis://redis:6379"
+
 # Faster builds with compose
 COMPOSE_BAKE=true

.devcontainer/docker-compose.yml

@@ -25,6 +25,9 @@ services:
     command: ["mise", "run", "e2e:serve"]
     environment:
       BUNDLE_PATH: /home/sentry/bundle
+    depends_on:
+      redis:
+        condition: service_healthy
     volumes:
       - ..:/workspace/sentry:cached
       - bundle-gems:/home/sentry/bundle
@@ -38,6 +41,11 @@ services:
       - ALLOW_EMPTY_PASSWORD=yes
     ports:
       - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 2s
+      timeout: 3s
+      retries: 10
 
 volumes:
   bundle-gems:

.github/workflows/e2e_tests.yml

@@ -17,18 +17,24 @@ concurrency:
 
 jobs:
   e2e-tests:
-    name: e2e tests
+    name: e2e tests (ruby ${{ matrix.ruby.flavor }}, ${{ matrix.adapter }})
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 8
 
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - ruby_version: "3.4.9"
+        ruby:
+          - version: "3.4.9"
             flavor: "3.4"
-          - ruby_version: "4.0.3"
+          - version: "4.0.3"
             flavor: "4.0"
+        adapter:
+          - async
+          - inline
+          - sidekiq
+          - resque
+          - delayed_job
 
     steps:
       - name: Checkout code
@@ -42,9 +48,10 @@ jobs:
         run: |
           cd .devcontainer
           cp .env.example .env
-          echo "RUBY_VERSION=${{ matrix.ruby_version }}" >> .env
-          echo "DOCKER_IMAGE=ghcr.io/getsentry/sentry-ruby-devcontainer-${{ matrix.flavor }}" >> .env
+          echo "RUBY_VERSION=${{ matrix.ruby.version }}" >> .env
+          echo "DOCKER_IMAGE=ghcr.io/getsentry/sentry-ruby-devcontainer-${{ matrix.ruby.flavor }}" >> .env
           echo "DOCKER_TAG=${{ steps.devcontainer-version.outputs.version }}" >> .env
+          echo "SENTRY_E2E_ACTIVE_JOB_ADAPTER=${{ matrix.adapter }}" >> .env
 
       - name: Log in to GHCR
         uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3
@@ -54,7 +61,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Pull test container image
-        run: docker pull ghcr.io/getsentry/sentry-ruby-devcontainer-${{ matrix.flavor }}:${{ steps.devcontainer-version.outputs.version }}
+        run: docker pull ghcr.io/getsentry/sentry-ruby-devcontainer-${{ matrix.ruby.flavor }}:${{ steps.devcontainer-version.outputs.version }}
 
       - name: Restore node_modules cache
         uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3
@@ -114,7 +121,7 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: e2e-test-logs-ruby-${{ matrix.ruby_version }}
+          name: e2e-test-logs-ruby-${{ matrix.ruby.version }}-${{ matrix.adapter }}
           path: |
             log/sentry_debug_events.log
           retention-days: 7

.mise.toml

@@ -14,6 +14,10 @@ run = "cd spec/apps/rails-mini && bundle exec ruby app.rb"
 description = "Start the svelte-mini e2e app"
 run = "cd spec/apps/svelte-mini && npm run dev"
 
+[tasks."e2e:worker"]
+description = "Start the rails-mini ActiveJob worker (sidekiq/resque/delayed_job; idles for async/inline)"
+run = "cd spec/apps/rails-mini && bundle exec ruby worker.rb"
+
 [tasks."e2e:serve"]
 description = "Start all e2e apps in parallel"
-depends = ["e2e:rails", "e2e:svelte"]
+depends = ["e2e:rails", "e2e:svelte", "e2e:worker"]

sentry-rails/Gemfile

@@ -56,6 +56,53 @@ end
 
 gem "mini_magick"
 
+# Sidekiq is a dev-only dependency, used by the common ActiveJob spec
+# suite to verify the AJ tracing extension works against the :sidekiq
+# adapter independent of sentry-sidekiq's native middleware.
+#
+# Gated on Ruby/Rails/platform because:
+#   - The sidekiq_adapter_spec only passes on Rails > 7.0.
+#   - Sidekiq 8 requires Ruby >= 3.2; older Rubies fall back to Sidekiq 7.
+#   - Sidekiq does not reliably support JRuby.
+#
+# sidekiq_adapter_spec.rb also rescues LoadError and re-checks the
+# Rails version, so matrices that don't bundle Sidekiq skip the spec
+# cleanly without any other gating.
+unless RUBY_PLATFORM.include?("java")
+  if rails_version > Gem::Version.new("7.0.0") && ruby_version >= Gem::Version.new("3.2")
+    gem "sidekiq", "~> 8.0"
+  elsif rails_version > Gem::Version.new("7.0.0")
+    gem "sidekiq", "~> 7.0"
+  end
+end
+
+# delayed_job and resque are dev-only dependencies, used by the common
+# ActiveJob spec suite to verify the AJ tracing extension works against
+# the :delayed_job and :resque adapters independent of the dedicated
+# sentry-delayed_job / sentry-resque integrations.
+#
+# Both spec files rescue LoadError and skip cleanly on matrices that
+# don't bundle the gem, so the gating below only needs to keep
+# `bundle install` resolvable — it doesn't have to be exact.
+unless RUBY_PLATFORM.include?("java")
+  # delayed_job is backed by ActiveRecord here (delayed_job_active_record),
+  # reusing the dummy app's SQLite database. It supports every Ruby/Rails
+  # combination in the matrix.
+  gem "delayed_job"
+  gem "delayed_job_active_record"
+
+  # resque has no in-memory test mode, so the spec drives it through
+  # mock_redis instead of a live Redis (mirroring how the sidekiq context
+  # uses Sidekiq's fake mode). resque 3 / resque-scheduler 5 / mock_redis
+  # all require Ruby >= 3.0, so gate them on that — older matrices skip
+  # the resque spec via its LoadError rescue.
+  if ruby_version >= Gem::Version.new("3.0")
+    gem "resque"
+    gem "resque-scheduler", "~> 5.0"
+    gem "mock_redis"
+  end
+end
+
 gem "sprockets-rails"
 
 gem "benchmark-ips"

sentry-rails/lib/sentry/rails/active_job.rb

@@ -5,13 +5,62 @@
 module Sentry
   module Rails
     module ActiveJobExtensions
+      SENTRY_PAYLOAD_KEY = "_sentry"
+
+      USER_FIELDS_ALLOWLIST = %w[id email username].freeze
+
+      def self.prepended(base)
+        base.attr_accessor :_sentry
+      end
+
       def perform_now
         if !Sentry.initialized? || already_supported_by_sentry_integration?
           super
         else
-          SentryReporter.record(self) do
-            super
+          data = _sentry || {}
+          SentryReporter.record(
+            self,
+            trace_headers: data["trace_propagation_headers"],
+            user: data["user"]
+          ) { super }
+        end
+      end
+
+      def serialize
+        payload = super
+        return payload if !Sentry.initialized? || already_supported_by_sentry_integration?
+
+        begin
+          sentry_data = {}
+          if Sentry.configuration.rails.active_job_propagate_traces
+            headers = Sentry.get_trace_propagation_headers
+            sentry_data["trace_propagation_headers"] = headers if headers && !headers.empty?
           end
+
+          if Sentry.configuration.send_default_pii
+            user = Sentry.get_current_scope.user || {}
+            allowed = user.each_with_object({}) do |(k, v), acc|
+              acc[k.to_s] = v if USER_FIELDS_ALLOWLIST.include?(k.to_s)
+            end
+            sentry_data["user"] = allowed unless allowed.empty?
+          end
+
+          payload[SENTRY_PAYLOAD_KEY] = sentry_data unless sentry_data.empty?
+        rescue StandardError => e
+          Sentry.sdk_logger&.error("sentry-rails: failed to inject _sentry payload: #{e.class}: #{e.message}\n  #{Array(e.backtrace).first(5).join("\n  ")}")
+        end
+
+        payload
+      end
+
+      def deserialize(job_data)
+        super
+        return if !Sentry.initialized? || already_supported_by_sentry_integration?
+
+        begin
+          self._sentry = job_data[SENTRY_PAYLOAD_KEY]
+        rescue StandardError => e
+          Sentry.sdk_logger&.error("sentry-rails: failed to extract _sentry payload: #{e.class}: #{e.message}\n  #{Array(e.backtrace).first(5).join("\n  ")}")
         end
       end
 
@@ -28,19 +77,67 @@ class SentryReporter
         }
 
         class << self
-          def record(job, &block)
+          def producer_callback_registered?
+            @producer_callback_registered ||= false
+          end
+
+          def producer_callback_registered!
+            @producer_callback_registered = true
+          end
+
+          def record_producer_span(job)
+            return yield if !Sentry.initialized? || job.already_supported_by_sentry_integration?
+
+            Sentry.with_child_span(op: "queue.publish", description: job.class.name) do |span|
+              if span
+                span.set_origin(SPAN_ORIGIN)
+                span.set_data(Sentry::Span::DataConventions::MESSAGING_MESSAGE_ID, job.job_id)
+                span.set_data(Sentry::Span::DataConventions::MESSAGING_DESTINATION_NAME, job.queue_name)
+              end
+              yield
+            end
+          end
+
+          def record(job, trace_headers: nil, user: nil, &block)
+            # Always give this thread a fresh hub cloned from the main hub so
+            # the job's events are fully isolated.  Save and restore whatever
+            # hub was on the thread before (e.g. the Rack request hub set by
+            # CaptureExceptions, or a stale hub left by a recycled thread-pool
+            # thread) so the outer context continues working correctly after
+            # the job finishes.
+            original_hub = Thread.current.thread_variable_get(Sentry::THREAD_LOCAL)
+            Sentry.clone_hub_to_current_thread
+
             Sentry.with_scope do |scope|
               begin
+                scope.set_user(user) if user && !user.empty?
                 scope.set_transaction_name(job.class.name, source: :task)
+                scope.set_tags(queue: job.queue_name)
+                scope.set_contexts(active_job: {
+                  job_class: job.class.name,
+                  job_id: job.job_id,
+                  queue: job.queue_name,
+                  provider_job_id: job.provider_job_id
+                })
 
-                transaction = Sentry.start_transaction(
+                transaction_options = {
                   name: scope.transaction_name,
                   source: scope.transaction_source,
                   op: OP_NAME,
                   origin: SPAN_ORIGIN
-                )
+                }
+
+                transaction = if trace_headers && !trace_headers.empty?
+                  continued = Sentry.continue_trace(trace_headers, **transaction_options)
+                  Sentry.start_transaction(transaction: continued, **transaction_options)
+                else
+                  Sentry.start_transaction(**transaction_options)
+                end
 
-                scope.set_span(transaction) if transaction
+                if transaction
+                  set_messaging_data(transaction, job)
+                  scope.set_span(transaction)
+                end
 
                 yield.tap do
                   finish_sentry_transaction(transaction, 200)
@@ -53,6 +150,25 @@ def record(job, &block)
                 raise
               end
             end
+          ensure
+            Thread.current.thread_variable_set(Sentry::THREAD_LOCAL, original_hub)
+          end
+
+          def set_messaging_data(transaction, job)
+            transaction.set_data(Sentry::Span::DataConventions::MESSAGING_MESSAGE_ID, job.job_id)
+            transaction.set_data(Sentry::Span::DataConventions::MESSAGING_DESTINATION_NAME, job.queue_name)
+            transaction.set_data(Sentry::Span::DataConventions::MESSAGING_MESSAGE_RETRY_COUNT, [job.executions.to_i - 1, 0].max)
+
+            if (latency = compute_latency(job))
+              transaction.set_data(Sentry::Span::DataConventions::MESSAGING_MESSAGE_RECEIVE_LATENCY, latency)
+            end
+          end
+
+          def compute_latency(job)
+            return unless job.respond_to?(:enqueued_at) && job.enqueued_at
+
+            enqueued_time = job.enqueued_at.is_a?(String) ? Time.parse(job.enqueued_at) : job.enqueued_at
+            ((Time.now.to_f - enqueued_time.to_f) * 1000).round
           end
 
           def capture_exception(job, e)

sentry-rails/lib/sentry/rails/configuration.rb

@@ -172,6 +172,11 @@ class Configuration
       # Set this option to true if you want Sentry to capture each retry failure
       attr_accessor :active_job_report_on_retry_error
 
+      # Whether we should inject trace propagation headers into the serialized job
+      # payload in order to have a connected trace between producer and consumer.
+      # Defaults to true. Set to false to opt out.
+      attr_accessor :active_job_propagate_traces
+
       # Configuration for structured logging feature
       # @return [StructuredLoggingConfiguration]
       attr_reader :structured_logging
@@ -193,6 +198,7 @@ def initialize
         @db_query_source_threshold_ms = 100
         @active_support_logger_subscription_items = Sentry::Rails::ACTIVE_SUPPORT_LOGGER_SUBSCRIPTION_ITEMS_DEFAULT.dup
         @active_job_report_on_retry_error = false
+        @active_job_propagate_traces = true
         @structured_logging = StructuredLoggingConfiguration.new
       end
     end

sentry-rails/lib/sentry/rails/railtie.rb

@@ -21,6 +21,13 @@ class Railtie < ::Rails::Railtie
       ActiveSupport.on_load(:active_job) do
         require "sentry/rails/active_job"
         prepend Sentry::Rails::ActiveJobExtensions
+
+        unless Sentry::Rails::ActiveJobExtensions::SentryReporter.producer_callback_registered?
+          around_enqueue do |job, block|
+            Sentry::Rails::ActiveJobExtensions::SentryReporter.record_producer_span(job, &block)
+          end
+          Sentry::Rails::ActiveJobExtensions::SentryReporter.producer_callback_registered!
+        end
       end
     end
 

sentry-rails/spec/active_job/delayed_job_adapter_spec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+# delayed_job 4.2+ ships an ActiveJob adapter that inherits from
+# ActiveJob::QueueAdapters::AbstractAdapter, which only exists in Rails 7.2+.
+# On older Rails, instantiating the adapter raises NameError, so skip the
+# whole file. Bail out before loading delayed_job so old matrices don't trip
+# on the gem either.
+return if RAILS_VERSION < 7.2
+
+# delayed_job is gated in the Gemfile by platform (skipped on JRuby).
+# Matrices that don't bundle it won't have it available — rescue LoadError
+# and skip the whole file so they don't blow up on the
+# `include_context "delayed_job adapter"` below.
+begin
+  require "delayed_job"
+  require "delayed_job_active_record"
+rescue LoadError
+  return
+end
+
+RSpec.describe "Sentry + ActiveJob on the delayed_job adapter", type: :job do
+  include ActiveSupport::Testing::TimeHelpers
+  include_context "active_job backend harness", adapter: :delayed_job
+  include_context "delayed_job adapter"
+
+  it_behaves_like "a Sentry-instrumented ActiveJob backend"
+  it_behaves_like "an ActiveJob backend that supports distributed tracing"
+end