Add MetaspanTagging module and Metastruct data class

Author: y9v

Diff for: lib/datadog/tracing/metadata.rb

@@ -2,7 +2,7 @@
 
 require_relative 'metadata/analytics'
 require_relative 'metadata/tagging'
-require_relative 'metadata/metastruct'
+require_relative 'metadata/metastruct_tagging'
 require_relative 'metadata/errors'
 
 module Datadog
@@ -11,6 +11,7 @@ module Tracing
     module Metadata
       def self.included(base)
         base.include(Metadata::Tagging)
+        base.include(Metadata::MetastructTagging)
         base.include(Metadata::Errors)
 
         # Additional extensions

Diff for: lib/datadog/tracing/metadata/metastruct.rb

@@ -5,59 +5,31 @@
 module Datadog
   module Tracing
     module Metadata
-      # Adds complex structures tagging behavior through metastruct
+      # This class is a data structure that is used to store
+      # complex metadata, such as an array of objects.
+      #
+      # It is serialized to MessagePack format when sent to the agent.
       class Metastruct
         extend Forwardable
 
-        MERGER = proc do |_, v1, v2|
-          if v1.is_a?(Hash) && v2.is_a?(Hash)
-            v1.merge(v2, &MERGER)
-          elsif v1.is_a?(Array) && v2.is_a?(Array)
-            v1.concat(v2)
-          elsif v2.nil?
-            v1
-          else
-            v2
-          end
-        end
+        def_delegators :@metastruct, :[], :[]=
 
-        def self.empty
-          new({})
+        def initialize
+          @metastruct = {}
         end
 
-        def initialize(metastruct)
-          @metastruct = metastruct
-        end
+        def to_msgpack(packer = nil)
+          # JRuby doesn't pass the packer
+          packer ||= MessagePack::Packer.new
 
-        # Deep merge two metastructs
-        # If the types are not both Arrays or Hashes, the second one will overwrite the first one
-        #
-        # Example with same types:
-        # metastruct = { a: { b: [1, 2] } }
-        # second = { a: { b: [3, 4], c: 5 } }
-        # result = { a: { b: [1, 2, 3, 4], c: 5 } }
-        #
-        # Example with different types:
-        # metastruct = { a: { b: 1 } }
-        # second = { a: { b: [2, 3] } }
-        # result = { a: { b: [2, 3] } }
-        def deep_merge!(second)
-          @metastruct.merge!(second.to_h, &MERGER)
+          packer.write(@metastruct.transform_values(&:to_msgpack))
         end
 
-        def_delegators :@metastruct, :[], :[]=, :dig, :to_h
-
         def pretty_print(q)
           q.seplist @metastruct.each do |key, value|
-            q.text "#{key} => #{value}\n"
+            q.text "#{key} => #{value}"
           end
         end
-
-        def to_msgpack(packer = nil)
-          packer ||= MessagePack::Packer.new
-
-          packer.write(@metastruct.transform_values(&:to_msgpack))
-        end
       end
     end
   end

Diff for: lib/datadog/tracing/metadata/metastruct_tagging.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'metastruct'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # Adds data storage for the `meta_struct` field.
+      #
+      # This field is used to send more complex data like an array of objects
+      # in MessagePack format to the agent, and has no size limitations.
+      #
+      # The agent fully supports meta_struct from version v7.35.0 (April 2022).
+      #
+      # On versions older than v7.35.0, sending traces containing meta_struct
+      # has no unexpected side-effects; traces are sent to the backend as expected,
+      # while the meta_struct field is stripped.
+      module MetastructTagging
+        # Set the given key / value tag pair on the metastruct.
+        #
+        # A valid example is:
+        #
+        #   span.set_metastruct_tag('_dd.stack', [])
+        def set_metastruct_tag(key, value)
+          metastruct[key] = value
+        end
+
+        # Return the metastruct tag value for the given key,
+        # returns nil if the key doesn't exist.
+        def get_metastruct_tag(key)
+          metastruct[key]
+        end
+
+        private
+
+        def metastruct
+          @metastruct ||= Metastruct.new
+        end
+      end
+    end
+  end
+end

Diff for: lib/datadog/tracing/span.rb

@@ -21,7 +21,6 @@ class Span
         :end_time,
         :id,
         :meta,
-        :metastruct,
         :metrics,
         :name,
         :parent_id,
@@ -34,6 +33,9 @@ class Span
         :status,
         :trace_id
 
+      attr_reader \
+        :metastruct
+
       attr_writer \
         :duration
 
@@ -54,8 +56,8 @@ def initialize(
         end_time: nil,
         id: nil,
         meta: nil,
-        metastruct: {},
         metrics: nil,
+        metastruct: nil,
         parent_id: 0,
         resource: name,
         service: nil,
@@ -77,8 +79,8 @@ def initialize(
         @trace_id = trace_id || Tracing::Utils.next_id
 
         @meta = meta || {}
-        @metastruct = Tracing::Metadata::Metastruct.new(metastruct)
         @metrics = metrics || {}
+        @metastruct = metastruct || {}
         @status = status || 0
 
         # start_time and end_time track wall clock. In Ruby, wall clock
@@ -147,7 +149,7 @@ def to_hash
           error: @status,
           meta: @meta,
           metrics: @metrics,
-          meta_struct: @metastruct.to_h,
+          meta_struct: @metastruct,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -195,9 +197,8 @@ def pretty_print(q)
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metastruct: ') do
-            q.breakable
-            q.pp metastruct
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
           end
         end
       end

Diff for: lib/datadog/tracing/span_operation.rb

@@ -36,8 +36,7 @@ class SpanOperation
         :service,
         :start_time,
         :trace_id,
-        :type,
-        :metastruct
+        :type
       attr_accessor :links, :status, :span_events
 
       def initialize(
@@ -89,7 +88,6 @@ def initialize(
 
         # Set tags if provided.
         set_tags(tags) if tags
-        @metastruct = Tracing::Metadata::Metastruct.empty
 
         # Some other SpanOperation-specific behavior
         @events = events || Events.new
@@ -291,7 +289,7 @@ def to_hash
           id: @id,
           meta: meta,
           metrics: metrics,
-          metastruct: @metastruct.to_h,
+          metastruct: metastruct,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -337,9 +335,8 @@ def pretty_print(q)
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metastruct: ') do
-            q.breakable
-            q.pp metastruct
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
           end
         end
       end
@@ -463,7 +460,7 @@ def build_span
           id: @id,
           meta: Core::Utils::SafeDup.frozen_or_dup(meta),
           metrics: Core::Utils::SafeDup.frozen_or_dup(metrics),
-          metastruct: @metastruct.to_h.dup,
+          metastruct: Core::Utils::SafeDup.frozen_or_dup(metastruct),
           parent_id: @parent_id,
           resource: @resource,
           service: @service,

Diff for: lib/datadog/tracing/trace_operation.rb

@@ -4,7 +4,7 @@
 require_relative '../core/utils'
 require_relative 'tracer'
 require_relative 'event'
-require_relative 'metadata'
+require_relative 'metadata/tagging'
 require_relative 'sampling/ext'
 require_relative 'span_operation'
 require_relative 'trace_digest'
@@ -25,7 +25,7 @@ module Tracing
     #
     # @public_api
     class TraceOperation
-      include Metadata
+      include Metadata::Tagging
 
       DEFAULT_MAX_LENGTH = 100_000
 
@@ -46,8 +46,7 @@ class TraceOperation
         :max_length,
         :parent_span_id,
         :trace_state,
-        :trace_state_unknown_fields,
-        :metastruct
+        :trace_state_unknown_fields
 
       attr_writer \
         :name,
@@ -74,7 +73,6 @@ def initialize(
         profiling_enabled: nil,
         tags: nil,
         metrics: nil,
-        metastruct: {},
         trace_state: nil,
         trace_state_unknown_fields: nil,
         remote_parent: false,
@@ -107,7 +105,6 @@ def initialize(
         # Generic tags
         set_tags(tags) if tags
         set_tags(metrics) if metrics
-        @metastruct = Tracing::Metadata::Metastruct.new(metastruct)
 
         # State
         @root_span = nil
@@ -372,7 +369,6 @@ def fork_clone
           trace_state_unknown_fields: (@trace_state_unknown_fields && @trace_state_unknown_fields.dup),
           tags: meta.dup,
           metrics: metrics.dup,
-          metastruct: @metastruct.to_h.dup,
           remote_parent: @remote_parent
         )
       end
@@ -512,7 +508,6 @@ def build_trace(spans, partial = false)
           service: service,
           tags: meta,
           metrics: metrics,
-          metastruct: @metastruct.to_h.dup,
           root_span_id: !partial ? root_span && root_span.id : nil,
           profiling_enabled: @profiling_enabled,
         )

Diff for: lib/datadog/tracing/trace_segment.rb

@@ -58,7 +58,6 @@ def initialize(
         service: nil,
         tags: nil,
         metrics: nil,
-        metastruct: {},
         profiling_enabled: nil
       )
         @id = id
@@ -69,7 +68,6 @@ def initialize(
         # The caller is expected to have done that
         @meta = (tags && tags.dup) || {}
         @metrics = (metrics && metrics.dup) || {}
-        @metastruct = Tracing::Metadata::Metastruct.new(metastruct)
 
         # Set well-known tags, defaulting to getting the values from tags
         @agent_sample_rate = agent_sample_rate || agent_sample_rate_tag
@@ -148,8 +146,7 @@ def high_order_tid
       attr_reader \
         :root_span_id,
         :meta,
-        :metrics,
-        :metastruct
+        :metrics
 
       private
 

Diff for: lib/datadog/tracing/transport/serializable_trace.rb

@@ -118,7 +118,6 @@ def to_msgpack(packer = nil)
           packer.write('metrics')
           packer.write(span.metrics)
           packer.write('meta_struct')
-          # We encapsulate the resulting msgpack in a binary msgpack
           packer.write(span.metastruct)
           packer.write('span_links')
           packer.write(span.links.map(&:to_hash))

Diff for: lib/datadog/tracing/transport/trace_formatter.rb

@@ -43,7 +43,6 @@ def format!
           # Apply generic trace tags. Any more specific value will be overridden
           # by the subsequent calls below.
           set_trace_tags!
-          set_metastruct!
 
           set_resource!
 
@@ -90,12 +89,6 @@ def set_trace_tags!
           root_span.set_tags(trace.send(:metrics))
         end
 
-        def set_metastruct!
-          return if partial?
-
-          root_span.metastruct.deep_merge!(trace.send(:metastruct))
-        end
-
         def tag_agent_sample_rate!
           return unless trace.agent_sample_rate
 

Diff for: lib/datadog/tracing/transport/traces.rb

@@ -127,7 +127,6 @@ def initialize(apis, default_api)
           end
 
           def send_traces(traces)
-            # object that extends Datadog::Core::Encoding::Encoder (MsgpackEncoder or JSONEncoder)
             encoder = current_api.encoder
             chunker = Datadog::Tracing::Transport::Traces::Chunker.new(
               encoder,

Diff for: sig/datadog/tracing/metadata.rbs

@@ -2,6 +2,7 @@ module Datadog
   module Tracing
     module Metadata
       include Metadata::Tagging
+      include Metadata::MetastructTagging
       include Metadata::Errors
       prepend Metadata::Analytics
     end