[DX-2789] add tracing and tracing-to-profiles to v2 workflow engine/syncer

.github/workflows/ci-core.yml

@@ -547,7 +547,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
-          fetch-depth: 0
       - name: Setup Go
         uses: ./.github/actions/setup-go
         with:

core/config/cre_config.go

@@ -9,6 +9,9 @@ type CRE interface {
 	UseLocalTimeProvider() bool
 	EnableDKGRecipient() bool
 	Linking() CRELinking
+	// DebugMode returns true if debug mode is enabled for workflow engines.
+	// When enabled, additional OTel tracing and logging is performed.
+	DebugMode() bool
 	LocalSecrets() map[string]string
 }
 

core/config/docs/core.toml

@@ -753,6 +753,8 @@ GoroutineThreshold = 5000 # Default
 ServerAddress = 'http://localhost:4040' # Example
 # Environment sets the target environment tag in which profiles will be added to.
 Environment = 'mainnet' # Default
+# LinkTracesToProfiles enables linking traces to profiles in Grafana when Pyroscope and Tracing are enabled.
+LinkTracesToProfiles = false # Default
 
 [Sentry]
 # **ADVANCED**
@@ -951,6 +953,8 @@ IgnoreJoblessBridges = false # Default
 UseLocalTimeProvider = true # Default
 # EnableDKGRecipient should be set to true if the DON runs a capability that uses a DKG result package.
 EnableDKGRecipient = false # Default
+# DebugMode enables additional tracing and logging for workflow engines.
+DebugMode = false # Default
 
 # Sharding holds settings for node sharding configuration.
 [Sharding]

core/config/pyroscope_config.go

@@ -4,4 +4,5 @@ type Pyroscope interface {
 	AuthToken() string
 	ServerAddress() string
 	Environment() string
+	LinkTracesToProfiles() bool
 }

core/config/toml/types.go

@@ -1640,8 +1640,9 @@ func (p *AutoPprof) setFrom(f *AutoPprof) {
 }
 
 type Pyroscope struct {
-	ServerAddress *string
-	Environment   *string
+	ServerAddress        *string
+	Environment          *string
+	LinkTracesToProfiles *bool
 }
 
 func (p *Pyroscope) setFrom(f *Pyroscope) {
@@ -1651,6 +1652,9 @@ func (p *Pyroscope) setFrom(f *Pyroscope) {
 	if v := f.Environment; v != nil {
 		p.Environment = v
 	}
+	if v := f.LinkTracesToProfiles; v != nil {
+		p.LinkTracesToProfiles = v
+	}
 }
 
 type Sentry struct {
@@ -1886,6 +1890,12 @@ type CreConfig struct {
 	UseLocalTimeProvider *bool                  `toml:",omitempty"`
 	EnableDKGRecipient   *bool                  `toml:",omitempty"`
 	Linking              *LinkingConfig         `toml:",omitempty"`
+	// DebugMode enables additional tracing and logging for workflow engines.
+	// When enabled, OTel traces are created for workflow execution and syncer events.
+	// Requires [Tracing].Enabled = true for traces to be exported (trace export is gated by
+	// Tracing.Enabled in initGlobals; Telemetry.Enabled is optional—traces work with or without it).
+	// WARNING: This is not suitable for production use due to performance overhead.
+	DebugMode *bool `toml:",omitempty"`
 }
 
 // WorkflowFetcherConfig holds the configuration for fetching workflow files
@@ -1942,6 +1952,10 @@ func (c *CreConfig) setFrom(f *CreConfig) {
 			c.Linking.TLSEnabled = v
 		}
 	}
+
+	if f.DebugMode != nil {
+		c.DebugMode = f.DebugMode
+	}
 }
 
 func (w *WorkflowFetcherConfig) ValidateConfig() error {

core/logger/pyroscope.go

@@ -1,6 +1,7 @@
 package logger
 
 import (
+	"os"
 	"runtime"
 
 	"github.com/grafana/pyroscope-go"
@@ -19,6 +20,10 @@ func StartPyroscope(pyroConfig config.Pyroscope, pprofConfig PprofConfig) (*pyro
 	runtime.SetBlockProfileRate(pprofConfig.BlockProfileRate())
 	runtime.SetMutexProfileFraction(pprofConfig.MutexProfileFraction())
 
+	// Increase memory profiling sample rate for better granularity
+	// Default is 512KB (524288 bytes) per sample
+	// runtime.MemProfileRate = 512 * 1024 // 512KB per sample
+
 	sha, ver := static.Short()
 
 	return pyroscope.Start(pyroscope.Config{
@@ -31,11 +36,15 @@ func StartPyroscope(pyroConfig config.Pyroscope, pprofConfig PprofConfig) (*pyro
 		// We disable logging the profiling info, it will be in the Pyroscope instance anyways...
 		Logger: nil,
 
-		Tags: map[string]string{
-			"SHA":         sha,
-			"Version":     ver,
-			"Environment": pyroConfig.Environment(),
-		},
+		Tags: func() map[string]string {
+			hostname, _ := os.Hostname()
+			return map[string]string{
+				"SHA":         sha,
+				"Version":     ver,
+				"Environment": pyroConfig.Environment(),
+				"hostname":    hostname, // set hostname, so we can distinguish between nodes in the same environment
+			}
+		}(),
 
 		ProfileTypes: []pyroscope.ProfileType{
 			// these profile types are enabled by default:

core/scripts/cre/environment/README.md

@@ -1745,6 +1745,73 @@ go run . env start --with-beholder
 go run . env beholder start
 ```
 
+### OTel Tracing Configuration
+
+To enable OpenTelemetry (OTel) tracing for workflow engines and see traces in Tempo/Grafana, **multiple configuration toggles must be set**:
+
+| Toggle | Location | Required Value | Purpose |
+|--------|----------|----------------|---------|
+| `Tracing.Enabled` | Node TOML | `true` | **Required.** Gates trace export; without this, no traces are exported even when `Telemetry.Enabled` is true |
+| `Tracing.CollectorTarget` | Node TOML | e.g., `host.docker.internal:4318` | OTLP endpoint for trace export. Must differ from `Telemetry.Endpoint` when both [Tracing] and [Telemetry] are enabled |
+| `Tracing.SamplingRatio` | Node TOML | `> 0` (e.g., `1.0`) | Sampling rate for the span exporter (0 = no traces, 1 = 100%) |
+| `Telemetry.Enabled` | Node TOML | `true` | Enables the OTel/beholder exporter |
+| `Telemetry.TraceSampleRatio` | Node TOML | `> 0` (e.g., `1.0`) | Sampling rate used by the beholder client (0 = no traces, 1 = 100%) |
+| `CRE.DebugMode` | Node TOML | `true` | Enables detailed tracing in workflow engines and syncer |
+| `OTEL_SERVICE_NAME` | Environment variable | e.g., `chainlink-node` | Sets the service name for traces in Tempo |
+| `Pyroscope.LinkTracesToProfiles` | Node TOML | `true` | Enables traces-to-profiles linking in Grafana (requires Pyroscope) |
+
+**Sampling ratios:** When both [Tracing] and [Telemetry] are enabled, `Tracing.SamplingRatio` controls the span exporter and `Telemetry.TraceSampleRatio` is used by the beholder client. Set both to the same value (e.g., `1.0`) for consistent trace capture.
+
+**Example TOML configuration:**
+
+```toml
+[Tracing]
+Enabled = true
+CollectorTarget = 'host.docker.internal:4318'  # Must differ from Telemetry.Endpoint when both enabled
+SamplingRatio = 1.0  # 100% sampling - adjust for production
+Mode = 'unencrypted'  # Use for local dev; use 'tls' in production
+
+[Telemetry]
+Enabled = true
+Endpoint = 'host.docker.internal:4317'  # Must differ from Tracing.CollectorTarget
+InsecureConnection = true
+TraceSampleRatio = 1.0  # 100% sampling - adjust for production
+
+[CRE]
+DebugMode = true  # WARNING: Not suitable for production due to overhead
+
+[Pyroscope]
+ServerAddress = 'http://host.docker.internal:4040'
+LinkTracesToProfiles = true  # Enables traces-to-profiles in Grafana
+```
+
+**Note:** When both [Tracing] and [Telemetry] are enabled, config validation requires `Tracing.CollectorTarget` and `Telemetry.Endpoint` to be different. For a single OTel collector, use different ports (e.g., gRPC on 4317, HTTP on 4318) if your collector exposes both.
+
+**Example environment variable (in nodeset config):**
+
+```toml
+[[nodesets]]
+  env_vars = { OTEL_SERVICE_NAME = "chainlink-node" }
+```
+
+**Common issues:**
+
+| Symptom | Likely Cause |
+|---------|--------------|
+| No traces at all | `Tracing.Enabled = false`, `Telemetry.Enabled = false`, or sampling ratios set to `0` |
+| No workflow engine traces | `CRE.DebugMode = false` |
+| Traces show `unknown_service:chainlink` | Missing `OTEL_SERVICE_NAME` env var |
+| Traces not exported | Telemetry endpoint unreachable (check `go run . obs up -f `) |
+| No traces-to-profiles link in Grafana | `Pyroscope.LinkTracesToProfiles = false` or Pyroscope not running |
+
+**Important notes:**
+
+- `CRE.DebugMode` adds performance overhead and should only be enabled during development/debugging, not in production environments.
+- **Tracing is only implemented for V2 components:**
+  - **V2 Syncer**: Only used when workflow registry contracts are v2.x. If you're using v1.x contracts, the V1 syncer is used and has no tracing.
+  - **V2 Engine (NoDAG)**: Only used by V2/NoDAG workflows. V1/DAG workflows use the V1 engine which has no tracing.
+- To use tracing, ensure your environment is configured with **v2 workflow registry contracts** and you're deploying **V2 workflows**.
+
 ### Expected Error Messages
 
 If these telemetry services are not running, you will see frequent "expected" error messages in the logs due to connection failures:

core/scripts/cre/environment/configs/examples/workflow-don-overrides.toml

@@ -30,7 +30,7 @@
   override_mode = "all"
   http_port_range_start = 10100
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["ocr3", "custom-compute", "web-api-target", "web-api-trigger", "vault", "write-evm-1337", "evm-2337"]
 
   # Capability config overrides for this DON.

core/scripts/cre/environment/configs/workflow-don-solana.toml

@@ -45,7 +45,7 @@
   supported_evm_chains = [1337]
   supported_sol_chains = ["22222222222222222222222222222222222222222222"]
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["ocr3", "custom-compute", "web-api-trigger", "cron", "http-action", "http-trigger", "consensus", "don-time"]
   registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
 
@@ -77,7 +77,7 @@
   supported_evm_chains = [1337]
   supported_sol_chains = ["22222222222222222222222222222222222222222222"]
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["web-api-target", "vault", "solana"]
 
   [nodesets.db]
@@ -100,7 +100,7 @@
   override_mode = "each"
   http_port_range_start = 10300
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   supported_evm_chains = [1337]
 
   [nodesets.db]

core/scripts/cre/environment/configs/workflow-don-tron.toml

@@ -38,7 +38,7 @@
   override_mode = "all"
   http_port_range_start = 10100
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["ocr3", "custom-compute", "web-api-target", "web-api-trigger", "vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "write-evm-1337", "write-evm-3360022319", "read-contract-1337", "read-contract-3360022319"]
   registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
 
@@ -63,7 +63,7 @@
   override_mode = "each"
   http_port_range_start = 10300
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   supported_evm_chains = [1337, 2337]
 
   [nodesets.db]

core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml

@@ -43,7 +43,7 @@
   # because bootstrap job for capability DON will be created on the boostrap node from this DON
   supported_evm_chains = [1337, 2337]
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["ocr3", "custom-compute", "web-api-trigger", "cron", "http-action", "http-trigger", "consensus", "don-time", "write-evm-1337", "read-contract-1337", "evm-1337"]
   registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
 
@@ -74,7 +74,7 @@
   # to identify nodes in the gateway configuration (required by both web-api-target and vault capabilities)
   supported_evm_chains = [1337, 2337]
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["web-api-target", "vault", "write-evm-2337", "read-contract-2337", "evm-2337"]
 
   [nodesets.db]
@@ -97,7 +97,7 @@
   override_mode = "each"
   http_port_range_start = 10300
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   supported_evm_chains = [1337, 2337]
 
   [nodesets.db]

core/scripts/cre/environment/configs/workflow-gateway-don-grpc-source.toml

@@ -44,7 +44,7 @@
   override_mode = "all"
   http_port_range_start = 10100
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["ocr3", "custom-compute", "web-api-target", "web-api-trigger", "vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "write-evm-1337", "write-evm-2337", "read-contract-1337", "read-contract-2337", "evm-1337", "evm-2337"]
   registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
 
@@ -77,7 +77,7 @@ Name = 'mock-private-registry'
   override_mode = "each"
   http_port_range_start = 10300
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   supported_evm_chains = [1337, 2337]
 
   [nodesets.db]

core/scripts/cre/environment/configs/workflow-gateway-don.toml

@@ -37,9 +37,10 @@
   name = "workflow"
   don_types = ["workflow"]
   override_mode = "all"
-  http_port_range_start = 10100
+  http_port_range_start = 10000
+  p2p_port_range_start = 12000
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   capabilities = ["ocr3", "custom-compute", "web-api-target", "web-api-trigger", "vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "write-evm-1337", "write-evm-2337", "evm-1337", "evm-2337", "read-contract-1337", "read-contract-2337"]
   registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
 
@@ -61,14 +62,15 @@
   name = "bootstrap-gateway"
   don_types = ["bootstrap", "gateway"]
   override_mode = "each"
-  http_port_range_start = 10300
+  http_port_range_start = 10100
+  p2p_port_range_start = 12100
 
-  env_vars = { CL_EVM_CMD = "" }
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" }
   supported_evm_chains = [1337, 2337]
 
   [nodesets.db]
     image = "postgres:12.0"
-    port = 13200
+    port = 13100
 
   [[nodesets.node_specs]]
     roles = ["bootstrap", "gateway"]