fix(logs): split child spans off block output so oversized state still spills

Author: waleedlatif1

apps/sim/executor/execution/block-executor.ts

@@ -379,14 +379,15 @@ export class BlockExecutor {
         normalizedOutput = await redactObjectStrings(normalizedOutput, redactionOptions)
       }
 
-      normalizedOutput = (await compactBlockOutput(normalizedOutput, {
+      const compacted = await compactBlockOutput(normalizedOutput, {
         workspaceId: blockCtx.workspaceId,
         workflowId: blockCtx.workflowId,
         executionId: blockCtx.executionId,
         userId: blockCtx.userId,
         preserveUserFileBase64: blockCtx.includeFileBase64 === true,
         requireDurable: true,
-      })) as NormalizedBlockOutput
+      })
+      normalizedOutput = compacted.output
 
       const endedAt = new Date().toISOString()
       const duration = performance.now() - startTime
@@ -396,8 +397,8 @@ export class BlockExecutor {
         blockLog.durationMs = duration
         blockLog.success = true
         blockLog.output = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block })
-        if (normalizedOutput.childTraceSpans && Array.isArray(normalizedOutput.childTraceSpans)) {
-          blockLog.childTraceSpans = normalizedOutput.childTraceSpans
+        if (Array.isArray(compacted.childTraceSpans)) {
+          blockLog.childTraceSpans = compacted.childTraceSpans
         }
         const childExecutionId = normalizedOutput[CHILD_EXECUTION_ID_OUTPUT_KEY]
         if (typeof childExecutionId === 'string' && childExecutionId) {
@@ -409,7 +410,6 @@ export class BlockExecutor {
       }
 
       const {
-        childTraceSpans: _traces,
         [CHILD_EXECUTION_ID_OUTPUT_KEY]: _childExecutionId,
         [CHILD_TRACE_DISABLED_OUTPUT_KEY]: _childTraceDisabled,
         ...outputForState

apps/sim/lib/execution/payloads/serializer.test.ts

@@ -360,12 +360,23 @@ describe('compacting span trees', () => {
     mockRegisterLargeValueOwner.mockResolvedValue(true)
   })
 
-  it('keeps a block output child span tree shaped as a tree', async () => {
+  const childWorkflowLog = (overrides: Partial<BlockLog>): BlockLog => ({
+    blockId: 'child-workflow',
+    blockType: 'workflow',
+    startedAt: '2026-09-29T00:00:00.000Z',
+    endedAt: '2026-09-29T00:00:00.002Z',
+    durationMs: 2,
+    success: true,
+    ...overrides,
+  })
+
+  it('splits a block output child span tree off shaped as a tree', async () => {
     const compacted = await compactBlockOutput(
       { result: 'done', childTraceSpans: childWorkflowSpans() },
       options
     )
 
+    expect(compacted.output).toEqual({ result: 'done' })
     expectSpanTree(compacted.childTraceSpans)
   })
 
@@ -380,34 +391,46 @@ describe('compacting span trees', () => {
     expect(isLargeValueRef(spilled?.output?.result)).toBe(true)
   })
 
-  it('keeps block log child span trees shaped as trees', async () => {
-    const log = {
-      blockId: 'child-workflow',
-      blockType: 'workflow',
-      startedAt: '2026-09-29T00:00:00.000Z',
-      endedAt: '2026-09-29T00:00:00.002Z',
-      durationMs: 2,
-      success: true,
-      childTraceSpans: childWorkflowSpans(),
-    } as BlockLog
-
-    const [compacted] = (await compactBlockLogs([log], options)) ?? []
-
-    expectSpanTree(compacted?.childTraceSpans)
-  })
-
-  it('keeps the output a record when its other fields together exceed the threshold', async () => {
+  it('still spills a block output whose fields together exceed the threshold', async () => {
     const compacted = await compactBlockOutput(
       { first: 'a'.repeat(600), second: 'b'.repeat(600), childTraceSpans: childWorkflowSpans() },
       options
     )
 
-    expect(isLargeValueRef(compacted)).toBe(false)
-    expect(compacted.first).toBe('a'.repeat(600))
-    expect(compacted.second).toBe('b'.repeat(600))
+    expect(isLargeValueRef(compacted.output)).toBe(true)
     expectSpanTree(compacted.childTraceSpans)
   })
 
+  it('keeps block log child span trees shaped as trees', async () => {
+    const [compacted] =
+      (await compactBlockLogs(
+        [childWorkflowLog({ childTraceSpans: childWorkflowSpans() })],
+        options
+      )) ?? []
+
+    expectSpanTree(compacted?.childTraceSpans)
+  })
+
+  it('keeps a block log output carrying child spans a record', async () => {
+    const [compacted] =
+      (await compactBlockLogs(
+        [
+          childWorkflowLog({
+            output: {
+              first: 'a'.repeat(600),
+              second: 'b'.repeat(600),
+              childTraceSpans: childWorkflowSpans(),
+            },
+          }),
+        ],
+        options
+      )) ?? []
+
+    expect(isLargeValueRef(compacted?.output)).toBe(false)
+    expect(compacted?.output?.first).toBe('a'.repeat(600))
+    expectSpanTree(compacted?.output?.childTraceSpans)
+  })
+
   it('keeps a nested child workflow span tree shaped as a tree', async () => {
     const nestedWorkflowSpan: TraceSpan = {
       id: 'nested-workflow',
@@ -416,16 +439,33 @@ describe('compacting span trees', () => {
       duration: 2,
       startTime: '2026-09-29T00:00:00.000Z',
       endTime: '2026-09-29T00:00:00.002Z',
-      output: { result: 'done', childTraceSpans: childWorkflowSpans() },
+      output: {
+        first: 'a'.repeat(600),
+        second: 'b'.repeat(600),
+        childTraceSpans: childWorkflowSpans(),
+      },
     }
 
-    const compacted = await compactBlockOutput(
-      { result: 'done', childTraceSpans: [nestedWorkflowSpan] },
-      options
-    )
+    const compacted = await compactBlockOutput({ childTraceSpans: [nestedWorkflowSpan] }, options)
 
     const [nested] = compacted.childTraceSpans as TraceSpan[]
-    expect(nested.output?.result).toBe('done')
+    expect(nested.output?.first).toBe('a'.repeat(600))
     expectSpanTree(nested.output?.childTraceSpans)
   })
+
+  it('terminates on a cyclic span tree', async () => {
+    const span: TraceSpan = {
+      id: 'cyclic',
+      name: 'Cyclic',
+      type: 'function',
+      duration: 1,
+      startTime: '2026-09-29T00:00:00.000Z',
+      endTime: '2026-09-29T00:00:00.001Z',
+    }
+    span.children = [span]
+
+    const compacted = await compactBlockOutput({ childTraceSpans: [span] }, options)
+
+    expect((compacted.childTraceSpans as TraceSpan[])[0].id).toBe('cyclic')
+  })
 })

apps/sim/lib/execution/payloads/serializer.ts

@@ -258,59 +258,96 @@ export async function compactSubflowResults<T>(
 /**
  * Compacts a trace span tree without collapsing its structure. Readers walk
  * `children` and `output.childTraceSpans` as arrays, so those stay arrays and
- * only each span's payload fields are spilled when oversized. Size of the tree
- * as a whole is bounded where the log is stored, not here.
+ * only each span's payload fields are spilled when oversized. Spans are log
+ * data: the tree as a whole is bounded where the log is stored, not here.
  */
 async function compactTraceSpanTree(
   spans: unknown,
-  options: CompactExecutionPayloadOptions
+  options: CompactExecutionPayloadOptions,
+  seen: WeakSet<object>
 ): Promise<unknown> {
   if (!Array.isArray(spans)) {
     return compactExecutionPayload(spans, options)
   }
-  return Promise.all(spans.map((span) => compactTraceSpan(span, options)))
+  return Promise.all(spans.map((span) => compactTraceSpan(span, options, seen)))
 }
 
 async function compactTraceSpan(
   span: unknown,
-  options: CompactExecutionPayloadOptions
+  options: CompactExecutionPayloadOptions,
+  seen: WeakSet<object>
 ): Promise<unknown> {
   if (!isRecordLike(span)) {
     return compactExecutionPayload(span, options)
   }
+  if (seen.has(span)) {
+    return span
+  }
+  seen.add(span)
   return Object.fromEntries(
     await Promise.all(
       Object.entries(span).map(async ([key, value]) => [
         key,
         key === 'children'
-          ? await compactTraceSpanTree(value, options)
+          ? await compactTraceSpanTree(value, options, seen)
           : key === 'output'
-            ? await compactBlockOutput(value, options)
+            ? await compactLoggedOutput(value, options, seen)
             : await compactExecutionPayload(value, options),
       ])
     )
   )
 }
 
 /**
- * Compacts a block output, keeping any `childTraceSpans` shaped as a span tree
- * (see {@link compactTraceSpanTree}). Everything else compacts as a normal
- * execution payload, except that an output carrying spans keeps its root so
- * the spans stay attached to it; its fields still spill individually.
+ * Compacts a span or block log output. One carrying `childTraceSpans` keeps its
+ * root so the spans stay attached; its other fields spill individually.
+ */
+async function compactLoggedOutput(
+  output: unknown,
+  options: CompactExecutionPayloadOptions,
+  seen: WeakSet<object>
+): Promise<unknown> {
+  if (!isRecordLike(output) || !('childTraceSpans' in output)) {
+    return compactExecutionPayload(output, options)
+  }
+  return Object.fromEntries(
+    await Promise.all(
+      Object.entries(output).map(async ([key, value]) => [
+        key,
+        key === 'childTraceSpans'
+          ? await compactTraceSpanTree(value, options, seen)
+          : await compactExecutionPayload(value, options),
+      ])
+    )
+  )
+}
+
+export interface CompactedBlockOutput<T> {
+  /** The output without `childTraceSpans`, compacted as execution state. */
+  output: T
+  /** The output's child span tree, compacted as log data. */
+  childTraceSpans?: unknown
+}
+
+/**
+ * Compacts a block output for execution state and splits off its
+ * `childTraceSpans`, which belong to the block log rather than state. The
+ * output compacts as any execution payload, so an oversized one still spills
+ * whole; the spans compact as a tree (see {@link compactTraceSpanTree}).
  */
 export async function compactBlockOutput<T>(
   output: T,
   options: CompactExecutionPayloadOptions = {}
-): Promise<T> {
+): Promise<CompactedBlockOutput<T>> {
   if (!isRecordLike(output) || !('childTraceSpans' in output)) {
-    return compactExecutionPayload(output, options)
+    return { output: await compactExecutionPayload(output, options) }
   }
   const { childTraceSpans, ...rest } = output
-  const [compactedRest, compactedSpans] = await Promise.all([
-    compactExecutionPayload(rest, { ...options, preserveRoot: true }),
-    compactTraceSpanTree(childTraceSpans, options),
+  const [compactedOutput, compactedSpans] = await Promise.all([
+    compactExecutionPayload(rest, options),
+    compactTraceSpanTree(childTraceSpans, options, new WeakSet<object>()),
   ])
-  return { ...compactedRest, childTraceSpans: compactedSpans } as T
+  return { output: compactedOutput as T, childTraceSpans: compactedSpans }
 }
 
 export async function compactBlockLogs(
@@ -335,12 +372,17 @@ export async function compactBlockLogs(
         compactedLog.input = await compactExecutionPayload(compactedLog.input, options)
       }
       if ('output' in compactedLog) {
-        compactedLog.output = await compactBlockOutput(compactedLog.output, options)
+        compactedLog.output = (await compactLoggedOutput(
+          compactedLog.output,
+          options,
+          new WeakSet<object>()
+        )) as BlockLog['output']
       }
       if ('childTraceSpans' in compactedLog) {
         compactedLog.childTraceSpans = (await compactTraceSpanTree(
           compactedLog.childTraceSpans,
-          options
+          options,
+          new WeakSet<object>()
         )) as BlockLog['childTraceSpans']
       }
       compactedLogs[index] = compactedLog