fix(logs): keep compacted child span trees shaped as trees

Block output compaction spilled oversized child span lists to large-value
references, so a span's children could stop being an array and trace span
building threw while finalizing the run. That left pauses unpersisted and
runs unfinalized.

- Compact child span trees structurally, spilling only each span's payload fields
- Drop non-list child spans with a warning when building trace spans
- Finalize without spans if building them fails, so logs, pauses, and billing settle

Author: waleedlatif1

apps/sim/executor/execution/block-executor.ts

@@ -7,7 +7,7 @@ import { isTimeoutAbortReason } from '@/lib/core/execution-limits/types'
 import { redactApiKeys } from '@/lib/core/security/redaction'
 import { normalizeStringArray } from '@/lib/core/utils/arrays'
 import { getBaseUrl } from '@/lib/core/utils/urls'
-import { compactExecutionPayload } from '@/lib/execution/payloads/serializer'
+import { compactBlockOutput } from '@/lib/execution/payloads/serializer'
 import { redactLargeValueRefsInValue } from '@/lib/logs/execution/pii-large-values'
 import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction'
 import {
@@ -379,7 +379,7 @@ export class BlockExecutor {
         normalizedOutput = await redactObjectStrings(normalizedOutput, redactionOptions)
       }
 
-      normalizedOutput = (await compactExecutionPayload(normalizedOutput, {
+      normalizedOutput = (await compactBlockOutput(normalizedOutput, {
         workspaceId: blockCtx.workspaceId,
         workflowId: blockCtx.workflowId,
         executionId: blockCtx.executionId,

apps/sim/lib/execution/payloads/serializer.test.ts

@@ -15,8 +15,14 @@ import {
   getLargeValueMaterializationError,
   isLargeValueRef,
 } from '@/lib/execution/payloads/large-value-ref'
-import { compactExecutionPayload, compactSubflowResults } from '@/lib/execution/payloads/serializer'
-import type { UserFile } from '@/executor/types'
+import {
+  compactBlockLogs,
+  compactBlockOutput,
+  compactExecutionPayload,
+  compactSubflowResults,
+} from '@/lib/execution/payloads/serializer'
+import type { TraceSpan } from '@/lib/logs/types'
+import type { BlockLog, UserFile } from '@/executor/types'
 
 const { mockDownloadFile, mockUploadFile } = storageServiceMockFns
 
@@ -295,3 +301,119 @@ describe('compactExecutionPayload', () => {
     expect(error.message).not.toContain('lv_CQcekP8gSJI5')
   })
 })
+
+/**
+ * A child workflow's spans as the workflow block reports them: a loop whose one
+ * iteration holds two block spans. With a 1 KiB threshold each block span stays
+ * inline, but the iteration's `children` array together is over it — the shape
+ * generic compaction turned into a manifest.
+ */
+function childWorkflowSpans(): TraceSpan[] {
+  const blockSpan = (id: string): TraceSpan => ({
+    id,
+    name: id,
+    type: 'function',
+    duration: 1,
+    startTime: '2026-09-29T00:00:00.000Z',
+    endTime: '2026-09-29T00:00:00.001Z',
+    output: { result: 'x'.repeat(600) },
+  })
+  return [
+    {
+      id: 'loop',
+      name: 'Loop',
+      type: 'loop',
+      duration: 2,
+      startTime: '2026-09-29T00:00:00.000Z',
+      endTime: '2026-09-29T00:00:00.002Z',
+      children: [
+        {
+          id: 'iteration-0',
+          name: 'Iteration 0',
+          type: 'loop-iteration',
+          duration: 2,
+          startTime: '2026-09-29T00:00:00.000Z',
+          endTime: '2026-09-29T00:00:00.002Z',
+          children: [blockSpan('span-a'), blockSpan('span-b')],
+        },
+      ],
+    },
+  ]
+}
+
+/** Asserts the loop → iteration → block span nesting survived with every `children` an array. */
+function expectSpanTree(spans: unknown): void {
+  expect(Array.isArray(spans)).toBe(true)
+  const [loop] = spans as TraceSpan[]
+  expect(Array.isArray(loop.children)).toBe(true)
+  const [iteration] = loop.children ?? []
+  expect(Array.isArray(iteration.children)).toBe(true)
+  expect(iteration.children?.map((span) => span.id)).toEqual(['span-a', 'span-b'])
+}
+
+describe('compacting span trees', () => {
+  const options = { thresholdBytes: 1024, requireDurable: true, ...TEST_EXECUTION_CONTEXT }
+
+  beforeEach(() => {
+    clearLargeValueCacheForTests()
+    mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey }))
+    mockRegisterLargeValueOwner.mockResolvedValue(true)
+  })
+
+  it('keeps a block output child span tree shaped as a tree', async () => {
+    const compacted = await compactBlockOutput(
+      { result: 'done', childTraceSpans: childWorkflowSpans() },
+      options
+    )
+
+    expectSpanTree(compacted.childTraceSpans)
+  })
+
+  it('still spills an oversized span payload', async () => {
+    const spans = childWorkflowSpans()
+    const iteration = spans[0].children?.[0]
+    if (iteration?.children) iteration.children[0].output = { result: 'y'.repeat(4096) }
+
+    const compacted = await compactBlockOutput({ childTraceSpans: spans }, options)
+
+    const spilled = (compacted.childTraceSpans as TraceSpan[])[0].children?.[0].children?.[0]
+    expect(isLargeValueRef(spilled?.output?.result)).toBe(true)
+  })
+
+  it('keeps block log child span trees shaped as trees', async () => {
+    const log = {
+      blockId: 'child-workflow',
+      blockType: 'workflow',
+      startedAt: '2026-09-29T00:00:00.000Z',
+      endedAt: '2026-09-29T00:00:00.002Z',
+      durationMs: 2,
+      success: true,
+      childTraceSpans: childWorkflowSpans(),
+    } as BlockLog
+
+    const [compacted] = (await compactBlockLogs([log], options)) ?? []
+
+    expectSpanTree(compacted?.childTraceSpans)
+  })
+
+  it('keeps a nested child workflow span tree shaped as a tree', async () => {
+    const nestedWorkflowSpan: TraceSpan = {
+      id: 'nested-workflow',
+      name: 'Nested Workflow',
+      type: 'workflow',
+      duration: 2,
+      startTime: '2026-09-29T00:00:00.000Z',
+      endTime: '2026-09-29T00:00:00.002Z',
+      output: { result: 'done', childTraceSpans: childWorkflowSpans() },
+    }
+
+    const compacted = await compactBlockOutput(
+      { result: 'done', childTraceSpans: [nestedWorkflowSpan] },
+      options
+    )
+
+    const [nested] = compacted.childTraceSpans as TraceSpan[]
+    expect(nested.output?.result).toBe('done')
+    expectSpanTree(nested.output?.childTraceSpans)
+  })
+})

apps/sim/lib/execution/payloads/serializer.ts

@@ -1,3 +1,4 @@
+import { isRecordLike } from '@sim/utils/object'
 import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { isUserFileWithMetadata } from '@/lib/core/utils/user-file'
 import {
@@ -254,6 +255,66 @@ export async function compactSubflowResults<T>(
   return compactedResults
 }
 
+/**
+ * Compacts a trace span tree without collapsing its structure. Readers walk
+ * `children` and `output.childTraceSpans` as arrays, so those stay arrays and
+ * only each span's payload fields are spilled when oversized. Size of the tree
+ * as a whole is bounded where the log is stored, not here.
+ */
+async function compactTraceSpanTree(
+  spans: unknown,
+  options: CompactExecutionPayloadOptions
+): Promise<unknown> {
+  if (!Array.isArray(spans)) {
+    return compactExecutionPayload(spans, options)
+  }
+  return Promise.all(spans.map((span) => compactTraceSpan(span, options)))
+}
+
+async function compactTraceSpan(
+  span: unknown,
+  options: CompactExecutionPayloadOptions
+): Promise<unknown> {
+  if (!isRecordLike(span)) {
+    return compactExecutionPayload(span, options)
+  }
+  return Object.fromEntries(
+    await Promise.all(
+      Object.entries(span).map(async ([key, value]) => [
+        key,
+        key === 'children'
+          ? await compactTraceSpanTree(value, options)
+          : key === 'output'
+            ? await compactBlockOutput(value, options)
+            : await compactExecutionPayload(value, options),
+      ])
+    )
+  )
+}
+
+/**
+ * Compacts a block output, keeping any `childTraceSpans` shaped as a span tree
+ * (see {@link compactTraceSpanTree}). Everything else compacts as a normal
+ * execution payload.
+ */
+export async function compactBlockOutput<T>(
+  output: T,
+  options: CompactExecutionPayloadOptions = {}
+): Promise<T> {
+  if (!isRecordLike(output) || !('childTraceSpans' in output)) {
+    return compactExecutionPayload(output, options)
+  }
+  const { childTraceSpans, ...rest } = output
+  const [compactedRest, compactedSpans] = await Promise.all([
+    compactExecutionPayload(rest, options),
+    compactTraceSpanTree(childTraceSpans, options),
+  ])
+  if (!isRecordLike(compactedRest)) {
+    return compactedRest as T
+  }
+  return { ...compactedRest, childTraceSpans: compactedSpans } as T
+}
+
 export async function compactBlockLogs(
   logs: BlockLog[] | undefined,
   options: CompactExecutionPayloadOptions = {}
@@ -276,13 +337,13 @@ export async function compactBlockLogs(
         compactedLog.input = await compactExecutionPayload(compactedLog.input, options)
       }
       if ('output' in compactedLog) {
-        compactedLog.output = await compactExecutionPayload(compactedLog.output, options)
+        compactedLog.output = await compactBlockOutput(compactedLog.output, options)
       }
       if ('childTraceSpans' in compactedLog) {
-        compactedLog.childTraceSpans = await compactExecutionPayload(
+        compactedLog.childTraceSpans = (await compactTraceSpanTree(
           compactedLog.childTraceSpans,
           options
-        )
+        )) as BlockLog['childTraceSpans']
       }
       compactedLogs[index] = compactedLog
     }

apps/sim/lib/logs/execution/trace-spans/span-factory.ts

@@ -392,8 +392,11 @@ function resolveToolCallsList(output: NormalizedBlockOutput | undefined): BlockT
 
 /** Extracts and flattens child workflow trace spans into the parent span's children. */
 function attachChildWorkflowSpans(span: TraceSpan, log: ValidBlockLog): void {
-  const childTraceSpans = log.childTraceSpans ?? log.output?.childTraceSpans
-  if (!childTraceSpans?.length) return
+  const childTraceSpans = readChildSpans(
+    log.childTraceSpans ?? log.output?.childTraceSpans,
+    'childTraceSpans'
+  )
+  if (childTraceSpans.length === 0) return
 
   span.children = flattenWorkflowChildren(childTraceSpans)
   span.output = stripChildTraceSpansFromOutput(span.output)
@@ -404,10 +407,21 @@ function isSyntheticWorkflowWrapper(span: TraceSpan): boolean {
   return span.type === 'workflow' && !span.blockId
 }
 
+/**
+ * Reads a list of child spans, or `[]` when absent. Anything other than an
+ * array (for example a span list that was spilled to a large-value reference)
+ * is dropped with a warning so one malformed subtree cannot fail the trace.
+ */
+function readChildSpans(value: unknown, source: 'children' | 'childTraceSpans'): TraceSpan[] {
+  if (value === undefined || value === null) return []
+  if (Array.isArray(value)) return value
+  logger.warn('Dropping child spans that are not a list', { source, shape: typeof value })
+  return []
+}
+
 /** Reads nested `childTraceSpans` off a span's output, or `[]` if absent. */
 function extractOutputChildren(output: TraceSpan['output']): TraceSpan[] {
-  const nested = (output as { childTraceSpans?: TraceSpan[] } | undefined)?.childTraceSpans
-  return Array.isArray(nested) ? nested : []
+  return readChildSpans(output?.childTraceSpans, 'childTraceSpans')
 }
 
 /** Returns a copy of `output` with `childTraceSpans` removed, or undefined unchanged. */
@@ -430,21 +444,21 @@ export function flattenWorkflowChildren(spans: TraceSpan[]): TraceSpan[] {
 
   for (const span of spans) {
     if (isSyntheticWorkflowWrapper(span)) {
-      if (span.children?.length) {
-        flattened.push(...flattenWorkflowChildren(span.children))
-      }
+      flattened.push(...flattenWorkflowChildren(readChildSpans(span.children, 'children')))
       continue
     }
 
-    const directChildren = span.children ?? []
+    const directChildren = readChildSpans(span.children, 'children')
     const outputChildren = extractOutputChildren(span.output)
     const allChildren = [...directChildren, ...outputChildren]
 
     const nextSpan: TraceSpan = { ...span }
     if (allChildren.length > 0) {
       nextSpan.children = flattenWorkflowChildren(allChildren)
+    } else if (!Array.isArray(span.children)) {
+      nextSpan.children = undefined
     }
-    if (outputChildren.length > 0) {
+    if (span.output && 'childTraceSpans' in span.output) {
       nextSpan.output = stripChildTraceSpansFromOutput(nextSpan.output)
     }
 

apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts

@@ -1419,3 +1419,65 @@ describe('custom block invoked as an Agent tool', () => {
     expect(toolSpan?.output).not.toHaveProperty('_childTraceDisabled')
   })
 })
+
+describe('child span trees that were compacted into references', () => {
+  const spilledChildren = {
+    __simLargeValueRef: true,
+    version: 1,
+    id: 'lv_ABCDEFGHIJKLMNOPQRSTUV',
+    kind: 'array',
+    size: 9_000_000,
+  } as unknown as TraceSpan[]
+
+  function childWorkflowResult(childTraceSpans: TraceSpan[]): ExecutionResult {
+    return {
+      success: true,
+      output: {},
+      logs: [
+        {
+          blockId: 'workflow-1',
+          blockName: 'Child Workflow',
+          blockType: 'workflow',
+          startedAt: '2024-01-01T10:00:00.000Z',
+          endedAt: '2024-01-01T10:00:05.000Z',
+          durationMs: 5000,
+          success: true,
+          output: { success: true },
+          childTraceSpans,
+        },
+      ],
+    }
+  }
+
+  const loopSpan = (overrides: Partial<TraceSpan>): TraceSpan => ({
+    id: 'loop-1',
+    name: 'Loop',
+    type: 'loop',
+    blockId: 'loop-1',
+    duration: 1000,
+    startTime: '2024-01-01T10:00:01.000Z',
+    endTime: '2024-01-01T10:00:02.000Z',
+    status: 'success',
+    ...overrides,
+  })
+
+  it.concurrent('keeps the span when its children are a reference', () => {
+    const { traceSpans } = buildTraceSpans(
+      childWorkflowResult([loopSpan({ children: spilledChildren })])
+    )
+
+    expect(traceSpans[0].children?.map((span) => span.id)).toEqual(['loop-1'])
+    expect(traceSpans[0].children?.[0].children).toBeUndefined()
+  })
+
+  it.concurrent('keeps the span when its output child spans are a reference', () => {
+    const { traceSpans } = buildTraceSpans(
+      childWorkflowResult([
+        loopSpan({ output: { result: 'done', childTraceSpans: spilledChildren } }),
+      ])
+    )
+
+    expect(traceSpans[0].children?.map((span) => span.id)).toEqual(['loop-1'])
+    expect(traceSpans[0].children?.[0].output).toEqual({ result: 'done' })
+  })
+})