fix(logs): keep a child span tree whole or drop it, bounded as before

A structurally compacted tree had no whole-tree bound, so a large one stayed
inline in block logs and pause snapshots. A tree still over the threshold
after its payloads spill is now dropped (or rejected), as generic compaction
bounded it. Block log outputs compact generically again; new logs never carry
child spans there.

Author: waleedlatif1

apps/sim/executor/execution/block-executor.ts

@@ -397,7 +397,7 @@ export class BlockExecutor {
         blockLog.durationMs = duration
         blockLog.success = true
         blockLog.output = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block })
-        if (Array.isArray(compacted.childTraceSpans)) {
+        if (compacted.childTraceSpans) {
           blockLog.childTraceSpans = compacted.childTraceSpans
         }
         const childExecutionId = normalizedOutput[CHILD_EXECUTION_ID_OUTPUT_KEY]

apps/sim/lib/execution/payloads/serializer.test.ts

@@ -2,7 +2,7 @@ import {
   largeValueMetadataMock,
   largeValueMetadataMockFns,
 } from '@sim/testing/mocks/large-value-metadata.mock'
-import { storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock'
+import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock'
 import { uploadsMock } from '@sim/testing/mocks/uploads.mock'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache'
@@ -27,6 +27,7 @@ import type { BlockLog, UserFile } from '@/executor/types'
 const { mockDownloadFile, mockUploadFile } = storageServiceMockFns
 
 vi.mock('@/lib/uploads', () => uploadsMock)
+vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock)
 
 vi.mock('@/lib/execution/payloads/large-value-metadata', () => largeValueMetadataMock)
 
@@ -304,19 +305,17 @@ describe('compactExecutionPayload', () => {
 
 /**
  * A child workflow's spans as the workflow block reports them: a loop whose one
- * iteration holds two block spans. With a 1 KiB threshold each block span stays
- * inline, but the iteration's `children` array together is over it — the shape
- * generic compaction turned into a manifest.
+ * iteration holds two block spans, each with a `resultBytes` payload.
  */
-function childWorkflowSpans(): TraceSpan[] {
+function childWorkflowSpans(resultBytes: number): TraceSpan[] {
   const blockSpan = (id: string): TraceSpan => ({
     id,
     name: id,
     type: 'function',
     duration: 1,
     startTime: '2026-09-29T00:00:00.000Z',
     endTime: '2026-09-29T00:00:00.001Z',
-    output: { result: 'x'.repeat(600) },
+    output: { result: 'x'.repeat(resultBytes) },
   })
   return [
     {
@@ -341,6 +340,16 @@ function childWorkflowSpans(): TraceSpan[] {
   ]
 }
 
+/** Spans whose payloads each exceed the 4 KiB test threshold, so each spills on its own. */
+const spansWithLargePayloads = () => childWorkflowSpans(8192)
+
+/**
+ * Spans whose payloads each stay under the 4 KiB test threshold but whose
+ * iteration `children` together exceed it — the shape generic compaction
+ * turned into a manifest nested inside the tree.
+ */
+const spansTooLargeAsAWhole = () => childWorkflowSpans(2500)
+
 /** Asserts the loop → iteration → block span nesting survived with every `children` an array. */
 function expectSpanTree(spans: unknown): void {
   expect(Array.isArray(spans)).toBe(true)
@@ -352,7 +361,7 @@ function expectSpanTree(spans: unknown): void {
 }
 
 describe('compacting span trees', () => {
-  const options = { thresholdBytes: 1024, requireDurable: true, ...TEST_EXECUTION_CONTEXT }
+  const options = { thresholdBytes: 4096, requireDurable: true, ...TEST_EXECUTION_CONTEXT }
 
   beforeEach(() => {
     clearLargeValueCacheForTests()
@@ -370,65 +379,64 @@ describe('compacting span trees', () => {
     ...overrides,
   })
 
-  it('splits a block output child span tree off shaped as a tree', async () => {
+  it('splits a block output child span tree off, spilling each oversized payload', async () => {
     const compacted = await compactBlockOutput(
-      { result: 'done', childTraceSpans: childWorkflowSpans() },
+      { result: 'done', childTraceSpans: spansWithLargePayloads() },
       options
     )
 
     expect(compacted.output).toEqual({ result: 'done' })
     expectSpanTree(compacted.childTraceSpans)
+    const [loop] = compacted.childTraceSpans as TraceSpan[]
+    const spilled = loop.children?.[0].children?.[0]
+    expect(isLargeValueRef(spilled?.output?.result)).toBe(true)
   })
 
-  it('still spills an oversized span payload', async () => {
-    const spans = childWorkflowSpans()
-    const iteration = spans[0].children?.[0]
-    if (iteration?.children) iteration.children[0].output = { result: 'y'.repeat(4096) }
+  it('drops a block output child span tree too large as a whole', async () => {
+    const compacted = await compactBlockOutput(
+      { result: 'done', childTraceSpans: spansTooLargeAsAWhole() },
+      options
+    )
 
-    const compacted = await compactBlockOutput({ childTraceSpans: spans }, options)
+    expect(compacted.output).toEqual({ result: 'done' })
+    expect(compacted.childTraceSpans).toBeUndefined()
+  })
 
-    const spilled = (compacted.childTraceSpans as TraceSpan[])[0].children?.[0].children?.[0]
-    expect(isLargeValueRef(spilled?.output?.result)).toBe(true)
+  it('rejects a child span tree too large as a whole when large values are rejected', async () => {
+    await expect(
+      compactBlockOutput(
+        { childTraceSpans: spansTooLargeAsAWhole() },
+        { ...options, rejectLargeValues: true }
+      )
+    ).rejects.toThrow()
   })
 
   it('still spills a block output whose fields together exceed the threshold', async () => {
     const compacted = await compactBlockOutput(
-      { first: 'a'.repeat(600), second: 'b'.repeat(600), childTraceSpans: childWorkflowSpans() },
+      {
+        first: 'a'.repeat(2500),
+        second: 'b'.repeat(2500),
+        childTraceSpans: spansWithLargePayloads(),
+      },
       options
     )
 
     expect(isLargeValueRef(compacted.output)).toBe(true)
     expectSpanTree(compacted.childTraceSpans)
   })
 
-  it('keeps block log child span trees shaped as trees', async () => {
-    const [compacted] =
-      (await compactBlockLogs(
-        [childWorkflowLog({ childTraceSpans: childWorkflowSpans() })],
-        options
-      )) ?? []
-
-    expectSpanTree(compacted?.childTraceSpans)
-  })
-
-  it('keeps a block log output carrying child spans a record', async () => {
-    const [compacted] =
+  it('keeps block log child span trees whole or drops them', async () => {
+    const compacted =
       (await compactBlockLogs(
         [
-          childWorkflowLog({
-            output: {
-              first: 'a'.repeat(600),
-              second: 'b'.repeat(600),
-              childTraceSpans: childWorkflowSpans(),
-            },
-          }),
+          childWorkflowLog({ childTraceSpans: spansWithLargePayloads() }),
+          childWorkflowLog({ childTraceSpans: spansTooLargeAsAWhole() }),
         ],
         options
       )) ?? []
 
-    expect(isLargeValueRef(compacted?.output)).toBe(false)
-    expect(compacted?.output?.first).toBe('a'.repeat(600))
-    expectSpanTree(compacted?.output?.childTraceSpans)
+    expectSpanTree(compacted[0]?.childTraceSpans)
+    expect(compacted[1]?.childTraceSpans).toBeUndefined()
   })
 
   it('keeps a nested child workflow span tree shaped as a tree', async () => {
@@ -439,17 +447,13 @@ describe('compacting span trees', () => {
       duration: 2,
       startTime: '2026-09-29T00:00:00.000Z',
       endTime: '2026-09-29T00:00:00.002Z',
-      output: {
-        first: 'a'.repeat(600),
-        second: 'b'.repeat(600),
-        childTraceSpans: childWorkflowSpans(),
-      },
+      output: { result: 'done', childTraceSpans: spansWithLargePayloads() },
     }
 
     const compacted = await compactBlockOutput({ childTraceSpans: [nestedWorkflowSpan] }, options)
 
     const [nested] = compacted.childTraceSpans as TraceSpan[]
-    expect(nested.output?.first).toBe('a'.repeat(600))
+    expect(nested.output?.result).toBe('done')
     expectSpanTree(nested.output?.childTraceSpans)
   })
 
@@ -464,8 +468,6 @@ describe('compacting span trees', () => {
     }
     span.children = [span]
 
-    const compacted = await compactBlockOutput({ childTraceSpans: [span] }, options)
-
-    expect((compacted.childTraceSpans as TraceSpan[])[0].id).toBe('cyclic')
+    await expect(compactBlockOutput({ childTraceSpans: [span] }, options)).resolves.toBeDefined()
   })
 })

apps/sim/lib/execution/payloads/serializer.ts

@@ -1,3 +1,4 @@
+import { createLogger } from '@sim/logger'
 import { isRecordLike } from '@sim/utils/object'
 import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { isUserFileWithMetadata } from '@/lib/core/utils/user-file'
@@ -10,8 +11,11 @@ import {
   LARGE_VALUE_THRESHOLD_BYTES,
 } from '@/lib/execution/payloads/large-value-ref'
 import { type LargeValueStoreContext, storeLargeValue } from '@/lib/execution/payloads/store'
+import type { TraceSpan } from '@/lib/logs/types'
 import type { BlockLog } from '@/executor/types'
 
+const logger = createLogger('ExecutionPayloadSerializer')
+
 export interface CompactExecutionPayloadOptions extends LargeValueStoreContext {
   thresholdBytes?: number
   preserveUserFileBase64?: boolean
@@ -255,11 +259,22 @@ export async function compactSubflowResults<T>(
   return compactedResults
 }
 
+/** Maps each entry of a record concurrently, keeping its keys. */
+async function mapEntriesAsync(
+  record: Record<string, unknown>,
+  mapValue: (key: string, value: unknown) => Promise<unknown>
+): Promise<Record<string, unknown>> {
+  return Object.fromEntries(
+    await Promise.all(
+      Object.entries(record).map(async ([key, value]) => [key, await mapValue(key, value)])
+    )
+  )
+}
+
 /**
- * Compacts a trace span tree without collapsing its structure. Readers walk
- * `children` and `output.childTraceSpans` as arrays, so those stay arrays and
- * only each span's payload fields are spilled when oversized. Spans are log
- * data: the tree as a whole is bounded where the log is stored, not here.
+ * Compacts a trace span tree without collapsing its structure: `children` and
+ * `output.childTraceSpans` stay arrays and only each span's payload fields
+ * spill when oversized. See {@link compactChildTraceSpans} for the size bound.
  */
 async function compactTraceSpanTree(
   spans: unknown,
@@ -284,56 +299,77 @@ async function compactTraceSpan(
     return span
   }
   seen.add(span)
-  return Object.fromEntries(
-    await Promise.all(
-      Object.entries(span).map(async ([key, value]) => [
-        key,
-        key === 'children'
-          ? await compactTraceSpanTree(value, options, seen)
-          : key === 'output'
-            ? await compactLoggedOutput(value, options, seen)
-            : await compactExecutionPayload(value, options),
-      ])
-    )
-  )
+  return mapEntriesAsync(span, (key, value) => {
+    if (key === 'children') return compactTraceSpanTree(value, options, seen)
+    if (key === 'output') return compactSpanOutput(value, options, seen)
+    return compactExecutionPayload(value, options)
+  })
 }
 
 /**
- * Compacts a span or block log output. One carrying `childTraceSpans` keeps its
- * root so the spans stay attached; its other fields spill individually.
+ * Compacts a span's output. One carrying a nested child workflow's
+ * `childTraceSpans` keeps its root so the spans stay attached; its other
+ * fields spill individually.
  */
-async function compactLoggedOutput(
+async function compactSpanOutput(
   output: unknown,
   options: CompactExecutionPayloadOptions,
   seen: WeakSet<object>
 ): Promise<unknown> {
   if (!isRecordLike(output) || !('childTraceSpans' in output)) {
     return compactExecutionPayload(output, options)
   }
-  return Object.fromEntries(
-    await Promise.all(
-      Object.entries(output).map(async ([key, value]) => [
-        key,
-        key === 'childTraceSpans'
-          ? await compactTraceSpanTree(value, options, seen)
-          : await compactExecutionPayload(value, options),
-      ])
-    )
+  return mapEntriesAsync(output, (key, value) =>
+    key === 'childTraceSpans'
+      ? compactTraceSpanTree(value, options, seen)
+      : compactExecutionPayload(value, options)
   )
 }
 
+/**
+ * Compacts a block's child span tree for its log. Readers walk span trees as
+ * arrays, so a tree is kept whole or not at all: payload fields spill
+ * individually (see {@link compactTraceSpanTree}), and a tree still over the
+ * threshold as a whole is dropped, bounding it as generic compaction did.
+ */
+async function compactChildTraceSpans(
+  spans: unknown,
+  options: CompactExecutionPayloadOptions
+): Promise<TraceSpan[] | undefined> {
+  if (!Array.isArray(spans)) {
+    if (spans !== undefined) {
+      logger.warn('Dropping child trace spans that are not a list', { shape: typeof spans })
+    }
+    return undefined
+  }
+  const compacted = await compactTraceSpanTree(spans, options, new WeakSet<object>())
+  const measured = getJsonAndSize(compacted)
+  const maxBytes = options.thresholdBytes ?? LARGE_VALUE_THRESHOLD_BYTES
+  if (measured && measured.size <= maxBytes) {
+    return compacted as TraceSpan[]
+  }
+  if (measured && options.rejectLargeValues) {
+    throw largeValueLimitError(options, measured.size)
+  }
+  logger.warn('Dropping child trace spans too large to keep', {
+    observedBytes: measured?.size,
+    maxBytes,
+  })
+  return undefined
+}
+
 export interface CompactedBlockOutput<T> {
   /** The output without `childTraceSpans`, compacted as execution state. */
   output: T
-  /** The output's child span tree, compacted as log data. */
-  childTraceSpans?: unknown
+  /** The output's child span tree for the block log (see {@link compactChildTraceSpans}). */
+  childTraceSpans?: TraceSpan[]
 }
 
 /**
  * Compacts a block output for execution state and splits off its
  * `childTraceSpans`, which belong to the block log rather than state. The
  * output compacts as any execution payload, so an oversized one still spills
- * whole; the spans compact as a tree (see {@link compactTraceSpanTree}).
+ * whole.
  */
 export async function compactBlockOutput<T>(
   output: T,
@@ -345,7 +381,7 @@ export async function compactBlockOutput<T>(
   const { childTraceSpans, ...rest } = output
   const [compactedOutput, compactedSpans] = await Promise.all([
     compactExecutionPayload(rest, options),
-    compactTraceSpanTree(childTraceSpans, options, new WeakSet<object>()),
+    compactChildTraceSpans(childTraceSpans, options),
   ])
   return { output: compactedOutput as T, childTraceSpans: compactedSpans }
 }
@@ -372,18 +408,13 @@ export async function compactBlockLogs(
         compactedLog.input = await compactExecutionPayload(compactedLog.input, options)
       }
       if ('output' in compactedLog) {
-        compactedLog.output = (await compactLoggedOutput(
-          compactedLog.output,
-          options,
-          new WeakSet<object>()
-        )) as BlockLog['output']
+        compactedLog.output = await compactExecutionPayload(compactedLog.output, options)
       }
       if ('childTraceSpans' in compactedLog) {
-        compactedLog.childTraceSpans = (await compactTraceSpanTree(
+        compactedLog.childTraceSpans = await compactChildTraceSpans(
           compactedLog.childTraceSpans,
-          options,
-          new WeakSet<object>()
-        )) as BlockLog['childTraceSpans']
+          options
+        )
       }
       compactedLogs[index] = compactedLog
     }

apps/sim/lib/logs/execution/trace-spans/span-factory.ts

@@ -458,9 +458,7 @@ export function flattenWorkflowChildren(spans: TraceSpan[]): TraceSpan[] {
     } else if (!Array.isArray(span.children)) {
       nextSpan.children = undefined
     }
-    if (span.output && 'childTraceSpans' in span.output) {
-      nextSpan.output = stripChildTraceSpansFromOutput(nextSpan.output)
-    }
+    nextSpan.output = stripChildTraceSpansFromOutput(nextSpan.output)
 
     flattened.push(nextSpan)
   }