fix(webapp): harden progressive trace loading

Clicking a span in the run trace view no longer reloads the page data.
Traces past the view limit now show their first part with a "partially
displayed" notice instead of an empty view, and the "Errors only"
filter, live updates and the emergency span cap put less load on the
server. Progressive loading is now controlled by its feature flag alone.

Mono-RevId: 8d876197c351f77777bfd2871933212f359ff8d7

Author: Mariefay

.env.schema

@@ -41,8 +41,6 @@ RUN_REPLICATION_CLICKHOUSE_URL=${CLICKHOUSE_URL}
 RUN_REPLICATION_ENABLED=1
 # LOGS_SEARCH_PROJECTOR_ENABLED=1
 # LOGS_SEARCH_PROJECTOR_PREVIEW_ENABLED=1
-# Live tail for progressive trace loading; "1" turns it on, anything else turns it off.
-# INCREMENTAL_LIVE_TAIL_ENABLED=0
 # Store task run spans/traces in ClickHouse so the dashboard trace view is
 # populated in local dev. The local stack is ClickHouse-backed (see above), so
 # leaving this unset falls back to the "postgres" store and dev run traces show

apps/webapp/app/env.server.ts

@@ -1116,8 +1116,6 @@ const EnvironmentSchema = z
     EVENT_LOOP_UTILIZATION_MONITOR_ENABLED: z.string().default("1"),
     MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
     MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
-    // Live tail for progressive trace loading; "1" on, anything else off.
-    INCREMENTAL_LIVE_TAIL_ENABLED: z.string().default("0"),
     MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000),
     // Emergency circuit breaker: when set, clamps the trace summary and detailed
     // summary span limits on both event store paths to this value. Unset = disabled.

apps/webapp/app/hooks/useProgressiveTrace.ts

@@ -42,6 +42,8 @@ type WireChunkResponse = {
   events: WireChunkEvent[];
   nextCursor: TraceChunkCursor | null;
   hasMore: boolean;
+  // The server capped the result: errors-only matches, or paging under the emergency cap.
+  isTruncated?: boolean;
   // Server time (ms) taken before the page was read.
   readAt?: number;
 };
@@ -53,7 +55,6 @@ type ProgressiveMeta = {
   hasMore: boolean;
   buildOptions: BuildTraceViewOptions;
   showDebug: boolean;
-  totalSpans?: number;
   maxSpans?: number;
   liveTailEnabled?: boolean;
   // Server time (ms) before the first chunk was read; the first tail reads back to it.
@@ -79,6 +80,8 @@ export type ProgressiveTraceState = {
   linkedRunIdBySpanId: Record<string, string>;
   isComplete: boolean;
   isTruncated: boolean;
+  // Errors-only matches were capped; separate from `isTruncated`, which stops live reload.
+  errorsTruncated: boolean;
   loadFailed: boolean;
 };
 
@@ -96,6 +99,7 @@ function initialState(trace: ProgressiveTraceInput): ProgressiveTraceState {
     linkedRunIdBySpanId: trace.linkedRunIdBySpanId ?? {},
     isComplete: !trace.progressive?.hasMore,
     isTruncated: false,
+    errorsTruncated: false,
     loadFailed: false,
   };
 }
@@ -192,6 +196,8 @@ export function useProgressiveTrace(
   const errorsFetchedRef = useRef(false);
   // The deep-link payload already merged, so a kept tree only merges new ones.
   const mergedSupplementaryRef = useRef<WireChunkEvent[] | null>(null);
+  // The first chunk already merged; a same-run revalidate merges only a new one.
+  const mergedFirstEventsRef = useRef<WireChunkEvent[] | null>(null);
   // Assembler the in-flight tail targets; a stale tail never touches the current one.
   const tailingRef = useRef<TraceChunkAssembler | null>(null);
   // Timestamp of the last tail read, for the size-aware cadence governor.
@@ -220,6 +226,7 @@ export function useProgressiveTrace(
       linkedRunIdBySpanId: view.linkedRunIdBySpanId,
       isComplete: prev.isComplete,
       isTruncated: prev.isTruncated,
+      errorsTruncated: prev.errorsTruncated,
       loadFailed: prev.loadFailed,
     }));
   }, []);
@@ -249,6 +256,7 @@ export function useProgressiveTrace(
 
     const assembler = new TraceChunkAssembler();
     assembler.mergeChunk(meta.firstEvents.map(toChunkEvent));
+    mergedFirstEventsRef.current = meta.firstEvents;
     if (meta.supplementaryFirstEvents?.length) {
       assembler.mergeChunk(meta.supplementaryFirstEvents.map(toChunkEvent), {
         source: "deeplink",
@@ -338,7 +346,7 @@ export function useProgressiveTrace(
         requestRebuild();
         cursor = data.hasMore ? data.nextCursor : null;
 
-        if (assembler.size > maxSpans) {
+        if (assembler.size > maxSpans || data.isTruncated) {
           markComplete(true);
           return;
         }
@@ -360,9 +368,12 @@ export function useProgressiveTrace(
     };
   }, [identity, chunkPath, retryGeneration, rebuild]);
 
+  // Every chunk is loaded, so the client-side filter already sees every error.
+  const fullyLoaded = state.isComplete && !state.loadFailed && !state.isTruncated;
+
   useEffect(() => {
     const meta = latestTraceRef.current.progressive;
-    if (!errorsOnly || !meta || errorsFetchedRef.current) {
+    if (!errorsOnly || !meta || errorsFetchedRef.current || fullyLoaded) {
       return;
     }
 
@@ -374,12 +385,15 @@ export function useProgressiveTrace(
       errorsFetchedRef.current = true;
       assembler.mergeChunk(data.events.map(toChunkEvent), { source: "errors" });
       rebuild(meta, assembler);
+      if (data.isTruncated) {
+        setState((prev) => (prev.errorsTruncated ? prev : { ...prev, errorsTruncated: true }));
+      }
     })();
 
     return () => {
       cancelled = true;
     };
-  }, [errorsOnly, identity, chunkPath, retryGeneration, rebuild]);
+  }, [errorsOnly, fullyLoaded, identity, chunkPath, retryGeneration, rebuild]);
 
   const runTail = useCallback(() => {
     const meta = latestTraceRef.current.progressive;
@@ -477,6 +491,20 @@ export function useProgressiveTrace(
     rebuild(meta, assembler);
   }, [supplementary, rebuild]);
 
+  // A kept tree merges a revalidated first chunk, e.g. the root's final row after the
+  // tail stopped at the view ceiling.
+  const firstEvents = progressive?.firstEvents;
+  useEffect(() => {
+    const meta = latestTraceRef.current.progressive;
+    const assembler = assemblerRef.current;
+    if (!meta?.liveTailEnabled || !assembler || !firstEvents) return;
+    if (firstEvents === mergedFirstEventsRef.current) return;
+    mergedFirstEventsRef.current = firstEvents;
+    assembler.mergeChunk(firstEvents.map(toChunkEvent), { source: "revalidate" });
+    if (!assembler.changedSinceRender) return;
+    rebuild(meta, assembler);
+  }, [firstEvents, rebuild]);
+
   // Stable trigger so the route's SSE effect doesn't re-fire on chunkPath changes.
   useEffect(() => {
     tailLiveRef.current = runTail;

apps/webapp/app/hooks/useRunStatusBackstop.ts

@@ -4,8 +4,10 @@ import type { RunStatusData } from "~/routes/resources.orgs.$organizationSlug.pr
 
 const DEFAULT_POLL_MS = 15_000;
 
-// Polls the run status until it finishes, then calls `onFinished` once. Failed polls
-// are ignored and retried on the next interval.
+// Polls the run status and calls `onFinished` on each poll that sees the run finished,
+// until the caller disables it. The reload `onFinished` triggers can read a lagging
+// replica or be interrupted, so polling continues until the page data shows the run
+// finished. Failed polls are ignored and retried on the next interval.
 export function useRunStatusBackstop({
   enabled,
   statusPath,
@@ -37,8 +39,6 @@ export function useRunStatusBackstop({
       if (stopped || fetchError || !response.ok) return;
       const [parseError, data] = await tryCatch(response.json() as Promise<RunStatusData>);
       if (stopped || parseError || (!data.isFinished && data.completedAt === null)) return;
-      stopped = true;
-      window.clearInterval(id);
       onFinishedRef.current();
     };
     const id = window.setInterval(() => void poll(), pollMs);

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -224,7 +224,7 @@ export class RunPresenter {
     const firstChunkPromise = progressivePromise.then((progressiveEnabled) =>
       isRootRunView &&
       progressiveEnabled &&
-      canLiveTail(env.INCREMENTAL_LIVE_TAIL_ENABLED, run.taskEventStore)
+      canLiveTail(run.taskEventStore, env.TRACE_VIEW_EMERGENCY_SPAN_CAP)
         ? repository.getTraceChunk(
             getTaskEventStoreTableForRun(run),
             environment.id,
@@ -252,39 +252,6 @@ export class RunPresenter {
       isAdmin: user?.admin ?? false,
     };
 
-    let totalSpans: number | undefined;
-    if (isRootRunView && firstChunk?.hasMore) {
-      const spanCount = await repository.getTraceSpanCount(
-        getTaskEventStoreTableForRun(run),
-        environment.id,
-        run.traceId,
-        traceTimeBounds.startCreatedAt,
-        traceTimeBounds.endCreatedAt,
-        { includeDebugLogs: showDebug }
-      );
-      totalSpans = typeof spanCount === "number" ? spanCount : undefined;
-
-      if (typeof spanCount === "number" && spanCount > repository.maximumTraceViewCount) {
-        return {
-          run: runData,
-          trace: {
-            events: [],
-            duration: 0,
-            rootStartedAt: undefined,
-            rootSpanStatus: "completed" as const,
-            startedAt: run.startedAt,
-            queuedDuration,
-            overridesBySpanId: {},
-            linkedRunIdBySpanId: {},
-            isTruncated: true,
-            missingAnchor: true,
-            progressive: undefined,
-          },
-          maximumLiveReloadingSetting: repository.maximumLiveReloadingSetting,
-        };
-      }
-    }
-
     if (firstChunk && hasWriteTimes(firstChunk.events)) {
       const firstEvents = stripAdminOnlyEventRows(firstChunk.events, buildOptions.isAdmin);
 
@@ -334,7 +301,6 @@ export class RunPresenter {
               hasMore: firstChunk.hasMore,
               buildOptions,
               showDebug,
-              totalSpans,
               maxSpans: repository.maximumTraceViewCount,
               liveTailEnabled: true,
               firstChunkReadAt,

apps/webapp/app/presenters/v3/TraceChunkPresenter.server.ts

@@ -38,7 +38,7 @@ export class TraceChunkPresenter {
     filter?: "errors";
     /** Live tail: only return rows written at/after this write time (ms since epoch). */
     tailInsertedAtSinceMs?: number;
-  }): Promise<TraceChunk | undefined> {
+  }): Promise<(TraceChunk & { isTruncated?: boolean }) | undefined> {
     const run = await runStore.findRun(
       { friendlyId: runFriendlyId },
       {
@@ -98,21 +98,22 @@ export class TraceChunkPresenter {
     const endCreatedAt = run.completedAt ?? undefined;
 
     if (filter === "errors") {
-      const events = await repository.getTraceErrorEvents(
+      const errors = await repository.getTraceErrorEvents(
         storeTable,
         environment.id,
         run.traceId,
         startCreatedAt,
         endCreatedAt,
         { includeDebugLogs: showDebug }
       );
-      if (!events) {
+      if (!errors) {
         return undefined;
       }
       return {
-        events: stripAdminOnlyEventRows(events, isAdmin),
+        events: stripAdminOnlyEventRows(errors.events, isAdmin),
         nextCursor: null,
         hasMore: false,
+        isTruncated: errors.isTruncated,
       };
     }
 

apps/webapp/app/presenters/v3/liveTailGate.ts

@@ -1,8 +1,8 @@
-// Progressive loading only runs with the live tail: without it every stream ping would
-// reset and re-download the tree. The env switch turns both off, and only the v2 store
-// has the write time the tail pages by.
-export function canLiveTail(envSwitch: string, taskEventStore: string): boolean {
-  return envSwitch === "1" && taskEventStore === "clickhouse_v2";
+// Progressive loading only runs with the live tail (without it every stream ping would
+// reset and re-download the tree), and only the v2 store has the write time it pages by.
+// The emergency span cap turns both off, so new loads take the summary path that honours it.
+export function canLiveTail(taskEventStore: string, emergencySpanCap: number | undefined): boolean {
+  return taskEventStore === "clickhouse_v2" && emergencySpanCap === undefined;
 }
 
 export function hasWriteTimes(events: { insertedAt?: string }[]): boolean {

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx

@@ -104,10 +104,9 @@ import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.se
 import { buildSyntheticRunHeader } from "~/v3/mollifier/syntheticRunHeader.server";
 import { buildSyntheticTraceForBufferedRun } from "~/v3/mollifier/syntheticTrace.server";
 import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
-import { getImpersonationId } from "~/services/impersonation.server";
 import { logger } from "~/services/logger.server";
 import { getResizableSnapshot } from "~/services/resizablePanel.server";
-import { requireUserId } from "~/services/session.server";
+import { requireUser } from "~/services/session.server";
 import { rbac } from "~/services/rbac.server";
 import { runAgentPageContext } from "~/components/dashboard-agent/suggested-prompts";
 import { WhenAgentUnavailable } from "~/components/dashboard-agent/WhenAgentUnavailable";
@@ -128,6 +127,7 @@ import {
 } from "~/utils/pathBuilder";
 import type { SpanOverride } from "~/v3/eventRepository/eventRepository.types";
 import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
+import { shouldRevalidateRunPage } from "./shouldRevalidateRunPage";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route";
 import { pageMeta } from "~/utils/pageTitle";
 
@@ -278,24 +278,28 @@ async function runWritePermissions(request: Request, userId: string, organizatio
   return { canReplayRun: canWriteRun, canCancelRun: canWriteRun };
 }
 
+export { shouldRevalidateRunPage as shouldRevalidate };
+
 export const handle: Handle = {
   agentPageContext: (data) => runAgentPageContext(data),
 };
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
-  const userId = await requireUserId(request);
-  const impersonationId = await getImpersonationId(request);
+  const user = await requireUser(request);
+  const userId = user.id;
   const { projectParam, organizationSlug, envParam, runParam } = v3RunParamsSchema.parse(params);
 
   const url = new URL(request.url);
-  const showDebug = url.searchParams.get("showDebug") === "true";
+  // Same rule as the trace-chunk route, so the first chunk and later chunks agree.
+  const showDebug =
+    url.searchParams.get("showDebug") === "true" && (user.admin || user.isImpersonating);
   const selectedSpanId = url.searchParams.get("span") ?? undefined;
 
   const presenter = new RunPresenter();
   const [error, result] = await tryCatch(
     presenter.call({
       userId,
-      showDeletedLogs: !!impersonationId,
+      showDeletedLogs: user.isImpersonating,
       projectSlug: projectParam,
       runFriendlyId: runParam,
       environmentSlug: envParam,
@@ -677,6 +681,7 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Trace
     linkedRunIdBySpanId,
     isComplete,
     isTruncated: progressiveIsTruncated,
+    errorsTruncated,
     loadFailed,
     tailLive,
     liveTailEnabled,
@@ -725,21 +730,26 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Trace
     }
   }, [streamedEvents, liveTailEnabled, tailLive]);
 
-  // Tail path: refresh run-row-fed controls once the root span goes terminal.
+  // The page data still shows the run unfinished, so a finish reload is useful.
+  const runDataUnfinished = !run.isFinished && run.completedAt === null;
+
+  // Tail path: refresh run-row-fed controls once the root span goes terminal, unless a
+  // reload already brought the finished run (e.g. the backstop's).
   const prevRootSpanRef = useRef({ runId: run.friendlyId, status: rootSpanStatus });
   useEffect(() => {
     const previous = prevRootSpanRef.current;
     prevRootSpanRef.current = { runId: run.friendlyId, status: rootSpanStatus };
     if (!liveTailEnabled || previous.runId !== run.friendlyId) return;
-    if (previous.status === "executing" && rootSpanStatus !== "executing") {
+    if (previous.status === "executing" && rootSpanStatus !== "executing" && runDataUnfinished) {
       revalidatorRef.current.revalidate();
     }
-  }, [run.friendlyId, rootSpanStatus, liveTailEnabled]);
+  }, [run.friendlyId, rootSpanStatus, liveTailEnabled, runDataUnfinished]);
 
-  // Tail path backstop: reconcile once the run finishes, even if the tail missed the
-  // root's final row. Polls whatever the stream is doing.
+  // Tail path backstop: reload once the run finishes, even if the tail missed the
+  // root's final row. Polls whatever the stream is doing, and keeps reloading until the
+  // page data shows the run finished (a reload can read a lagging replica).
   useRunStatusBackstop({
-    enabled: liveTailEnabled && run.completedAt === null,
+    enabled: liveTailEnabled && runDataUnfinished,
     statusPath,
     shouldSkip: () => document.visibilityState === "hidden",
     onFinished: () => {
@@ -812,6 +822,13 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Trace
                 </Callout>
               </div>
             )}
+            {errorsOnly && errorsTruncated && !traceIsTruncated && (
+              <div className="shrink-0 border-b border-grid-bright px-3 py-2">
+                <Callout variant="info" className="text-sm">
+                  This trace is large, so Errors only may not show every error.
+                </Callout>
+              </div>
+            )}
             <div className="min-h-0 flex-1">
               <TasksTreeView
                 selectedId={selectedSpanId}