improvement(webhooks): cut inline dispatch latency + add trigger-age instrumentation

- Skip the redundant second preprocessExecution pass for inline (Slack/generic)
  webhook execution: thread the route's already-resolved workflowRecord +
  executionTimeout into the in-process job (PreprocessedWebhookContext) so it
  reuses them instead of redoing ~4 DB reads. Scoped to the in-process inline
  call only; the persisted/polling Trigger.dev payload still re-validates fresh.
- Add dispatch-latency / trigger-age instrumentation: capture webhook receipt
  time + Slack x-slack-request-timestamp at the route and log structured
  dispatchLatencyMs + triggerAgeMs, surfacing the pre-execution latency that
  per-block timings cannot see (Slack trigger_id expires at 3s).
- Guard the effective-env fetch in verifyProviderAuth to only run when the
  handler verifies auth AND the providerConfig references env vars ({{VAR}}),
  avoiding a needless DB read/decrypt on the synchronous pre-ack path.

Author: waleedlatif1

apps/sim/app/api/webhooks/trigger/[path]/route.ts

@@ -69,6 +69,17 @@ async function handleWebhookPost(
   request: NextRequest,
   context: { params: Promise<{ path: string }> }
 ): Promise<NextResponse> {
+  const receivedAt = Date.now()
+  /**
+   * Slack signs every interactive request with the originating interaction time.
+   * Capturing it lets the executor surface the true trigger_id age (the window
+   * that expires at 3s) instead of only the in-workflow block timings.
+   */
+  const slackRequestTimestamp = request.headers.get('x-slack-request-timestamp')
+  const triggerTimestampMs = slackRequestTimestamp
+    ? Number(slackRequestTimestamp) * 1000
+    : undefined
+
   const requestId = generateRequestId()
   const parsed = await parseRequest(webhookTriggerPostContract, request, context)
   if (!parsed.success) return parsed.response
@@ -200,6 +211,9 @@ async function handleWebhookPost(
       actorUserId: preprocessResult.actorUserId,
       executionId: preprocessResult.executionId,
       correlation: preprocessResult.correlation,
+      receivedAt,
+      triggerTimestampMs: Number.isFinite(triggerTimestampMs) ? triggerTimestampMs : undefined,
+      preprocessed: preprocessResult.preprocessed,
     })
     responses.push(response)
   }

apps/sim/background/webhook-execution.ts

@@ -18,6 +18,7 @@ import {
   WebhookAttachmentProcessor,
 } from '@/lib/webhooks/attachment-processor'
 import { resolveWebhookRecordProviderConfig } from '@/lib/webhooks/env-resolver'
+import type { PreprocessedWebhookContext } from '@/lib/webhooks/processor'
 import { getProviderHandler } from '@/lib/webhooks/providers'
 import {
   executeWorkflowCore,
@@ -236,6 +237,12 @@ export type WebhookExecutionPayload = {
   blockId?: string
   workspaceId?: string
   credentialId?: string
+  /** Epoch ms when the webhook HTTP request was first received (for dispatch-latency metrics). */
+  webhookReceivedAt?: number
+  /** Epoch ms of the originating provider interaction (e.g. Slack x-slack-request-timestamp). */
+  triggerTimestampMs?: number
+  /** Route-level preprocessing result; when present the job skips the redundant second pass. */
+  preprocessed?: PreprocessedWebhookContext
 }
 
 export async function executeWebhookJob(payload: WebhookExecutionPayload) {
@@ -350,25 +357,43 @@ async function executeWebhookJobInternal(
     requestId
   )
 
-  const preprocessResult = await preprocessExecution({
-    workflowId: payload.workflowId,
-    userId: payload.userId,
-    triggerType: 'webhook',
-    executionId,
-    requestId,
-    triggerData: { correlation },
-    checkRateLimit: false,
-    checkDeployment: false,
-    skipUsageLimits: true,
-    workspaceId: payload.workspaceId,
-    loggingSession,
-  })
+  /**
+   * The webhook route already ran full preprocessing (workflow validation,
+   * billing actor, ban/usage/rate-limit gates, admission reservation) moments
+   * ago and handed back the resolved record + timeout. Reuse it to skip a
+   * redundant second pass (≈4 DB reads) on the latency-critical dispatch path.
+   * Fall back to a fresh pass when the prefetched context is absent (e.g. other
+   * callers, or a payload that predates this field).
+   */
+  let workflowRecord: PreprocessedWebhookContext['workflowRecord'] | undefined
+  let executionTimeout: PreprocessedWebhookContext['executionTimeout'] | undefined
+
+  if (payload.preprocessed) {
+    workflowRecord = payload.preprocessed.workflowRecord
+    executionTimeout = payload.preprocessed.executionTimeout
+  } else {
+    const preprocessResult = await preprocessExecution({
+      workflowId: payload.workflowId,
+      userId: payload.userId,
+      triggerType: 'webhook',
+      executionId,
+      requestId,
+      triggerData: { correlation },
+      checkRateLimit: false,
+      checkDeployment: false,
+      skipUsageLimits: true,
+      workspaceId: payload.workspaceId,
+      loggingSession,
+    })
 
-  if (!preprocessResult.success) {
-    throw new Error(preprocessResult.error?.message || 'Preprocessing failed in background job')
+    if (!preprocessResult.success) {
+      throw new Error(preprocessResult.error?.message || 'Preprocessing failed in background job')
+    }
+
+    workflowRecord = preprocessResult.workflowRecord
+    executionTimeout = preprocessResult.executionTimeout
   }
 
-  const { workflowRecord, executionTimeout } = preprocessResult
   if (!workflowRecord) {
     throw new Error(`Workflow ${payload.workflowId} not found during preprocessing`)
   }
@@ -564,6 +589,25 @@ async function executeWebhookJobInternal(
 
     const triggerInput = input || {}
 
+    /**
+     * Surface the pre-execution latency that per-block timings cannot see: the
+     * gap between webhook receipt and the first block running, and — for
+     * trigger_id-bound providers like Slack — the true age of the interaction
+     * against its 3s expiry window. Logged structured so it is queryable/alarmable.
+     */
+    if (payload.webhookReceivedAt !== undefined || payload.triggerTimestampMs !== undefined) {
+      const now = Date.now()
+      logger.info(`[${requestId}] Webhook dispatch latency`, {
+        workflowId: payload.workflowId,
+        provider: payload.provider,
+        dispatchLatencyMs:
+          payload.webhookReceivedAt !== undefined ? now - payload.webhookReceivedAt : undefined,
+        triggerAgeMs:
+          payload.triggerTimestampMs !== undefined ? now - payload.triggerTimestampMs : undefined,
+        reusedPreprocessing: Boolean(payload.preprocessed),
+      })
+    }
+
     const snapshot = new ExecutionSnapshot(
       metadata,
       workflowRecord,

apps/sim/lib/webhooks/processor.ts

@@ -32,20 +32,37 @@ import { isPollingWebhookProvider } from '@/triggers/constants'
 
 const logger = createLogger('WebhookProcessor')
 
+/**
+ * Preprocessing outputs resolved at the route level (active workflow record and
+ * resolved execution timeout) that the background job can reuse to avoid a
+ * redundant second {@link preprocessExecution} pass on the dispatch path.
+ */
+export interface PreprocessedWebhookContext {
+  workflowRecord: typeof workflow.$inferSelect
+  executionTimeout: { sync: number; async: number }
+}
+
 export interface WebhookProcessorOptions {
   requestId: string
   path?: string
   webhookId?: string
   actorUserId?: string
   executionId?: string
   correlation?: AsyncExecutionCorrelation
+  /** Epoch ms when the webhook HTTP request was first received (for dispatch-latency metrics). */
+  receivedAt?: number
+  /** Epoch ms of the originating provider interaction (e.g. Slack x-slack-request-timestamp). */
+  triggerTimestampMs?: number
+  /** Route-level preprocessing result reused by the background job. */
+  preprocessed?: PreprocessedWebhookContext
 }
 
 export interface WebhookPreprocessingResult {
   error: NextResponse | null
   actorUserId?: string
   executionId?: string
   correlation?: AsyncExecutionCorrelation
+  preprocessed?: PreprocessedWebhookContext
 }
 
 async function verifyCredentialSetBilling(credentialSetId: string): Promise<{
@@ -406,6 +423,16 @@ function resolveEnvVars(value: string, envVars: Record<string, string>): string
   return resolveEnvVarReferences(value, envVars) as string
 }
 
+/** True when any string value in the provider config contains an env-var reference (`{{VAR}}`). */
+function providerConfigReferencesEnvVars(config: Record<string, unknown>): boolean {
+  for (const value of Object.values(config)) {
+    if (typeof value === 'string' && value.includes('{{')) {
+      return true
+    }
+  }
+  return false
+}
+
 function resolveProviderConfigEnvVars(
   config: Record<string, unknown>,
   envVars: Record<string, string>
@@ -432,22 +459,30 @@ export async function verifyProviderAuth(
   rawBody: string,
   requestId: string
 ): Promise<NextResponse | null> {
+  const handler = getProviderHandler(foundWebhook.provider)
+  const rawProviderConfig = (foundWebhook.providerConfig as Record<string, unknown>) || {}
+
+  /**
+   * Only fetch + decrypt the effective env when there is auth to verify AND the
+   * provider config actually references env vars (`{{VAR}}`). This avoids a DB
+   * read and decryption on the synchronous pre-ack path for the common case.
+   */
   let decryptedEnvVars: Record<string, string> = {}
-  try {
-    decryptedEnvVars = await getEffectiveDecryptedEnv(
-      foundWorkflow.userId,
-      foundWorkflow.workspaceId
-    )
-  } catch (error) {
-    logger.error(`[${requestId}] Failed to fetch environment variables`, {
-      error,
-    })
+  if (handler.verifyAuth && providerConfigReferencesEnvVars(rawProviderConfig)) {
+    try {
+      decryptedEnvVars = await getEffectiveDecryptedEnv(
+        foundWorkflow.userId,
+        foundWorkflow.workspaceId
+      )
+    } catch (error) {
+      logger.error(`[${requestId}] Failed to fetch environment variables`, {
+        error,
+      })
+    }
   }
 
-  const rawProviderConfig = (foundWebhook.providerConfig as Record<string, unknown>) || {}
   const providerConfig = resolveProviderConfigEnvVars(rawProviderConfig, decryptedEnvVars)
 
-  const handler = getProviderHandler(foundWebhook.provider)
   if (handler.verifyAuth) {
     const authResult = await handler.verifyAuth({
       webhook: foundWebhook,
@@ -515,6 +550,13 @@ export async function checkWebhookPreprocessing(
       actorUserId: preprocessResult.actorUserId,
       executionId,
       correlation,
+      preprocessed:
+        preprocessResult.workflowRecord && preprocessResult.executionTimeout
+          ? {
+              workflowRecord: preprocessResult.workflowRecord,
+              executionTimeout: preprocessResult.executionTimeout,
+            }
+          : undefined,
     }
   } catch (preprocessError) {
     logger.error(`[${requestId}] Error during webhook preprocessing:`, preprocessError)
@@ -611,6 +653,10 @@ export async function queueWebhookExecution(
       blockId: foundWebhook.blockId,
       workspaceId: foundWorkflow.workspaceId,
       ...(credentialId ? { credentialId } : {}),
+      ...(options.receivedAt !== undefined ? { webhookReceivedAt: options.receivedAt } : {}),
+      ...(options.triggerTimestampMs !== undefined
+        ? { triggerTimestampMs: options.triggerTimestampMs }
+        : {}),
     }
 
     const isPolling = isPollingWebhookProvider(payload.provider)
@@ -641,10 +687,20 @@ export async function queueWebhookExecution(
         `[${options.requestId}] Queued ${foundWebhook.provider} webhook execution ${jobId} via inline backend`
       )
 
+      /**
+       * Reuse the route's preprocessing ONLY for this in-process execution, where
+       * no time passes and no serialization occurs. The enqueued/persisted copy
+       * (and the polling Trigger.dev path) deliberately omit it so any deferred
+       * re-run still performs a fresh ban/archived re-validation.
+       */
+      const inlinePayload = options.preprocessed
+        ? { ...payload, preprocessed: options.preprocessed }
+        : payload
+
       void (async () => {
         try {
           await jobQueue.startJob(jobId)
-          const output = await executeWebhookJob(payload)
+          const output = await executeWebhookJob(inlinePayload)
           await jobQueue.completeJob(jobId, output)
         } catch (error) {
           const errorMessage = toError(error).message