fix(webapp,webhook-engine,redis): recover webhook ingest when a Redis cluster primary goes silent

Webhook ingest recovers within seconds when a webhooks Redis cluster
primary fails over. The cluster client refreshes its slot map on an
interval, times out commands to a node that stopped answering (so ingest
returns 503 with `Retry-After` instead of hanging), and verifies TLS
against the configured host for nodes addressed by IP.

Mono-RevId: 5464ec3e301de8602ffa833b37c891ade0851074

Author: ericallam

apps/webapp/app/v3/webhookEngine.server.ts

@@ -27,25 +27,33 @@ import { meter, tracer } from "./tracer.server";
 
 export const webhookEngine = singleton("WebhookEngine", createWebhookEngine);
 
-/** The webhooks Redis cluster, seeded from one node; ioredis discovers the rest. */
+/**
+ * The webhooks Redis cluster, seeded from one node; ioredis discovers the rest. A primary that stops
+ * answering without closing its sockets (an ElastiCache failover reboots the old one) never replies
+ * MOVED, so the slot map is refreshed on an interval to find the promoted replica, and a command
+ * waiting on the silent node times out instead of holding an ingest request. Redirects can name a
+ * node by IP, which AWS certificates don't cover, so TLS checks every node against the seed host.
+ */
 function webhookRedisConnection(): WebhookRedisOptions {
+  const host = env.WEBHOOK_REDIS_HOST ?? "localhost";
   const redisOptions = {
     username: env.WEBHOOK_REDIS_USERNAME,
     password: env.WEBHOOK_REDIS_PASSWORD,
     enableAutoPipelining: true,
-    ...(env.WEBHOOK_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    commandTimeout: 3_000,
+    connectTimeout: 3_000,
+    ...(env.WEBHOOK_REDIS_TLS_DISABLED === "true" ? {} : { tls: { servername: host } }),
   };
   return {
     cluster: {
-      nodes: [
-        { host: env.WEBHOOK_REDIS_HOST ?? "localhost", port: env.WEBHOOK_REDIS_PORT ?? 6379 },
-      ],
+      nodes: [{ host, port: env.WEBHOOK_REDIS_PORT ?? 6379 }],
       redisOptions,
       clusterOptions: {
         keyPrefix: "webhook:",
         dnsLookup: (address: string, callback: (err: Error | null, address: string) => void) =>
           callback(null, address),
-        slotsRefreshTimeout: 10_000,
+        slotsRefreshInterval: 2_000,
+        slotsRefreshTimeout: 2_000,
       },
     },
   };

internal-packages/redis/src/index.ts

@@ -141,5 +141,11 @@ export function createRedisClusterClient(
     }
   });
 
+  // A failed slot refresh only surfaces as ClusterAllFailedError; each node's own failure, and
+  // which node it was, arrives here.
+  client.on("node error", (error: Error, node: string) => {
+    logger.warn(`Redis cluster node error:`, { node, error: error.message });
+  });
+
   return client;
 }

internal-packages/webhook-engine/src/engine/errors.test.ts

@@ -19,6 +19,7 @@ describe("isInfrastructureError", () => {
     expect(isInfrastructureError(Object.assign(new Error("x"), { code: "ECONNRESET" }))).toBe(true);
     expect(isInfrastructureError(new Error("CLUSTERDOWN The cluster is down"))).toBe(true);
     expect(isInfrastructureError(new Error("Connection is closed."))).toBe(true);
+    expect(isInfrastructureError(new Error("Command timed out"))).toBe(true);
     expect(isInfrastructureError(new Error("None of startup nodes is available"))).toBe(true);
     expect(
       isInfrastructureError(

internal-packages/webhook-engine/src/engine/errors.ts

@@ -36,7 +36,7 @@ const NETWORK_ERROR_CODES = new Set([
  * redirections. Last error: <Name>: <message>`, so that wrapper counts too.
  */
 const REDIS_UNAVAILABLE =
-  /^(?:Too many Cluster redirections\. Last error: \w*Error: )?(?:(?:CLUSTERDOWN|TRYAGAIN|LOADING|READONLY|MASTERDOWN|OOM|MOVED|ASK)\b.*|Connection is closed\.|Stream isn't writeable and enableOfflineQueue options is false|Failed to refresh slots cache\.|None of startup nodes is available)$/s;
+  /^(?:Too many Cluster redirections\. Last error: \w*Error: )?(?:(?:CLUSTERDOWN|TRYAGAIN|LOADING|READONLY|MASTERDOWN|OOM|MOVED|ASK)\b.*|Connection is closed\.|Command timed out|Stream isn't writeable and enableOfflineQueue options is false|Failed to refresh slots cache\.|None of startup nodes is available)$/s;
 
 /**
  * Whether an error, or anything in its cause chain, is an infrastructure failure. Everything else,