Add socketioxide (Rust Socket.io) as a comparison target

.gitignore

@@ -18,3 +18,4 @@ anycable-go-darwin-*
 anycable-go-windows-*
 anycable-go-mrb-*
 anycable-thruster-*
+.DS_Store

README.md

@@ -8,6 +8,8 @@ The repo behind [anycable.io/compare/nodejs-websocket](https://anycable.io/compa
 
 Setups under test: default Socket.io, Socket.io + Connection State Recovery, uWebSockets.js, AnyCable OSS, AnyCable Pro.
 
+Sixth target, [socketioxide](https://github.com/totodore/socketioxide) (Rust Socket.io server), benchmarked head-to-head with AnyCable on the same Railway hardware. Results in [its own section below](#socketioxide-rust-socketio); deep dive in [`docs/socketioxide-comparison.md`](./docs/socketioxide-comparison.md).
+
 Methodology, traps, and the bugs we caught in our own setup: [`docs/methodology.md`](./docs/methodology.md). Below: the numbers and how to rerun them.
 
 ## Headlines
@@ -77,6 +79,37 @@ Three knobs that shape the numbers. Full reasoning in [`docs/methodology.md`](./
 - **CSR runs with the in-memory adapter.** Simplest opt-in path. Redis Streams or MongoDB shift the tail by adding network RTT; structural picture holds. CSR is documented as incompatible with the Redis pub/sub adapter, so the "Redis adapter" most teams reach for first is the one CSR can't use.
 - **AnyCable's jitter-row RAM is the tradeoff for parallel replay.** Its history buffer is per-stream so `history` parallelises across streams; that costs more RAM during jittery runs. Page-level RAM-per-connection comes from the idle test, where the per-connection footprint is what's measured.
 
+## socketioxide (Rust Socket.io)
+
+[socketioxide](https://github.com/totodore/socketioxide) speaks the Socket.io wire protocol in Rust. Its author asked us to benchmark it, so we ran it head-to-head with AnyCable on the same Railway hardware, AnyCable alongside as a same-window control. It answers a sharp question: which of Socket.io's problems are about the runtime language, and which are about the architecture?
+
+**Latency (steady network).** Comparable to AnyCable. Nothing a user feels.
+
+| | 1K p50 / p99 | 10K p50 / p99 | Delivery |
+| --- | --- | --- | --- |
+| socketioxide | 23 / 66 ms | 289 / 972 ms | 100% |
+| AnyCable OSS | 16 / 46 ms | 232 / 731 ms | 100% |
+
+**Delivery under jitter.** At-most-once, no replay. In the Socket.io band to 1K, then it falls off a cliff.
+
+| Clients | socketioxide | AnyCable |
+| --- | --- | --- |
+| 200 (local) | 91.6% | 100% |
+| 1,000 | 89.4% | 100% |
+| 10,000 | **41% then 33%** (two runs) | 100% |
+
+**Avalanche (app deploy).** Every connection dies on the deploy (in-process WS goes down with the app), and recovery collapses at scale, tracking Node Socket.io almost exactly.
+
+| Clients | Reconnected | Recovery |
+| --- | --- | --- |
+| 5,000 | 100% | 2.9 s |
+| 10,000 | 96% | 67 s |
+| 20,000 | **0%** | never |
+
+**Idle capacity.** Held **600K+** idle connections at ~37 KB each (comparable to AnyCable's ~39 KB), roughly 5x past Node Socket.io's ~120K event-loop ceiling. We could not find its true ceiling: the load-generation fleet caps near ~12K connections per shard (ephemeral ports), so the harness ran out before socketioxide did.
+
+**The takeaway.** Rust fixes Socket.io's capacity ceiling, the single-event-loop wall that caps Node around 120K. It leaves two things untouched: at-most-once delivery (no replay protocol) and deploy fragility (the WS layer still dies with its app). Both live in the protocol and the topology, so swapping the language to Rust leaves them intact, and socketioxide collapses under jitter and deploy storms at scale the same way Node Socket.io does. AnyCable holds 100% on both because the WS layer is a separate process with replay. Full numbers and the deploy story: [`docs/socketioxide-comparison.md`](./docs/socketioxide-comparison.md).
+
 ## Repository layout
 
 ```

backend/results/socketioxide-local-2026-06-23.json

@@ -0,0 +1,23 @@
+{
+  "run": "socketioxide vs anycable-go, local head-to-head (control canary)",
+  "date": "2026-06-23T01:23:00Z",
+  "environment": {
+    "host": "local single machine (macOS, M-series)",
+    "scale": "small (200 clients) — NOT Railway 10K headline scale",
+    "note": "AnyCable run in the same window as a control; its expected shape (100% delivery, multi-second replay tail under jitter) confirms the environment was sound for the socketioxide measurement.",
+    "socketioxide": "0.18.4 (rustc 1.95.0, release build, features v4+tracing+state)",
+    "anycable_go": "1.6.14 (--broker=memory --presets=broker)",
+    "node": "v26.0.0",
+    "publish_path": "per-message HTTP POST to /_broadcast (symmetric for both)"
+  },
+  "latency": {
+    "params": { "n": 200, "totalMessages": 100, "intervalMs": 200, "durationSec": 60, "jitter": "disabled" },
+    "socketioxide": { "deliveryRatePct": 100, "lost": 0, "p50": 4, "p95": 12, "p99": 18, "max": 27 },
+    "anycable":     { "deliveryRatePct": 100, "lost": 0, "p50": 5, "p95": 12, "p99": 22, "max": 29 }
+  },
+  "jitter": {
+    "params": { "n": 200, "totalMessages": 60, "intervalMs": 500, "durationSec": 90, "jitterIntervalSec": 15, "jitterDurationMs": 1000 },
+    "socketioxide": { "deliveryRatePct": 91.61, "lost": 928, "received": 10993, "jitterEvents": 872, "p50": 4, "p95": 12, "p99": 16, "max": 26, "protocol": "at-most-once (no replay)" },
+    "anycable":     { "deliveryRatePct": 100,   "lost": 0,   "received": 12000, "jitterEvents": 931, "p50": 7, "p95": 3621, "p99": 5623, "max": 8455, "protocol": "replay (history on reconnect)" }
+  }
+}

backend/results/socketioxide-railway-2026-06-23.json

@@ -0,0 +1,95 @@
+{
+  "run": "socketioxide vs anycable-go OSS, Railway phase 1 (10K)",
+  "date": "2026-06-23",
+  "infra": "Railway project gentle-commitment/production; bench-runner over internal network; anycable-go OSS as same-window canary (held expected shape on every test)",
+  "socketioxide": "0.18.4 (rust:1-slim release, [::]:3000)",
+  "anycable_go": "1.6.14 (broker=memory, presets=broker)",
+  "latency": {
+    "1k": {
+      "socketioxide": {
+        "deliv": 100,
+        "p50": 23,
+        "p99": 66
+      },
+      "anycable": {
+        "deliv": 100,
+        "p50": 16,
+        "p99": 46
+      }
+    },
+    "10k": {
+      "socketioxide": {
+        "deliv": 100,
+        "p50": 289,
+        "p99": 972,
+        "max": 3627
+      },
+      "anycable": {
+        "deliv": 100,
+        "p50": 232,
+        "p99": 731,
+        "max": 7798
+      }
+    }
+  },
+  "jitter_delivery_pct": {
+    "200_local": {
+      "socketioxide": 91.6,
+      "anycable": 100
+    },
+    "1k_railway": {
+      "socketioxide": 89.4,
+      "anycable": 100
+    },
+    "10k_railway": {
+      "socketioxide_run1": 40.6,
+      "socketioxide_run2": 32.7,
+      "anycable": 100
+    }
+  },
+  "finding": "socketioxide is at-most-once: in the band with default Socket.io / uWS up to 1K, then collapses under the 10K reconnect storm (41% then 33%, reproducible, no crash, 0 connect failures). AnyCable holds 100% (separate process + replay). Rust runtime does not rescue the in-process at-most-once architecture at scale.",
+  "idle_600k": {
+    "note": "harness-limited at ~600K (shards capped ~12K each); neither target saturated",
+    "socketioxide": {
+      "connected": 600091,
+      "peakMemGB": 21.4,
+      "ramKbPerConn": 37,
+      "peakCpuPct": 1.8
+    },
+    "anycable_go": {
+      "connected": 600084,
+      "peakMemGB": 22.1,
+      "ramKbPerConn": 39,
+      "peakCpuPct": 9.0
+    },
+    "finding": "socketioxide held 600K+ (5x past Node Socket.io ~120K event-loop ceiling); RAM/conn comparable to anycable. Rust runtime fixes the capacity wall, not the at-most-once delivery limit."
+  },
+  "avalanche": {
+    "5k": {
+      "reconnectPct": 100,
+      "recoveryMs": 2922,
+      "neverBack": 0
+    },
+    "10k": {
+      "reconnectPct": 96,
+      "recoveryMs": 67271,
+      "neverBack": 411
+    },
+    "20k": {
+      "reconnectPct": 0,
+      "recoveryMs": 610836,
+      "neverBack": "all",
+      "note": "client side capped ~12K; 0% recovery unambiguous"
+    },
+    "finding": "socketioxide tracks Node Socket.io avalanche cliff: 5K recovers, 10K ~67s/96%, 20K collapses to 0%. In-process WS dies on deploy regardless of language."
+  },
+  "idle_1M_attempt": {
+    "per_shard_cap": 12002,
+    "cap_cause": "ephemeral-port exhaustion to single host:port from one source IP (not memory; nofile=122880)",
+    "fleet_grown_to": 85,
+    "theoretical_capacity": 1020000,
+    "result": "49/85 shards clean (588000, 0 failures); 36 shards errored/timed out under coordinator fan-out",
+    "conclusion": "harness-limited, not server-limited; socketioxide held every connection thrown (588K, 0 failures) with memory headroom. True ceiling unmeasured; needs more source IPs or a lighter idle client."
+  },
+  "teardown": "all 87 services (85 shards + anycable-go + socketioxide-server) stopped; both targets downsized 32GB->0.5GB/1vCPU; verified offline"
+}

backend/src/bench/tests-manifest.ts

@@ -93,6 +93,14 @@ const TARGETS = {
   anycablePro: "ws://anycable-go-pro.railway.internal:8080/cable",
   anycableProBroadcast:
     "http://anycable-go-pro.railway.internal:8080/_broadcast",
+  // socketioxide is the Rust implementation of the Socket.io protocol.
+  // Wire-compatible with socket.io-client, so the existing bench-runner
+  // bench-jitter-socketio / bench-idle-socketio / bench-avalanche-socketio
+  // endpoints work with ?serverUrl=. No CSR variant yet — the library
+  // doesn't appear to ship Connection State Recovery as of 0.18.3.
+  // See docs/socketioxide-comparison.md for the open question to the
+  // library author.
+  socketioxide: "http://socketioxide-server.railway.internal:3000",
 };
 
 // Common knobs reused across tests. Keep these explicit so the manifest
@@ -202,6 +210,27 @@ export const tests: TestSpec[] = [
     params: { n: 10000, ...LATENCY_10K, cableUrl: TARGETS.anycablePro, broadcastUrl: TARGETS.anycableProBroadcast },
     baseline: { "latencyRawMs.p50": 234, "latencyRawMs.p99": 694, deliveryRatePct: 100 },
   },
+  // socketioxide: same Socket.io wire protocol, Rust implementation. Uses
+  // the bench-jitter-socketio endpoint with ?serverUrl=<socketioxide>. No
+  // baselines yet — first run pending. See docs/socketioxide-comparison.md.
+  {
+    id: "latency-socketioxide-1k",
+    description: "Roundtrip latency, socketioxide (Rust), 1K subs",
+    category: "latency",
+    endpoint: "bench-jitter-socketio",
+    mode: "sync",
+    params: { n: 1000, ...LATENCY_1K, serverUrl: TARGETS.socketioxide },
+    baseline: {},
+  },
+  {
+    id: "latency-socketioxide-10k",
+    description: "Roundtrip latency, socketioxide (Rust), 10K subs",
+    category: "latency",
+    endpoint: "bench-jitter-socketio",
+    mode: "sync",
+    params: { n: 10000, ...LATENCY_10K, serverUrl: TARGETS.socketioxide },
+    baseline: {},
+  },
 
   // -------------------------------------------------------------------------
   // Reliability (jitter under WiFi-drop pattern)
@@ -265,6 +294,18 @@ export const tests: TestSpec[] = [
     params: { n: 10000, ...JITTER_10K, cableUrl: TARGETS.anycablePro, broadcastUrl: TARGETS.anycableProBroadcast, samplesCap: 5000 },
     baseline: { deliveryRatePct: 100, lostDeliveries: 0, "latencyRawMs.p95": 4100, "latencyRawMs.p99": 6200 },
   },
+  // socketioxide jitter row. Expected to land in the at-most-once band
+  // with default Socket.io and uWS, since socketioxide doesn't appear to
+  // ship CSR. Confirms the architectural claim across runtimes.
+  {
+    id: "jitter-socketioxide-10k",
+    description: "Reliability under WiFi jitter, socketioxide (Rust), 10K",
+    category: "jitter",
+    endpoint: "bench-jitter-socketio",
+    mode: "async",
+    params: { n: 10000, ...JITTER_10K, serverUrl: TARGETS.socketioxide, samplesCap: 5000 },
+    baseline: {},
+  },
 
   // -------------------------------------------------------------------------
   // Whispers (1K × 10 rooms, 100 peers/room)
@@ -441,6 +482,20 @@ export const tests: TestSpec[] = [
     baseline: { connected: 1000000, ramKbPerConnected: 5 },
     driftThresholdPct: 60,
   },
+  // socketioxide idle: same multi-shard fan-out, targets the Rust service.
+  {
+    id: "idle-socketioxide",
+    description: "Idle connections held, socketioxide (Rust), 1M target",
+    category: "idle",
+    endpoint: "bench-idle-socketio",
+    mode: "multi-shard",
+    numShards: 50,
+    perShardN: 20000,
+    params: { hold: 120, ramp: 200, stream: "idle-rebaseline", serverUrl: TARGETS.socketioxide },
+    targetServiceId: "41f1ac22-2ea6-4d04-974e-4c148be426ff",
+    baseline: {},
+    driftThresholdPct: 60,
+  },
 
   // -------------------------------------------------------------------------
   // Avalanche (in-process WS layer restart under N held connections).
@@ -536,4 +591,42 @@ export const tests: TestSpec[] = [
     baseline: { reconnectRatePct: 0 },
     driftThresholdPct: 100,
   },
+  // socketioxide avalanche escalation: mirror the Socket.io ladder (5K, 10K,
+  // 15K, 20K, 25K). Same redeploy mechanism, just pointed at the Rust service.
+  // The interesting question is whether Rust's event loop pushes the cliff out
+  // further than Node's, or whether the architectural problem (in-process WS
+  // dies with the app) holds the shape across languages.
+  {
+    id: "avalanche-socketioxide-5k",
+    description: "Avalanche: 5K socketioxide clients, server redeploy",
+    category: "avalanche",
+    endpoint: "bench-avalanche-socketio",
+    mode: "avalanche",
+    redeployServiceName: "socketioxide-server",
+    params: { n: 5000, ramp: 200, prearm: 90, recoveryWait: 180, stream: "avalanche-sox-5k", serverUrl: TARGETS.socketioxide },
+    baseline: {},
+    driftThresholdPct: 100,
+  },
+  {
+    id: "avalanche-socketioxide-10k",
+    description: "Avalanche: 10K socketioxide clients, server redeploy",
+    category: "avalanche",
+    endpoint: "bench-avalanche-socketio",
+    mode: "avalanche",
+    redeployServiceName: "socketioxide-server",
+    params: { n: 10000, ramp: 200, prearm: 120, recoveryWait: 240, stream: "avalanche-sox-10k", serverUrl: TARGETS.socketioxide },
+    baseline: {},
+    driftThresholdPct: 100,
+  },
+  {
+    id: "avalanche-socketioxide-20k",
+    description: "Avalanche: 20K socketioxide clients, server redeploy",
+    category: "avalanche",
+    endpoint: "bench-avalanche-socketio",
+    mode: "avalanche",
+    redeployServiceName: "socketioxide-server",
+    params: { n: 20000, ramp: 200, prearm: 240, recoveryWait: 600, stream: "avalanche-sox-20k", serverUrl: TARGETS.socketioxide },
+    baseline: {},
+    driftThresholdPct: 100,
+  },
 ];