feat: headless-host metrics report

rust/crates/truapi-host-cli/e2e/fleet.sh

@@ -17,6 +17,9 @@
 #   SIGNER_BOT_SVC_TOKEN bearer token; sent only if set (a local dev bot needs none)
 #   RUN_ID               shared run id (default fleet-<epoch>)
 #   METRICS_JSONL        shared metrics sink (default /tmp/fleet-metrics.jsonl)
+#   THINK_MS             base pause (ms) between signer calls in the flow script (default 0)
+#   THINK_JITTER_MS      extra uniform-random pause (ms), desyncs VUs (default 0)
+#   RAMP_JITTER          extra random seconds (0..N) added to each VU's ramp delay (default 0)
 #
 # Each VU pairs against the bot, which auto-provisions an attested user and
 # signs, so scale is bounded by the bot's per-user attestation, not the host.
@@ -29,12 +32,17 @@ BIN="$ROOT/target/debug/truapi-host"
 SCRIPT="${SCRIPT:-$ROOT/rust/crates/truapi-host-cli/js/scripts/battery.ts}"
 VUS="${VUS:-3}"
 RAMP="${RAMP:-3}"
+RAMP_JITTER="${RAMP_JITTER:-0}"
 PRODUCT_ID="${PRODUCT_ID:-headless-playground.dot}"
 BASE_PORT="${BASE_PORT:-9955}"
 BOT="${SIGNER_BOT_BASE_URL:-http://localhost:3737}"
 NETWORK="${SIGNER_BOT_NETWORK:-paseo-next-v2}"
 export RUN_ID="${RUN_ID:-fleet-$(date +%s)}"
 export METRICS_JSONL="${METRICS_JSONL:-/tmp/fleet-metrics.jsonl}"
+# Think-time knobs: exported so each VU's host process (and the script it
+# spawns) inherits them; default 0 keeps flow scripts pause-free.
+export THINK_MS="${THINK_MS:-0}"
+export THINK_JITTER_MS="${THINK_JITTER_MS:-0}"
 
 [ -x "$BIN" ] || { echo "missing $BIN — build first (cargo build -p truapi-host-cli)" >&2; exit 2; }
 
@@ -87,6 +95,7 @@ for i in $(seq 0 $((VUS - 1))); do
   run_vu "$i" "$((BASE_PORT + i))" &
   pids+=($!)
   sleep "$RAMP"
+  [ "$RAMP_JITTER" -gt 0 ] && sleep $((RANDOM % (RAMP_JITTER + 1)))
 done
 for p in "${pids[@]}"; do wait "$p" || true; done
 echo "fleet complete -> $METRICS_JSONL"

rust/crates/truapi-host-cli/js/scripts/battery.ts

@@ -13,6 +13,19 @@ export {}; // module marker so top-level `await` is allowed
 const GENESIS_HASH = `0x${"11".repeat(32)}` as const;
 const account = host.productAccount();
 
+// Load-shaping knobs (both default 0 -> think() is a no-op, byte-identical run).
+//   THINK_MS         base pause (ms) between signer-critical calls
+//   THINK_JITTER_MS  extra uniform-random pause (ms), desyncs concurrent VUs
+const THINK_MS = Number(process.env.THINK_MS) || 0;
+const THINK_JITTER_MS = Number(process.env.THINK_JITTER_MS) || 0;
+const THINK_ENABLED = THINK_MS + THINK_JITTER_MS > 0;
+
+async function think() {
+  if (!THINK_ENABLED) return;
+  const ms = THINK_MS + Math.random() * THINK_JITTER_MS;
+  await new Promise((r) => setTimeout(r, ms));
+}
+
 interface Case {
   name: string;
   ok: boolean;
@@ -40,6 +53,7 @@ if (!(login.isOk() && login.value === "Success")) {
   throw new Error("login did not succeed");
 }
 
+await think();
 await record("account.getAccount", async () => {
   const result = await truapi.account.getAccount({ productAccountId: account });
   return result.match(
@@ -51,6 +65,7 @@ await record("account.getAccount", async () => {
   );
 });
 
+await think();
 await record("signing.signRaw(bytes)", async () => {
   const result = await truapi.signing.signRaw({
     account,
@@ -65,6 +80,7 @@ await record("signing.signRaw(bytes)", async () => {
   );
 });
 
+await think();
 await record("signing.signRaw(message)", async () => {
   const result = await truapi.signing.signRaw({
     account,
@@ -76,6 +92,7 @@ await record("signing.signRaw(message)", async () => {
   );
 });
 
+await think();
 await record("signing.signPayload", async () => {
   const result = await truapi.signing.signPayload({
     account,
@@ -103,6 +120,7 @@ await record("signing.signPayload", async () => {
   );
 });
 
+await think();
 await record("signing.createTransaction", async () => {
   const result = await truapi.signing.createTransaction({
     signer: account,
@@ -120,6 +138,7 @@ await record("signing.createTransaction", async () => {
   );
 });
 
+await think();
 await record("entropy.derive", async () => {
   const result = await truapi.entropy.derive({ context: "0x6d792d6b6579" });
   return result.match(

rust/crates/truapi-host-cli/src/frame_server.rs

@@ -173,7 +173,11 @@ fn record_frame<E: std::fmt::Display>(
     let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
     let (outcome, error_class) = match result {
         Err(err) => (Outcome::Error, Some(err.to_string())),
-        Ok(()) => match outcomes.lock().ok().and_then(|mut m| m.remove(&class.request_id)) {
+        Ok(()) => match outcomes
+            .lock()
+            .ok()
+            .and_then(|mut m| m.remove(&class.request_id))
+        {
             Some(Outcome::Error) => (Outcome::Error, Some(RESPONSE_ERROR_CLASS.to_string())),
             Some(other) => (other, None),
             None => (Outcome::Success, None),

rust/crates/truapi-host-cli/src/main.rs

@@ -17,6 +17,7 @@ mod frame_server;
 mod metrics;
 mod network;
 mod platform;
+mod report;
 mod script_runner;
 
 use std::future::Future;
@@ -67,6 +68,8 @@ enum Command {
     /// specified, and can accept pairing deeplinks. With `--script`, exits with
     /// the script's status; otherwise stays interactive.
     SigningHost(SigningHostArgs),
+    /// Aggregate a metrics JSONL into one comparable report.
+    MetricsReport(MetricsReportArgs),
     /// Probe the People chain for a mnemonic's registered identity/username.
     IdentityCheck {
         /// BIP-39 mnemonic to probe.
@@ -159,6 +162,18 @@ struct SigningHostArgs {
     auto_accept: bool,
 }
 
+#[derive(clap::Args)]
+struct MetricsReportArgs {
+    /// Metrics JSONL produced by a host or fleet run (`METRICS_JSONL`).
+    file: PathBuf,
+    /// Older run to compare against; adds per-op delta columns.
+    #[arg(long)]
+    baseline: Option<PathBuf>,
+    /// Emit the structured report as JSON instead of a table.
+    #[arg(long)]
+    json: bool,
+}
+
 #[tokio::main]
 async fn main() -> Result<()> {
     // Install a rustls crypto provider so `wss://` chain connections work;
@@ -176,6 +191,28 @@ async fn main() -> Result<()> {
     match Cli::parse().command {
         Command::PairingHost(args) => run_pairing_host(args).await,
         Command::SigningHost(args) => run_signing_host(args).await,
+        Command::MetricsReport(args) => {
+            let current = report::load_report(&args.file)?;
+            let text = match args.baseline {
+                Some(baseline) => {
+                    let cmp = report::compare(current, report::load_report(&baseline)?);
+                    if args.json {
+                        serde_json::to_string_pretty(&cmp)?
+                    } else {
+                        report::render_compare_table(&cmp)
+                    }
+                }
+                None => {
+                    if args.json {
+                        serde_json::to_string_pretty(&current)?
+                    } else {
+                        report::render_table(&current)
+                    }
+                }
+            };
+            println!("{text}");
+            Ok(())
+        }
         Command::IdentityCheck { mnemonic, network } => {
             let entropy = bip39::Mnemonic::parse(mnemonic.trim())
                 .context("invalid BIP-39 mnemonic")?

rust/crates/truapi-host-cli/src/metrics.rs

@@ -9,15 +9,14 @@ use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 /// What kind of host operation a record measures.
 ///
 /// `Frame` is the coarse label for a whole product request frame at the
 /// WebSocket boundary; the fine-grained variants below are decoded from the
-/// wire id. The `#[allow(dead_code)]` covers variants not yet emitted.
-#[allow(dead_code)]
-#[derive(Debug, Clone, Copy, Serialize)]
+/// wire id.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum Category {
     Frame,
@@ -33,8 +32,7 @@ pub enum Category {
 }
 
 /// Terminal outcome of a measured operation.
-#[allow(dead_code)]
-#[derive(Debug, Clone, Copy, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum Outcome {
     Success,
@@ -43,7 +41,7 @@ pub enum Outcome {
 }
 
 /// One per-operation host metric event. Serialises to camelCase JSON.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct HostMetricRecord {
     pub ts: String,
@@ -174,10 +172,7 @@ pub fn classify_frame(frame: &[u8]) -> FrameClass {
             WireKind::Request(r) => (false, r.request_id == id || r.response_id == id),
             WireKind::Subscription(s) => (
                 true,
-                s.start_id == id
-                    || s.stop_id == id
-                    || s.interrupt_id == id
-                    || s.receive_id == id,
+                s.start_id == id || s.stop_id == id || s.interrupt_id == id || s.receive_id == id,
             ),
         };
         if matches {
@@ -230,8 +225,8 @@ pub fn response_outcome(frame: &[u8]) -> Option<(String, Outcome)> {
 }
 
 /// Map a method name to a coarse metric category by its namespace prefix.
-/// The buckets are provisional and unused in v1 (only `Frame` is emitted);
-/// revisit them when the fine-grained category path goes live.
+/// A few mappings are provisional (e.g. `statement` under Signing) and can be
+/// refined later without touching the wire.
 fn category_for_method(method: &str) -> Category {
     match method.split('_').next().unwrap_or("") {
         "signing" | "entropy" | "statement" => Category::Signing,
@@ -253,11 +248,26 @@ mod tests {
 
     #[test]
     fn category_maps_by_namespace() {
-        assert!(matches!(category_for_method("signing_sign_raw"), Category::Signing));
-        assert!(matches!(category_for_method("chain_call"), Category::ChainRpc));
-        assert!(matches!(category_for_method("local_storage_write"), Category::Storage));
-        assert!(matches!(category_for_method("account_request_login"), Category::Pairing));
-        assert!(matches!(category_for_method("mystery_method"), Category::Frame));
+        assert!(matches!(
+            category_for_method("signing_sign_raw"),
+            Category::Signing
+        ));
+        assert!(matches!(
+            category_for_method("chain_call"),
+            Category::ChainRpc
+        ));
+        assert!(matches!(
+            category_for_method("local_storage_write"),
+            Category::Storage
+        ));
+        assert!(matches!(
+            category_for_method("account_request_login"),
+            Category::Pairing
+        ));
+        assert!(matches!(
+            category_for_method("mystery_method"),
+            Category::Frame
+        ));
     }
 
     fn sample() -> HostMetricRecord {
@@ -319,7 +329,10 @@ mod tests {
         let frame = |request_id: &str, value: Vec<u8>| {
             ProtocolMessage {
                 request_id: request_id.to_string(),
-                payload: Payload { id: response_id, value },
+                payload: Payload {
+                    id: response_id,
+                    value,
+                },
             }
             .encode()
         };
@@ -329,9 +342,26 @@ mod tests {
         assert_eq!(id, "req-ok");
         assert!(matches!(outcome, Outcome::Success));
 
-        let (id, outcome) = response_outcome(&frame("req-err", encode_versioned_err_payload(0u32, 1)))
-            .expect("response frame is classified");
+        let (id, outcome) =
+            response_outcome(&frame("req-err", encode_versioned_err_payload(0u32, 1)))
+                .expect("response frame is classified");
         assert_eq!(id, "req-err");
         assert!(matches!(outcome, Outcome::Error));
     }
+
+    #[test]
+    fn record_round_trips_through_json() {
+        let line = r#"{"ts":"2026-07-20T10:00:00Z","runId":"fleet-1","vuIndex":2,"category":"signing","op":"signing_sign_raw","latencyMs":12.5,"outcome":"error","errorClass":"NoAllowance"}"#;
+        let rec: HostMetricRecord = serde_json::from_str(line).expect("deserializes");
+        assert_eq!(rec.run_id, "fleet-1");
+        assert_eq!(rec.vu_index, 2);
+        assert_eq!(rec.category, Category::Signing);
+        assert_eq!(rec.outcome, Outcome::Error);
+        assert_eq!(rec.error_class.as_deref(), Some("NoAllowance"));
+        let back = serde_json::to_string(&rec).expect("serializes");
+        assert_eq!(
+            serde_json::from_str::<HostMetricRecord>(&back).unwrap().op,
+            rec.op
+        );
+    }
 }