feat(bench): add sealed holdout corpus protocol

PROJECT_STATUS.md

@@ -140,7 +140,13 @@ validation.
   OpenCode, and Antigravity configuration and execution are unsupported.
 - Linux and macOS lack live platform evidence.
 - The accepted economic results are calibrations, not a powered corpus or
-  general savings claim.
+  general savings claim. The public v4 benchmark manifest is answer-free and
+  synthetic; the offline protocol and synthetic evaluator fixture are complete,
+  but real private material, a non-synthetic validated PowerPlan, and a
+  provider run are absent. App provider execution remains fail-closed until an
+  isolated executor/broker consumes only ArmLaunch. This does not prove
+  filesystem ACLs or process isolation; callers must keep private bundles
+  unmounted and inaccessible to the runner identity.
 - No public beta, support channel, or compatibility window exists.
 
 ## Next milestone
@@ -150,8 +156,10 @@ Release readiness requires all of the following:
 1. provider-backed calibration of authoritative claim reuse;
 2. live end-to-end verified-change validation (`prepare_change` -> `verify_change` / patcher-verifier);
 3. a second live platform;
-4. a frozen multi-task corpus with independent oracles;
-5. a powered paired analysis and confidence interval;
+4. a frozen answer-free multi-task corpus bound to maintainer-owned sealed
+   evaluator material;
+5. a validated production PowerPlan, powered paired analysis, and confidence
+   interval;
 6. stable developer packaging and a compatibility policy;
 7. a dedicated security and publication review.
 

benchmarks/README.md

@@ -11,7 +11,8 @@ savings claim.
 
 ```text
 benchmarks/
-  corpus/router-cache/   frozen tasks, oracles, campaign, and cost model
+  corpus/router-cache/   answer-free manifest, schedule, synthetic plan, campaign, and cost model
+    oracles/             legacy synthetic evaluator fixtures only (not public handles)
   fixtures/              structured protocol and worker fixtures
   results/historical/    accepted observations from an earlier product boundary
   results/live/          accepted provider-backed current-path calibration
@@ -65,4 +66,12 @@ are excluded from published benchmark evidence and do not support benchmark
 claims.
 
 The evidence index records observations; it does not approve or schedule
-provider runs.
+provider runs. Public corpus material is answer-free and synthetic. The
+evaluator-owned sealed oracle index and its bytes are supplied out of band by
+maintainers; no external bundle path is present in public manifests or arm
+launch projections. A real sealed bundle and validated production PowerPlan
+are absent from this checkout, so offline preflight remains fail-closed. The
+synthetic bundle is complete for evaluator tests but cannot enable a provider
+run. A future caller must keep private evaluator material unmounted and
+inaccessible to the runner identity; this protocol does not claim ACL or
+process-isolation proof.

benchmarks/corpus/router-cache/campaign.json

@@ -1,9 +1,6 @@
 {
-  "schema": "needle.multi-task-campaign/1",
-  "task_ids": [
-    "ripgrep-glob-case-insensitive-locate-calibration",
-    "ripgrep-crlf-trace-calibration"
-  ],
+  "schema": "needle.multi-task-campaign/2",
+  "schedule_digest": "b3:c853b4ed84d8e9119013d788a7c769c63fd77f63a947706fbe4c128b65ed51d5",
   "paid_arms": [
     "frontier_direct",
     "needle_miss"
@@ -18,7 +15,6 @@
     "native_subagent",
     "escalation"
   ],
-  "repetitions_per_task": 1,
   "automatic_retries": false,
   "statistical_claim": false,
   "bootstrap_resamples": 10000,

benchmarks/corpus/router-cache/legacy-offline-campaign.json

@@ -0,0 +1,26 @@
+{
+  "schema": "needle.multi-task-campaign/1",
+  "task_ids": [
+    "ripgrep-glob-case-insensitive-locate-calibration",
+    "ripgrep-crlf-trace-calibration"
+  ],
+  "paid_arms": ["frontier_direct", "needle_miss"],
+  "offline_cache_arms": ["exact_hit", "partial_hit", "irrelevant_mutation", "relevant_mutation"],
+  "deferred_diagnostic_arms": ["native_subagent", "escalation"],
+  "repetitions_per_task": 1,
+  "automatic_retries": false,
+  "statistical_claim": false,
+  "bootstrap_resamples": 10000,
+  "one_sided_alpha_basis_points": 500,
+  "target_power_basis_points": 9000,
+  "budget_reserve": {
+    "main_turn_microcredits": 2633875,
+    "extra_main_turns_per_needle_observation": 2,
+    "worker_microcredits": 2034130,
+    "extra_workers_per_needle_observation": 0,
+    "evidence": [
+      "benchmarks/results/live/routing-and-cache-calibration.md",
+      "benchmarks/results/historical/cache-mutation-calibration.md"
+    ]
+  }
+}

benchmarks/corpus/router-cache/legacy-offline-manifest.json

@@ -0,0 +1,70 @@
+{
+  "schema": "needle.frozen-corpus/3",
+  "frozen_unix_ms": 1785412789576,
+  "arms": [
+    "frontier_direct",
+    "native_subagent",
+    "needle_miss",
+    "exact_hit",
+    "partial_hit",
+    "escalation",
+    "irrelevant_mutation",
+    "relevant_mutation"
+  ],
+  "cost_model_path": "cost-model.json",
+  "cost_model_digest": "b3:a517364a8e5cdf1af49ea728c150c2bd2cff13503888175bf5da2471cfe8a74a",
+  "next_pilot_path": "minimal-live-pilot.json",
+  "next_pilot_digest": "b3:18a434227eb9abf2381eb0a65348f0dba66301ccba7dbdeb57c7933b078180cb",
+  "campaign_path": "legacy-offline-campaign.json",
+  "campaign_digest": "b3:883d16a6d66049e7a68fe3d9d70dbf7e46186bca5f9670eecc48e52f6abc0e26",
+  "tasks": [
+    {
+      "id": "ripgrep-glob-case-insensitive-locate-calibration",
+      "route": "locate_implementation",
+      "split": "calibration",
+      "repository_url": "https://github.com/BurntSushi/ripgrep.git",
+      "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
+      "prompt": "Where is ripgrep's --glob-case-insensitive option implemented, and which focused test demonstrates its behavior? Give the primary location and only the supporting evidence needed to continue.",
+      "oracle_path": "oracles/ripgrep-glob-case-insensitive-locate.json",
+      "oracle_digest": "b3:efc9f2b932008a42393bad9145d1db1dc420bade8778889d7fcf83d6c9734b7f",
+      "test_identifier": "misc::glob_always_case_insensitive",
+      "focused_command": ["cargo", "test", "--offline", "--test", "integration", "misc::glob_always_case_insensitive", "--", "--exact"]
+    },
+    {
+      "id": "ripgrep-crlf-trace-calibration",
+      "route": "trace_state_flow",
+      "split": "calibration",
+      "repository_url": "https://github.com/BurntSushi/ripgrep.git",
+      "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
+      "prompt": "Trace how ripgrep's --crlf option changes matching and search line terminators, and name a focused test that proves the default scenario.",
+      "oracle_path": "oracles/ripgrep-crlf-trace.json",
+      "oracle_digest": "b3:db68c75eebfab64593825a1563facc1e67634ad15e32dcccbd23c83d655b5059",
+      "test_identifier": "feature::f416_crlf",
+      "focused_command": ["cargo", "test", "--offline", "--test", "integration", "feature::f416_crlf", "--", "--exact"]
+    },
+    {
+      "id": "ripgrep-no-ignore-vcs-locate-holdout",
+      "route": "locate_implementation",
+      "split": "holdout",
+      "repository_url": "https://github.com/BurntSushi/ripgrep.git",
+      "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
+      "prompt": "Find the implementation that makes ripgrep's --no-ignore-vcs option stop applying Git ignore rules, and identify a focused test for that behavior.",
+      "oracle_path": "oracles/ripgrep-no-ignore-vcs-locate.json",
+      "oracle_digest": "b3:4eac2f4e630f8a457de9bcfa1cfdd3adef60fd46e399d24ac7479c2b1906398d",
+      "test_identifier": "feature::f68_no_ignore_vcs",
+      "focused_command": ["cargo", "test", "--offline", "--test", "integration", "feature::f68_no_ignore_vcs", "--", "--exact"]
+    },
+    {
+      "id": "ripgrep-null-data-trace-holdout",
+      "route": "trace_state_flow",
+      "split": "holdout",
+      "repository_url": "https://github.com/BurntSushi/ripgrep.git",
+      "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
+      "prompt": "Trace how ripgrep's --null-data option changes matching and searching from newline-delimited input to NUL-delimited input, with a focused test for the behavior.",
+      "oracle_path": "oracles/ripgrep-null-data-trace.json",
+      "oracle_digest": "b3:e8efb461add472614a36eb920a0c0aa1857e13fe4834aa12cc2582d9e8df4661",
+      "test_identifier": "feature::f993_null_data",
+      "focused_command": ["cargo", "test", "--offline", "--test", "integration", "feature::f993_null_data", "--", "--exact"]
+    }
+  ]
+}

benchmarks/corpus/router-cache/manifest.json

@@ -1,5 +1,5 @@
 {
-  "schema": "needle.frozen-corpus/3",
+  "schema": "needle.frozen-corpus/4",
   "frozen_unix_ms": 1785412789576,
   "arms": [
     "frontier_direct",
@@ -16,91 +16,73 @@
   "next_pilot_path": "minimal-live-pilot.json",
   "next_pilot_digest": "b3:18a434227eb9abf2381eb0a65348f0dba66301ccba7dbdeb57c7933b078180cb",
   "campaign_path": "campaign.json",
-  "campaign_digest": "b3:207fb6ef4a29606c76054b3dcd27641cb10ae114c35501b7b798fb46e7c1a597",
+  "campaign_digest": "b3:082239af1ebe785555801af74c817c5e49c143759cd3e62ef6607f54dadf5e1d",
+  "schedule_path": "schedule.json",
+  "schedule_digest": "b3:c853b4ed84d8e9119013d788a7c769c63fd77f63a947706fbe4c128b65ed51d5",
+  "power_plan_path": "power-plan.json",
+  "power_plan_digest": "b3:8605f040792744907aa94d3ce017a211b774c16521dbe744acb7a1b587647630",
+  "sealed_bundle_schema": "needle.sealed-oracle-index/1",
+  "sealed_bundle_digest": "b3:3750e933fa1226fed1892dc4ba3ddad4f84fbf6bd58135d8f26355103dc681c7",
   "tasks": [
     {
       "id": "ripgrep-glob-case-insensitive-locate-calibration",
       "route": "locate_implementation",
       "split": "calibration",
       "repository_url": "https://github.com/BurntSushi/ripgrep.git",
       "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
-      "prompt": "Where is ripgrep's --glob-case-insensitive option implemented, and which focused test demonstrates its behavior? Give the primary location and only the supporting evidence needed to continue.",
-      "oracle_path": "oracles/ripgrep-glob-case-insensitive-locate.json",
-      "oracle_digest": "b3:efc9f2b932008a42393bad9145d1db1dc420bade8778889d7fcf83d6c9734b7f",
-      "test_identifier": "misc::glob_always_case_insensitive",
-      "focused_command": [
-        "cargo",
-        "test",
-        "--offline",
-        "--test",
-        "integration",
-        "misc::glob_always_case_insensitive",
-        "--",
-        "--exact"
-      ]
+      "prompt": "Where is ripgrep's glob-case-insensitive option implemented, and which focused test demonstrates its behavior? Give the primary location and only the supporting evidence needed to continue.",
+      "material_class": "synthetic",
+      "focused_test_policy": {
+        "identity": "ripgrep.synthetic.glob-case-insensitive.v1",
+        "commitment": "b3:a7d0811cef052068f9640398720e65b19e7f1a9e0990facd03e42a9e97e26f4f"
+      },
+      "oracle_schema": "needle.sealed-oracle/1",
+      "oracle_digest": "b3:7a6787e9e7aaa0bcc147e69a7dd32b226390dabc2e854ed7afa2f1d7b36c49b6"
     },
     {
       "id": "ripgrep-crlf-trace-calibration",
       "route": "trace_state_flow",
       "split": "calibration",
       "repository_url": "https://github.com/BurntSushi/ripgrep.git",
       "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
-      "prompt": "Trace how ripgrep's --crlf option changes matching and search line terminators, and name a focused test that proves the default scenario.",
-      "oracle_path": "oracles/ripgrep-crlf-trace.json",
-      "oracle_digest": "b3:db68c75eebfab64593825a1563facc1e67634ad15e32dcccbd23c83d655b5059",
-      "test_identifier": "feature::f416_crlf",
-      "focused_command": [
-        "cargo",
-        "test",
-        "--offline",
-        "--test",
-        "integration",
-        "feature::f416_crlf",
-        "--",
-        "--exact"
-      ]
+      "prompt": "Trace how ripgrep's crlf option changes matching and search line terminators, and name a focused test that proves the default scenario.",
+      "material_class": "synthetic",
+      "focused_test_policy": {
+        "identity": "ripgrep.synthetic.crlf.v1",
+        "commitment": "b3:27d2e8e1cecfba3b600bcb411ffea48547319aa3fefd6704c46dd7c13fa5c250"
+      },
+      "oracle_schema": "needle.sealed-oracle/1",
+      "oracle_digest": "b3:4e4053553759f507b41bd3b3834691f53b2b743da9b0aa866c6f53764044213d"
     },
     {
       "id": "ripgrep-no-ignore-vcs-locate-holdout",
       "route": "locate_implementation",
       "split": "holdout",
       "repository_url": "https://github.com/BurntSushi/ripgrep.git",
       "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
-      "prompt": "Find the implementation that makes ripgrep's --no-ignore-vcs option stop applying Git ignore rules, and identify a focused test for that behavior.",
-      "oracle_path": "oracles/ripgrep-no-ignore-vcs-locate.json",
-      "oracle_digest": "b3:4eac2f4e630f8a457de9bcfa1cfdd3adef60fd46e399d24ac7479c2b1906398d",
-      "test_identifier": "feature::f68_no_ignore_vcs",
-      "focused_command": [
-        "cargo",
-        "test",
-        "--offline",
-        "--test",
-        "integration",
-        "feature::f68_no_ignore_vcs",
-        "--",
-        "--exact"
-      ]
+      "prompt": "Find the implementation that makes ripgrep's no-ignore-vcs option stop applying repository ignore rules, and identify a focused test for that behavior.",
+      "material_class": "synthetic",
+      "focused_test_policy": {
+        "identity": "ripgrep.synthetic.no-ignore-vcs.v1",
+        "commitment": "b3:ff0c45be0d9ea0e25a4c7a4f6dcac9b153a05b08c7f23d723011e5bf4e58de57"
+      },
+      "oracle_schema": "needle.sealed-oracle/1",
+      "oracle_digest": "b3:96c6568343f73fb983c8cae8a7f483d339e14f42d5e6af0c36b2dfdc942f9571"
     },
     {
       "id": "ripgrep-null-data-trace-holdout",
       "route": "trace_state_flow",
       "split": "holdout",
       "repository_url": "https://github.com/BurntSushi/ripgrep.git",
       "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c",
-      "prompt": "Trace how ripgrep's --null-data option changes matching and searching from newline-delimited input to NUL-delimited input, with a focused test for the behavior.",
-      "oracle_path": "oracles/ripgrep-null-data-trace.json",
-      "oracle_digest": "b3:e8efb461add472614a36eb920a0c0aa1857e13fe4834aa12cc2582d9e8df4661",
-      "test_identifier": "feature::f993_null_data",
-      "focused_command": [
-        "cargo",
-        "test",
-        "--offline",
-        "--test",
-        "integration",
-        "feature::f993_null_data",
-        "--",
-        "--exact"
-      ]
+      "prompt": "Trace how ripgrep's null-data option changes matching and searching from newline-delimited input to NUL-delimited input, with a focused test for the behavior.",
+      "material_class": "synthetic",
+      "focused_test_policy": {
+        "identity": "ripgrep.synthetic.null-data.v1",
+        "commitment": "b3:2cc53eb4760e8106d5638ec54c545d7447fda90a10a35668e7707ae5cfe1b21e"
+      },
+      "oracle_schema": "needle.sealed-oracle/1",
+      "oracle_digest": "b3:6900cdf524dfaab61c434864f9eaede519737cc9fd30d7b418ef5003d5042beb"
     }
   ]
 }

benchmarks/corpus/router-cache/oracles/README.md

@@ -0,0 +1,12 @@
+# Synthetic evaluator fixtures
+
+These answer-bearing JSON documents are retained solely for deterministic
+legacy/evaluator tests. They are not referenced by the public
+`needle.frozen-corpus/4` manifest, are not sealed production material, and
+must never be used as independent provider evidence.
+
+The checked-in `../synthetic-sealed/` index and four documents exercise the
+sealed evaluator contract and exact byte commitments. They are synthetic and
+remain provider-ineligible. Private production bundles must stay unmounted and
+inaccessible to the runner identity; this fixture does not claim ACL or
+process-isolation proof.

benchmarks/corpus/router-cache/power-plan.json

@@ -0,0 +1,32 @@
+{
+  "schema": "needle.power-plan/1",
+  "plan_id": "synthetic-router-cache-v1",
+  "manifest_digest": "b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a",
+  "campaign_digest": "b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864",
+  "calibration_input_digest": "b3:4d27a8427bd3655d59e88c37efadcccfca4a7d82cd4a4324708180aea31289c0",
+  "estimator_revision": "issue-7-structural-v1",
+  "alpha_basis_points": 500,
+  "target_power_basis_points": 9000,
+  "routes": [
+    {
+      "route": "locate_implementation",
+      "baseline_arm": "frontier_direct",
+      "treatment_arm": "needle_miss",
+      "pair_key": "task_id:repetition",
+      "observed_log_ratio_mean": -0.4,
+      "observed_log_ratio_stddev": 0.2,
+      "required_pairs": 1
+    },
+    {
+      "route": "trace_state_flow",
+      "baseline_arm": "frontier_direct",
+      "treatment_arm": "needle_miss",
+      "pair_key": "task_id:repetition",
+      "observed_log_ratio_mean": -0.4,
+      "observed_log_ratio_stddev": 0.2,
+      "required_pairs": 1
+    }
+  ],
+  "validated": true,
+  "synthetic": true
+}

benchmarks/corpus/router-cache/schedule.json

@@ -0,0 +1,28 @@
+{
+  "schema": "needle.corpus-schedule/1",
+  "manifest_digest": "b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a",
+  "power_plan_digest": "b3:8605f040792744907aa94d3ce017a211b774c16521dbe744acb7a1b587647630",
+  "automatic_retries": false,
+  "entries": [
+    {"task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"frontier_direct","repetition":0},
+    {"task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"needle_miss","repetition":0},
+    {"task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"frontier_direct","repetition":0},
+    {"task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"needle_miss","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"frontier_direct","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"native_subagent","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"needle_miss","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"exact_hit","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"partial_hit","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"escalation","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"irrelevant_mutation","repetition":0},
+    {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"relevant_mutation","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"frontier_direct","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"native_subagent","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"needle_miss","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"exact_hit","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"partial_hit","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"escalation","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"irrelevant_mutation","repetition":0},
+    {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"relevant_mutation","repetition":0}
+  ]
+}