Record the optimizer harness release each reported result used, replicable with --pin-harness

harness-opt-bench/CONFIGURATION.md

@@ -140,6 +140,27 @@ stopped and final scoring still runs on the candidate it had submitted, so a run
 cut off by the clock is scored rather than lost. The sandbox limit is the hard
 ceiling: reaching it loses the run.
 
+### Optimizer harness versions
+
+The optimizer harness is installed from its public registry at the start of
+every run, so by default a run uses the current release. Each build records
+the releases its reported results were produced with under
+`optimizer_harness_versions`; pass `--pin-harness` to `vero harbor run` to
+install those instead and replicate a result with the same harness.
+
+| Harness | Release |
+| --- | --- |
+| claude-code | 2.1.220 |
+| codex | 0.146.0 |
+| kimi-cli | 1.49.0 |
+| goose | 1.45.0 |
+| mini-swe-agent | 2.4.6 |
+| opencode | 1.18.9 (OfficeQA), 1.18.10 (BrowseComp-Plus, Terminal-Bench), 1.18.11 (GAIA) |
+
+opencode released three patch versions while the original rounds ran, so the
+recorded release is the one most of that benchmark's rounds used. Pass
+`--ak version=<release>` on the command line to run any specific release.
+
 ## Configuration rules
 
 ### Baselines

harness-opt-bench/browsecomp-plus/baseline/build.yaml

@@ -73,6 +73,13 @@ optimizer_sandbox_timeout_seconds: 86400
 optimizer_sandbox_idle_timeout_seconds: 3600
 optimizer_agent_timeout_seconds: 72000
 optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.10
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
 
 secrets:
   - MODAL_TOKEN_ID

harness-opt-bench/gaia/baseline/build.shell.e2e.yaml

@@ -92,6 +92,13 @@ optimizer_sandbox_idle_timeout_seconds: 1800
 # which is the path this variant exists to exercise.
 optimizer_agent_timeout_seconds: 1500
 optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.11
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
 
 secrets:
   - MODAL_TOKEN_ID

harness-opt-bench/gaia/baseline/build.shell.yaml

@@ -84,6 +84,13 @@ optimizer_sandbox_timeout_seconds: 86400
 optimizer_sandbox_idle_timeout_seconds: 3600
 optimizer_agent_timeout_seconds: 72000
 optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.11
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
 
 secrets:
   - MODAL_TOKEN_ID

harness-opt-bench/gaia/baseline/build.yaml

@@ -75,6 +75,13 @@ optimizer_sandbox_timeout_seconds: 86400
 optimizer_sandbox_idle_timeout_seconds: 3600
 optimizer_agent_timeout_seconds: 72000
 optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.11
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
 
 secrets:
   - MODAL_TOKEN_ID

harness-opt-bench/officeqa/baseline/build.yaml

@@ -76,6 +76,13 @@ optimizer_sandbox_timeout_seconds: 86400
 optimizer_sandbox_idle_timeout_seconds: 3600
 optimizer_agent_timeout_seconds: 72000
 optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.9
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
 
 secrets:
   - MODAL_TOKEN_ID

harness-opt-bench/skills/run-benchmark/SKILL.md

@@ -62,6 +62,12 @@ uv run vero harbor run \
 gateway. Some harnesses normalize model names, so verify the wire value before a
 large run.
 
+The optimizer harness is installed at launch, at its current release. To
+replicate a reported result with the harness it was produced with, add
+`--pin-harness`, which installs the release recorded in the build's
+`optimizer_harness_versions`. `--ak version=<release>` selects any other
+release.
+
 ## Preflight
 
 Compile before launching:

harness-opt-bench/terminal-bench/baseline/build.routed.yaml

@@ -83,6 +83,13 @@ optimizer_sandbox_timeout_seconds: 86400
 optimizer_sandbox_idle_timeout_seconds: 3600
 optimizer_agent_timeout_seconds: 72000
 optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.10
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
 
 secrets:
   - MODAL_TOKEN_ID

harness-opt-bench/terminal-bench/baseline/build.yaml

@@ -83,6 +83,13 @@ optimizer_sandbox_timeout_seconds: 86400
 optimizer_sandbox_idle_timeout_seconds: 3600
 optimizer_agent_timeout_seconds: 72000
 optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.10
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
 
 secrets:
   - MODAL_TOKEN_ID

vero/src/vero/harbor/build/config.py

@@ -338,6 +338,11 @@ def validate_optimizer_harbor_args(cls, value: list[str]) -> list[str]:
 _HARBOR_ONLY_FIELDS = frozenset(_HarborEvaluationFields.model_fields)
 
 
+# An exact release: `1.18.10`, `v1.45.0`, `2.1.220-beta.1`. Not `latest`, `*`
+# or a range, which harbor would forward to the registry as a moving target.
+_RELEASE_VERSION = re.compile(r"v?\d+\.\d+\.\d+(?:[-+.][0-9A-Za-z.+-]+)?")
+
+
 class _OptimizerTrialFields(StrictModel):
     """Limits on the outer trial the optimizer itself runs in.
 
@@ -364,6 +369,12 @@ class _OptimizerTrialFields(StrictModel):
             declarations for the outer sandbox (``[environment]``). None leaves
             the field undeclared, which is Modal's default: a reservation of
             0.125 cores and 128 MiB that bursts to whatever the host has.
+        optimizer_harness_versions: Release of each optimizer harness the
+            build's reported results were produced with (harbor agent name ->
+            version). Harbor installs the harness at trial start from its
+            public registry, so this is a record, not a constraint: a plain
+            `vero harbor run` installs the current release, and
+            ``--pin-harness`` installs the recorded one to replicate a result.
     """
 
     # Limits on the outer optimizer trial. Every default below is what ran
@@ -375,6 +386,19 @@ class _OptimizerTrialFields(StrictModel):
     optimizer_cpus: int | None = Field(default=None, ge=1)
     optimizer_memory_mb: int | None = Field(default=None, ge=1)
     optimizer_storage_mb: int | None = Field(default=None, ge=1)
+    optimizer_harness_versions: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("optimizer_harness_versions")
+    @classmethod
+    def validate_optimizer_harness_versions(cls, value: dict[str, str]) -> dict[str, str]:
+        for agent, version in value.items():
+            if not agent.strip() or not _RELEASE_VERSION.fullmatch(version):
+                raise ValueError(
+                    f"optimizer_harness_versions[{agent!r}] must be an exact release, "
+                    f"got {version!r}"
+                )
+        return value
+
     @model_validator(mode="after")
     def validate_optimizer_clocks(self):
         agent = self.optimizer_agent_timeout_seconds

vero/src/vero/harbor/cli.py

@@ -403,6 +403,53 @@ def _outer_app_name_args(
     return ["--ek", f"app_name={slug or 'vero'}"]
 
 
+def _harness_version_args(
+    agent: str, config, extra: tuple[str, ...], pin: bool
+) -> list[str]:
+    """Install the optimizer harness at the release the build records, on request.
+
+    Harbor's installed-agent adapters take ``version`` as an agent kwarg and
+    install exactly that release (npm ``@version``, PyPI ``==version``, or the
+    claude installer's version argument). Left unset they install the current
+    one, which is the default here. ``--pin-harness`` asks for the release in
+    the build's ``optimizer_harness_versions`` instead, so a result can be
+    replicated with the harness it was produced with; a harness the build does
+    not record is refused rather than silently unpinned. A caller's own
+    ``--ak version=`` wins either way.
+
+    goose is the odd one out: harbor fetches ``download_cli.sh`` from the
+    release tag named by ``version``, but that script installs the *stable*
+    release unless ``GOOSE_VERSION`` is set, so the pin is delivered both ways.
+    """
+    overrides = [
+        value.removeprefix("version=")
+        for flag, value in zip(extra, extra[1:])
+        if flag == "--ak" and value.startswith("version=")
+    ]
+    if overrides:
+        version, args = overrides[-1], []
+    elif not pin:
+        return []
+    else:
+        versions = getattr(config, "optimizer_harness_versions", None) or {}
+        version = versions.get(agent)
+        if version is None:
+            raise click.ClickException(
+                f"the build records no release for optimizer harness {agent!r}; "
+                "add it to optimizer_harness_versions or pass --ak version=<release>"
+            )
+        args = ["--ak", f"version={version}"]
+    if agent == "goose" and not any(
+        flag == "--ae" and value.startswith("GOOSE_VERSION=")
+        for flag, value in zip(extra, extra[1:])
+    ):
+        tag = version if version.startswith("v") else f"v{version}"
+        if not overrides:
+            args = ["--ak", f"version={tag}"]
+        args += ["--ae", f"GOOSE_VERSION={tag}"]
+    return args
+
+
 def _outer_sandbox_args(
     environment: str, config, extra: tuple[str, ...]
 ) -> list[str]:
@@ -924,6 +971,14 @@ def _preflight_models(config) -> None:
 )
 @click.option("--agent", required=True, help="Harbor optimizer agent.")
 @click.option("--model", help="Model used by the optimizer agent.")
+@click.option(
+    "--pin-harness",
+    is_flag=True,
+    help=(
+        "Install the optimizer harness at the release recorded in the build's "
+        "optimizer_harness_versions instead of the current one."
+    ),
+)
 @click.option("--environment", default="modal", show_default=True)
 @click.option(
     "--env-file",
@@ -937,7 +992,9 @@ def _preflight_models(config) -> None:
 )
 @_PARAM_OPTION
 @click.argument("extra", nargs=-1, type=click.UNPROCESSED)
-def run_command(config_path, agent, model, environment, params, env_file, extra):
+def run_command(
+    config_path, agent, model, pin_harness, environment, params, env_file, extra
+):
     """Compile to a temporary directory and invoke `harbor run`."""
     from vero.harbor.build import (
         compile_harbor_task,
@@ -994,6 +1051,7 @@ def run_command(config_path, agent, model, environment, params, env_file, extra)
         ]
         if model is not None:
             command.extend(["-m", model])
+        command.extend(_harness_version_args(agent, config, extra, pin_harness))
         # Forward the build's declared agent env to the optimizer agent's shell.
         # Harbor's `--ae KEY=VALUE` populates the agent's extra_env, which harbor
         # injects into the agent's setup/install exec (scoped_exec_env). Sorted

vero/tests/test_v05_benchmark_configs.py

@@ -17,6 +17,7 @@
 
 from __future__ import annotations
 
+import re
 from pathlib import Path
 
 import pytest
@@ -287,3 +288,15 @@ def test_gaia_shell_variant_shares_the_measurement_substrate_and_stays_a_shell()
             f"the gaia shell seed calls {call} -- it is no longer a shell, and its "
             f"baseline_reward of 0.0 is no longer true"
         )
+
+
+def test_every_build_pins_every_optimizer_harness():
+    """A harness installed at trial start drifts unless the build names its release."""
+    harnesses = {"claude-code", "codex", "opencode", "kimi-cli", "goose", "mini-swe-agent"}
+    builds = sorted(BENCHMARK_ROOT.glob("*/baseline/build*.yaml"))
+    assert builds
+    for path in builds:
+        pins = yaml.safe_load(path.read_text(encoding="utf-8")).get("optimizer_harness_versions") or {}
+        assert set(pins) == harnesses, f"{path} pins {sorted(pins)}"
+        for version in pins.values():
+            assert re.fullmatch(r"\d+\.\d+\.\d+", str(version)), f"{path}: {version!r}"

vero/tests/test_v05_cli.py

@@ -934,3 +934,38 @@ def test_harbor_run_refuses_when_a_declared_credential_is_unset(tmp_path, monkey
     assert result.exit_code != 0
     assert "declared task credentials are missing: DEFINITELY_UNSET_CREDENTIAL" in result.output
     assert compiled == []  # refused before compiling, so before any cost
+
+
+def test_harness_version_args_pin_the_recorded_release_only_on_request():
+    import click
+    import pytest
+    from types import SimpleNamespace
+
+    from vero.harbor.cli import _harness_version_args
+
+    cfg = SimpleNamespace(optimizer_harness_versions={"opencode": "1.18.10", "goose": "1.45.0"})
+    # the default installs the current release: the record is documentation
+    assert _harness_version_args("opencode", cfg, (), pin=False) == []
+    assert _harness_version_args("goose", cfg, (), pin=False) == []
+    # --pin-harness installs the recorded one
+    assert _harness_version_args("opencode", cfg, (), pin=True) == ["--ak", "version=1.18.10"]
+    # goose's installer script ignores the release tag it was fetched from
+    # unless GOOSE_VERSION is set, so the pin is delivered both ways.
+    assert _harness_version_args("goose", cfg, (), pin=True) == [
+        "--ak", "version=v1.45.0", "--ae", "GOOSE_VERSION=v1.45.0",
+    ]
+    # a caller's own --ak version= wins; a stray version= under another flag does not
+    assert _harness_version_args("opencode", cfg, ("--ak", "version=1.18.30"), pin=True) == []
+    assert _harness_version_args("opencode", cfg, ("--ae", "version=1.18.30"), pin=True) == [
+        "--ak", "version=1.18.10",
+    ]
+    # goose keeps the env delivery when the caller picks the release, pinned or not
+    assert _harness_version_args("goose", cfg, ("--ak", "version=1.50.0"), pin=False) == [
+        "--ae", "GOOSE_VERSION=v1.50.0",
+    ]
+    assert _harness_version_args(
+        "goose", cfg, ("--ak", "version=v1.50.0", "--ae", "GOOSE_VERSION=v1.50.0"), pin=True
+    ) == []
+    # pinning a harness the build does not record is refused, not silently unpinned
+    with pytest.raises(click.ClickException, match="records no release for optimizer harness 'codex'"):
+        _harness_version_args("codex", cfg, (), pin=True)

vero/tests/test_v05_harbor_build.py

@@ -1797,3 +1797,16 @@ def test_vero_requirement_must_match_the_compiling_vero(tmp_path):
         _config(tmp_path / "b", vero_requirement="scaleapi-vero[harbor]>=0.5")
     with pytest.raises(ValueError, match="pin the VeRO distribution"):
         _config(tmp_path / "c", vero_requirement="some-other-package==1.0")
+
+
+def test_harness_versions_must_be_exact_releases(tmp_path):
+    config = _config(
+        tmp_path / "a", optimizer_harness_versions={"opencode": "1.18.10", "codex": "0.146.0"}
+    )
+    assert config.optimizer_harness_versions["opencode"] == "1.18.10"
+    with pytest.raises(ValueError, match="exact release"):
+        _config(tmp_path / "b", optimizer_harness_versions={"opencode": ""})
+    for i, bad in enumerate((">= 1.18", "latest", "*", "1.18", "^1.18.10")):
+        with pytest.raises(ValueError, match="exact release"):
+            _config(tmp_path / f"bad{i}", optimizer_harness_versions={"opencode": bad})
+    assert _config(tmp_path / "d", optimizer_harness_versions={"goose": "v1.45.0"})