Fix preset verification rejecting a tool's synthetic dataset name

src/dstack/_internal/cli/models/presets.py

@@ -1,6 +1,6 @@
 import re
 from datetime import datetime
-from typing import Annotated, Literal, Optional, Union
+from typing import Annotated, Literal, Optional
 
 from pydantic import (
     Field,
@@ -24,16 +24,19 @@
 
 class PresetWorkload(CoreModel):
     api: Literal["chat_completions", "completions"]
-    dataset: str
+    dataset: Optional[str] = None
+    """The benchmark tool's own name for the data it served. It is the requested
+    dataset when the configuration named one; for the synthetic `random` workload it
+    is whatever the tool calls the data it generates (`random`,
+    `generated-shared-prefix`, ...), recorded but never compared with dstack's own
+    `random`."""
     num_requests: PositiveInt
     input_tokens: PositiveInt
     output_tokens: Annotated[int, Field(ge=2)]
     concurrency: PositiveInt
-
-
-class PresetRandomWorkload(PresetWorkload):
-    dataset: Literal["random"] = "random"
     shared_prefix_tokens: Annotated[int, Field(ge=0)] = 0
+    """How many leading tokens every measured request shared. Only a synthetic
+    workload has one: a named dataset defines its own requests."""
 
 
 class PresetBenchmarkLatency(CoreModel):
@@ -64,9 +67,7 @@ class PresetBenchmark(CoreModel):
     tool: Annotated[str, Field(min_length=1)]
     tool_version: Annotated[str, Field(min_length=1)]
     command: Annotated[str, Field(min_length=1)]
-    # The subclass first: a report without `dataset` is a random workload, and a
-    # base-typed field would reject its `shared_prefix_tokens` as unknown.
-    workload: Union[PresetRandomWorkload, PresetWorkload]
+    workload: PresetWorkload
     metrics: PresetBenchmarkMetrics
 
     @property

src/dstack/_internal/cli/services/presets/create.py

@@ -77,7 +77,6 @@
 from dstack._internal.core.models.envs import Env, EnvSentinel
 from dstack._internal.core.models.fleets import FleetStatus
 from dstack._internal.core.models.presets import (
-    DEFAULT_DATASET,
     PresetConfiguration,
     PresetConstraints,
     PresetDatasetConstraints,
@@ -613,7 +612,7 @@ async def _create_preset(
         user_prompt=setup.user_prompt,
         baseline=configuration.effective_baseline,
         previous=setup.previous,
-        custom_dataset=configuration.effective_dataset != DEFAULT_DATASET,
+        custom_dataset=configuration.has_custom_dataset,
     )
     if setup.write_constraints:
         if setup.user_prompt:
@@ -914,8 +913,7 @@ def _build_constraints(
     build_name: str,
     allowed_fleets: Sequence[str],
 ) -> str:
-    dataset = configuration.effective_dataset
-    if dataset == DEFAULT_DATASET:
+    if not configuration.has_custom_dataset:
         constraints: PresetConstraints = PresetRandomConstraints(
             run_name_prefix=build_name,
             model=configuration.model,
@@ -938,7 +936,7 @@ def _build_constraints(
             max_ttft=configuration.max_ttft,
             trials_num=configuration.trials,
             concurrency=configuration.concurrency,
-            dataset=dataset,
+            dataset=configuration.effective_dataset,
             baseline=configuration.effective_baseline,
             fleets=list(allowed_fleets),
             env=list(configuration.env),

src/dstack/_internal/cli/services/presets/output.py

@@ -8,7 +8,6 @@
     VerifiedPreset,
 )
 from dstack._internal.cli.utils.common import add_row_from_dict, console
-from dstack._internal.core.models.presets import DEFAULT_DATASET
 from dstack._internal.utils.common import pretty_date, pretty_resources
 
 _STATUS_DISPLAY = {
@@ -291,7 +290,7 @@ def format_preset_objective(
     configuration = preset.configuration
     workload = preset.benchmark.workload
     parts = []
-    if configuration.effective_dataset != DEFAULT_DATASET:
+    if configuration.has_custom_dataset:
         parts.append(f"data={configuration.effective_dataset}")
     else:
         input_tokens = configuration.effective_input_tokens

src/dstack/_internal/cli/services/presets/resources/system_prompt.md

@@ -361,7 +361,7 @@ trial benchmarks in `trials/<n>/trial.json`, the final benchmark as
   "tool_version": "0.11.0",
   "command": "vllm bench serve ...",<!--?if dataset-->
   "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "dataset": "sharegpt"},<!--?else-->
-  "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "shared_prefix_tokens": 768},<!--?end-->
+  "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "shared_prefix_tokens": 768, "dataset": "random"},<!--?end-->
   "metrics": {
     "successful_requests": 16, "failed_requests": 0, "duration_seconds": 4.0,
     "total_input_tokens": 16384, "total_output_tokens": 2048,
@@ -376,6 +376,11 @@ trial benchmarks in `trials/<n>/trial.json`, the final benchmark as
 Set `workload.dataset` to `dataset` from `constraints.json`, and compute
 `workload.input_tokens` and `workload.output_tokens` as the measured mean
 input and output token counts of the benchmark, rounded to whole tokens.
+<!--?else-->
+Set `workload.shared_prefix_tokens` to `shared_prefix_tokens` from
+`constraints.json`, and `workload.dataset` to the name the benchmark tool gives
+the data it generated, whatever that name is. It records what ran and is never
+required to be a particular value.
 <!--?end-->
 Compute `output_tok_per_s` as `total_output_tokens / duration_seconds` and
 `per_user_tok_per_s` as `output_tok_per_s / workload.concurrency`. These are

src/dstack/_internal/cli/services/presets/verify.py

@@ -11,6 +11,7 @@
 )
 from dstack._internal.cli.models.presets import (
     PresetVerificationReplicaGroup,
+    PresetWorkload,
     VerifiedPreset,
 )
 from dstack._internal.cli.services.presets.agent import (
@@ -134,18 +135,47 @@ def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfig
 def _check_report_answers_request(
     report: PresetAgentSuccess, configuration: PresetConfiguration
 ) -> None:
-    """The report must answer what the configuration asked: the same dataset, and
-    the requested model — exactly when it was exact, any variant of the base
-    otherwise."""
-    if report.benchmark.workload.dataset != configuration.effective_dataset:
-        raise CLIError("Claude final benchmark dataset does not match the requested dataset")
+    """The report must answer what the configuration asked: the same benchmark
+    workload, and the requested model — exactly when it was exact, any variant of
+    the base otherwise."""
+    _check_workload_answers_request(report.benchmark.workload, configuration)
     if configuration.model.allows_variant_selection:
         if report.base != configuration.model.api_model_name:
             raise CLIError("Claude final report base does not match the requested model")
     elif report.model != configuration.model.exact_repo:
         raise CLIError("Claude changed an exact model request")
 
 
+def _check_workload_answers_request(
+    workload: PresetWorkload, configuration: PresetConfiguration
+) -> None:
+    """Only what the configuration specifies exactly is compared. A named dataset is
+    compared by name, because the request and the report both use the dataset's own
+    name. A synthetic workload has no such shared name: `random` is dstack's name for
+    it, while the report carries the benchmark tool's, so the shared prefix it was run
+    with is compared instead. `input_tokens` and `output_tokens` are what the
+    benchmark measured rather than an echo of the request, so they are not
+    compared."""
+    if configuration.has_custom_dataset:
+        if workload.dataset != configuration.effective_dataset:
+            raise CLIError(
+                f"Claude final benchmark dataset {workload.dataset!r} does not match the"
+                f" requested dataset {configuration.effective_dataset!r}"
+            )
+    else:
+        shared_prefix_tokens = configuration.shared_prefix_tokens or 0
+        if workload.shared_prefix_tokens != shared_prefix_tokens:
+            raise CLIError(
+                f"Claude final benchmark shared prefix of {workload.shared_prefix_tokens}"
+                f" tokens does not match the requested {shared_prefix_tokens}"
+            )
+    if configuration.concurrency is not None and workload.concurrency != configuration.concurrency:
+        raise CLIError(
+            f"Claude final benchmark concurrency of {workload.concurrency} does not match the"
+            f" requested concurrency of {configuration.concurrency}"
+        )
+
+
 def _portable_service(
     service: ServiceConfiguration,
     configuration: PresetConfiguration,

src/dstack/_internal/core/models/presets.py

@@ -268,6 +268,14 @@ def effective_baseline(self) -> bool:
     def effective_dataset(self) -> str:
         return self.dataset if self.dataset is not None else DEFAULT_DATASET
 
+    @property
+    def has_custom_dataset(self) -> bool:
+        """Whether a named dataset provides the benchmark requests. The default
+        `random` is not one: it is dstack's own name for synthetic prompts shaped by
+        `input_tokens` and `output_tokens`, and every benchmark tool has its own name
+        for the data it generates."""
+        return self.effective_dataset != DEFAULT_DATASET
+
     @field_validator("dataset")
     @classmethod
     def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]:

src/tests/_internal/cli/common.py

@@ -79,6 +79,20 @@ def run_dstack_cli(
     return exit_code
 
 
+# The trial workload from the SGLang session in dstackai/dstack#4198: a synthetic
+# shared-prefix benchmark run with a tool that does not call its generated data
+# `random`. It used to match neither of the two workload models the schema offered.
+SHARED_PREFIX_WORKLOAD = {
+    "api": "completions",
+    "dataset": "generated-shared-prefix",
+    "num_requests": 16,
+    "input_tokens": 131072,
+    "output_tokens": 512,
+    "concurrency": 4,
+    "shared_prefix_tokens": 130048,
+}
+
+
 def get_preset_benchmark() -> PresetBenchmark:
     benchmark = PresetBenchmark(
         tool="vllm bench serve",

src/tests/_internal/cli/models/test_presets.py

@@ -4,7 +4,7 @@
 from dstack._internal.cli.models.presets import (
     PresetBenchmark,
 )
-from tests._internal.cli.common import get_preset_benchmark
+from tests._internal.cli.common import SHARED_PREFIX_WORKLOAD, get_preset_benchmark
 
 pytestmark = pytest.mark.windows
 
@@ -31,3 +31,24 @@ def test_rejects_tool_specific_metrics(self):
         # permitted" rather than "extra fields not permitted"). What matters is the rejection.
         with pytest.raises(ValidationError):
             PresetBenchmark.model_validate(data)
+
+    def test_keeps_both_a_tool_dataset_name_and_a_shared_prefix(self):
+        # A synthetic workload has two facts to state: the shared prefix it was run
+        # with and the name the benchmark tool gave the data it generated.
+        data = get_preset_benchmark().model_dump()
+        data["workload"] = dict(SHARED_PREFIX_WORKLOAD)
+
+        benchmark = PresetBenchmark.model_validate(data)
+
+        assert benchmark.workload.dataset == "generated-shared-prefix"
+        assert benchmark.workload.shared_prefix_tokens == 130048
+
+    def test_reads_a_workload_stored_without_a_dataset(self):
+        # Every preset written before the workload could carry a tool dataset name.
+        data = get_preset_benchmark().model_dump()
+        del data["workload"]["dataset"]
+
+        benchmark = PresetBenchmark.model_validate(data)
+
+        assert benchmark.workload.dataset is None
+        assert benchmark.workload.shared_prefix_tokens == 0

src/tests/_internal/cli/services/presets/test_create.py

@@ -99,7 +99,8 @@ def creation_context(tmp_path, monkeypatch):
         base="Qwen/Qwen3.5-27B",
         min_context_length=8192,
         max_ttft=5000,
-        concurrency=8,
+        # Matches the fixture report's benchmark concurrency, which verification compares.
+        concurrency=1,
         trials=1,
         fleets=["gpu-fleet"],
         env={"LICENSE": "license-secret", "TOKENIZERS_PARALLELISM": "false"},
@@ -109,7 +110,7 @@ def creation_context(tmp_path, monkeypatch):
         base="Qwen/Qwen3.5-27B",
         min_context_length=8192,
         max_ttft=5000,
-        concurrency=8,
+        concurrency=1,
         trials=1,
         fleets=["gpu-fleet"],
         env=["LICENSE", "TOKENIZERS_PARALLELISM=false"],

src/tests/_internal/cli/services/presets/test_prompt.py

@@ -59,6 +59,16 @@ def test_a_random_dataset_session_never_hears_of_datasets(self):
         assert "shared_prefix_tokens" in text
         assert "`dataset`" not in text
 
+    def test_asks_a_random_dataset_session_for_the_prefix_and_the_tool_s_name(self):
+        text = get_preset_agent_system_prompt(
+            user_prompt=None, baseline=False, previous=(), custom_dataset=False
+        )
+
+        # Both facts fit the benchmark schema, so the prompt asks for both: the
+        # prefix the request fixed, and whatever the tool calls the data it made.
+        assert "Set `workload.shared_prefix_tokens` to `shared_prefix_tokens`" in text
+        assert "`workload.dataset` to the name the benchmark tool gives" in text
+
     def test_fails_loudly_when_the_prompt_has_no_directives(self, tmp_path, monkeypatch):
         plain = tmp_path / "system_prompt.md"
         plain.write_text("# Objective\n\nA prompt without directives.\n")

src/tests/_internal/cli/services/presets/test_store.py

@@ -187,7 +187,8 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path):
             group.name for group in preset.service.replica_groups
         ]
         assert preset.verified_on[0].replicas[0].gpu.name == ["MI300X"]
-        assert preset.benchmark.workload.dataset == "random"
+        # The old format never recorded a dataset name for a synthetic workload.
+        assert preset.benchmark.workload.dataset is None
         assert store.list() == [preset]
 
     def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path):