test(e2e): pin the contracts a README walkthrough expects (EAI-8024)

tests/e2e-cucumber/expectations.toml

@@ -162,3 +162,181 @@ bug = "EAI-7423"
 reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model."
 serve_timeout_secs = 90
 flaky = true
+
+# ── Contracts found by walking the public README end to end ─────────────────
+#
+# The rows below were all filed from one pass through README.md against a
+# nightly build on an MI300X pod (EAI-8024 tracks the set). Each scenario asserts
+# the behaviour the README leads a reader to expect, and each row here says the
+# product does not do that yet — so the day one is fixed its row goes stale and
+# deletes itself.
+
+# --- EAI-8011: the help's own worked examples name a model the CLI cannot
+# resolve (`qwen2.5-7b-instruct` — not a catalog alias, and not an `owner/repo`
+# reference either). Pure help text, so it holds on every platform. Remove this
+# row when the examples name a resolvable model. ---
+[["help-serve-example-names-a-resolvable-model"]]
+when = {}
+bug = "EAI-8011"
+reason = "The help offers `rocm serve qwen2.5-7b-instruct`, which is neither a name the model listing knows nor an owner/repo reference."
+
+# --- EAI-8011: the help says running `rocm` with no subcommand opens the
+# dashboard, which is what it also says `rocm dash` does; the README documents
+# the plain command as opening the launcher instead, and that is what it really
+# does. Pure help text. Remove when the two are described distinctly. ---
+[["help-describes-the-default-command"]]
+when = {}
+bug = "EAI-8011"
+reason = "The help describes running the CLI with no subcommand as opening the dashboard, the same thing it says `rocm dash` does."
+
+# --- EAI-8010: `--dry-run` is declared as requiring `--apply`, so previewing an
+# update without asking for one is rejected as a usage error, though the README
+# documents the two as independent. Argument handling only, so platform-
+# independent. Remove when `rocm update --dry-run` is accepted. ---
+[["update-preview-without-applying"]]
+when = {}
+bug = "EAI-8010"
+reason = "`rocm update --dry-run` is refused as a usage error unless --apply is also given."
+
+# --- EAI-8019: `serve --help` advertises `cpu_only` among its device policies
+# and the command then rejects it outright. Scoped to the no-GPU lane, which is
+# where the scenario runs (`@requires-no-gpu`) — there every policy meets the
+# same "no GPU on this machine" refusal, which makes a refusal aimed at the
+# policy itself cleanly visible. Remove when the advertised list and the accepted
+# list are the same list. ---
+[["serve-rejects-no-advertised-device-policy"]]
+when = {}
+bug = "EAI-8019"
+reason = "`serve` advertises cpu_only as a device policy and then refuses it for being cpu_only."
+
+# --- EAI-8020: `diagnose` and `fix` each carry their own copy of the
+# render-group remedy, and the verification step has drifted between them
+# (`diagnose` adds `ls -l /dev/kfd`). Both strings are constants, so this holds
+# wherever the catalog runs at all; `@requires-os:linux` on the scenario keeps it
+# off Windows, where the recipe does not apply. Remove when the two are one. ---
+[["diagnose-and-fix-agree-on-how-to-verify"]]
+when = {}
+bug = "EAI-8020"
+reason = "The diagnosis and the fix preview print different verification steps for the same remedy."
+
+# --- EAI-8009: with no `--model`, the local chat provider only looks for the
+# built-in Lemonade assistant model, so a ready managed service running anything
+# else is ignored and the user is told nothing is running. The scenario plants
+# its own ready service, so this needs no hardware and holds on every lane.
+# Remove when chatting locally uses whichever local server is ready. ---
+[["chat-local-uses-the-running-server"]]
+when = {}
+bug = "EAI-8009"
+reason = "The local chat provider only accepts the built-in assistant model, so a ready local server running any other model is not used."
+
+# --- EAI-7423, not EAI-8007. This scenario's premise is a REAL serve on whatever
+# engine the host defaults to, so on a lemonade LINUX host it inherits the same
+# managed-serve bug as its siblings above: the serve reaches ready and is shut
+# down ~0.08s later, so the endpoint never becomes reachable and the precondition
+# — not the contract under test — is what fails. Measured on Strix Halo Ubuntu:
+# passed in run 196, failed in run 210 with "endpoint did not serve model
+# Qwen3-0.6B after 2 attempt(s) of 300s", which is that signature exactly.
+#
+# Scoped and flagged identically to the other EAI-7423 rows: os=linux because
+# native Windows lemonade serve works (it passes on Strix Windows), gfx* because
+# a real GPU host is needed at all, and `flaky = true` because which of these
+# pass varies run to run — a deterministic-XPASS row would fail that lane on
+# unrelated PRs. Delete this row with the other EAI-7423 ones, not with EAI-8007.
+[["services-stop-reports-what-it-stopped"]]
+when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" }
+bug = "EAI-7423"
+reason = "Lemonade managed serve reaches ready then shuts down immediately, so the service this scenario needs to stop never comes up."
+serve_timeout_secs = 90
+flaky = true
+
+# For the contract this scenario actually exists to hold:
+# EAI-8007 has NO row on purpose. `services stop` reported stopping 0 processes
+# for a service it did in fact stop on the pod, but neither fixture reproduces it
+# here: a plain child process registered as a managed service is counted
+# correctly (no-GPU lane), and so is a real vLLM serve (MI300X, run 188). Its
+# scenario therefore ships as a guard rather than an expected failure — a row
+# would be a stale XPASS on both lanes and would fail them. The ticket keeps the
+# pod observation; what distinguishes that host is still unknown.
+
+# --- EAI-8014: `uninstall` removes the managed files and reports completion
+# without stopping the servers it manages — and takes the service records with
+# it, so the supported way to stop them goes too. It warns that it does not stop
+# them, which is the behaviour being disputed, not a defence of it. No hardware
+# needed. Remove when uninstall stops what it manages first. ---
+[["uninstall-stops-what-it-manages"]]
+when = {}
+bug = "EAI-8014"
+reason = "Uninstall reports completion while leaving the servers it manages running, and removes the records needed to stop them."
+
+# --- EAI-8015: MEASURE. When the device's owning group has no entry in the local
+# group database, `stat` answers `UNKNOWN` and the diagnosis passes that straight
+# into a `usermod -a -G UNKNOWN,video` the user cannot run; only an EMPTY group
+# name is filtered. The scenario substitutes that `stat` answer, so it needs a
+# real device node but not a real broken host. GPU lane only — with no
+# `/dev/kfd` the inspection stops before it ever asks `stat`. ---
+[["diagnose-commands-name-a-real-group"]]
+when = {}
+bug = "EAI-8015"
+reason = "A diagnosis names the group-lookup failure `UNKNOWN` as a group for the user to join."
+
+# --- EAI-8020: MEASURE. The other half of the same divergence: `diagnose` names
+# the device's actual owning group while the `fix` recipe is hard-coded to
+# `render`, so the two propose different changes wherever the device's group is
+# not `render`. Needs a real device node, hence the GPU lane. ---
+[["diagnose-and-fix-agree-on-the-remedy-command"]]
+when = {}
+bug = "EAI-8020"
+reason = "The diagnosis proposes adding the user to the device's own group while the fix is hard-coded to `render`."
+
+# --- EAI-8021: MEASURE. The render-group check only ever ADDS score for missing
+# group membership and never credits access that demonstrably works, so a device
+# the user can already read and write is still diagnosed as a permission problem.
+# The scenario substitutes a world-writable device answer and asserts the
+# inspection agrees before relying on it. Where the runner's user already belongs
+# to the conventional groups the finding scores zero anyway and this XPASSes —
+# which is what the first GPU run is for. ---
+[["diagnose-credits-a-usable-device"]]
+when = {}
+bug = "EAI-8021"
+reason = "A device the user can already read and write is still offered as a group-permission cause."
+
+# --- EAI-8006: `--gpu auto` reports `selected none` on a machine that has a GPU,
+# because the CLI cannot get an answer out of `amd-smi` — on the pod the bundled
+# copy could not execute at all (its `#!/usr/bin/env python3` shebang found no
+# interpreter), even though the CLI has a working interpreter recorded in its own
+# config.
+#
+# os = "windows" is MEASURED, not guessed, and the three lanes separate the two
+# candidate explanations cleanly:
+#   MI300X            gfx943  linux    PASSES (runs 190/191/192)
+#   Strix Halo Ubuntu gfx1151 linux    PASSES (run 196)
+#   Strix Halo Windows gfx1151 windows FAILS  (run 196) — `gpu: auto (selected none)`
+# Same GPU family on both Strix lanes, so this is the OS, not the family — which
+# is why the row is not scoped with therock_family the way
+# `examine-both-forms-agree-on-gpu` is. Remove it when a Windows host names the
+# device it picked. ---
+[["serve-auto-gpu-selection-names-a-device"]]
+when = { os = "windows" }
+bug = "EAI-8006"
+reason = "Letting the CLI choose a GPU reports that it selected none on a Windows machine that has one."
+serve_timeout_secs = 180
+
+# --- EAI-8016: MEASURE. The serve path has no address-availability check: the
+# default port is used whether or not something already holds it, so a second
+# server is handed the address the first is on. The scenario holds the address
+# itself rather than starting a second real server. ---
+[["serve-second-server-gets-a-free-port"]]
+when = {}
+bug = "EAI-8016"
+reason = "A server started without an address is given the default one even when it is already in use."
+serve_timeout_secs = 120
+
+# EAI-8017 and EAI-8018 have NO rows and NO scenarios, because neither can be
+# exercised on any lane available here. Both need a working Lemonade install,
+# and Lemonade's own launcher cannot start on either lane: on the MI300X runner
+# `lemond` wants GLIBC_2.38 / GLIBCXX_3.4.32, which that OS does not have
+# (measured, runs 190/191), and in a plain container it wants
+# libdrm_amdgpu.so.1. Both die before reaching the behaviour under test, so a
+# scenario would pass vacuously — worse than no scenario, because it would read
+# as coverage. They were reproduced by hand on the pod (ubuntu 24.04) and stay
+# on their tickets until a lane can run Lemonade.

tests/e2e-cucumber/features/chat.feature

@@ -75,3 +75,14 @@ Feature: Chat and endpoint detection
     And the model is registered with the CLI
     When the user sends a one-shot chat prompt through the CLI
     Then the CLI prints the assistant's reply
+
+  # Expected to FAIL. Scenario 7 names the model and works; this one leaves the
+  # choice to the CLI and is told nothing is running — on the very machine whose
+  # services list shows a ready local server. Asking to chat locally should use
+  # the local server that is there, whichever model the user chose to serve.
+  @id:chat-local-uses-the-running-server
+  Scenario: 8 - Chatting locally uses the local server that is already running
+    Given a model is being served
+    And the model is registered with the CLI
+    When the user sends a one-shot chat prompt without naming a model
+    Then the CLI prints the assistant's reply

tests/e2e-cucumber/features/diagnose.feature

@@ -78,3 +78,43 @@ Feature: Diagnosing failures and listing fixes
     When the user asks the CLI to apply it without agreeing to the change
     Then the CLI refuses and explains that it needs agreement
     And the file the fix would have changed is untouched
+
+  # Expected to FAIL. `diagnose` and `fix` are two views of the same remedy: the
+  # first tells the user what will put the machine right, while the second is the
+  # command they are told to run. They must not leave the user with two different
+  # definitions of what proves the remedy worked.
+  @id:diagnose-and-fix-agree-on-how-to-verify @requires-os:linux
+  Scenario: 9 - Diagnosing a problem and previewing its fix agree on how to verify it
+    Given a user who hit a device-permission failure
+    When the user compares the diagnosis with the matching fix preview
+    Then both give the same way to verify that the fix worked
+
+  # Expected to FAIL on a bare-metal GPU host. When the device belongs to a
+  # group that is not named in the local group database, the diagnosis prints
+  # that lookup failure as if it were a group the user could join. A proposed
+  # remedy has to name a group that actually exists on the machine.
+  @id:diagnose-commands-name-a-real-group @requires-gpu @requires-bare-metal @requires-os:linux
+  Scenario: 10 - Every group named in a diagnosis is one the machine recognises
+    Given the GPU device belongs to a group the machine cannot name
+    When the user asks the CLI to diagnose a device-permission failure
+    Then every group named in the remedy exists on the machine
+
+  # Expected to FAIL on a bare-metal GPU host whose device group is not the
+  # hard-coded default. The diagnosis promises one command and its matching fix
+  # previews another. This is distinct from scenario 9: even after the verify
+  # text agrees, the actual change must agree too.
+  @id:diagnose-and-fix-agree-on-the-remedy-command @requires-gpu @requires-bare-metal @requires-os:linux
+  Scenario: 11 - Diagnosing a problem and previewing its fix agree on the remedy command
+    Given the GPU device belongs to a recognised non-default group
+    When the user compares the diagnosis with the matching fix preview
+    Then both give the same command for applying the remedy
+
+  # Expected to FAIL on a bare-metal GPU host where direct access already works.
+  # Membership in one conventional group is only a means to access the device,
+  # not the outcome. A diagnosis should credit the observable access the user
+  # has instead of recommending a permission repair for a usable device.
+  @id:diagnose-credits-a-usable-device @requires-gpu @requires-bare-metal @requires-os:linux
+  Scenario: 12 - A usable GPU device is not diagnosed as a permission failure
+    Given the user can already read and write the GPU device
+    When the user asks the CLI to diagnose the machine
+    Then adding the user to a device group is not the leading remedy

tests/e2e-cucumber/features/examine.feature

@@ -17,6 +17,25 @@ Feature: GPU detection and system inspection
     When the user asks for help
     Then the subcommands are listed in alphabetical order
 
+  # Expected to FAIL. The help's own worked examples are the first thing a new
+  # user copies, so a model named there has to be one this CLI can actually
+  # serve. The check accepts either form the README documents — a name the model
+  # listing knows, or an explicit `owner/repo` reference — and so does not
+  # prescribe which model the examples should use.
+  @id:help-serve-example-names-a-resolvable-model
+  Scenario: 11 - Every model the help offers as an example is one the CLI can resolve
+    When the user reads the serve examples the help offers
+    Then every model named there is one the CLI can resolve
+
+  # Expected to FAIL. Running the CLI with no subcommand and running its
+  # dashboard command are two different things, and the help describes them as
+  # the same one — so a reader has no way to learn what the plain command does
+  # or that there is anything else to reach.
+  @id:help-describes-the-default-command
+  Scenario: 12 - The help tells the two ways of opening a screen apart
+    When the user asks for help
+    Then running the CLI with no subcommand is not described as the dashboard command
+
   @id:examine-detects-gpu-and-driver @requires-gpu
   Scenario: 3 - System inspection detects the GPU and driver
     Given a machine with an AMD GPU

tests/e2e-cucumber/features/model_serving.feature

@@ -144,6 +144,70 @@ Feature: Model serving
     Then serving is refused before any engine starts
     And the user is told no AMD GPU was detected
 
+  # Expected to FAIL. `serve` advertises a set of device policies and then
+  # refuses one of them outright, so the list the user is offered is not the list
+  # the command accepts. Runs on the no-GPU lane, where a refusal that names the
+  # policy itself is cleanly distinguishable from the ordinary "this machine has
+  # no GPU" refusal every policy gets there.
+  @id:serve-rejects-no-advertised-device-policy @requires-no-gpu
+  Scenario: 15 - Every device policy the serve command offers is one it accepts
+    When the user serves a model under each device policy the command offers
+    Then no policy is refused for being that policy
+
+  # A guard, not a finding. Stopping a running server was reported as having
+  # stopped nothing on the pod this set came from, but neither fixture tried here
+  # reproduces that: a plain process registered as a managed service is counted
+  # correctly (measured on the no-GPU lane), and so is a real vLLM serve
+  # (measured on MI300X, run 188). So this carries no expected-failure row — it
+  # holds the contract the pod violated, and goes red if CI ever meets it.
+  # @merge-queue: this serves a real model, and the Strix Halo lanes already run
+  # at 32 of their 35 allotted minutes on main alone. Three new real serves would
+  # put them over, so they follow scenarios 6/6b/8 onto the merge-queue path,
+  # where the budget for heavy serves lives.
+  @id:services-stop-reports-what-it-stopped @requires-gpu @merge-queue
+  Scenario: 16 - Stopping a running server reports that it stopped it
+    Given a managed runtime is active
+    And a model is being served on GPU
+    When the user stops the server that is running
+    Then the CLI reports that it stopped a process
+
+  # Expected to FAIL. Removing the CLI's managed files reports completion while
+  # the server it was managing is left running — and with the records gone, the
+  # supported way to stop it has been removed along with them. Nothing here
+  # touches the installed program: the removal is scoped to this scenario's own
+  # directories and keeps the binaries.
+  @id:uninstall-stops-what-it-manages @requires-os:linux
+  Scenario: 17 - Removing the CLI's managed files stops the servers it manages
+    Given a local server this machine manages is running
+    When the user removes the CLI's managed files
+    Then the removal is reported as complete
+    And the server is no longer running
+
+  # Expected to FAIL on Windows. Asking the CLI to choose a GPU on a machine that
+  # has one must end with a device chosen: reporting that it selected none and
+  # carrying on leaves the user unable to tell which GPU their model will run on,
+  # or whether it will run on one at all. Both Linux lanes name the device, so
+  # this scenario also guards them.
+  # @merge-queue for the serve-cost reason on scenario 16.
+  @id:serve-auto-gpu-selection-names-a-device @requires-gpu @merge-queue
+  Scenario: 18 - Letting the CLI choose the GPU names the device it chose
+    Given a managed runtime is active
+    And a machine with an AMD GPU
+    When the user serves a model letting the CLI choose the GPU
+    Then the plan names the device it chose
+
+  # Expected to FAIL on a GPU host. A second server started while the usual
+  # address is already taken is handed that same address anyway, so it collides
+  # with the server already there. Either outcome is fine — pick a free address,
+  # or say the usual one is busy — but silently reusing it is not.
+  # @merge-queue for the serve-cost reason on scenario 16.
+  @id:serve-second-server-gets-a-free-port @requires-gpu @merge-queue
+  Scenario: 19 - A second server does not take an address already in use
+    Given a managed runtime is active
+    And the address a new server would use is already taken
+    When the user serves a model without choosing an address
+    Then the new server does not try to use the taken address
+
   # Honest device selection: a `--gpu` index that does not exist on the host is
   # rejected outright, never silently remapped to another device (no device-0
   # fallback). Runs on GPU hardware: on a no-GPU host the GPU-required pre-flight