Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 69 additions & 1 deletion tests/e2e-cucumber/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,34 @@ pub mod model_id;
pub mod panic_capture;
pub mod serve_log;

/// Render everything known about a failed `rocm` invocation.
///
/// Both streams are always shown, each labelled and each with an explicit
/// `(empty)` marker. A bare `(empty)` is a finding in itself — it says the CLI
/// died without explaining itself — whereas an omitted section just looks like
/// the harness lost the output.
///
/// Exists because a step that asserts on the exit code while printing only
/// stdout leaves a failed step undiagnosable: the panic reads `rocm serve
/// failed:` followed by nothing at all, which is what EAI-8031 hit on the
/// MI300X lane. The CLI reports its errors on stderr.
pub fn cli_failure_report(args: &[&str], rc: i32, stdout: &str, stderr: &str) -> String {
fn section(label: &str, body: &str) -> String {
let body = body.trim_end();
if body.is_empty() {
format!("--- {label}: (empty) ---")
} else {
format!("--- {label} ---\n{body}")
}
}
format!(
"`rocm {}` failed (rc={rc})\n{}\n{}",
args.join(" "),
section("stdout", stdout),
section("stderr", stderr),
)
}

pub fn chat_response_is_successful(response: &serde_json::Value) -> bool {
response
.get("choices")
Expand All @@ -27,7 +55,47 @@ pub use e2e_report as report;

#[cfg(test)]
mod tests {
use super::chat_response_is_successful;
use super::{chat_response_is_successful, cli_failure_report};

/// The regression this whole helper exists for: a serve that dies with
/// nothing on stdout must still show the reason, which is on stderr.
#[test]
fn failure_report_shows_stderr_when_stdout_is_empty() {
let report = cli_failure_report(
&["serve", "unsloth/Qwen3-0.6B-GGUF:Q4_0", "--managed"],
1,
"",
"error: no llama-server backend found",
);
assert!(
report.contains("no llama-server backend found"),
"the reason must survive into the panic message:\n{report}"
);
assert!(
report.contains("rc=1"),
"exit code must be shown:\n{report}"
);
assert!(
report.contains("serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --managed"),
"the failing invocation must be identifiable:\n{report}"
);
}

/// An empty stream is labelled rather than omitted: "the CLI said nothing"
/// and "the harness dropped the output" are different diagnoses.
#[test]
fn failure_report_marks_empty_streams_explicitly() {
let report = cli_failure_report(&["examine"], 2, "", "");
assert!(report.contains("stdout: (empty)"), "{report}");
assert!(report.contains("stderr: (empty)"), "{report}");
}

#[test]
fn failure_report_keeps_both_streams_when_both_are_present() {
let report = cli_failure_report(&["install", "sdk"], 3, "plan line", "boom");
assert!(report.contains("plan line"), "{report}");
assert!(report.contains("boom"), "{report}");
}

#[test]
fn chat_success_requires_non_empty_choices_array() {
Expand Down
16 changes: 16 additions & 0 deletions tests/e2e-cucumber/tests/e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
use std::path::PathBuf;

use cucumber::{World as _, WriterExt as _};
use e2e_cucumber::cli_failure_report;
use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with};
use tempfile::TempDir;

Expand Down Expand Up @@ -491,6 +492,21 @@ pub fn run_rocm(world: &E2eWorld, args: &[&str]) -> (String, String, i32) {
)
}

/// Run `rocm`, returning stdout, and panic with the full diagnostic bundle
/// ([`cli_failure_report`]) if it exits non-zero.
///
/// Use this instead of asserting on [`run_rocm`]'s `rc` by hand — that idiom is
/// what left a failed step undiagnosable in EAI-8031.
pub fn run_rocm_ok(world: &E2eWorld, args: &[&str]) -> String {
let (stdout, stderr, rc) = run_rocm(world, args);
assert!(
rc == 0,
"{}",
cli_failure_report(args, rc, &stdout, &stderr)
);
stdout
}

/// Like [`run_rocm`], but with extra environment variables set on the child.
///
/// Used by scenarios that must control the device environment the CLI and engine
Expand Down
6 changes: 2 additions & 4 deletions tests/e2e-cucumber/tests/e2e/runtime_steps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ async fn setup_active_runtime(world: &mut E2eWorld) {
world.use_shared_runtimes();
let (stdout, _, _) = crate::run_rocm(world, &["runtimes", "list"]);
if stdout.contains("installed: none") {
let (install_out, _, rc) = crate::run_rocm(world, &["install", "sdk"]);
assert!(rc == 0, "rocm install sdk failed (rc={rc}):\n{install_out}");
crate::run_rocm_ok(world, &["install", "sdk"]);
}
let (stdout, _, _) = crate::run_rocm(world, &["runtimes", "list"]);
assert!(
Expand All @@ -50,8 +49,7 @@ async fn setup_active_runtime(world: &mut E2eWorld) {

#[when("the user installs the SDK")]
async fn user_installs_sdk(world: &mut E2eWorld) {
let (stdout, _, rc) = crate::run_rocm(world, &["install", "sdk"]);
assert!(rc == 0, "rocm install sdk failed (rc={rc}):\n{stdout}");
let stdout = crate::run_rocm_ok(world, &["install", "sdk"]);
world.cli_output = Some(stdout);
}

Expand Down
31 changes: 21 additions & 10 deletions tests/e2e-cucumber/tests/e2e/serving_steps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -502,11 +502,10 @@ async fn setup_lemonade_model(world: &mut E2eWorld) {
// serve+inference coverage. Qwen3-0.6B-GGUF is the smallest lemonade recipe.
let model = "Qwen3-0.6B-GGUF";
ensure_serve_port_free().await;
let (stdout, _, rc) = crate::run_rocm(
crate::run_rocm_ok(
world,
&["serve", model, "--engine", "lemonade", "--managed"],
);
assert!(rc == 0, "rocm serve failed:\n{stdout}");
world.endpoint = Some("http://127.0.0.1:11435/v1".to_string());
world.model_name = Some(model.to_string());
// Wait for this lemonade model specifically (see setup_gpu_model): guards
Expand All @@ -528,11 +527,10 @@ async fn setup_lemonade_hf_checkpoint_model(world: &mut E2eWorld) {
// GGUF is already warm in the HF cache when both scenarios run in one job.
let model = "unsloth/Qwen3-0.6B-GGUF:Q4_0";
ensure_serve_port_free().await;
let (stdout, _, rc) = crate::run_rocm(
crate::run_rocm_ok(
world,
&["serve", model, "--engine", "lemonade", "--managed"],
);
assert!(rc == 0, "rocm serve failed:\n{stdout}");
world.endpoint = Some("http://127.0.0.1:11435/v1".to_string());
world.model_name = Some(model.to_string());
wait_for_model(
Expand Down Expand Up @@ -569,9 +567,7 @@ async fn setup_large_gpu_model(world: &mut E2eWorld) {
("Qwen/Qwen3.6-27B", "vllm", "Qwen3.6-27B")
};
ensure_serve_port_free().await;
let (stdout, _, rc) =
crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]);
assert!(rc == 0, "rocm serve failed:\n{stdout}");
crate::run_rocm_ok(world, &["serve", model, "--engine", engine, "--managed"]);
world.endpoint = Some("http://127.0.0.1:11435/v1".to_string());
world.model_name = Some(model.to_string());
wait_for_model(
Expand Down Expand Up @@ -660,13 +656,16 @@ async fn user_serves_default_engine(world: &mut E2eWorld) {
// fn's doc comment.
let model = default_engine_serve_target();
ensure_serve_port_free().await;
let (stdout, _, rc) = crate::run_rocm(world, &["serve", model, "--managed"]);
let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", model, "--managed"]);
// The model the CLI resolved (what actually gets served) can differ from the
// requested id, so downstream reachability/readiness checks must look for the
// resolved model on the shared port; fall back to the requested id.
let served = resolved_model(&stdout).unwrap_or(model).to_string();
let ready_substr = ready_substr_for(&served).to_string();
world.cli_output = Some(stdout);
// rc is asserted by a later Then step, so stderr has to survive with it —
// otherwise that deferred assertion reports a failure it cannot explain.
world.cli_stderr = Some(stderr);
world.cli_rc = Some(rc);
world.endpoint = Some("http://127.0.0.1:11435/v1".to_string());
world.model_name = Some(served);
Expand All @@ -690,9 +689,12 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) {
// platform. Qwen2.5-0.5B is the smallest vLLM-preferred catalog entry. Omit
// `--engine` so the CLI's own default selection is what's exercised.
ensure_serve_port_free().await;
let (stdout, _, rc) =
let (stdout, stderr, rc) =
crate::run_rocm(world, &["serve", "Qwen/Qwen2.5-0.5B-Instruct", "--managed"]);
world.cli_output = Some(stdout);
// See the note in user_serves_default_engine: the rc is asserted later, so
// stderr must be carried along to explain a non-zero one.
world.cli_stderr = Some(stderr);
world.cli_rc = Some(rc);
}

Expand Down Expand Up @@ -936,7 +938,16 @@ async fn assert_vllm_default(world: &mut E2eWorld) {
// non-zero rc after a good plan-print (e.g. the engine fails to start) would
// otherwise go undetected since this scenario only inspected the plan line.
let rc = world.cli_rc.expect("no serve rc recorded");
assert!(rc == 0, "rocm serve failed (rc={rc}):\n{output}");
assert!(
rc == 0,
"{}",
e2e_cucumber::cli_failure_report(
&["serve", "<default engine>", "--managed"],
rc,
output,
world.cli_stderr.as_deref().unwrap_or(""),
)
);
let engine = selected_engine(output);
assert_eq!(
engine, "vllm",
Expand Down
Loading