fix(lemonade): retry interrupted backend setup

apps/rocm/Cargo.toml

@@ -10,6 +10,9 @@ publish.workspace = true
 [lints]
 workspace = true
 
+[features]
+e2e-test-hooks = ["rocm-engine-lemonade/e2e-test-hooks"]
+
 [dependencies]
 anyhow.workspace = true
 clap.workspace = true

apps/rocm/src/main.rs

@@ -4630,8 +4630,13 @@ fn serve(args: ServeArgs) -> Result<()> {
     // BEFORE preparing or launching any engine (no wasted engine download, and an
     // actionable message instead of a late engine crash). The engine enforces the
     // same rule as a backstop. Skipped for cpu_only; permissive when availability
-    // cannot be probed on this platform (probe returns `None`).
+    // cannot be probed on this platform (probe returns `None`). The E2E-only
+    // backend-failure scenario bypasses this host precondition so the black-box
+    // test reaches Lemonade's backend boundary without real GPU hardware.
+    let scripted_backend_failure = cfg!(feature = "e2e-test-hooks")
+        && std::env::var_os("ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE").is_some();
     if !cpu_only
+        && !scripted_backend_failure
         && let Some(usable) = rocm_core::usable_amd_gpu_indices()
         && usable.is_empty()
     {

engines/lemonade/Cargo.toml

@@ -10,6 +10,9 @@ publish.workspace = true
 [lints]
 workspace = true
 
+[features]
+e2e-test-hooks = []
+
 [[bin]]
 name = "rocm-engine-lemonade"
 path = "src/main.rs"

engines/lemonade/src/lib.rs

@@ -38,6 +38,8 @@ const DEFAULT_MODEL_REPO_DIR: &str = "models--unsloth--Qwen3-4B-Instruct-2507-GG
 const DEFAULT_MODEL_GGUF: &str = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf";
 const LLAMACPP_RECIPE: &str = "llamacpp";
 const ROCM_BACKEND_NAME: &str = "rocm";
+#[cfg(feature = "e2e-test-hooks")]
+const BACKEND_INSTALL_FAILURE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE";
 /// Preferred llama.cpp backends, best first. Lemonade reports per-GPU support;
 /// we pick the highest-priority backend it considers supported on this host.
 /// GPU backends only — `cpu` is intentionally excluded so the router path never
@@ -424,6 +426,14 @@ fn detect_response() -> DetectResponse {
 fn install_response(request: InstallRequest) -> Result<InstallResponse> {
     let paths = AppPaths::discover()?;
     paths.ensure()?;
+    // Debug builds expose a deterministic failure seam for the black-box CLI
+    // scenario that pins retry count and terminal recovery guidance. Keep it at
+    // the backend phase: the real defect happens after the embeddable is ready,
+    // and exercising it must not download or alter a runtime on the test host.
+    #[cfg(feature = "e2e-test-hooks")]
+    if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() {
+        install_llamacpp_backend_with_retry(|| bail!("scripted Lemonade backend install failure"))?;
+    }
     eprintln!(
         "Preparing Lemonade embeddable {}...",
         rocm_deps::LEMONADE_VERSION
@@ -1132,6 +1142,25 @@ fn install_best_llamacpp_backend(manifest: &mut LemonadeInstallManifest) -> Resu
 
 /// Ask Lemonade which llama.cpp backends it supports on this GPU, choose the best
 /// one (`LLAMACPP_BACKEND_PRIORITY`), install it if necessary, and return its name.
+/// Retry the backend install itself once, rather than the whole Lemonade runtime
+/// preparation. The embeddable download already has bounded transport retries;
+/// repeating that outer operation would redo deterministic failures and may
+/// re-extract a healthy runtime. A backend subprocess can instead fail after a
+/// completed download when its connection to lemond is interrupted, and a second
+/// call can reuse the backend cache immediately without a delay.
+fn install_llamacpp_backend_with_retry(mut install: impl FnMut() -> Result<()>) -> Result<()> {
+    match install() {
+        Ok(()) => Ok(()),
+        Err(first_error) => {
+            eprintln!("Lemonade backend installation failed; retrying once: {first_error:#}");
+            install().with_context(|| {
+                "Lemonade backend installation failed again; run `rocm engines install \
+                 lemonade --reinstall` and then retry `rocm serve`"
+            })
+        }
+    }
+}
+
 fn ensure_best_llamacpp_backend(
     manifest: &LemonadeInstallManifest,
     host: &str,
@@ -1153,7 +1182,9 @@ fn ensure_best_llamacpp_backend(
         eprintln!("Using installed Lemonade {LLAMACPP_RECIPE}:{backend} backend.");
     } else {
         eprintln!("Installing Lemonade {LLAMACPP_RECIPE}:{backend} backend...");
-        run_lemonade_backend_install(manifest, host, port, &backend, process_env)?;
+        install_llamacpp_backend_with_retry(|| {
+            run_lemonade_backend_install(manifest, host, port, &backend, process_env)
+        })?;
     }
     Ok(backend)
 }
@@ -3857,6 +3888,65 @@ mod tests {
         dir
     }
 
+    #[test]
+    fn backend_install_succeeds_without_retry() {
+        let mut attempts = 0;
+
+        install_llamacpp_backend_with_retry(|| {
+            attempts += 1;
+            Ok(())
+        })
+        .unwrap();
+
+        assert_eq!(attempts, 1);
+    }
+
+    #[test]
+    fn backend_install_recovers_on_the_second_attempt() {
+        let mut attempts = 0;
+
+        install_llamacpp_backend_with_retry(|| {
+            attempts += 1;
+            if attempts == 1 {
+                bail!("first backend connection was interrupted");
+            }
+            Ok(())
+        })
+        .unwrap();
+
+        assert_eq!(attempts, 2);
+    }
+
+    #[test]
+    fn backend_install_stops_after_one_retry_with_reinstall_guidance() {
+        let mut attempts = 0;
+
+        let error = install_llamacpp_backend_with_retry(|| {
+            attempts += 1;
+            if attempts == 1 {
+                bail!("first backend connection was interrupted");
+            }
+            bail!("second backend connection was interrupted");
+        })
+        .unwrap_err();
+        let rendered = format!("{error:#}");
+
+        assert_eq!(attempts, 2);
+        assert!(
+            rendered.contains("second backend connection was interrupted"),
+            "{rendered}"
+        );
+        assert!(
+            !rendered.contains("first backend connection was interrupted"),
+            "the terminal error must be the retry's failure: {rendered}"
+        );
+        assert!(
+            rendered.contains("rocm engines install lemonade --reinstall"),
+            "{rendered}"
+        );
+        assert!(rendered.contains("retry `rocm serve`"), "{rendered}");
+    }
+
     fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest {
         LemonadeInstallManifest {
             env_id: "test".to_owned(),

tests/e2e-cucumber/features/model_serving.feature

@@ -154,3 +154,15 @@ Feature: Model serving
     When the user serves a model pinned to a GPU index that does not exist
     Then serving is refused before any engine starts
     And the user is told that GPU index is unavailable
+
+  # The failure is injected at Lemonade's backend-install boundary in debug/test
+  # builds, after the CLI has selected Lemonade but before any runtime download or
+  # machine mutation. That makes the user-visible retry and final recovery command
+  # deterministic on the blocking no-GPU lane rather than relying on a real 3 GiB
+  # transfer to fail at just the right moment.
+  @id:serve-lemonade-preparation-recovery @requires-no-gpu
+  Scenario: 15 - Repeated Lemonade preparation failure gives the user a recovery path
+    Given Lemonade preparation cannot complete
+    When the user serves a model with Lemonade
+    Then serving stops after one automatic retry
+    And the user is told how to reinstall Lemonade and retry serving

tests/e2e-cucumber/tests/e2e.rs

@@ -63,6 +63,10 @@ pub struct E2eWorld {
     /// a run whose failure is already the expected outcome — see the relaunch
     /// budget in `setup_gpu_model`.
     pub expect_xfail: bool,
+    /// Extra environment for this scenario's next `rocm` invocation. A Given step
+    /// records a behavioral precondition here; the When step remains a plain user
+    /// action and consumes the fixture without exposing its mechanism in Gherkin.
+    pub command_env: Vec<(&'static str, std::ffi::OsString)>,
     /// The interactive dash/chat TUI spawned under a pseudo-terminal for this
     /// scenario, if any (see `e2e::tui_driver`). Torn down in `Drop` before the
     /// mock server and isolated directory so the child process never outlives
@@ -177,6 +181,7 @@ impl Default for E2eWorld {
             legacy_rocm_path: None,
             serve_timeout_override: None,
             expect_xfail: false,
+            command_env: Vec::new(),
             tui: None,
             chat_use_mock: false,
             lifecycle: None,
@@ -522,6 +527,29 @@ pub fn run_rocm_with_env(
     )
 }
 
+/// Run `rocm` with the behavioral fixture established by a Given step, then
+/// consume it so it cannot leak into a later action in the same scenario.
+pub fn run_rocm_with_scenario_env(world: &mut E2eWorld, args: &[&str]) -> (String, String, i32) {
+    let binary = rocm_binary();
+    let mut cmd = std::process::Command::new(&binary);
+    cmd.args(args);
+    world.isolate_cmd(&mut cmd);
+    for (key, value) in std::mem::take(&mut world.command_env) {
+        cmd.env(key, value);
+    }
+    let output = cmd
+        .output()
+        .unwrap_or_else(|e| panic!("failed to run {binary}: {e}"));
+    let rc = output.status.code().unwrap_or(-1);
+    let stdout = String::from_utf8_lossy(&output.stdout).to_string();
+    record_command(world.current_scenario.as_deref(), args, rc, &stdout);
+    (
+        stdout,
+        String::from_utf8_lossy(&output.stderr).to_string(),
+        rc,
+    )
+}
+
 /// Append one `rocm` invocation to `results/commands.jsonl` so the consolidated
 /// report can build a command × platform coverage table tied to real results.
 /// Best-effort: a recording failure must never fail a scenario.

tests/e2e-cucumber/tests/e2e/serving_steps.rs

@@ -696,6 +696,31 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) {
     world.cli_rc = Some(rc);
 }
 
+#[given("Lemonade preparation cannot complete")]
+async fn lemonade_preparation_cannot_complete(world: &mut E2eWorld) {
+    world.command_env.push((
+        "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE",
+        "repeated".into(),
+    ));
+}
+
+#[when("the user serves a model with Lemonade")]
+async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) {
+    let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(
+        world,
+        &[
+            "serve",
+            "Qwen3-0.6B-GGUF",
+            "--engine",
+            "lemonade",
+            "--managed",
+        ],
+    );
+    world.cli_output = Some(stdout);
+    world.cli_stderr = Some(stderr);
+    world.cli_rc = Some(rc);
+}
+
 #[when("the user sends a chat completion request")]
 async fn user_sends_completion(world: &mut E2eWorld) {
     crate::send_chat(world).await;
@@ -755,6 +780,34 @@ async fn when_cli_reports_ready(world: &mut E2eWorld) {
 
 // ── Then ───────────────────────────────────────────────────────────
 
+#[then("serving stops after one automatic retry")]
+async fn assert_lemonade_preparation_retry_is_bounded(world: &mut E2eWorld) {
+    let output = serve_output(world);
+    assert_ne!(
+        world.cli_rc,
+        Some(0),
+        "serve unexpectedly succeeded:\n{output}"
+    );
+    assert_eq!(
+        output.matches("retrying once").count(),
+        1,
+        "expected exactly one retry announcement:\n{output}"
+    );
+}
+
+#[then("the user is told how to reinstall Lemonade and retry serving")]
+async fn assert_lemonade_recovery_guidance(world: &mut E2eWorld) {
+    let output = serve_output(world);
+    assert!(
+        output.contains("rocm engines install lemonade --reinstall"),
+        "expected a forced-reinstall recovery command:\n{output}"
+    );
+    assert!(
+        output.contains("retry `rocm serve`"),
+        "expected guidance to retry serving:\n{output}"
+    );
+}
+
 #[then("an inference request succeeds immediately")]
 async fn assert_inference_succeeds_now(world: &mut E2eWorld) {
     // No extra wait: the CLI already reported ready, so inference must work now.

xtask/src/e2e.rs

@@ -43,7 +43,16 @@ pub fn run(args: &[String]) -> Result<()> {
         }
     } else {
         let status = Command::new(&cargo)
-            .args(["build", "--release", "-p", "rocm", "-p", "rocmd"])
+            .args([
+                "build",
+                "--release",
+                "-p",
+                "rocm",
+                "-p",
+                "rocmd",
+                "--features",
+                "rocm/e2e-test-hooks",
+            ])
             .current_dir(&root)
             .status()
             .context("failed to run `cargo build --release -p rocm`")?;