feat(simulate): fail CI only on new baseline regressions

README.md

@@ -455,6 +455,20 @@ The above simulates 5 concurrent rooms, where each room has:
 
 Once the specified duration is over (or if the load test is manually stopped), the load test statistics will be displayed in the form of a table.
 
+## Agent simulations in CI
+
+Use a finished simulation run as a baseline so known failures are reported without failing CI, while regressions still return a nonzero exit code:
+
+```shell
+lk agent simulate \
+  --scenarios scenarios.yaml \
+  --baseline "$SIMULATION_BASELINE_RUN_ID" \
+  --run-id-file "$RUNNER_TEMP/simulation-run-id"
+```
+
+The CLI writes the new run ID to `--run-id-file` as soon as the run is created, even if the simulation later fails. Store the ID in your CI provider's variable or artifact store. Only a successful run on the main branch should replace the stored baseline; pull requests and failed main runs should leave it unchanged.
+
+For the first run, omit `--baseline`, inspect and accept its results, then store the ID written to the file. A missing, unfinished, or inaccessible baseline fails CI rather than silently using strict comparison.
 
 ## Browsing documentation
 

autocomplete/fish_autocomplete

@@ -226,6 +226,8 @@ complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcomma
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l yes -s y -d 'Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l view -r -d 'Open a pre-existing simulation'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l export -r -d 'Print the run with run `ID` and its exact per-job chat contexts as JSON. Nothing is run or polled: the run must already be finished'
+complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l baseline -r -d 'Compare failures against the finished run with run `ID`: only scenarios that pass there and fail here fail the exit code. Non-interactive (CI) runs only'
+complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l run-id-file -r -d 'Write the simulation run ID to `FILE` as soon as it is available. Non-interactive (CI) runs only'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l agent-name -r -d 'Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or "" to target the project\'s default agent (the one that auto-joins every room). Requires --scenarios.'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l help -s h -d 'show help'
 complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from audio' -a 'audio' -d 'Simulate speech-to-speech interactions using the agent\'s full audio pipeline'

cmd/lk/simulate.go

@@ -101,6 +101,14 @@ var simulateCommand = &cli.Command{
 			Name:  "export",
 			Usage: "Print the run with run `ID` and its exact per-job chat contexts as JSON. Nothing is run or polled: the run must already be finished",
 		},
+		&cli.StringFlag{
+			Name:  "baseline",
+			Usage: "Compare failures against the finished run with run `ID`: only scenarios that pass there and fail here fail the exit code. Non-interactive (CI) runs only",
+		},
+		&cli.StringFlag{
+			Name:  "run-id-file",
+			Usage: "Write the simulation run ID to `FILE` as soon as it is available. Non-interactive (CI) runs only",
+		},
 		&cli.StringFlag{
 			Name:  "agent-name",
 			Usage: "Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or \"\" to target the project's default agent (the one that auto-joins every room). Requires --scenarios.",
@@ -214,6 +222,8 @@ type simulateConfig struct {
 	scenarioGroup  *livekit.ScenarioGroup
 	scenariosPath  string   // path to the --scenarios file (empty when generating from source)
 	viewModeRunID  string   // non-empty when --view opens a pre-existing run
+	baselineRunID  string   // --baseline: failures this run also has don't fail CI
+	runIDFile      string   // --run-id-file: machine-readable handoff to CI
 	liveAgent      bool     // --agent-name: run against an already-running agent, don't spawn one
 	warnings       []string // config-level warnings surfaced at setup (e.g. ignored flags)
 
@@ -416,6 +426,8 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 		scenarioGroup:  scenarioGroup,
 		scenariosPath:  scenariosPath,
 		viewModeRunID:  runID,
+		baselineRunID:  cmd.String("baseline"),
+		runIDFile:      cmd.String("run-id-file"),
 		liveAgent:      liveAgent,
 		warnings:       simulateConfigWarnings(mode, numSimulations),
 	}
@@ -429,6 +441,12 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 	if !isInteractive() {
 		return runSimulateCI(ctx, simCfg)
 	}
+	if simCfg.baselineRunID != "" {
+		return fmt.Errorf("--baseline only applies to non-interactive (CI) runs; the TUI already shows every failure")
+	}
+	if simCfg.runIDFile != "" {
+		return fmt.Errorf("--run-id-file only applies to non-interactive (CI) runs; the TUI already shows the run ID")
+	}
 	return runSimulateTUI(simCfg)
 }
 

cmd/lk/simulate_ci.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"os"
 	"os/signal"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -119,6 +120,11 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
 		report.EndSetup()
 		return err
 	}
+	if err := writeSimulationRunID(config.runIDFile, runID); err != nil {
+		report.SetupFailed(err)
+		report.EndSetup()
+		return err
+	}
 	report.SimulationCreated(time.Since(start))
 
 	if config.mode == modeGenerateFromSource {
@@ -217,23 +223,118 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
 		out.Statusf("Dashboard:  %s", url)
 	}
 
-	return runFailureError(run)
+	return baselineFailureError(ctx, config, run)
+}
+
+func writeSimulationRunID(path, runID string) error {
+	if path == "" {
+		return nil
+	}
+	if err := os.WriteFile(path, []byte(runID+"\n"), 0o644); err != nil {
+		return fmt.Errorf("write simulation run ID to %s: %w", path, err)
+	}
+	return nil
+}
+
+// baselineFailureError fetches the --baseline run when one was given and
+// reports which failures it already had before deciding the exit error. A
+// baseline that can't be fetched fails CI loudly rather than silently
+// falling back to strict comparison.
+func baselineFailureError(ctx context.Context, config *simulateConfig, run *livekit.SimulationRun) error {
+	var baseline *livekit.SimulationRun
+	if config.baselineRunID != "" {
+		fetchCtx, cancel := context.WithTimeout(ctx, simulationAPITimeout)
+		defer cancel()
+		var err error
+		baseline, err = getSimulationRun(fetchCtx, config.client, config.baselineRunID, config.pc.ProjectId)
+		if err != nil {
+			return fmt.Errorf("fetch baseline run %s: %w", config.baselineRunID, err)
+		}
+		if !isTerminalRunStatus(baseline.GetStatus()) {
+			return fmt.Errorf("baseline run %s is still in progress", config.baselineRunID)
+		}
+		if cmp := compareToBaseline(run, baseline); len(cmp.knownFailures) > 0 {
+			out.Statusf("%d failure(s) already failing in baseline %s (not failing CI): %s",
+				len(cmp.knownFailures), config.baselineRunID, strings.Join(cmp.knownFailures, ", "))
+		}
+	}
+	return runFailureError(run, baseline)
 }
 
 // runFailureError converts a terminal run's failures into the CI exit error;
 // the error is printed by main and reports the failure — the counts line /
 // full dump above already carries the detail. Returns nil when everything
-// passed.
-func runFailureError(run *livekit.SimulationRun) error {
+// passed. With a baseline run, scenario failures the baseline already had
+// don't fail CI; a run-level STATUS_FAILED is systemic (the server only sets
+// it when generation/submission breaks, never for scenario failures) and
+// fails regardless of baseline.
+func runFailureError(run, baseline *livekit.SimulationRun) error {
 	_, _, _, failed := simulationJobCounts(run)
-	if failed > 0 || run.Status == livekit.SimulationRun_STATUS_FAILED {
-		if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 {
+	if failed == 0 && run.Status != livekit.SimulationRun_STATUS_FAILED {
+		return nil
+	}
+	if run.Status == livekit.SimulationRun_STATUS_FAILED {
+		if len(run.Jobs) == 0 {
 			return fmt.Errorf("simulation failed: %s", run.Error)
 		}
 		return fmt.Errorf("%d of %d simulations failed", failed, len(run.Jobs))
 	}
+	if baseline == nil {
+		return fmt.Errorf("%d of %d simulations failed", failed, len(run.Jobs))
+	}
 
-	return nil
+	cmp := compareToBaseline(run, baseline)
+	if len(cmp.newFailures) == 0 {
+		return nil
+	}
+	return fmt.Errorf("%d new simulation failure(s) not failing in the baseline: %s",
+		len(cmp.newFailures), strings.Join(cmp.newFailures, ", "))
+}
+
+// baselineComparison splits the run's failed scenarios by whether the
+// baseline run already failed them.
+type baselineComparison struct {
+	newFailures   []string
+	knownFailures []string
+}
+
+func compareToBaseline(run, baseline *livekit.SimulationRun) baselineComparison {
+	known := make(map[string]bool)
+	for _, key := range failedScenarioKeys(baseline) {
+		known[key] = true
+	}
+	var cmp baselineComparison
+	for _, key := range failedScenarioKeys(run) {
+		if known[key] {
+			cmp.knownFailures = append(cmp.knownFailures, key)
+		} else {
+			cmp.newFailures = append(cmp.newFailures, key)
+		}
+	}
+	return cmp
+}
+
+// failedScenarioKeys returns each failed scenario once, in job order. The
+// label (scenario name) identifies a scenario across runs; generated jobs may
+// carry only instructions. Repeats of one scenario (--num-simulations) share
+// a key, so any failed repeat marks the scenario failed.
+func failedScenarioKeys(run *livekit.SimulationRun) []string {
+	seen := make(map[string]bool)
+	var keys []string
+	for _, job := range run.GetJobs() {
+		if job.GetStatus() != livekit.SimulationRun_Job_STATUS_FAILED {
+			continue
+		}
+		key := job.GetLabel()
+		if key == "" {
+			key = job.GetInstructions()
+		}
+		if !seen[key] {
+			seen[key] = true
+			keys = append(keys, key)
+		}
+	}
+	return keys
 }
 
 // runSimulateCIView handles --view in non-interactive mode: it fetches the
@@ -246,6 +347,9 @@ func runSimulateCIView(ctx context.Context, config *simulateConfig) error {
 
 	report := newSimLog(out.ResultWriter(), out.StatusWriter())
 	runID := config.viewModeRunID
+	if err := writeSimulationRunID(config.runIDFile, runID); err != nil {
+		return err
+	}
 
 	ticker := time.NewTicker(simulationPollInterval)
 	defer ticker.Stop()
@@ -281,5 +385,5 @@ func runSimulateCIView(ctx context.Context, config *simulateConfig) error {
 		out.Statusf("Dashboard:  %s", url)
 	}
 
-	return runFailureError(run)
+	return baselineFailureError(ctx, config, run)
 }