feat(eval): pin injected docker-agent image to host CLI version by default

cmd/root/eval.go

@@ -52,6 +52,8 @@ func newEvalCmd() *cobra.Command {
 	cmd.Flags().StringVar(&flags.outputDir, "output", "", "Directory for results and logs (default: <eval-dir>/results)")
 	cmd.Flags().StringSliceVar(&flags.Only, "only", nil, "Only run evaluations with file names matching these patterns (can be specified multiple times)")
 	cmd.Flags().StringVar(&flags.BaseImage, "base-image", "", "Custom base image for running evaluations")
+	cmd.Flags().StringVar(&flags.AgentImage, "agent-image", "",
+		"docker-agent image to inject into eval containers (default: pinned to this CLI's own release version, e.g. docker/docker-agent:1.2.3; falls back to docker/docker-agent:edge for dev builds); pass \"none\" to skip injection and trust the base image's own binary")
 	cmd.Flags().StringVar(&flags.ContainerRuntime, "container-runtime", evaluation.DefaultContainerRuntime, "Container runtime executable for building and running evaluations")
 	cmd.Flags().BoolVar(&flags.KeepContainers, "keep-containers", false, "Keep containers after evaluation (don't use --rm)")
 	cmd.Flags().StringSliceVarP(&flags.EnvVars, "env", "e", nil, "Environment variables to pass to container (KEY or KEY=VALUE)")
@@ -118,6 +120,11 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) (commandEr
 	fmt.Fprintf(logFile, "Judge model: %s\n", f.JudgeModel)
 	fmt.Fprintf(logFile, "Concurrency: %d\n", f.Concurrency)
 	fmt.Fprintf(logFile, "Container runtime: %s\n", f.ContainerRuntime)
+	if agentImage := evaluation.ResolvedAgentImage(f.Config); agentImage != "" {
+		fmt.Fprintf(logFile, "Agent image: %s\n", agentImage)
+	} else {
+		fmt.Fprintf(logFile, "Agent image: (none, trusting base image binary)\n")
+	}
 	fmt.Fprintf(logFile, "\n")
 
 	// Create tee writer to write to both console and log file

cmd/root/eval_test.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/docker/docker-agent/pkg/evaluation"
 )
 
 func TestEvalContainerRuntimeFlagDefaultsToDocker(t *testing.T) {
@@ -27,3 +29,35 @@ func TestEvalContainerRuntimeFlagAcceptsCustomExecutable(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "podman", value)
 }
+
+func TestEvalAgentImageFlagDefaultsToEmpty(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+
+	flag := cmd.Flags().Lookup("agent-image")
+	require.NotNil(t, flag, "eval must expose --agent-image")
+	assert.Empty(t, flag.DefValue, "unset --agent-image must defer to the version-derived default")
+}
+
+func TestEvalAgentImageFlagAcceptsOverride(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+	require.NoError(t, cmd.Flags().Parse([]string{"--agent-image", "docker/docker-agent:1.2.3"}))
+
+	value, err := cmd.Flags().GetString("agent-image")
+	require.NoError(t, err)
+	assert.Equal(t, "docker/docker-agent:1.2.3", value)
+}
+
+func TestEvalAgentImageFlagAcceptsNoneToSkipInjection(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+	require.NoError(t, cmd.Flags().Parse([]string{"--agent-image", "none"}))
+
+	value, err := cmd.Flags().GetString("agent-image")
+	require.NoError(t, err)
+	assert.Equal(t, evaluation.NoAgentImage, value)
+}

docs/features/cli/index.md

@@ -523,6 +523,7 @@ $ docker agent eval <agent-file>|<registry-ref> [<eval-dir>|./evals] [flags]
 | `--output <dir>`    | `<eval-dir>/results`                 | Directory for results, logs, and session databases                         |
 | `--only <pattern>`  | (all)                                | Only run evals with file names matching these patterns (repeatable)        |
 | `--base-image`      | (default)                            | Custom base image for eval containers                                      |
+| `--agent-image`     | (this CLI's version)                 | docker-agent image injected into eval containers; `none` skips injection   |
 | `--container-runtime` | `docker`                           | Container runtime executable for building and running evaluations (e.g. `podman`) |
 | `--keep-containers` | `false`                              | Keep containers after evaluation (don't remove with `--rm`)                |
 | `-e, --env`         | (none)                               | Environment variables to pass to container (`KEY` or `KEY=VALUE`, repeatable) |

docs/features/evaluation/index.md

@@ -218,6 +218,7 @@ $ docker agent eval <agent-file>|<registry-ref> [<eval-dir>|./evals]
 | `--output`          | `<eval-dir>/results`  | Directory for results, logs, and session databases                |
 | `--only`            | (all)                       | Only run evals with file names matching these patterns            |
 | `--base-image`      | (default)                   | Custom base image for eval containers (see [Custom Base Images](#custom-base-images)) |
+| `--agent-image`     | (this CLI's version)        | docker-agent image injected into eval containers; `none` skips injection (see [Custom Base Images](#custom-base-images)) |
 | `--container-runtime` | `docker`                  | Container runtime executable for building and running evaluations (e.g. `podman`) |
 | `--keep-containers` | `false`                     | Keep containers after evaluation (don't remove with `--rm`)       |
 | `-e, --env`         | (none)                      | Environment variables to pass to container (`KEY` or `KEY=VALUE`) |
@@ -299,7 +300,7 @@ evaluated agent run fails to authenticate.
 
 When `--base-image` is set, the eval harness builds a derived image on top of your base image at evaluation time. Two things happen automatically:
 
-1. **The docker-agent binary is injected** — it is copied from `docker/docker-agent:edge` into the derived image at build time, so you don't need to include it in your base image.
+1. **The docker-agent binary is injected** — by default it is copied from `docker/docker-agent:<version>`, pinned to this CLI's own release version, so eval results are reproducible against a known build rather than a moving target. A dev build (compiled from `main`, without a release version) falls back to `docker/docker-agent:edge`. This injection applies to every eval run, not just those using `--base-image`. Use `--agent-image <ref>` to inject a specific image instead — for example to pin CI to an older release, or to test against `docker/docker-agent:edge` deliberately. Pass `--agent-image none` to skip injection entirely and trust whatever `/docker-agent` binary is already present in your base image.
 2. **The entrypoint is overridden** — Docker Agent replaces your base image's entrypoint with its own `/run.sh` wrapper.
 
 Your base image therefore only needs to provide the runtime environment: language runtimes, installed dependencies, test fixtures, the appropriate working directory, and so on. Any `ENTRYPOINT` or `CMD` defined in your base image is ignored.

pkg/evaluation/Dockerfile.custom.template

@@ -2,8 +2,8 @@
 
 FROM {{.BaseImage}}
 LABEL "io.docker.agent.evals.image"="custom"
-COPY --from=docker/docker-agent:edge /docker-agent /
-RUN printf '#!/usr/bin/env sh\nset -euo pipefail\nexec "$@"\n' > /run.sh && chmod +x /run.sh
+{{if .AgentImage}}COPY --from={{.AgentImage}} /docker-agent /
+{{end}}RUN printf '#!/usr/bin/env sh\nset -euo pipefail\nexec "$@"\n' > /run.sh && chmod +x /run.sh
 WORKDIR /working_dir
 ENV TELEMETRY_ENABLED=false
 ENV DOCKER_AGENT_HIDE_TELEMETRY_BANNER=1

pkg/evaluation/Dockerfile.template

@@ -2,8 +2,8 @@
 
 FROM alpine:latest
 LABEL "io.docker.agent.evals.image"="default"
-COPY --from=docker/docker-agent:edge /docker-agent /
-RUN printf '#!/usr/bin/env sh\nset -euo pipefail\nexec "$@"\n' > /run.sh && chmod +x /run.sh
+{{if .AgentImage}}COPY --from={{.AgentImage}} /docker-agent /
+{{end}}RUN printf '#!/usr/bin/env sh\nset -euo pipefail\nexec "$@"\n' > /run.sh && chmod +x /run.sh
 WORKDIR /working_dir
 ENV TELEMETRY_ENABLED=false
 ENV DOCKER_AGENT_HIDE_TELEMETRY_BANNER=1

pkg/evaluation/agent_image.go

@@ -0,0 +1,53 @@
+package evaluation
+
+import (
+	"fmt"
+
+	"github.com/Masterminds/semver/v3"
+
+	"github.com/docker/docker-agent/pkg/version"
+)
+
+// defaultAgentImageRepo is the Docker Hub repository CI publishes the
+// docker-agent binary to, tagged both with release semvers and a rolling
+// :edge tracking main.
+const defaultAgentImageRepo = "docker/docker-agent"
+
+// edgeAgentImage is injected when the host CLI isn't a release build (e.g.
+// compiled from main or a PR), so there is no matching version tag to pin to.
+const edgeAgentImage = defaultAgentImageRepo + ":edge"
+
+// NoAgentImage is the Config.AgentImage sentinel that skips injecting a
+// docker-agent binary into the eval image entirely, trusting whatever
+// /docker-agent is already present in the (custom) base image.
+const NoAgentImage = "none"
+
+// DefaultAgentImage returns the docker-agent image injected into eval
+// containers when no explicit --agent-image override is given: the release
+// image matching the host CLI's own version, so eval results are
+// reproducible against a pinned build rather than a moving main-HEAD binary.
+// Falls back to the rolling :edge image when the host binary isn't a release
+// build (version.Version isn't a valid semantic version, e.g. "dev" or "pr").
+func DefaultAgentImage() string {
+	v, err := semver.NewVersion(version.Version)
+	if err != nil {
+		return edgeAgentImage
+	}
+	return fmt.Sprintf("%s:%s", defaultAgentImageRepo, v.String())
+}
+
+// ResolvedAgentImage returns the docker-agent image to inject into eval
+// containers for the given config: the explicit cfg.AgentImage override when
+// set, DefaultAgentImage() when unset, or "" when cfg.AgentImage is
+// NoAgentImage, meaning injection is skipped and the base image's own
+// /docker-agent binary is trusted instead.
+func ResolvedAgentImage(cfg Config) string {
+	switch cfg.AgentImage {
+	case "":
+		return DefaultAgentImage()
+	case NoAgentImage:
+		return ""
+	default:
+		return cfg.AgentImage
+	}
+}

pkg/evaluation/agent_image_test.go

@@ -0,0 +1,64 @@
+package evaluation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/docker/docker-agent/pkg/version"
+)
+
+// withVersion temporarily overrides version.Version for the duration of the
+// test, restoring it afterwards. version.Version is a package-level var (set
+// via -ldflags at release build time), so tests mutate it directly rather
+// than plumbing it through as a parameter.
+func withVersion(t *testing.T, v string) {
+	t.Helper()
+	original := version.Version
+	version.Version = v
+	t.Cleanup(func() { version.Version = original })
+}
+
+func TestDefaultAgentImage(t *testing.T) {
+	tests := []struct {
+		name    string
+		version string
+		want    string
+	}{
+		{name: "release version with v prefix", version: "v1.133.0", want: "docker/docker-agent:1.133.0"},
+		{name: "release version without v prefix", version: "1.133.0", want: "docker/docker-agent:1.133.0"},
+		{name: "dev build", version: "dev", want: edgeAgentImage},
+		{name: "main build", version: "main", want: edgeAgentImage},
+		{name: "pr build", version: "pr", want: edgeAgentImage},
+		{name: "empty version", version: "", want: edgeAgentImage},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withVersion(t, tt.version)
+			assert.Equal(t, tt.want, DefaultAgentImage())
+		})
+	}
+}
+
+func TestResolvedAgentImage(t *testing.T) {
+	withVersion(t, "v1.133.0")
+
+	tests := []struct {
+		name       string
+		agentImage string
+		want       string
+	}{
+		{name: "default", agentImage: "", want: "docker/docker-agent:1.133.0"},
+		{name: "skip injection", agentImage: NoAgentImage, want: ""},
+		{name: "explicit override", agentImage: "docker/docker-agent:1.100.0", want: "docker/docker-agent:1.100.0"},
+		{name: "explicit override, different registry", agentImage: "myregistry.example.com/docker-agent:custom", want: "myregistry.example.com/docker-agent:custom"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := Config{AgentImage: tt.agentImage}
+			assert.Equal(t, tt.want, ResolvedAgentImage(cfg))
+		})
+	}
+}

pkg/evaluation/build.go

@@ -86,7 +86,9 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria
 	var data struct {
 		CopyWorkingDir bool
 		BaseImage      string
+		AgentImage     string
 	}
+	data.AgentImage = ResolvedAgentImage(r.Config)
 
 	if evals.WorkingDir == "" {
 		buildContext = r.EvalsDir

pkg/evaluation/dockerfile_template_test.go

@@ -10,15 +10,17 @@ import (
 
 // renderTemplate is a small helper to render the embedded Dockerfile
 // templates with the same fields buildEvalImage populates.
-func renderTemplate(t *testing.T, custom, copyWorkingDir bool, baseImage string) string {
+func renderTemplate(t *testing.T, custom, copyWorkingDir bool, baseImage, agentImage string) string {
 	t.Helper()
 
 	var data struct {
 		CopyWorkingDir bool
 		BaseImage      string
+		AgentImage     string
 	}
 	data.CopyWorkingDir = copyWorkingDir
 	data.BaseImage = baseImage
+	data.AgentImage = agentImage
 
 	tmpl := dockerfileTemplate
 	if custom {
@@ -40,11 +42,11 @@ func renderTemplate(t *testing.T, custom, copyWorkingDir bool, baseImage string)
 func TestDockerfileCustomTemplateParity(t *testing.T) {
 	t.Parallel()
 
-	out := renderTemplate(t, true /* custom */, true /* copyWorkingDir */, "python:3.12")
+	out := renderTemplate(t, true /* custom */, true /* copyWorkingDir */, "python:3.12", "docker/docker-agent:1.2.3")
 
 	assert.Contains(t, out, "FROM python:3.12",
 		"custom template must use the provided base image")
-	assert.Contains(t, out, "COPY --from=docker/docker-agent:edge /docker-agent /",
+	assert.Contains(t, out, "COPY --from=docker/docker-agent:1.2.3 /docker-agent /",
 		"custom template must copy the docker-agent binary into the eval image")
 	assert.Contains(t, out, `ENTRYPOINT ["/run.sh", "/docker-agent", "run", "--exec", "--yolo", "--json"]`,
 		"custom template must set the /run.sh docker-agent run entrypoint")
@@ -59,7 +61,7 @@ func TestDockerfileTemplatesRender(t *testing.T) {
 
 	for _, custom := range []bool{false, true} {
 		for _, copyWorkingDir := range []bool{false, true} {
-			out := renderTemplate(t, custom, copyWorkingDir, "alpine:latest")
+			out := renderTemplate(t, custom, copyWorkingDir, "alpine:latest", "docker/docker-agent:edge")
 			assert.Contains(t, out, "ENTRYPOINT [")
 			if copyWorkingDir {
 				assert.Contains(t, out, "COPY . ./")
@@ -69,3 +71,19 @@ func TestDockerfileTemplatesRender(t *testing.T) {
 		}
 	}
 }
+
+// TestDockerfileTemplatesSkipInjectionWhenAgentImageEmpty covers the "none"
+// --agent-image mode (resolved to an empty AgentImage), where the eval image
+// must not copy in a docker-agent binary at all and instead trusts whatever
+// /docker-agent is already present in the base image.
+func TestDockerfileTemplatesSkipInjectionWhenAgentImageEmpty(t *testing.T) {
+	t.Parallel()
+
+	for _, custom := range []bool{false, true} {
+		out := renderTemplate(t, custom, false, "alpine:latest", "")
+		assert.NotContains(t, out, "COPY --from=",
+			"no docker-agent binary should be injected when AgentImage is empty")
+		assert.Contains(t, out, `ENTRYPOINT ["/run.sh", "/docker-agent", "run", "--exec", "--yolo", "--json"]`,
+			"entrypoint must still run whatever /docker-agent the base image provides")
+	}
+}

pkg/evaluation/save.go

@@ -431,6 +431,7 @@ func SaveRunSessionsJSON(run *EvalRun, outputDir string) (string, error) {
 			Concurrency:      run.Config.Concurrency,
 			EvalsDir:         run.Config.EvalsDir,
 			BaseImage:        run.Config.BaseImage,
+			AgentImage:       ResolvedAgentImage(run.Config),
 			ContainerRuntime: run.Config.ContainerRuntime,
 		},
 		Summary:  run.Summary,

pkg/evaluation/save_test.go

@@ -333,6 +333,66 @@ func TestSaveRunSessionsJSONContainerRuntime(t *testing.T) {
 	})
 }
 
+// TestSaveRunSessionsJSONAgentImage covers the RunOutputConfig.AgentImage
+// cases: it records the resolved image (not the raw Config.AgentImage), and
+// unlike sibling fields it is never omitted, even when empty, so an explicit
+// --agent-image none (skip injection) is distinguishable from a run
+// predating this field.
+func TestSaveRunSessionsJSONAgentImage(t *testing.T) {
+	t.Parallel()
+
+	withVersion(t, "v1.133.0")
+
+	save := func(t *testing.T, cfg Config) []byte {
+		t.Helper()
+		run := &EvalRun{
+			Name:      "test-agent-image-001",
+			Timestamp: time.Now(),
+			Config:    cfg,
+		}
+
+		sessionsPath, err := SaveRunSessionsJSON(run, t.TempDir())
+		require.NoError(t, err)
+
+		data, err := os.ReadFile(sessionsPath)
+		require.NoError(t, err)
+		return data
+	}
+
+	t.Run("records the version-derived default", func(t *testing.T) {
+		t.Parallel()
+
+		data := save(t, Config{})
+
+		var output RunOutput
+		require.NoError(t, json.Unmarshal(data, &output))
+		assert.Equal(t, "docker/docker-agent:1.133.0", output.Config.AgentImage)
+	})
+
+	t.Run("records an explicit override", func(t *testing.T) {
+		t.Parallel()
+
+		data := save(t, Config{AgentImage: "docker/docker-agent:1.100.0"})
+
+		var output RunOutput
+		require.NoError(t, json.Unmarshal(data, &output))
+		assert.Equal(t, "docker/docker-agent:1.100.0", output.Config.AgentImage)
+	})
+
+	t.Run("present but empty when injection is skipped", func(t *testing.T) {
+		t.Parallel()
+
+		data := save(t, Config{AgentImage: NoAgentImage})
+
+		var raw struct {
+			Config map[string]any `json:"config"`
+		}
+		require.NoError(t, json.Unmarshal(data, &raw))
+		require.Contains(t, raw.Config, "agent_image")
+		assert.Empty(t, raw.Config["agent_image"])
+	})
+}
+
 func TestSaveRunSessionsWithCost(t *testing.T) {
 	t.Parallel()