POC: agentInject agent-only checkpoint storage (do not merge)

deploy/helm/charts/platform/components/operator/templates/operator-config.yaml

@@ -180,9 +180,18 @@ data:
       {{- $pvc := .pvc | default dict }}
       storage:
         type: {{ .type | quote }}
+        {{- if .accessMode }}
+        accessMode: {{ .accessMode | quote }}
+        {{- end }}
         {{- if eq .type "pvc" }}
         pvc:
-          pvcName: {{ required "checkpoint.storage.pvc.pvcName is required when checkpoint.storage.type=pvc" $pvc.pvcName | quote }}
+          {{- if eq (.accessMode | default "") "agentInject" }}
+          {{- if $pvc.pvcName }}
+          pvcName: {{ $pvc.pvcName | quote }}
+          {{- end }}
+          {{- else }}
+          pvcName: {{ required "checkpoint.storage.pvc.pvcName is required when checkpoint.storage.type=pvc and accessMode is not agentInject" $pvc.pvcName | quote }}
+          {{- end }}
           basePath: {{ required "checkpoint.storage.pvc.basePath is required when checkpoint.storage.type=pvc" $pvc.basePath | quote }}
           {{- if $pvc.create }}
           create: true

deploy/helm/charts/snapshot/templates/configmap.yaml

@@ -6,8 +6,8 @@ kind: ConfigMap
 {{- if ne .Values.storage.type "pvc" }}
 {{- fail (printf "snapshot.storage.type=%q is not supported yet; only pvc is currently implemented" .Values.storage.type) }}
 {{- end }}
-{{- if not (has .Values.storage.accessMode (list "agentMount" "podMount")) }}
-{{- fail (printf "snapshot.storage.accessMode=%q is not supported; expected agentMount or podMount" .Values.storage.accessMode) }}
+{{- if not (has .Values.storage.accessMode (list "agentMount" "podMount" "agentInject")) }}
+{{- fail (printf "snapshot.storage.accessMode=%q is not supported; expected agentMount, podMount, or agentInject" .Values.storage.accessMode) }}
 {{- end }}
 metadata:
   name: {{ include "snapshot.fullname" . }}-config

deploy/helm/charts/snapshot/templates/daemonset.yaml

@@ -125,8 +125,9 @@ spec:
             - name: config
               mountPath: /etc/snapshot
               readOnly: true
-            {{- if and (eq .Values.storage.type "pvc") (eq .Values.storage.accessMode "agentMount") }}
-            # Mount the checkpoint PVC directly in namespace-local agent mode
+            {{- if and (eq .Values.storage.type "pvc") (has .Values.storage.accessMode (list "agentMount" "agentInject")) }}
+            # Mount the checkpoint PVC directly. agentMount serves it to the pod;
+            # agentInject clones it into the target container via move_mount.
             - name: checkpoints
               mountPath: {{ .Values.storage.pvc.basePath }}
             {{- end }}
@@ -182,7 +183,7 @@ spec:
             path: /var/lib/kubelet/seccomp
             type: DirectoryOrCreate
         {{- end }}
-        {{- if and (eq .Values.storage.type "pvc") (eq .Values.storage.accessMode "agentMount") }}
+        {{- if and (eq .Values.storage.type "pvc") (has .Values.storage.accessMode (list "agentMount" "agentInject")) }}
         - name: checkpoints
           persistentVolumeClaim:
             claimName: {{ .Values.storage.pvc.name }}

deploy/helm/charts/snapshot/templates/pvc.yaml

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-{{- if and (eq .Values.storage.type "pvc") (eq .Values.storage.accessMode "agentMount") .Values.storage.pvc.create }}
+{{- if and (eq .Values.storage.type "pvc") (has .Values.storage.accessMode (list "agentMount" "agentInject")) .Values.storage.pvc.create }}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:

deploy/helm/charts/snapshot/values.yaml

@@ -46,6 +46,12 @@ storage:
   #   DaemonSet in an infrastructure namespace. Because the DaemonSet no longer
   #   mounts the PVC from every GPU node, suitable ReadWriteOnce storage
   #   classes can be used for sequential checkpoint/restore workflows.
+  # - agentInject: workload pods NEVER mount the checkpoint PVC. The agent mounts
+  #   it (like agentMount) and, for restore, grafts the checkpoint dir into the
+  #   target container's mount namespace via open_tree(OPEN_TREE_CLONE)+move_mount
+  #   before nsrestore reads it. Set the operator's checkpoint.storage.accessMode
+  #   to agentInject to match (the operator then skips injecting the PVC into
+  #   workload pods).
   accessMode: agentMount
 
   # PVC configuration (when type=pvc)

deploy/operator/api/config/v1alpha1/types.go

@@ -390,6 +390,12 @@ func (c *CheckpointConfiguration) EffectiveSeccompProfile() string {
 type CheckpointStorageConfiguration struct {
 	// Type is the storage backend type. Only pvc is implemented today.
 	Type string `json:"type"`
+	// AccessMode selects how the snapshot-agent reaches storage and must match the
+	// snapshot chart's storage.accessMode. "agentInject" tells the operator to
+	// stamp checkpoint metadata but NOT mount the PVC into workload pods (the agent
+	// grafts the checkpoint into the container instead); pvcName is then not
+	// required. Empty/"agentMount"/"podMount" preserve the existing behavior.
+	AccessMode string `json:"accessMode,omitempty"`
 	// PVC configuration for pvc-based settings.
 	PVC CheckpointPVCConfig `json:"pvc"`
 	// Deprecated: S3 is retained for compatibility and ignored.

deploy/operator/internal/checkpoint/checkpoint_test.go

@@ -165,6 +165,32 @@ func TestStorageFromConfig(t *testing.T) {
 		assert.Equal(t, "/snapshots", storage.BasePath)
 	})
 
+	t.Run("agentInject resolves storage without a pvcName", func(t *testing.T) {
+		storage, ok, err := StorageFromConfig(configv1alpha1.CheckpointStorageConfiguration{
+			Type:       snapshotprotocol.StorageTypePVC,
+			AccessMode: snapshotprotocol.StorageAccessModeAgentInject,
+			PVC: configv1alpha1.CheckpointPVCConfig{
+				BasePath: "/checkpoints",
+			},
+		})
+		require.NoError(t, err)
+		require.True(t, ok)
+		assert.Equal(t, snapshotprotocol.StorageTypePVC, storage.Type)
+		assert.Equal(t, "", storage.PVCName, "agentInject must not carry a workload PVC name")
+		assert.Equal(t, "/checkpoints", storage.BasePath)
+		assert.Equal(t, snapshotprotocol.StorageAccessModeAgentInject, storage.AccessMode)
+	})
+
+	t.Run("non-agentInject still requires a pvcName", func(t *testing.T) {
+		_, _, err := StorageFromConfig(configv1alpha1.CheckpointStorageConfiguration{
+			Type: snapshotprotocol.StorageTypePVC,
+			PVC: configv1alpha1.CheckpointPVCConfig{
+				BasePath: "/checkpoints",
+			},
+		})
+		require.Error(t, err)
+	})
+
 	t.Run("pvc config normalizes clean base path", func(t *testing.T) {
 		storage, ok, err := StorageFromConfig(configv1alpha1.CheckpointStorageConfiguration{
 			Type: snapshotprotocol.StorageTypePVC,

deploy/operator/internal/checkpoint/podspec.go

@@ -73,6 +73,7 @@ func ApplyRestorePodMetadataWithStorageConfig(
 		delete(annotations, snapshotprotocol.TargetContainersAnnotation)
 		delete(annotations, snapshotprotocol.CheckpointStorageTypeAnnotation)
 		delete(annotations, snapshotprotocol.CheckpointStorageBasePathAnnotation)
+		delete(annotations, snapshotprotocol.CheckpointStorageAccessModeAnnotation)
 		delete(annotations, commonconsts.CheckpointRestoreCandidateAnnotation)
 		delete(annotations, commonconsts.CheckpointNameAnnotation)
 		delete(annotations, commonconsts.CheckpointStartupPolicyAnnotation)

deploy/operator/internal/checkpoint/storage.go

@@ -23,12 +23,14 @@ import (
 
 func StorageFromConfig(config configv1alpha1.CheckpointStorageConfiguration) (snapshotprotocol.Storage, bool, error) {
 	storageType := strings.TrimSpace(config.Type)
+	storageAccessMode := strings.TrimSpace(config.AccessMode)
+	agentInject := storageAccessMode == snapshotprotocol.StorageAccessModeAgentInject
 	pvcName := strings.TrimSpace(config.PVC.PVCName)
 	basePath := strings.TrimSpace(config.PVC.BasePath)
 	size := strings.TrimSpace(config.PVC.Size)
 	storageClassName := strings.TrimSpace(config.PVC.StorageClassName)
 	accessMode := strings.TrimSpace(config.PVC.AccessMode)
-	hasPVCConfig := pvcName != "" || basePath != "" || config.PVC.Create || size != "" || storageClassName != "" || accessMode != ""
+	hasPVCConfig := pvcName != "" || basePath != "" || config.PVC.Create || size != "" || storageClassName != "" || accessMode != "" || storageAccessMode != ""
 	if storageType == "" && !hasPVCConfig {
 		return snapshotprotocol.Storage{}, false, nil
 	}
@@ -46,22 +48,32 @@ func StorageFromConfig(config configv1alpha1.CheckpointStorageConfiguration) (sn
 	if storageType != snapshotprotocol.StorageTypePVC {
 		return snapshotprotocol.Storage{}, false, fmt.Errorf("checkpoint storage type %q is not supported; only pvc is implemented today", storageType)
 	}
-	if pvcName == "" || basePath == "" {
-		return snapshotprotocol.Storage{}, false, fmt.Errorf("checkpoint.storage.pvc.pvcName and checkpoint.storage.pvc.basePath are required when checkpoint storage is configured")
+	// agentInject: the workload pod never mounts the checkpoint PVC, so pvcName is
+	// not required (and is deliberately left empty so no PVC is injected into the
+	// pod). Only basePath is needed to locate the artifact.
+	if basePath == "" || (!agentInject && pvcName == "") {
+		return snapshotprotocol.Storage{}, false, fmt.Errorf("checkpoint.storage.pvc.basePath (and pvcName unless accessMode=agentInject) are required when checkpoint storage is configured")
 	}
 	basePath, err := normalizeStorageBasePath(basePath)
 	if err != nil {
 		return snapshotprotocol.Storage{}, false, err
 	}
-	if config.PVC.Create {
+	// agentInject does not create a workload-namespace PVC; skip PVC access-mode
+	// validation, which only governs operator-created claims.
+	if config.PVC.Create && !agentInject {
 		if _, err := storagePVCAccessMode(accessMode); err != nil {
 			return snapshotprotocol.Storage{}, false, err
 		}
 	}
+	resolvedPVCName := pvcName
+	if agentInject {
+		resolvedPVCName = ""
+	}
 	return snapshotprotocol.Storage{
-		Type:     snapshotprotocol.StorageTypePVC,
-		PVCName:  pvcName,
-		BasePath: basePath,
+		Type:       snapshotprotocol.StorageTypePVC,
+		PVCName:    resolvedPVCName,
+		BasePath:   basePath,
+		AccessMode: storageAccessMode,
 	}, true, nil
 }
 
@@ -78,6 +90,11 @@ func EnsureStoragePVC(
 	if !ok {
 		return nil
 	}
+	// agentInject leaves PVCName empty: the workload pod never mounts a PVC, so
+	// there is no namespace-local claim for the operator to ensure or create.
+	if storage.PVCName == "" {
+		return nil
+	}
 	if kubeClient == nil {
 		return fmt.Errorf("checkpoint storage client is required")
 	}

deploy/operator/internal/controller/checkpoint_job.go

@@ -78,8 +78,13 @@ func buildCheckpointJob(
 	if storage, ok, err := checkpoint.StorageFromConfig(config.Checkpoint.Storage); err != nil {
 		return nil, err
 	} else if ok {
-		snapshotprotocol.InjectCheckpointVolume(&podTemplate.Spec, storage.PVCName)
-		snapshotprotocol.InjectCheckpointVolumeMount(targetContainer, storage.BasePath)
+		// agentInject leaves PVCName empty: skip injecting the PVC into the
+		// checkpoint Job pod. The agent writes the dump through its own mount, so
+		// only the storage metadata (type/basePath/accessMode) needs stamping.
+		if storage.PVCName != "" {
+			snapshotprotocol.InjectCheckpointVolume(&podTemplate.Spec, storage.PVCName)
+			snapshotprotocol.InjectCheckpointVolumeMount(targetContainer, storage.BasePath)
+		}
 		if podTemplate.Annotations == nil {
 			podTemplate.Annotations = map[string]string{}
 		}

deploy/snapshot/cmd/nsrestore/main.go

@@ -20,6 +20,7 @@ func main() {
 	cudaDeviceMap := flag.String("cuda-device-map", "", "CUDA device map for cuda-checkpoint-helper restore")
 	cgroupRoot := flag.String("cgroup-root", "", "CRIU cgroup root remap path")
 	targetPodIP := flag.String("target-pod-ip", "", "Restore pod IP for CRIU TCP socket remapping")
+	checkpointFD := flag.Int("checkpoint-fd", -1, "Inherited fd of a detached checkpoint mount (agentInject); grafted at --checkpoint-path before restore")
 	flag.Parse()
 
 	if *checkpointPath == "" {
@@ -31,6 +32,7 @@ func main() {
 		CUDADeviceMap:  *cudaDeviceMap,
 		CgroupRoot:     *cgroupRoot,
 		TargetPodIP:    *targetPodIP,
+		CheckpointFD:   *checkpointFD,
 	}
 
 	result, err := executor.RestoreInNamespace(context.Background(), opts, log)

deploy/snapshot/internal/controller/controller.go

@@ -583,6 +583,7 @@ func (w *NodeController) runRestore(ctx context.Context, pod *corev1.Pod, contai
 		TargetPodIP:                 pod.Status.PodIP,
 		ContainerName:               containerName,
 		Clientset:                   w.clientset,
+		AccessMode:                  w.effectiveAccessMode(pod),
 	}
 	placeholderHostPID, err := executor.Restore(restoreCtx, w.runtime, log, req)
 	if err != nil {
@@ -696,6 +697,19 @@ func chooseActiveContent(objs []interface{}) string {
 	return chosen.Name
 }
 
+// effectiveAccessMode returns the operator-stamped storage access mode from the
+// pod (the single source of truth), falling back to the agent's own configured
+// mode when the annotation is absent (legacy pods). Making the stamped value
+// authoritative prevents the operator and agent configs from silently disagreeing.
+func (w *NodeController) effectiveAccessMode(pod *corev1.Pod) string {
+	if pod != nil {
+		if m := strings.TrimSpace(pod.Annotations[snapshotprotocol.CheckpointStorageAccessModeAnnotation]); m != "" {
+			return m
+		}
+	}
+	return strings.TrimSpace(w.config.Storage.AccessMode)
+}
+
 func (w *NodeController) checkpointLocationsFromPod(pod *corev1.Pod, checkpointID string, hostPID int) (checkpointLocations, error) {
 	rawBasePath, hasBasePathAnnotation := pod.Annotations[snapshotprotocol.CheckpointStorageBasePathAnnotation]
 	basePath := strings.TrimSpace(rawBasePath)

deploy/snapshot/internal/executor/nsrestore.go

@@ -21,6 +21,11 @@ type RestoreOptions struct {
 	CUDADeviceMap  string
 	CgroupRoot     string
 	TargetPodIP    string
+	// CheckpointFD, when >= 0, is an inherited fd of a detached checkpoint mount
+	// (agentInject mode). nsrestore grafts it at CheckpointPath inside this
+	// container's mount namespace before reading the checkpoint. -1 means the
+	// checkpoint is already visible (agentMount/podMount).
+	CheckpointFD int
 }
 
 type RestoreInNamespaceResult struct {
@@ -38,8 +43,20 @@ func RestoreInNamespace(ctx context.Context, opts RestoreOptions, log logr.Logge
 		"has_cuda_map", opts.CUDADeviceMap != "",
 		"cgroup_root", opts.CgroupRoot,
 		"target_pod_ip_present", opts.TargetPodIP != "",
+		"checkpoint_fd", opts.CheckpointFD,
 	)
 
+	// agentInject: graft the agent-supplied detached checkpoint mount into this
+	// container's mount namespace at CheckpointPath before anything reads it. This
+	// must precede ReadManifest/ApplyRootfsDiff, which both read the checkpoint dir.
+	if opts.CheckpointFD >= 0 {
+		detach, err := snapshotruntime.AttachCheckpointTree(opts.CheckpointFD, opts.CheckpointPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to graft checkpoint mount: %w", err)
+		}
+		defer detach()
+	}
+
 	manifestReadStart := time.Now()
 	m, err := types.ReadManifest(opts.CheckpointPath)
 	if err != nil {

deploy/snapshot/internal/executor/restore.go

@@ -35,6 +35,10 @@ type RestoreRequest struct {
 	TargetPodIP                 string
 	ContainerName               string
 	Clientset                   kubernetes.Interface
+	// AccessMode is the operator-stamped storage access mode. For agentInject the
+	// workload pod has no checkpoint PVC mount, so the agent clones its own
+	// checkpoint dir and grafts it into the container ns via nsrestore.
+	AccessMode string
 }
 
 // Restore performs external restore for the given request.
@@ -218,7 +222,24 @@ func execNSRestore(ctx context.Context, log logr.Logger, req RestoreRequest, sna
 		args = append(args, "--target-pod-ip", req.TargetPodIP)
 	}
 
+	// agentInject: the workload pod does not mount the checkpoint PVC. Clone the
+	// agent's own checkpoint dir into a detached mount and hand it to nsrestore,
+	// which grafts it into the container's mount namespace at --checkpoint-path.
+	var extraFiles []*os.File
+	if req.AccessMode == types.StorageAccessModeAgentInject {
+		tree, err := snapshotruntime.OpenCheckpointTree(req.CheckpointLocation)
+		if err != nil {
+			return nil, fmt.Errorf("agentInject checkpoint clone: %w", err)
+		}
+		defer tree.Close()
+		extraFiles = append(extraFiles, tree)
+		// First ExtraFiles entry lands at fd 3 in nsenter and is inherited by
+		// nsrestore across the execve chain (nsenter does not close it).
+		args = append(args, "--checkpoint-fd", "3")
+	}
+
 	cmd := exec.CommandContext(ctx, "nsenter", args...)
+	cmd.ExtraFiles = extraFiles
 	// Inherit the agent environment so nsrestore uses the same logger settings.
 	cmd.Env = os.Environ()
 	log.V(1).Info("Executing nsenter + nsrestore", "cmd", cmd.String())

deploy/snapshot/internal/runtime/nsmount.go

@@ -0,0 +1,43 @@
+package runtime
+
+import (
+	"fmt"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// OpenCheckpointTree creates a detached clone of the mount subtree rooted at
+// path (the agent's own checkpoint dir) and returns it as an *os.File so it can
+// be handed to a child process via exec.Cmd.ExtraFiles.
+//
+// The clone is an anonymous mount (attached to no mount namespace), which is the
+// only thing that can be grafted into a different mount namespace. A plain bind
+// of /proc/self/fd/N fails with EINVAL from inside another namespace because the
+// source mount belongs to this (the agent's) namespace; open_tree(OPEN_TREE_CLONE)
+// plus move_mount is the kernel API built for exactly this transfer (Linux 5.2+).
+func OpenCheckpointTree(path string) (*os.File, error) {
+	fd, err := unix.OpenTree(unix.AT_FDCWD, path,
+		uint(unix.OPEN_TREE_CLONE|unix.AT_RECURSIVE|unix.OPEN_TREE_CLOEXEC))
+	if err != nil {
+		return nil, fmt.Errorf("open_tree(%s): %w", path, err)
+	}
+	return os.NewFile(uintptr(fd), "checkpoint-tree:"+path), nil
+}
+
+// AttachCheckpointTree grafts the detached mount referred to by treeFD onto
+// target inside the CURRENT mount namespace, creating target if absent. It
+// returns a cleanup func that lazily detaches the mount. It is meant to run from
+// inside the target container's mount namespace (i.e. from nsrestore), so the
+// checkpoint dir becomes visible to CRIU without the workload pod ever mounting
+// the PVC.
+func AttachCheckpointTree(treeFD int, target string) (func(), error) {
+	if err := os.MkdirAll(target, 0o755); err != nil {
+		return nil, fmt.Errorf("create checkpoint mount target %s: %w", target, err)
+	}
+	if err := unix.MoveMount(treeFD, "", unix.AT_FDCWD, target,
+		unix.MOVE_MOUNT_F_EMPTY_PATH); err != nil {
+		return nil, fmt.Errorf("move_mount -> %s: %w", target, err)
+	}
+	return func() { _ = unix.Unmount(target, unix.MNT_DETACH) }, nil
+}

deploy/snapshot/internal/types/config.go

@@ -26,6 +26,11 @@ const (
 	// StorageAccessModePodMount means workload pods mount the checkpoint PVC,
 	// and snapshot-agent reaches it through /host/proc/<pid>/root.
 	StorageAccessModePodMount = "podMount"
+	// StorageAccessModeAgentInject means workload pods do NOT mount the checkpoint
+	// PVC. The snapshot-agent mounts it (like agentMount) and, for restore, grafts
+	// the checkpoint dir into the target container's mount namespace via
+	// open_tree(OPEN_TREE_CLONE)+move_mount before nsrestore reads it.
+	StorageAccessModeAgentInject = "agentInject"
 )
 
 func (c *AgentConfig) LoadEnvOverrides() {
@@ -58,11 +63,11 @@ func (c *AgentConfig) Validate() error {
 		accessMode = StorageAccessModeAgentMount
 	}
 	switch accessMode {
-	case StorageAccessModeAgentMount, StorageAccessModePodMount:
+	case StorageAccessModeAgentMount, StorageAccessModePodMount, StorageAccessModeAgentInject:
 	default:
 		return &ConfigError{
 			Field:   "storage.accessMode",
-			Message: fmt.Sprintf("unsupported access mode %q; expected %q or %q", c.Storage.AccessMode, StorageAccessModeAgentMount, StorageAccessModePodMount),
+			Message: fmt.Sprintf("unsupported access mode %q; expected %q, %q, or %q", c.Storage.AccessMode, StorageAccessModeAgentMount, StorageAccessModePodMount, StorageAccessModeAgentInject),
 		}
 	}
 	c.Storage.AccessMode = accessMode