Merge branch '23.03-cherry-picks' into 'release-23.03'

Cherry-picks for 23.3.0 release

See merge request nvidia/kubernetes/gpu-operator!669

Author: cdesiniotis

assets/state-container-toolkit/0400_configmap.yaml

@@ -33,4 +33,4 @@ data:
     #
     sleep 5
 
-    exec nvidia-toolkit /usr/local/nvidia
+    exec nvidia-toolkit

assets/state-container-toolkit/0500_daemonset.yaml

@@ -59,6 +59,8 @@ spec:
         args:
           - /bin/entrypoint.sh
         env:
+        - name: ROOT
+          value: "/usr/local/nvidia"
         - name: RUNTIME_ARGS
           value: ""
         - name: NVIDIA_CONTAINER_RUNTIME_MODES_CDI_DEFAULT_KIND

assets/state-device-plugin/0400_configmap.yaml

@@ -9,14 +9,28 @@ data:
   entrypoint.sh: |-
     #!/bin/bash
 
-    driver_root=/run/nvidia/driver
-    driver_root_ctr_path=$driver_root
-    if [[ -f /run/nvidia/validations/host-driver-ready ]]; then
-      driver_root=/
-      driver_root_ctr_path=/host
-    fi;
+    driver_root=""
+    container_driver_root=""
+    while true; do
+      if [[ -f /run/nvidia/validations/host-driver-ready ]]; then
+        driver_root=/
+        container_driver_root=/host
+        break
+      elif [[ -f /run/nvidia/validations/driver-ready ]]; then
+        driver_root=/run/nvidia/driver
+        container_driver_root=$driver_root
+        break
+      else
+        echo "waiting for the driver validations to be ready..."
+        sleep 5
+      fi
+    done
 
     export NVIDIA_DRIVER_ROOT=$driver_root
-    export DRIVER_ROOT_CTR_PATH=$driver_root_ctr_path
+    echo "NVIDIA_DRIVER_ROOT=$NVIDIA_DRIVER_ROOT"
 
+    export CONTAINER_DRIVER_ROOT=$container_driver_root
+    echo "CONTAINER_DRIVER_ROOT=$CONTAINER_DRIVER_ROOT"
+
+    echo "Starting nvidia-device-plugin"
     exec nvidia-device-plugin

assets/state-mig-manager/0420_configmap.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nvidia-mig-manager-entrypoint
+  namespace: "FILLED BY THE OPERATOR"
+  labels:
+    app: nvidia-mig-manager
+data:
+  entrypoint.sh: |-
+    #!/bin/bash
+
+    host_driver=""
+    driver_root=""
+    driver_root_ctr_path=""
+    while true; do
+      if [[ -f /run/nvidia/validations/host-driver-ready ]]; then
+        host_driver=true
+        driver_root="/"
+        driver_root_ctr_path="/host"
+        break
+      elif [[ -f /run/nvidia/validations/driver-ready ]]; then
+        host_driver=false
+        driver_root="/run/nvidia/driver"
+        driver_root_ctr_path="/run/nvidia/driver"
+        break
+      else
+        echo "waiting for the driver validations to be ready..."
+        sleep 5
+      fi
+    done
+
+    export WITH_SHUTDOWN_HOST_GPU_CLIENTS=$host_driver
+    echo "WITH_SHUTDOWN_HOST_GPU_CLIENTS=$WITH_SHUTDOWN_HOST_GPU_CLIENTS"
+
+    export DRIVER_ROOT=$driver_root
+    echo "DRIVER_ROOT=$DRIVER_ROOT"
+
+    export DRIVER_ROOT_CTR_PATH=$driver_root_ctr_path
+    echo "DRIVER_ROOT_CTR_PATH=$DRIVER_ROOT_CTR_PATH"
+
+    echo "Starting nvidia-mig-manager"
+    exec nvidia-mig-manager

assets/state-mig-manager/0600_daemonset.yaml

@@ -40,7 +40,8 @@ spec:
         image: "FILLED BY THE OPERATOR"
         imagePullPolicy: IfNotPresent
         command: [bash, -c]
-        args: [" [[ -f /run/nvidia/validations/host-driver-ready ]] && host_driver=true || host_driver=false; export WITH_SHUTDOWN_HOST_GPU_CLIENTS=$host_driver; exec nvidia-mig-manager"]
+        args:
+          - /bin/entrypoint.sh
         env:
         - name: NODE_NAME
           valueFrom:
@@ -57,6 +58,10 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - name: nvidia-mig-manager-entrypoint
+          readOnly: true
+          mountPath: /bin/entrypoint.sh
+          subPath: entrypoint.sh
         - mountPath: /sys
           name: host-sys
         - mountPath: /mig-parted-config
@@ -69,7 +74,13 @@ spec:
         - name: run-nvidia
           mountPath: /run/nvidia
           mountPropagation: HostToContainer
+        - name: cdi-root
+          mountPath: /var/run/cdi
       volumes:
+      - name: nvidia-mig-manager-entrypoint
+        configMap:
+          name: nvidia-mig-manager-entrypoint
+          defaultMode: 448
       - name: host-sys
         hostPath:
           path: /sys
@@ -87,3 +98,7 @@ spec:
       - name: gpu-clients
         configMap:
           name: "FILLED_BY_OPERATOR"
+      - name: cdi-root
+        hostPath:
+          path: /var/run/cdi
+          type: DirectoryOrCreate

bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml

@@ -40,6 +40,7 @@ metadata:
                   "timeoutSeconds": 300
                 },
                 "maxParallelUpgrades": 1,
+                "maxUnavailable": "25%",
                 "podDeletion": {
                   "deleteEmptyDir": false,
                   "force": false,

bundle/manifests/nvidia.com_clusterpolicies.yaml

@@ -1085,6 +1085,18 @@ spec:
                           will be upgraded in parallel
                         minimum: 0
                         type: integer
+                      maxUnavailable:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        default: 25%
+                        description: 'MaxUnavailable is the maximum number of nodes
+                          with the driver installed, that can be unavailable during
+                          the upgrade. Value can be an absolute number (ex: 5) or
+                          a percentage of total nodes at the start of upgrade (ex:
+                          10%). Absolute number is calculated from percentage by rounding
+                          up. By default, a fixed value of 1 is used.'
+                        x-kubernetes-int-or-string: true
                       podDeletion:
                         description: PodDeletionSpec describes configuration for deletion
                           of pods using special resources during automatic upgrade

config/crd/bases/nvidia.com_clusterpolicies.yaml

@@ -1085,6 +1085,18 @@ spec:
                           will be upgraded in parallel
                         minimum: 0
                         type: integer
+                      maxUnavailable:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        default: 25%
+                        description: 'MaxUnavailable is the maximum number of nodes
+                          with the driver installed, that can be unavailable during
+                          the upgrade. Value can be an absolute number (ex: 5) or
+                          a percentage of total nodes at the start of upgrade (ex:
+                          10%). Absolute number is calculated from percentage by rounding
+                          up. By default, a fixed value of 1 is used.'
+                        x-kubernetes-int-or-string: true
                       podDeletion:
                         description: PodDeletionSpec describes configuration for deletion
                           of pods using special resources during automatic upgrade

controllers/object_controls.go

@@ -45,6 +45,10 @@ const (
 	DefaultDockerConfigFile = "/etc/docker/daemon.json"
 	// DefaultDockerSocketFile indicates default docker socket file
 	DefaultDockerSocketFile = "/var/run/docker.sock"
+	// DefaultCRIOConfigFile indicates default config file path for cri-o.
+	// Note, config files in the drop-in directory, /etc/crio/crio.conf.d,
+	// have a higher priority than the default /etc/crio/crio.conf file.
+	DefaultCRIOConfigFile = "/etc/crio/crio.conf.d/99-nvidia.conf"
 	// TrustedCAConfigMapName indicates configmap with custom user CA injected
 	TrustedCAConfigMapName = "gpu-operator-trusted-ca"
 	// TrustedCABundleFileName indicates custom user ca certificate filename
@@ -111,18 +115,26 @@ const (
 	ServiceMonitorCRDName = "servicemonitors.monitoring.coreos.com"
 	// DefaultToolkitInstallDir is the default toolkit installation directory on the host
 	DefaultToolkitInstallDir = "/usr/local/nvidia"
+	// ToolkitInstallDirEnvName is the name of the toolkit container env for configuring where NVIDIA Container Toolkit is installed
+	ToolkitInstallDirEnvName = "ROOT"
 	// VgpuDMDefaultConfigMapName indicates name of ConfigMap containing default vGPU devices configuration
 	VgpuDMDefaultConfigMapName = "default-vgpu-devices-config"
 	// VgpuDMDefaultConfigName indicates name of default configuration in the vGPU devices config file
 	VgpuDMDefaultConfigName = "default"
 	// NvidiaCtrRuntimeModeEnvName is the name of the toolkit container env for configuring the NVIDIA Container Runtime mode
 	NvidiaCtrRuntimeModeEnvName = "NVIDIA_CONTAINER_RUNTIME_MODE"
+	// NvidiaCtrRuntimeCDIPrefixesEnvName is the name of toolkit container env for configuring the CDI annotation prefixes
+	NvidiaCtrRuntimeCDIPrefixesEnvName = "NVIDIA_CONTAINER_RUNTIME_MODES_CDI_ANNOTATION_PREFIXES"
 	// CDIEnabledEnvName is the name of the envvar used to enable CDI in the operands
 	CDIEnabledEnvName = "CDI_ENABLED"
 	// NvidiaCTKPathEnvName is the name of the envvar specifying the path to the 'nvidia-ctk' binary
 	NvidiaCTKPathEnvName = "NVIDIA_CTK_PATH"
 	// CrioConfigModeEnvName is the name of the envvar controlling how the toolkit container updates the cri-o configuration
 	CrioConfigModeEnvName = "CRIO_CONFIG_MODE"
+	// DeviceListStrategyEnvName is the name of the envvar for configuring the device-list-strategy in the device-plugin
+	DeviceListStrategyEnvName = "DEVICE_LIST_STRATEGY"
+	// CDIAnnotationPrefixEnvName is the name of the device-plugin envvar for configuring the CDI annotation prefix
+	CDIAnnotationPrefixEnvName = "CDI_ANNOTATION_PREFIX"
 )
 
 // RepoConfigPathMap indicates standard OS specific paths for repository configuration files
@@ -1047,6 +1059,7 @@ func TransformToolkit(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n
 	// update env required for CDI support
 	if config.CDI.IsEnabled() {
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
+		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCtrRuntimeCDIPrefixesEnvName, "nvidia.cdi.k8s.io/")
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CrioConfigModeEnvName, "config")
 		if config.CDI.IsDefault() {
 			setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCtrRuntimeModeEnvName, "cdi")
@@ -1055,13 +1068,8 @@ func TransformToolkit(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n
 
 	// set install directory for the toolkit
 	if config.Toolkit.InstallDir != "" && config.Toolkit.InstallDir != DefaultToolkitInstallDir {
-		// set args for the toolkit
-		toolkitArgStrFmt := "[[ -f /run/nvidia/validations/host-driver-ready ]] && driver_root=/ || driver_root=/run/nvidia/driver; export NVIDIA_DRIVER_ROOT=$driver_root; sleep 5; exec nvidia-toolkit %s"
-		toolkitArg := fmt.Sprintf(toolkitArgStrFmt, config.Toolkit.InstallDir)
-		args := []string{toolkitArg}
-		obj.Spec.Template.Spec.Containers[0].Args = args
+		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), ToolkitInstallDirEnvName, config.Toolkit.InstallDir)
 
-		// update install directory for the toolkit
 		for i, volume := range obj.Spec.Template.Spec.Volumes {
 			if volume.Name == "toolkit-install-dir" {
 				obj.Spec.Template.Spec.Volumes[i].HostPath.Path = config.Toolkit.InstallDir
@@ -1086,27 +1094,30 @@ func TransformToolkit(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "CONTAINERD_RUNTIME_CLASS", getRuntimeClass(config))
 	}
 
-	// setup mounts for runtime config file and socket file
-	if runtime == gpuv1.Docker.String() || runtime == gpuv1.Containerd.String() {
-		runtimeConfigFile := getRuntimeConfigFile(&(obj.Spec.Template.Spec.Containers[0]), runtime)
-		runtimeSocketFile := getRuntimeSocketFile(&(obj.Spec.Template.Spec.Containers[0]), runtime)
-
-		sourceSocketFileName := path.Base(runtimeSocketFile)
-		sourceConfigFileName := path.Base(runtimeConfigFile)
+	// setup mounts for runtime config file
+	runtimeConfigFile, err := getRuntimeConfigFile(&(obj.Spec.Template.Spec.Containers[0]), runtime)
+	if err != nil {
+		return fmt.Errorf("error getting path to runtime config file: %v", err)
+	}
+	sourceConfigFileName := path.Base(runtimeConfigFile)
+	runtimeArgs := "--config " + DefaultRuntimeConfigTargetDir + sourceConfigFileName
 
-		// docker needs socket file as runtime arg
-		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "RUNTIME_ARGS",
-			"--socket "+DefaultRuntimeSocketTargetDir+sourceSocketFileName+" --config "+DefaultRuntimeConfigTargetDir+sourceConfigFileName)
+	volMountConfigName := fmt.Sprintf("%s-config", runtime)
+	volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: DefaultRuntimeConfigTargetDir}
+	obj.Spec.Template.Spec.Containers[0].VolumeMounts = append(obj.Spec.Template.Spec.Containers[0].VolumeMounts, volMountConfig)
 
-		// setup config file mount
-		volMountConfigName := fmt.Sprintf("%s-config", runtime)
-		volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: DefaultRuntimeConfigTargetDir}
-		obj.Spec.Template.Spec.Containers[0].VolumeMounts = append(obj.Spec.Template.Spec.Containers[0].VolumeMounts, volMountConfig)
+	configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Dir(runtimeConfigFile), Type: newHostPathType(corev1.HostPathDirectoryOrCreate)}}}
+	obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
 
-		configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Dir(runtimeConfigFile)}}}
-		obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
+	// setup mounts for runtime socket file
+	if runtime == gpuv1.Docker.String() || runtime == gpuv1.Containerd.String() {
+		runtimeSocketFile, err := getRuntimeSocketFile(&(obj.Spec.Template.Spec.Containers[0]), runtime)
+		if err != nil {
+			return fmt.Errorf("error getting path to runtime socket: %v", err)
+		}
+		sourceSocketFileName := path.Base(runtimeSocketFile)
+		runtimeArgs += " --socket " + DefaultRuntimeSocketTargetDir + sourceSocketFileName
 
-		// setup socket file mount
 		volMountSocketName := fmt.Sprintf("%s-socket", runtime)
 		volMountSocket := corev1.VolumeMount{Name: volMountSocketName, MountPath: DefaultRuntimeSocketTargetDir}
 		obj.Spec.Template.Spec.Containers[0].VolumeMounts = append(obj.Spec.Template.Spec.Containers[0].VolumeMounts, volMountSocket)
@@ -1115,6 +1126,9 @@ func TransformToolkit(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n
 		obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, socketVol)
 	}
 
+	// update runtime args
+	setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "RUNTIME_ARGS", runtimeArgs)
+
 	// Update CRI-O hooks path to use default path for non OCP cases
 	if n.openshift == "" && n.runtime == gpuv1.CRIO {
 		for index, volume := range obj.Spec.Template.Spec.Volumes {
@@ -1187,6 +1201,8 @@ func TransformDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
 	// update env required for CDI support
 	if config.CDI.IsEnabled() {
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
+		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DeviceListStrategyEnvName, "envvar,cdi-annotations")
+		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIAnnotationPrefixEnvName, "nvidia.cdi.k8s.io/")
 		if config.Toolkit.IsEnabled() {
 			setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCTKPathEnvName, filepath.Join(config.Toolkit.InstallDir, "toolkit/nvidia-ctk"))
 		}
@@ -1506,6 +1522,11 @@ func TransformMIGManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,
 		break
 	}
 
+	// update env required for CDI support
+	if config.CDI.IsEnabled() {
+		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
+	}
+
 	return nil
 }
 
@@ -1856,35 +1877,50 @@ func TransformNodeStatusExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol
 }
 
 // get runtime(docker, containerd) config file path based on toolkit container env or default
-func getRuntimeConfigFile(c *corev1.Container, runtime string) (runtimeConfigFile string) {
-	if runtime == gpuv1.Docker.String() {
+func getRuntimeConfigFile(c *corev1.Container, runtime string) (string, error) {
+	var runtimeConfigFile string
+	switch runtime {
+	case gpuv1.Docker.String():
 		runtimeConfigFile = DefaultDockerConfigFile
-		if getContainerEnv(c, "DOCKER_CONFIG") != "" {
-			runtimeConfigFile = getContainerEnv(c, "DOCKER_CONFIG")
+		if value := getContainerEnv(c, "DOCKER_CONFIG"); value != "" {
+			runtimeConfigFile = value
 		}
-	} else if runtime == gpuv1.Containerd.String() {
+	case gpuv1.Containerd.String():
 		runtimeConfigFile = DefaultContainerdConfigFile
-		if getContainerEnv(c, "CONTAINERD_CONFIG") != "" {
-			runtimeConfigFile = getContainerEnv(c, "CONTAINERD_CONFIG")
+		if value := getContainerEnv(c, "CONTAINERD_CONFIG"); value != "" {
+			runtimeConfigFile = value
 		}
+	case gpuv1.CRIO.String():
+		runtimeConfigFile = DefaultCRIOConfigFile
+		if value := getContainerEnv(c, "CRIO_CONFIG"); value != "" {
+			runtimeConfigFile = value
+		}
+	default:
+		return "", fmt.Errorf("invalid runtime: %s", runtime)
 	}
-	return runtimeConfigFile
+
+	return runtimeConfigFile, nil
 }
 
 // get runtime(docker, containerd) socket file path based on toolkit container env or default
-func getRuntimeSocketFile(c *corev1.Container, runtime string) (runtimeSocketFile string) {
-	if runtime == gpuv1.Docker.String() {
+func getRuntimeSocketFile(c *corev1.Container, runtime string) (string, error) {
+	var runtimeSocketFile string
+	switch runtime {
+	case gpuv1.Docker.String():
 		runtimeSocketFile = DefaultDockerSocketFile
 		if getContainerEnv(c, "DOCKER_SOCKET") != "" {
 			runtimeSocketFile = getContainerEnv(c, "DOCKER_SOCKET")
 		}
-	} else if runtime == gpuv1.Containerd.String() {
+	case gpuv1.Containerd.String():
 		runtimeSocketFile = DefaultContainerdSocketFile
 		if getContainerEnv(c, "CONTAINERD_SOCKET") != "" {
 			runtimeSocketFile = getContainerEnv(c, "CONTAINERD_SOCKET")
 		}
+	default:
+		return "", fmt.Errorf("invalid runtime: %s", runtime)
 	}
-	return runtimeSocketFile
+
+	return runtimeSocketFile, nil
 }
 
 func getContainerEnv(c *corev1.Container, key string) string {