Merge branch 'make_dcgm_remote_optional' into 'master'

Make it optional to run DCGM hostengine as a separate container

See merge request nvidia/kubernetes/gpu-operator!283

Author: shivamerla

api/v1/clusterpolicy_types.go

@@ -529,6 +529,12 @@ type DCGMExporterMetricsConfig struct {
 
 // DCGMSpec defines the properties for DCGM deployment
 type DCGMSpec struct {
+	// Enabled indicates if deployment of DCGM hostengine as a separate pod is enabled.
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Enable DCGM hostengine as a separate Pod"
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
+	Enabled *bool `json:"enabled,omitempty"`
+
 	// DCGM image repository
 	// +kubebuilder:validation:Optional
 	Repository string `json:"repository"`
@@ -983,3 +989,12 @@ func (g *GPUDirectRDMASpec) IsEnabled() bool {
 	}
 	return *g.Enabled
 }
+
+// IsEnabled returns true if DCGM hostengine as a separate Pod is enabled through gpu-perator
+func (dcgm *DCGMSpec) IsEnabled() bool {
+	if dcgm.Enabled == nil {
+		// DCGM is enabled by default
+		return true
+	}
+	return *dcgm.Enabled
+}

api/v1/zz_generated.deepcopy.go

@@ -39,6 +39,7 @@ metadata:
               "version": "sha256:e37404194fa2bc2275827411049422b93d1493991fb925957f170b4b842846ff"
             },
             "dcgm": {
+              "enabled": true,
               "image": "dcgm",
               "imagePullSecrets": [],
               "repository": "nvcr.io/nvidia/cloud-native",

bundle/manifests/gpu-operator.clusterserviceversion.yaml

@@ -90,6 +90,10 @@ spec:
                     items:
                       type: string
                     type: array
+                  enabled:
+                    description: Enabled indicates if deployment of DCGM hostengine
+                      as a separate pod is enabled.
+                    type: boolean
                   env:
                     description: 'Optional: List of environment variables'
                     items:

bundle/manifests/nvidia.com_clusterpolicies.yaml

@@ -90,6 +90,10 @@ spec:
                     items:
                       type: string
                     type: array
+                  enabled:
+                    description: Enabled indicates if deployment of DCGM hostengine
+                      as a separate pod is enabled.
+                    type: boolean
                   env:
                     description: 'Optional: List of environment variables'
                     items:

config/crd/bases/nvidia.com_clusterpolicies.yaml

@@ -115,6 +115,7 @@ spec:
     resources: {}
 
   dcgm:
+    enabled: true
     repository: nvcr.io/nvidia/cloud-native
     image: dcgm
     version: 2.2.3-ubi8

config/samples/v1_clusterpolicy.yaml

@@ -767,13 +767,15 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
 			setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), env.Name, env.Value)
 		}
 	}
-	// set DCGM host engine env. NODE_IP will be substituted during pod runtime
-	dcgmHostPort := int32(DCGMDefaultHostPort)
-	if config.DCGM.HostPort != 0 {
-		dcgmHostPort = config.DCGM.HostPort
+	// check if DCGM hostengine is enabled as a separate Pod and setup env accordingly
+	if config.DCGM.IsEnabled() {
+		// set DCGM host engine env. NODE_IP will be substituted during pod runtime
+		dcgmHostPort := int32(DCGMDefaultHostPort)
+		if config.DCGM.HostPort != 0 {
+			dcgmHostPort = config.DCGM.HostPort
+		}
+		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMRemoteEngineEnvName, fmt.Sprintf("$(NODE_IP):%d", dcgmHostPort))
 	}
-	setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMRemoteEngineEnvName, fmt.Sprintf("$(NODE_IP):%d", dcgmHostPort))
-
 	// set RuntimeClass for supported runtimes
 	setRuntimeClass(&obj.Spec.Template.Spec, config.Operator.DefaultRuntime, config.Operator.RuntimeClass)
 

controllers/object_controls.go

@@ -317,7 +317,9 @@ func (n *ClusterPolicyController) init(reconciler *ClusterPolicyReconciler, clus
 		}
 		addState(n, "/opt/gpu-operator/state-operator-validation")
 		addState(n, "/opt/gpu-operator/state-device-plugin")
-		addState(n, "/opt/gpu-operator/state-dcgm")
+		if clusterPolicy.Spec.DCGM.IsEnabled() {
+			addState(n, "/opt/gpu-operator/state-dcgm")
+		}
 		addState(n, "/opt/gpu-operator/state-dcgm-exporter")
 
 		addState(n, "/opt/gpu-operator/gpu-feature-discovery")

controllers/state_manager.go

@@ -90,6 +90,10 @@ spec:
                     items:
                       type: string
                     type: array
+                  enabled:
+                    description: Enabled indicates if deployment of DCGM hostengine
+                      as a separate pod is enabled.
+                    type: boolean
                   env:
                     description: 'Optional: List of environment variables'
                     items:

deployments/gpu-operator/crds/nvidia.com_clusterpolicies_crd.yaml

@@ -209,6 +209,7 @@ spec:
     args: {{ toYaml .Values.devicePlugin.args | nindent 6 }}
     {{- end }}
   dcgm:
+    enabled: {{ .Values.dcgm.enabled }}
     {{- if .Values.dcgm.repository }}
     repository: {{ .Values.dcgm.repository }}
     {{- end }}

deployments/gpu-operator/templates/clusterpolicy.yaml

@@ -162,6 +162,7 @@ devicePlugin:
   resources: {}
 
 dcgm:
+  enabled: true
   repository: nvcr.io/nvidia/cloud-native
   image: dcgm
   version: 2.2.3-ubuntu20.04