Merge branch 'allow_custom_dcgm_metrics' into 'master'

Allow custom dcgm metrics

See merge request nvidia/kubernetes/gpu-operator!282

Author: shivamerla

api/v1/clusterpolicy_types.go

@@ -510,6 +510,21 @@ type DCGMExporterSpec struct {
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables"
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:text"
 	Env []corev1.EnvVar `json:"env,omitempty"`
+
+	// Optional: Custom metrics configuration for DCGM exporter
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Custom Metrics Configuration For DCGM Exporter"
+	MetricsConfig *DCGMExporterMetricsConfig `json:"config,omitempty"`
+}
+
+// DCGMExporterMetricsConfig defines metrics to be collected by DCGM Exporter
+type DCGMExporterMetricsConfig struct {
+	// ConfigMap name with file dcgm-metrics.csv for metrics to be collected by DCGM exporter
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="ConfigMap name with file dcgm-metrics.csv"
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:text"
+	Name string `json:"name,omitempty"`
 }
 
 // DCGMSpec defines the properties for DCGM deployment

api/v1/zz_generated.deepcopy.go

@@ -16,6 +16,9 @@ metadata:
               "image": "dcgm-exporter",
               "imagePullSecrets": [],
               "repository": "nvcr.io/nvidia/k8s",
+              "config": {
+                "name": ""
+              },
               "env": [
                 {
                   "name": "DCGM_EXPORTER_LISTEN",

bundle/manifests/gpu-operator.clusterserviceversion.yaml

@@ -405,6 +405,15 @@ spec:
                     items:
                       type: string
                     type: array
+                  config:
+                    description: 'Optional: Custom metrics configuration for DCGM
+                      exporter'
+                    properties:
+                      name:
+                        description: ConfigMap name with file dcgm-metrics.csv for
+                          metrics to be collected by DCGM exporter
+                        type: string
+                    type: object
                   env:
                     description: 'Optional: List of environment variables'
                     items:

bundle/manifests/nvidia.com_clusterpolicies.yaml

@@ -405,6 +405,15 @@ spec:
                     items:
                       type: string
                     type: array
+                  config:
+                    description: 'Optional: Custom metrics configuration for DCGM
+                      exporter'
+                    properties:
+                      name:
+                        description: ConfigMap name with file dcgm-metrics.csv for
+                          metrics to be collected by DCGM exporter
+                        type: string
+                    type: object
                   env:
                     description: 'Optional: List of environment variables'
                     items:

config/crd/bases/nvidia.com_clusterpolicies.yaml

@@ -77,12 +77,14 @@ rules:
 - apiGroups:
   - monitoring.coreos.com
   resources:
-  - servicemonitors
   - prometheusrule
+  - servicemonitors
   verbs:
   - create
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - nvidia.com

config/rbac/role.yaml

@@ -140,6 +140,9 @@ spec:
         value: "/etc/dcgm-exporter/dcp-metrics-included.csv"
     securityContext: {}
     resources: {}
+    # configmap name for custom dcgm metrics
+    config:
+      name: ""
 
   gfd:
     repository: nvcr.io/nvidia

config/samples/v1_clusterpolicy.yaml

@@ -18,13 +18,16 @@ package controllers
 
 import (
 	"context"
+
 	"github.com/go-logr/logr"
 	"github.com/prometheus/common/log"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 
+	"time"
+
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -34,7 +37,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
-	"time"
 
 	gpuv1 "github.com/NVIDIA/gpu-operator/api/v1"
 )
@@ -57,7 +59,7 @@ type ClusterPolicyReconciler struct {
 // +kubebuilder:rbac:groups="",resources=namespaces;serviceaccounts;pods;services;services/finalizers;endpoints,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims;events;configmaps;secrets;nodes,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=deployments;daemonsets;replicasets;statefulsets,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create
+// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrule,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=get;list;watch;create
 // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch

controllers/clusterpolicy_controller.go

@@ -77,6 +77,10 @@ const (
 	DCGMDefaultHostPort = 5555
 	// GPUDirectRDMAEnabledEnvName indicates if GPU direct RDMA is enabled through GPU operator
 	GPUDirectRDMAEnabledEnvName = "GPU_DIRECT_RDMA_ENABLED"
+	// MetricsConfigMountPath indicates mount path for custom dcgm metrics file
+	MetricsConfigMountPath = "/etc/dcgm-exporter/" + MetricsConfigFileName
+	// MetricsConfigFileName indicates custom dcgm metrics file name
+	MetricsConfigFileName = "dcgm-metrics.csv"
 )
 
 type controlFunc []func(n ClusterPolicyController) (gpuv1.State, error)
@@ -412,7 +416,6 @@ func parseOSRelease() (map[string]string, error) {
 			release[m[1]] = strings.Trim(m[2], `"`)
 		}
 	}
-
 	return release, nil
 }
 
@@ -774,16 +777,39 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
 	// set RuntimeClass for supported runtimes
 	setRuntimeClass(&obj.Spec.Template.Spec, config.Operator.DefaultRuntime, config.Operator.RuntimeClass)
 
-	kvers, osTag, _ := kernelFullVersion(n)
-	if kvers == "" {
-		return fmt.Errorf("ERROR: Could not find kernel full version: ('%s', '%s')", kvers, osTag)
+	// mount configmap for custom metrics if provided by user
+	if config.DCGMExporter.MetricsConfig != nil && config.DCGMExporter.MetricsConfig.Name != "" {
+		metricsConfigVolMount := corev1.VolumeMount{Name: "metrics-config", ReadOnly: true, MountPath: MetricsConfigMountPath, SubPath: MetricsConfigFileName}
+		obj.Spec.Template.Spec.Containers[0].VolumeMounts = append(obj.Spec.Template.Spec.Containers[0].VolumeMounts, metricsConfigVolMount)
+
+		metricsConfigVolumeSource := corev1.VolumeSource{
+			ConfigMap: &corev1.ConfigMapVolumeSource{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: config.DCGMExporter.MetricsConfig.Name,
+				},
+				Items: []corev1.KeyToPath{
+					{
+						Key:  MetricsConfigFileName,
+						Path: MetricsConfigFileName,
+					},
+				},
+			},
+		}
+		metricsConfigVol := corev1.Volume{Name: "metrics-config", VolumeSource: metricsConfigVolumeSource}
+		obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, metricsConfigVol)
 	}
 
-	if !strings.Contains(osTag, "rhel") && !strings.Contains(osTag, "rhcos") {
+	release, err := parseOSRelease()
+	if err != nil {
+		return fmt.Errorf("ERROR: failed to get os-release: %s", err)
+	}
+
+	// skip SELinux changes if not an OCP cluster
+	if _, ok := release["OPENSHIFT_VERSION"]; !ok {
 		return nil
 	}
 
-	// update init container config for per pod specific resources
+	// Add initContainer for OCP to set proper SELinux context on /var/lib/kubelet/pod-resources
 	initImage, err := gpuv1.ImagePath(&config.Operator.InitContainer)
 	if err != nil {
 		return err
@@ -801,7 +827,6 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
 		Privileged: &privileged,
 	}
 
-	// Add initContainer for OCP to set proper SELinux context on /var/lib/kubelet/pod-resources
 	initContainer.SecurityContext = securityContext
 
 	volMountSockName, volMountSockPath := "pod-gpu-resources", "/var/lib/kubelet/pod-resources"
@@ -1294,9 +1319,8 @@ func resolveDriverTag(n ClusterPolicyController, driverSpec *gpuv1.DriverSpec) (
 }
 
 func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
-	logger := n.rec.Log.WithValues("Daemonset", obj.Name)
 	for i, container := range obj.Spec.Template.Spec.Containers {
-		// skip if not nvidia-peermem
+		// skip if not nvidia-driver container
 		if !strings.Contains(container.Name, "nvidia-driver") {
 			continue
 		}
@@ -1394,41 +1418,28 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
 			obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, topologyConfigVol)
 		}
 
-		// Inject EUS kernel RPM's as an override to the entrypoint
-		// Add Env Vars needed by nvidia-driver to enable the right releasever and rpm repo
-		kvers, osTag, _ := kernelFullVersion(n)
-		if kvers == "" {
-			return fmt.Errorf("ERROR: Could not find kernel full version: ('%s', '%s')", kvers, osTag)
-		}
-
-		if !strings.Contains(osTag, "rhel") && !strings.Contains(osTag, "rhcos") {
-			return nil
-		}
-
 		release, err := parseOSRelease()
 		if err != nil {
 			return fmt.Errorf("ERROR: failed to get os-release: %s", err)
 		}
 
-		ocpV, err := OpenshiftVersion()
-		if err != nil {
-			// might be RHEL node using upstream k8s, don't error out.
-			logger.Info(fmt.Sprintf("ERROR: failed to get OpenShift version: %s", err))
+		// skip proxy and env settings if not ocp cluster
+		if _, ok := release["OPENSHIFT_VERSION"]; !ok {
+			return nil
 		}
 
+		// Add env vars needed by nvidia-driver to enable the right releasever and EUS rpm repos
 		rhelVersion := corev1.EnvVar{Name: "RHEL_VERSION", Value: release["RHEL_VERSION"]}
-		ocpVersion := corev1.EnvVar{Name: "OPENSHIFT_VERSION", Value: ocpV}
+		ocpVersion := corev1.EnvVar{Name: "OPENSHIFT_VERSION", Value: release["OPENSHIFT_VERSION"]}
 
 		obj.Spec.Template.Spec.Containers[i].Env = append(obj.Spec.Template.Spec.Containers[i].Env, rhelVersion)
 		obj.Spec.Template.Spec.Containers[i].Env = append(obj.Spec.Template.Spec.Containers[i].Env, ocpVersion)
 
-		if ocpV != "" {
-			// Automatically apply proxy settings for OCP and inject custom CA if configured by user
-			// https://docs.openshift.com/container-platform/4.6/networking/configuring-a-custom-pki.html
-			err = applyOCPProxySpec(n, &obj.Spec.Template.Spec)
-			if err != nil {
-				return err
-			}
+		// Automatically apply proxy settings for OCP and inject custom CA if configured by user
+		// https://docs.openshift.com/container-platform/4.6/networking/configuring-a-custom-pki.html
+		err = applyOCPProxySpec(n, &obj.Spec.Template.Spec)
+		if err != nil {
+			return err
 		}
 	}
 	return nil

controllers/object_controls.go

@@ -405,6 +405,15 @@ spec:
                     items:
                       type: string
                     type: array
+                  config:
+                    description: 'Optional: Custom metrics configuration for DCGM
+                      exporter'
+                    properties:
+                      name:
+                        description: ConfigMap name with file dcgm-metrics.csv for
+                          metrics to be collected by DCGM exporter
+                        type: string
+                    type: object
                   env:
                     description: 'Optional: List of environment variables'
                     items:

deployments/gpu-operator/crds/nvidia.com_clusterpolicies_crd.yaml

@@ -267,6 +267,9 @@ spec:
     {{- if .Values.dcgmExporter.args }}
     args: {{ toYaml .Values.dcgmExporter.args | nindent 6 }}
     {{- end }}
+    {{- if .Values.dcgmExporter.config }}
+    config: {{ toYaml .Values.dcgmExporter.config | nindent 6 }}
+    {{- end }}
   gfd:
     {{- if .Values.gfd.repository }}
     repository: {{ .Values.gfd.repository }}