[Prometheus] Add ray_clusters_created_total metric

Signed-off-by: win5923 <[email protected]>

Author: win5923

ray-operator/controllers/ray/common/metrics.go

@@ -1,64 +1,29 @@
 package common
 
 import (
+	"strconv"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 // Define all the prometheus counters for all clusters
 var (
-	clustersCreatedCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_created_total",
-			Help: "Counts number of clusters created",
-		},
-		[]string{"namespace"},
-	)
-	clustersDeletedCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_deleted_total",
-			Help: "Counts number of clusters deleted",
-		},
-		[]string{"namespace"},
-	)
-	clustersSuccessfulCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_successful_total",
-			Help: "Counts number of clusters successful",
-		},
-		[]string{"namespace"},
-	)
-	clustersFailedCount = promauto.NewCounterVec(
+	rayClustersCreatedCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_failed_total",
-			Help: "Counts number of clusters failed",
+			Name: "ray_clusters_created_total",
+			Help: "The total number of RayClusters created",
 		},
-		[]string{"namespace"},
+		[]string{"namespace", "created_by_ray_job", "created_by_ray_service"},
 	)
 )
 
 func init() {
 	// Register custom metrics with the global prometheus registry
-	metrics.Registry.MustRegister(clustersCreatedCount,
-		clustersDeletedCount,
-		clustersSuccessfulCount,
-		clustersFailedCount)
-}
-
-func CreatedClustersCounterInc(namespace string) {
-	clustersCreatedCount.WithLabelValues(namespace).Inc()
-}
-
-// TODO: We don't handle the delete events in new reconciler mode, how to emit deletion metrics?
-func DeletedClustersCounterInc(namespace string) {
-	clustersDeletedCount.WithLabelValues(namespace).Inc()
-}
-
-func SuccessfulClustersCounterInc(namespace string) {
-	clustersSuccessfulCount.WithLabelValues(namespace).Inc()
+	metrics.Registry.MustRegister(rayClustersCreatedCounter)
 }
 
-func FailedClustersCounterInc(namespace string) {
-	clustersFailedCount.WithLabelValues(namespace).Inc()
+func CreatedRayClustersCounterInc(namespace string, createdByRayJob bool, createdByRayService bool) {
+	rayClustersCreatedCounter.WithLabelValues(namespace, strconv.FormatBool(createdByRayJob), strconv.FormatBool(createdByRayService)).Inc()
 }

ray-operator/controllers/ray/common/metrics_test.go

@@ -0,0 +1,26 @@
+package common
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestCreatedRayClustersCounterInc(t *testing.T) {
+	CreatedRayClustersCounterInc("default", true, false)
+	CreatedRayClustersCounterInc("default", false, true)
+	CreatedRayClustersCounterInc("test", false, false)
+	CreatedRayClustersCounterInc("test", false, false)
+
+	expected := `
+	# HELP ray_clusters_created_total The total number of RayClusters created
+	# TYPE ray_clusters_created_total counter
+	ray_clusters_created_total{created_by_ray_job="true",created_by_ray_service="false",namespace="default"} 1
+	ray_clusters_created_total{created_by_ray_job="false",created_by_ray_service="true",namespace="default"} 1
+	ray_clusters_created_total{created_by_ray_job="false",created_by_ray_service="false",namespace="test"} 2
+	`
+	if err := testutil.CollectAndCompare(rayClustersCreatedCounter, strings.NewReader(expected)); err != nil {
+		t.Errorf("unexpected collecting result:\n%s", err)
+	}
+}

ray-operator/controllers/ray/raycluster_controller.go

@@ -734,12 +734,20 @@ func (r *RayClusterReconciler) reconcilePods(ctx context.Context, instance *rayv
 	} else if len(headPods.Items) == 0 {
 		// Create head Pod if it does not exist.
 		logger.Info("reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.")
-		common.CreatedClustersCounterInc(instance.Namespace)
+
+		creatorCRDType := getCreatorCRDType(*instance)
+		// Increase the counter for ray_clusters_created_total metric.
+		if creatorCRDType == utils.RayClusterCRD {
+			common.CreatedRayClustersCounterInc(instance.Namespace, false, false)
+		} else if creatorCRDType == utils.RayJobCRD {
+			common.CreatedRayClustersCounterInc(instance.Namespace, true, false)
+		} else if creatorCRDType == utils.RayServiceCRD {
+			common.CreatedRayClustersCounterInc(instance.Namespace, false, true)
+		}
+
 		if err := r.createHeadPod(ctx, *instance); err != nil {
-			common.FailedClustersCounterInc(instance.Namespace)
 			return errstd.Join(utils.ErrFailedCreateHeadPod, err)
 		}
-		common.SuccessfulClustersCounterInc(instance.Namespace)
 	} else if len(headPods.Items) > 1 { // This should never happen. This protects against the case that users manually create headpod.
 		correctHeadPodName := instance.Name + "-head"
 		headPodNames := make([]string, len(headPods.Items))

ray-operator/go.mod

@@ -63,6 +63,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect