[Prometheus] Add ray_cluster_provisioned_duration_seconds metric

Signed-off-by: win5923 <[email protected]>

Author: win5923

ray-operator/controllers/ray/common/metrics.go

@@ -1,64 +1,33 @@
 package common
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 // Define all the prometheus counters for all clusters
 var (
-	clustersCreatedCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_created_total",
-			Help: "Counts number of clusters created",
-		},
-		[]string{"namespace"},
-	)
-	clustersDeletedCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_deleted_total",
-			Help: "Counts number of clusters deleted",
-		},
-		[]string{"namespace"},
-	)
-	clustersSuccessfulCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_successful_total",
-			Help: "Counts number of clusters successful",
-		},
-		[]string{"namespace"},
-	)
-	clustersFailedCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "ray_operator_clusters_failed_total",
-			Help: "Counts number of clusters failed",
+	RayClusterProvisionedDurationSeconds = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "ray_cluster_provisioned_duration_seconds",
+			Help: "The time from RayClusters created to all ray pods are ready for the first time (RayClusterProvisioned) in seconds",
+			// It may not be applicable to all users, but default buckets cannot be used either.
+			// For reference, see: https://github.com/prometheus/client_golang/blob/331dfab0cc853dca0242a0d96a80184087a80c1d/prometheus/histogram.go#L271
+			Buckets: []float64{30, 60, 120, 180, 240, 300, 600, 900, 1800, 3600},
 		},
-		[]string{"namespace"},
+		[]string{"namespace", "name"},
 	)
 )
 
 func init() {
 	// Register custom metrics with the global prometheus registry
-	metrics.Registry.MustRegister(clustersCreatedCount,
-		clustersDeletedCount,
-		clustersSuccessfulCount,
-		clustersFailedCount)
-}
-
-func CreatedClustersCounterInc(namespace string) {
-	clustersCreatedCount.WithLabelValues(namespace).Inc()
-}
-
-// TODO: We don't handle the delete events in new reconciler mode, how to emit deletion metrics?
-func DeletedClustersCounterInc(namespace string) {
-	clustersDeletedCount.WithLabelValues(namespace).Inc()
-}
-
-func SuccessfulClustersCounterInc(namespace string) {
-	clustersSuccessfulCount.WithLabelValues(namespace).Inc()
+	metrics.Registry.MustRegister(RayClusterProvisionedDurationSeconds)
 }
 
-func FailedClustersCounterInc(namespace string) {
-	clustersFailedCount.WithLabelValues(namespace).Inc()
+// ObserveRayClusterProvisionedDuration observes the duration of RayCluster from creation to provisioned
+func ObserveRayClusterProvisionedDuration(namespace, name string, duration time.Duration) {
+	RayClusterProvisionedDurationSeconds.WithLabelValues(namespace, name).Observe(duration.Seconds())
 }

ray-operator/controllers/ray/common/metrics_test.go

@@ -0,0 +1,34 @@
+package common
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestObserveRayClusterProvisionedDuration(t *testing.T) {
+	ObserveRayClusterProvisionedDuration("default", "raycluster-sample", 2*time.Minute)
+
+	metric := `
+		# HELP ray_cluster_provisioned_duration_seconds The time from RayClusters created to all ray pods are ready for the first time (RayClusterProvisioned) in seconds
+		# TYPE ray_cluster_provisioned_duration_seconds histogram
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="30"} 0
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="60"} 0
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="120"} 1
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="180"} 1
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="240"} 1
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="300"} 1
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="600"} 1
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="900"} 1
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="1800"} 1
+		ray_cluster_provisioned_duration_seconds_bucket{name="raycluster-sample",namespace="default",le="3600"} 1
+		ray_cluster_provisioned_duration_seconds_sum{name="raycluster-sample",namespace="default"} 120
+		ray_cluster_provisioned_duration_seconds_count{name="raycluster-sample",namespace="default"} 1
+	`
+
+	if err := testutil.CollectAndCompare(RayClusterProvisionedDurationSeconds, strings.NewReader(metric), "ray_cluster_provisioned_duration_seconds"); err != nil {
+		t.Errorf("unexpected collecting result:\n%s", err)
+	}
+}

ray-operator/controllers/ray/raycluster_controller.go

@@ -734,12 +734,9 @@ func (r *RayClusterReconciler) reconcilePods(ctx context.Context, instance *rayv
 	} else if len(headPods.Items) == 0 {
 		// Create head Pod if it does not exist.
 		logger.Info("reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.")
-		common.CreatedClustersCounterInc(instance.Namespace)
 		if err := r.createHeadPod(ctx, *instance); err != nil {
-			common.FailedClustersCounterInc(instance.Namespace)
 			return errstd.Join(utils.ErrFailedCreateHeadPod, err)
 		}
-		common.SuccessfulClustersCounterInc(instance.Namespace)
 	} else if len(headPods.Items) > 1 { // This should never happen. This protects against the case that users manually create headpod.
 		correctHeadPodName := instance.Name + "-head"
 		headPodNames := make([]string, len(headPods.Items))
@@ -1336,6 +1333,11 @@ func (r *RayClusterReconciler) calculateStatus(ctx context.Context, instance *ra
 					Reason:  rayv1.AllPodRunningAndReadyFirstTime,
 					Message: "All Ray Pods are ready for the first time",
 				})
+
+				// Record ray_cluster_provisioned_duration_seconds duration metric
+				// Calculate the time between cluster creation and being fully provisioned
+				provisionDuration := time.Since(instance.CreationTimestamp.Time)
+				common.ObserveRayClusterProvisionedDuration(instance.Namespace, instance.Name, provisionDuration)
 			} else {
 				meta.SetStatusCondition(&newInstance.Status.Conditions, metav1.Condition{
 					Type:    string(rayv1.RayClusterProvisioned),

ray-operator/go.mod

@@ -63,6 +63,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect