feat: expire stale advanced metrics after not being updated for some time

Signed-off-by: Matthew McKeen <[email protected]>

Author: mmckeen

crd/api/v1alpha1/metricsconfiguration_types.go

@@ -41,6 +41,10 @@ type MetricsContextOptions struct {
 	// +optional
 	// +listType=set
 	AdditionalLabels []string `json:"additionalLabels,omitempty"`
+	// TTL represents the time-to-live of the metrics collected
+	// Metrics which have not been updated within the TTL will be removed from export
+	// +optional
+	TTL string `json:"ttl,omitempty"`
 }
 
 // MetricsNamespaces indicates the namespaces to include or exclude in metric collection

crd/api/v1alpha1/validations/validate_metricconfiguration.go

@@ -7,6 +7,7 @@ package validations
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/microsoft/retina/crd/api/v1alpha1"
 	"github.com/microsoft/retina/pkg/utils"
@@ -40,6 +41,15 @@ func MetricsSpec(metricsSpec v1alpha1.MetricsSpec) error {
 		if !utils.IsAdvancedMetric(contextOption.MetricName) {
 			return fmt.Errorf("%s is not a valid metric", contextOption.MetricName)
 		}
+		if contextOption.TTL != "" {
+			ttl, err := time.ParseDuration(contextOption.TTL)
+			if err != nil {
+				return fmt.Errorf("invalid TTL format for metric %s: %v", contextOption.MetricName, err)
+			}
+			if ttl < 0 {
+				return fmt.Errorf("TTL cannot be negative for metric %s", contextOption.MetricName)
+			}
+		}
 	}
 
 	err := MetricsNamespaces(metricsSpec.Namespaces)
@@ -152,10 +162,13 @@ func MetricsContextOptionsCompare(old, new []v1alpha1.MetricsContextOptions) boo
 			return false
 		}
 
-		if !utils.CompareStringSlice(oldContextOption.AdditionalLabels, newContextOption.AdditionalLabels) {
+		if oldContextOption.TTL != newContextOption.TTL {
 			return false
 		}
 
+		if !utils.CompareStringSlice(oldContextOption.AdditionalLabels, newContextOption.AdditionalLabels) {
+			return false
+		}
 	}
 
 	return true

crd/api/v1alpha1/zz_generated.deepcopy.go

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: captures.retina.sh
 spec:
   group: retina.sh
@@ -53,6 +53,14 @@ spec:
                           should continue for.
                         pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$
                         type: string
+                      interfaces:
+                        description: |-
+                          Interfaces specifies the network interfaces on which to capture packets.
+                          If specified, captures only on the listed interfaces.
+                          If empty, captures on all interfaces by default.
+                        items:
+                          type: string
+                        type: array
                       maxCaptureSize:
                         default: 100
                         description: MaxCaptureSize limits the capture file to MB
@@ -290,10 +298,14 @@ spec:
                         description: SecretName is the name of secret which stores
                           S3 compliant storage access key and secret key.
                         type: string
+                    required:
+                    - bucket
+                    - secretName
                     type: object
                 type: object
             required:
             - captureConfiguration
+            - outputConfiguration
             type: object
           status:
             description: CaptureStatus describes the status of the capture.
@@ -310,16 +322,8 @@ spec:
                 type: string
               conditions:
                 items:
-                  description: "Condition contains details for one aspect of the current
-                    state of this API Resource.\n---\nThis struct is intended for
-                    direct use as an array at the field path .status.conditions.  For
-                    example,\n\n\n\ttype FooStatus struct{\n\t    // Represents the
-                    observations of a foo's current state.\n\t    // Known .status.conditions.type
-                    are: \"Available\", \"Progressing\", and \"Degraded\"\n\t    //
-                    +patchMergeKey=type\n\t    // +patchStrategy=merge\n\t    // +listType=map\n\t
-                    \   // +listMapKey=type\n\t    Conditions []metav1.Condition `json:\"conditions,omitempty\"
-                    patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t
-                    \   // other fields\n\t}"
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
                   properties:
                     lastTransitionTime:
                       description: |-
@@ -360,12 +364,7 @@ spec:
                       - Unknown
                       type: string
                     type:
-                      description: |-
-                        type of condition in CamelCase or in foo.example.com/CamelCase.
-                        ---
-                        Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be
-                        useful (see .node.status.conditions), the ability to deconflict is important.
-                        The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
                       maxLength: 316
                       pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                       type: string

deploy/standard/manifests/controller/helm/retina/crds/retina.sh_captures.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: metricsconfigurations.retina.sh
 spec:
   group: retina.sh
@@ -75,6 +75,11 @@ spec:
                         type: string
                       type: array
                       x-kubernetes-list-type: set
+                    ttl:
+                      description: |-
+                        TTL represents the time-to-live of the metrics collected
+                        Metrics which have not been updated within the TTL will be removed from export
+                      type: string
                   required:
                   - metricName
                   type: object
@@ -136,6 +141,11 @@ spec:
                             type: string
                           type: array
                           x-kubernetes-list-type: set
+                        ttl:
+                          description: |-
+                            TTL represents the time-to-live of the metrics collected
+                            Metrics which have not been updated within the TTL will be removed from export
+                          type: string
                       required:
                       - metricName
                       type: object

deploy/standard/manifests/controller/helm/retina/crds/retina.sh_metricsconfigurations.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: retinaendpoints.retina.sh
 spec:
   group: retina.sh

deploy/standard/manifests/controller/helm/retina/crds/retina.sh_retinaendpoints.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.0
+    controller-gen.kubebuilder.io/version: v0.16.5
   name: tracesconfigurations.retina.sh
 spec:
   group: retina.sh

deploy/standard/manifests/controller/helm/retina/crds/retina.sh_tracesconfigurations.yaml

@@ -9,12 +9,18 @@ import (
 
 //go:generate go run go.uber.org/mock/[email protected] -source=interfaces.go -destination=mock_types.go -package=metrics
 
+type MetricVec interface {
+	DeleteLabelValues(lvs ...string) bool
+}
+
 type CounterVec interface {
+	MetricVec
 	WithLabelValues(lvs ...string) prometheus.Counter
 	GetMetricWithLabelValues(lvs ...string) (prometheus.Counter, error)
 }
 
 type GaugeVec interface {
+	MetricVec
 	WithLabelValues(lvs ...string) prometheus.Gauge
 	GetMetricWithLabelValues(lvs ...string) (prometheus.Gauge, error)
 }

pkg/metrics/interfaces.go

@@ -194,6 +194,13 @@ func InitializeMetrics() {
 		parsedPacketsCounterDescription,
 	)
 
+	MetricsExpiredCounter = exporter.CreatePrometheusCounterVecForControlPlaneMetric(
+		exporter.DefaultRegistry,
+		expiredMetricsCounterName,
+		expiredMetricsCounterDescription,
+		utils.Metric,
+	)
+
 	isInitialized = true
 	metricsLogger.Info("Metrics initialized")
 }