[nodeutilization]: prometheus usage client with prometheus metrics

Author: ingvagabund

README.md

@@ -129,14 +129,18 @@ These are top level keys in the Descheduler Policy that you can use to configure
 | `metricsProviders`                 | `[]object` | `nil`       | Enables various metrics providers like Kubernetes [Metrics Server](https://kubernetes-sigs.github.io/metrics-server/)      |
 | `evictionFailureEventNotification` | `bool`   | `false`       | Enables eviction failure event notification.                                                                               |
 | `gracePeriodSeconds`               | `int`    | `0`           | The duration in seconds before the object should be deleted. The value zero indicates delete immediately.                  |
+| `prometheus` |`object`| `nil` | Configures collection of Prometheus metrics for actual resource utilization |
+| `prometheus.url` |`string`| `nil` | Points to a Prometheus server url |
+| `prometheus.authToken` |`object`| `nil` | Sets Prometheus server authentication token. If not specified in cluster authentication token from the container's file system is read. |
+| `prometheus.authToken.secretReference` |`object`| `nil` | Read the authentication token from a kubernetes secret (the secret is expected to contain the token under `prometheusAuthToken` data key) |
+| `prometheus.authToken.secretReference.namespace` |`string`| `nil` | Authentication token kubernetes secret namespace (currently, the RBAC configuration permits retrieving secrets from the `kube-system` namespace. If the secret needs to be accessed from a different namespace, the existing RBAC rules must be explicitly extended. |
+| `prometheus.authToken.secretReference.name` |`string`| `nil` | Authentication token kubernetes secret name |
 
 The descheduler currently allows to configure a metric collection of Kubernetes Metrics through `metricsProviders` field.
-The previous way of setting `metricsCollector` field is deprecated. There is currently one source to configure:
-```
-metricsProviders:
-- source: KubernetesMetrics
-```
-The list can be extended with other metrics providers in the future.
+The previous way of setting `metricsCollector` field is deprecated. There are currently two sources to configure:
+- `KubernetesMetrics`: enables metrics collection from Kubernetes Metrics server
+- `Prometheus`: enables metrics collection from Prometheus server
+
 In general, each plugin can consume metrics from a different provider so multiple distinct providers can be configured in parallel.
 
 
@@ -174,9 +178,15 @@ maxNoOfPodsToEvictPerNode: 5000 # you don't need to set this, unlimited if not s
 maxNoOfPodsToEvictPerNamespace: 5000 # you don't need to set this, unlimited if not set
 maxNoOfPodsToEvictTotal: 5000 # you don't need to set this, unlimited if not set
 gracePeriodSeconds: 60 # you don't need to set this, 0 if not set
-# you don't need to set this, Kubernetes metrics are not collected if not set
+# you don't need to set this, metrics are not collected if not set
 metricsProviders:
-- source: KubernetesMetrics
+- source: Prometheus
+  prometheus:
+    url: http://prometheus-kube-prometheus-prometheus.prom.svc.cluster.local
+    authToken:
+      secretReference:
+        namespace: "kube-system"
+        name: "authtoken"
 profiles:
   - name: ProfileName
     pluginConfig:
@@ -303,6 +313,10 @@ like `kubectl top`) may differ from the calculated consumption, due to these com
 actual usage metrics. Metrics-based descheduling can be enabled by setting `metricsUtilization.metricsServer` field (deprecated)
 or `metricsUtilization.source` field to `KubernetesMetrics`.
 In order to have the plugin consume the metrics the metric provider needs to be configured as well.
+Alternatively, it is possible to create a prometheus client and configure a prometheus query to consume
+metrics outside of the kubernetes metrics server. The query is expected to return a vector of values for
+each node. The values are expected to be any real number within <0; 1> interval. During eviction only
+a single pod is evicted at most from each overutilized node. There's currently no support for evicting more.
 See `metricsProviders` field at [Top Level configuration](#top-level-configuration) for available options.
 
 **Parameters:**
@@ -318,6 +332,7 @@ See `metricsProviders` field at [Top Level configuration](#top-level-configurati
 |`metricsUtilization`|object|
 |`metricsUtilization.metricsServer` (deprecated)|bool|
 |`metricsUtilization.source`|string|
+|`metricsUtilization.prometheus.query`|string|
 
 
 **Example:**
@@ -340,6 +355,8 @@ profiles:
           "pods": 50
         metricsUtilization:
           source: KubernetesMetrics
+          # prometheus:
+          #   query: instance:node_cpu:rate:sum
         evictionLimits:
           node: 5
     plugins:

cmd/descheduler/app/options/options.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 	"time"
 
+	promapi "github.com/prometheus/client_golang/api"
 	"github.com/spf13/pflag"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -54,6 +55,7 @@ type DeschedulerServer struct {
 	Client            clientset.Interface
 	EventClient       clientset.Interface
 	MetricsClient     metricsclient.Interface
+	PrometheusClient  promapi.Client
 	SecureServing     *apiserveroptions.SecureServingOptionsWithLoopback
 	SecureServingInfo *apiserver.SecureServingInfo
 	DisableMetrics    bool

kubernetes/base/rbac.yaml

@@ -36,6 +36,15 @@ rules:
   resources: ["nodes", "pods"]
   verbs: ["get", "list"]
 ---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: descheduler-role
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "watch"]
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -54,3 +63,16 @@ subjects:
   - name: descheduler-sa
     kind: ServiceAccount
     namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: descheduler-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: descheduler-role
+subjects:
+  - name: descheduler-sa
+    kind: ServiceAccount
+    namespace: kube-system

pkg/api/types.go

@@ -115,6 +115,9 @@ const (
 	// KubernetesMetrics enables metrics from a Kubernetes metrics server.
 	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
 	KubernetesMetrics MetricsSource = "KubernetesMetrics"
+
+	// KubernetesMetrics enables metrics from a Prometheus metrics server.
+	PrometheusMetrics MetricsSource = "Prometheus"
 )
 
 // MetricsCollector configures collection of metrics about actual resource utilization
@@ -128,7 +131,31 @@ type MetricsCollector struct {
 type MetricsProvider struct {
 	// Source enables metrics from Kubernetes metrics server.
 	Source MetricsSource
+
+	// Prometheus enables metrics collection through Prometheus
+	Prometheus *Prometheus
 }
 
 // ReferencedResourceList is an adaption of v1.ResourceList with resources as references
 type ReferencedResourceList = map[v1.ResourceName]*resource.Quantity
+
+type Prometheus struct {
+	URL string
+	// authToken used for authentication with the prometheus server.
+	// If not set the in cluster authentication token from the container's file system is read.
+	AuthToken *AuthToken
+}
+
+type AuthToken struct {
+	// secretReference references an authentication token.
+	// secrets are expected to be created under the descheduler's namespace.
+	SecretReference *SecretReference
+}
+
+// SecretReference holds a reference to a Secret
+type SecretReference struct {
+	// namespace is the namespace of the secret.
+	Namespace string
+	// name is the name of the secret.
+	Name string
+}

pkg/api/v1alpha2/types.go

@@ -90,6 +90,9 @@ const (
 	// KubernetesMetrics enables metrics from a Kubernetes metrics server.
 	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
 	KubernetesMetrics MetricsSource = "KubernetesMetrics"
+
+	// KubernetesMetrics enables metrics from a Prometheus metrics server.
+	PrometheusMetrics MetricsSource = "Prometheus"
 )
 
 // MetricsCollector configures collection of metrics about actual resource utilization
@@ -103,4 +106,27 @@ type MetricsCollector struct {
 type MetricsProvider struct {
 	// Source enables metrics from Kubernetes metrics server.
 	Source MetricsSource `json:"source,omitempty"`
+
+	// Prometheus enables metrics collection through Prometheus
+	Prometheus *Prometheus `json:"prometheus,omitempty"`
+}
+
+type Prometheus struct {
+	URL string `json:"url,omitempty"`
+	// If not set the in cluster authentication token from the container's file system is read.
+	AuthToken *AuthToken `json:"authToken,omitempty"`
+}
+
+type AuthToken struct {
+	// secretReference references an authentication token.
+	// secrets are expected to be created under the descheduler's namespace.
+	SecretReference *SecretReference `json:"secretReference,omitempty"`
+}
+
+// SecretReference holds a reference to a Secret
+type SecretReference struct {
+	// namespace is the namespace of the secret.
+	Namespace string `json:"namespace,omitempty"`
+	// name is the name of the secret.
+	Name string `json:"name,omitempty"`
 }