[WIP] [TestLeaderElection] e2e: build a descheduler image and run the descheduler as a pod

test/e2e/e2e_leaderelection_test.go

@@ -19,7 +19,6 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -30,12 +29,61 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
 	clientset "k8s.io/client-go/kubernetes"
 	utilptr "k8s.io/utils/ptr"
-	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
+	"sigs.k8s.io/descheduler/pkg/api"
+	apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2"
 	"sigs.k8s.io/descheduler/pkg/descheduler"
+	"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
+	"sigs.k8s.io/descheduler/pkg/framework/plugins/podlifetime"
 )
 
+// const deploymentReplicas = 5 // May not be the best value
+
+func PodLifeTimePolicy(targetNamespace string, maxPodLifeTimeSeconds *uint) *apiv1alpha2.DeschedulerPolicy {
+	return &apiv1alpha2.DeschedulerPolicy{
+		Profiles: []apiv1alpha2.DeschedulerProfile{
+			{
+				Name: "PodLifeTimeProfile",
+				PluginConfigs: []apiv1alpha2.PluginConfig{
+					{
+						Name: podlifetime.PluginName,
+						Args: runtime.RawExtension{
+							Object: &podlifetime.PodLifeTimeArgs{
+								MaxPodLifeTimeSeconds: maxPodLifeTimeSeconds,
+								Namespaces: &api.Namespaces{
+									Include: []string{targetNamespace},
+								},
+							},
+						},
+					},
+					{
+						Name: defaultevictor.PluginName,
+						Args: runtime.RawExtension{
+							Object: &defaultevictor.DefaultEvictorArgs{
+								EvictLocalStoragePods: true,
+							},
+						},
+					},
+				},
+				Plugins: apiv1alpha2.Plugins{
+					Filter: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							defaultevictor.PluginName,
+						},
+					},
+					Deschedule: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							podlifetime.PluginName,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func TestLeaderElection(t *testing.T) {
 	descheduler.SetupPlugins()
 	ctx := context.Background()
@@ -59,17 +107,17 @@ func TestLeaderElection(t *testing.T) {
 	}
 	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace2.Name, metav1.DeleteOptions{})
 
-	deployment1, err := createDeployment(ctx, clientSet, ns1, 5, t)
+	deploymentObj1, err := createDeployment(ctx, clientSet, ns1, deploymentReplicas, t)
 	if err != nil {
 		t.Fatalf("create deployment 1: %v", err)
 	}
-	defer clientSet.AppsV1().Deployments(deployment1.Namespace).Delete(ctx, deployment1.Name, metav1.DeleteOptions{})
+	defer clientSet.AppsV1().Deployments(deploymentObj1.Namespace).Delete(ctx, deploymentObj1.Name, metav1.DeleteOptions{})
 
-	deployment2, err := createDeployment(ctx, clientSet, ns2, 5, t)
+	deploymentObj2, err := createDeployment(ctx, clientSet, ns2, deploymentReplicas, t)
 	if err != nil {
 		t.Fatalf("create deployment 2: %v", err)
 	}
-	defer clientSet.AppsV1().Deployments(deployment2.Namespace).Delete(ctx, deployment2.Name, metav1.DeleteOptions{})
+	defer clientSet.AppsV1().Deployments(deploymentObj2.Namespace).Delete(ctx, deploymentObj2.Name, metav1.DeleteOptions{})
 
 	waitForPodsRunning(ctx, t, clientSet, map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, 5, ns1)
 
@@ -79,32 +127,6 @@ func TestLeaderElection(t *testing.T) {
 
 	podListBOrg := getPodNameList(ctx, clientSet, ns2, t)
 
-	s1, err := options.NewDeschedulerServer()
-	if err != nil {
-		t.Fatalf("unable to initialize server: %v", err)
-	}
-	s1.Client = clientSet
-	s1.DeschedulingInterval = 5 * time.Second
-	s1.LeaderElection.LeaderElect = true
-	s1.LeaderElection.RetryPeriod = metav1.Duration{
-		Duration: time.Second,
-	}
-	s1.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG")
-	s1.PolicyConfigFile = "./policy_leaderelection_a.yaml"
-
-	s2, err := options.NewDeschedulerServer()
-	if err != nil {
-		t.Fatalf("unable to initialize server: %v", err)
-	}
-	s2.Client = clientSet
-	s2.DeschedulingInterval = 5 * time.Second
-	s2.LeaderElection.LeaderElect = true
-	s2.LeaderElection.RetryPeriod = metav1.Duration{
-		Duration: time.Second,
-	}
-	s2.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG")
-	s2.PolicyConfigFile = "./policy_leaderelection_b.yaml"
-
 	// Delete the descheduler lease
 	err = clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{})
 	if err != nil {
@@ -114,28 +136,10 @@ func TestLeaderElection(t *testing.T) {
 	}
 	t.Logf("Removed kube-system/descheduler lease")
 
-	t.Log("starting deschedulers")
-
-	go func() {
-		err := descheduler.Run(ctx, s1)
-		if err != nil {
-			t.Errorf("unable to start descheduler: %v", err)
-			return
-		}
-	}()
+	t.Log("starting deschedulers deployments with leader election enabled")
 
-	time.Sleep(1 * time.Second)
-
-	go func() {
-		err := descheduler.Run(ctx, s2)
-		if err != nil {
-			t.Errorf("unable to start descheduler: %v", err)
-			return
-		}
-	}()
-
-	defer clientSet.CoordinationV1().Leases(s1.LeaderElection.ResourceNamespace).Delete(ctx, s1.LeaderElection.ResourceName, metav1.DeleteOptions{})
-	defer clientSet.CoordinationV1().Leases(s2.LeaderElection.ResourceNamespace).Delete(ctx, s2.LeaderElection.ResourceName, metav1.DeleteOptions{})
+	createDeschedulerDeploymentWithPolicyConfigMap(t, ns1)
+	createDeschedulerDeploymentWithPolicyConfigMap(t, ns2)
 
 	// wait for a while so all the pods are 5 seconds older
 	time.Sleep(7 * time.Second)
@@ -166,7 +170,7 @@ func TestLeaderElection(t *testing.T) {
 }
 
 func createDeployment(ctx context.Context, clientSet clientset.Interface, namespace string, replicas int32, t *testing.T) (*appsv1.Deployment, error) {
-	deployment := &appsv1.Deployment{
+	deploymentObj := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "leaderelection",
 			Namespace: namespace,
@@ -209,18 +213,18 @@ func createDeployment(ctx context.Context, clientSet clientset.Interface, namesp
 		},
 	}
 
-	t.Logf("Creating deployment %v for namespace %s", deployment.Name, deployment.Namespace)
-	deployment, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{})
+	t.Logf("Creating deployment %v for namespace %s", deploymentObj.Name, deploymentObj.Namespace)
+	deploymentObj, err := clientSet.AppsV1().Deployments(deploymentObj.Namespace).Create(ctx, deploymentObj, metav1.CreateOptions{})
 	if err != nil {
 		t.Logf("Error creating deployment: %v", err)
-		if err = clientSet.AppsV1().Deployments(deployment.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
+		if err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
 			LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "leaderelection", "name": "test-leaderelection"})).String(),
 		}); err != nil {
 			t.Fatalf("Unable to delete deployment: %v", err)
 		}
 		return nil, fmt.Errorf("create deployment %v", err)
 	}
-	return deployment, nil
+	return deploymentObj, nil
 }
 
 func getPodNameList(ctx context.Context, clientSet clientset.Interface, namespace string, t *testing.T) []string {
@@ -235,3 +239,67 @@ func getPodNameList(ctx context.Context, clientSet clientset.Interface, namespac
 	}
 	return podNames
 }
+
+// createDeschedulerDeploymentWithPolicyConfigMap Reuse function for 2 testNamespaces
+func createDeschedulerDeploymentWithPolicyConfigMap(t *testing.T, targetNamespace string) *appsv1.Deployment {
+
+	descheduler.SetupPlugins()
+	ctx := context.Background()
+
+	clientSet, _, _, _ := initializeClient(ctx, t)
+
+	// test case
+	tc := struct {
+		name   string
+		policy *apiv1alpha2.DeschedulerPolicy
+	}{
+		name:   "leaderelection",
+		policy: PodLifeTimePolicy(targetNamespace, utilptr.To[uint](5)),
+	}
+
+	// Deploy the descheduler with the configured policy
+	deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tc.policy)
+	if err != nil {
+		t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
+	}
+	t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
+	_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
+	if err != nil {
+		t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
+	}
+
+	defer func() {
+		t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
+		err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
+		if err != nil {
+			t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
+		}
+	}()
+
+	deschedulerDeploymentObj := deschedulerDeployment("kube-system")
+	t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
+	_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
+	if err != nil {
+		t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
+	}
+
+	deschedulerPodName := ""
+	defer func() {
+		if deschedulerPodName != "" {
+			printPodLogs(ctx, t, clientSet, deschedulerPodName)
+		}
+
+		t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
+		err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
+		if err != nil {
+			t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
+		}
+		waitForDeschedulerPodAbsent(t, ctx, clientSet, "kube-system")
+	}()
+
+	t.Logf("Waiting for the descheduler pod running")
+	deschedulerPodName = waitForDeschedulerPodRunning(t, ctx, clientSet, "kube-system")
+
+	return deschedulerDeploymentObj
+
+}