diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 033a669cc..c46538911 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -58,6 +58,7 @@ const ( DaemonSetKind = "DaemonSet" StatefulSetKind = "StatefulSet" ConfigMapKind = "ConfigMap" + ServiceKind = "Service" NamespaceKind = "Namespace" ) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 857864b7d..74db142d7 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -689,7 +689,7 @@ func customizedCRPStatusUpdatedActual(crpName string, } } -func safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, failedWorkloadResourceIdentifier placementv1beta1.ResourceIdentifier, wantSelectedClusters []string, wantObservedResourceIndex string) func() error { +func safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, failedWorkloadResourceIdentifier placementv1beta1.ResourceIdentifier, wantSelectedClusters []string, wantObservedResourceIndex string, failedResourceObservedGeneration int64) func() error { return func() error { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crp := &placementv1beta1.ClusterResourcePlacement{} @@ -745,7 +745,7 @@ func safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResourceIdentifiers [ Type: string(placementv1beta1.ResourcesAvailableConditionType), Status: metav1.ConditionFalse, Reason: "ManifestNotAvailableYet", - ObservedGeneration: 2, + ObservedGeneration: failedResourceObservedGeneration, }, }, }, diff --git a/test/e2e/enveloped_object_placement_test.go b/test/e2e/enveloped_object_placement_test.go index d7f99f7bb..c33e65913 100644 --- a/test/e2e/enveloped_object_placement_test.go +++ b/test/e2e/enveloped_object_placement_test.go @@ -30,7 +30,6 @@ import ( var ( // pre loaded test manifests testConfigMap, testEnvelopConfigMap corev1.ConfigMap - testService corev1.Service testEnvelopeWebhook admv1.MutatingWebhookConfiguration testEnvelopeResourceQuota corev1.ResourceQuota ) @@ -304,10 +303,6 @@ func readEnvelopTestManifests() { err = utils.GetObjectFromManifest("resources/test-envelop-configmap.yaml", &testEnvelopConfigMap) Expect(err).Should(Succeed()) - By("Read testService resource") - err = utils.GetObjectFromManifest("resources/test-service.yaml", &testService) - Expect(err).Should(Succeed()) - By("Read EnvelopeWebhook") err = utils.GetObjectFromManifest("resources/webhook.yaml", &testEnvelopeWebhook) Expect(err).Should(Succeed()) diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 6694557e7..490b54184 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -63,7 +63,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { createWrappedResourcesForRollout(&testEnvelopeDeployment, &testDeployment, utils.DeploymentKind) }) - It("Create the CRP that select the name space", func() { + It("Create the CRP that select the namespace", func() { crp := &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ Name: crpName, @@ -173,27 +173,8 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { createDeploymentForRollout(&testDeployment) }) - It("create the CRP that select the name space", func() { - crp := &placementv1beta1.ClusterResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: crpName, - // Add a custom finalizer; this would allow us to better observe - // the behavior of the controllers. - Finalizers: []string{customDeletionBlockerFinalizer}, - }, - Spec: placementv1beta1.ClusterResourcePlacementSpec{ - ResourceSelectors: workResourceSelector(), - Strategy: placementv1beta1.RolloutStrategy{ - Type: placementv1beta1.RollingUpdateRolloutStrategyType, - RollingUpdate: &placementv1beta1.RollingUpdateConfig{ - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - }, - }, - }, - }, - } + It("create the CRP that select the namespace", func() { + crp := buildCRPForSafeRollout() Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") }) @@ -230,7 +211,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { Name: testDeployment.Name, Namespace: testDeployment.Namespace, } - crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDeploymentResourceIdentifier, allMemberClusterNames, "1") + crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDeploymentResourceIdentifier, allMemberClusterNames, "1", 2) Eventually(crpStatusActual, 2*time.Minute, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") }) @@ -270,27 +251,8 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { createWrappedResourcesForRollout(&testEnvelopeDaemonSet, &testDaemonSet, utils.DaemonSetKind) }) - It("create the CRP that select the name space", func() { - crp := &placementv1beta1.ClusterResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: crpName, - // Add a custom finalizer; this would allow us to better observe - // the behavior of the controllers. - Finalizers: []string{customDeletionBlockerFinalizer}, - }, - Spec: placementv1beta1.ClusterResourcePlacementSpec{ - ResourceSelectors: workResourceSelector(), - Strategy: placementv1beta1.RolloutStrategy{ - Type: placementv1beta1.RollingUpdateRolloutStrategyType, - RollingUpdate: &placementv1beta1.RollingUpdateConfig{ - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - }, - }, - }, - }, - } + It("create the CRP that select the namespace", func() { + crp := buildCRPForSafeRollout() Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") }) @@ -332,7 +294,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { Type: placementv1beta1.ConfigMapEnvelopeType, }, } - crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDaemonSetResourceIdentifier, allMemberClusterNames, "1") + crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDaemonSetResourceIdentifier, allMemberClusterNames, "1", 2) Eventually(crpStatusActual, 2*time.Minute, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") }) @@ -372,27 +334,8 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { createWrappedResourcesForRollout(&testEnvelopeStatefulSet, &testStatefulSet, utils.StatefulSetKind) }) - It("create the CRP that select the name space", func() { - crp := &placementv1beta1.ClusterResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: crpName, - // Add a custom finalizer; this would allow us to better observe - // the behavior of the controllers. - Finalizers: []string{customDeletionBlockerFinalizer}, - }, - Spec: placementv1beta1.ClusterResourcePlacementSpec{ - ResourceSelectors: workResourceSelector(), - Strategy: placementv1beta1.RolloutStrategy{ - Type: placementv1beta1.RollingUpdateRolloutStrategyType, - RollingUpdate: &placementv1beta1.RollingUpdateConfig{ - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - }, - }, - }, - }, - } + It("create the CRP that select the namespace", func() { + crp := buildCRPForSafeRollout() Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") }) @@ -434,7 +377,84 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { Type: placementv1beta1.ConfigMapEnvelopeType, }, } - crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedStatefulSetResourceIdentifier, allMemberClusterNames, "1") + crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedStatefulSetResourceIdentifier, allMemberClusterNames, "1", 2) + Eventually(crpStatusActual, 2*time.Minute, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + }) + }) + + Context("Test a CRP place workload objects successfully, block rollout based on service availability", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + workNamespaceName := appNamespace().Name + var wantSelectedResources []placementv1beta1.ResourceIdentifier + var testService corev1.Service + + BeforeAll(func() { + // Create the test resources. + readServiceTestManifest(&testService) + wantSelectedResources = []placementv1beta1.ResourceIdentifier{ + { + Kind: utils.NamespaceKind, + Name: workNamespaceName, + Version: corev1.SchemeGroupVersion.Version, + }, + { + Kind: utils.ServiceKind, + Name: testService.Name, + Version: corev1.SchemeGroupVersion.Version, + Namespace: workNamespaceName, + }, + } + }) + + It("create the service resource in the namespace", func() { + createServiceForRollout(&testService) + }) + + It("create the CRP that select the namespace", func() { + crp := buildCRPForSafeRollout() + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + }) + + It("should update CRP status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(wantSelectedResources, allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, 2*time.Minute, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") + }) + + It("should place the resources on all member clusters", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + workResourcesPlacedActual := waitForServiceToReady(memberCluster, &testService) + Eventually(workResourcesPlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName) + } + }) + + It("change service to LoadBalancer, to make it unavailable", func() { + Eventually(func() error { + var service corev1.Service + err := hubClient.Get(ctx, types.NamespacedName{Name: testService.Name, Namespace: testService.Namespace}, &service) + if err != nil { + return err + } + service.Spec.Type = corev1.ServiceTypeLoadBalancer + return hubClient.Update(ctx, &service) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to change the service type to LoadBalancer") + }) + + It("should update CRP status as expected", func() { + failedDeploymentResourceIdentifier := placementv1beta1.ResourceIdentifier{ + Group: corev1.SchemeGroupVersion.Group, + Version: corev1.SchemeGroupVersion.Version, + Kind: utils.ServiceKind, + Name: testService.Name, + Namespace: testService.Namespace, + } + // failedResourceObservedGeneration is set to 0 because generation is not populated for service. + crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDeploymentResourceIdentifier, allMemberClusterNames, "1", 0) Eventually(crpStatusActual, 2*time.Minute, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") }) @@ -463,6 +483,12 @@ func readStatefulSetTestManifest(testStatefulSet *appv1.StatefulSet) { Expect(err).Should(Succeed()) } +func readServiceTestManifest(testService *corev1.Service) { + By("Read the service resource") + err := utils.GetObjectFromManifest("resources/test-service.yaml", testService) + Expect(err).Should(Succeed()) +} + func readEnvelopeConfigMapTestManifest(testEnvelopeObj *corev1.ConfigMap) { By("Read testEnvelopConfigMap resource") err := utils.GetObjectFromManifest("resources/test-envelope-object.yaml", testEnvelopeObj) @@ -476,6 +502,13 @@ func createDeploymentForRollout(testDeployment *appv1.Deployment) { Expect(hubClient.Create(ctx, testDeployment)).To(Succeed(), "Failed to create test deployment %s", testDeployment.Name) } +func createServiceForRollout(testService *corev1.Service) { + ns := appNamespace() + Expect(hubClient.Create(ctx, &ns)).To(Succeed(), "Failed to create namespace %s", ns.Namespace) + testService.Namespace = ns.Name + Expect(hubClient.Create(ctx, testService)).To(Succeed(), "Failed to create test service %s", testService.Name) +} + // createWrappedResourcesForRollout creates an enveloped resource on the hub cluster with a workload object for testing purposes. func createWrappedResourcesForRollout(testEnvelopeObj *corev1.ConfigMap, obj metav1.Object, kind string) { ns := appNamespace() @@ -566,3 +599,45 @@ func waitForStatefulSetPlacementToReady(memberCluster *framework.Cluster, testSt return errors.New("statefulset is not ready") } } + +func waitForServiceToReady(memberCluster *framework.Cluster, testService *corev1.Service) func() error { + workNamespaceName := appNamespace().Name + return func() error { + if err := validateWorkNamespaceOnCluster(memberCluster, types.NamespacedName{Name: workNamespaceName}); err != nil { + return err + } + By("check the placedService") + placedService := &corev1.Service{} + if err := memberCluster.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: testService.Name}, placedService); err != nil { + return err + } + By("check the placedService is ready") + if placedService.Spec.ClusterIP != "" { + return nil + } + return errors.New("service is not ready") + } +} + +func buildCRPForSafeRollout() *placementv1beta1.ClusterResourcePlacement { + return &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()), + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + ResourceSelectors: workResourceSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.RollingUpdateRolloutStrategyType, + RollingUpdate: &placementv1beta1.RollingUpdateConfig{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + }, + } +} diff --git a/test/e2e/taint_toleration_test.go b/test/e2e/taint_toleration_test.go index 683a73bec..35c3e69f4 100644 --- a/test/e2e/taint_toleration_test.go +++ b/test/e2e/taint_toleration_test.go @@ -1,3 +1,8 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + package e2e import (