From 5706894d6bd082945697bf1275d9f0706c9387b1 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Fri, 6 Dec 2024 16:00:18 -0800 Subject: [PATCH] feat: Eviction controller (#942) --- .../v1alpha1/disruptionbudget_types.go | 10 +- apis/placement/v1alpha1/eviction_types.go | 13 + ...terresourceplacementdisruptionbudgets.yaml | 10 +- .../controller.go | 373 +++++ .../controller_intergration_test.go | 609 +++++++ .../controller_test.go | 1453 +++++++++++++++++ .../suite_test.go | 105 ++ pkg/controllers/rollout/controller.go | 15 +- pkg/utils/binding/binding.go | 21 + pkg/utils/binding/binding_test.go | 136 ++ .../api_validation_integration_test.go | 5 + test/apis/placement/v1alpha1/suite_test.go | 5 + 12 files changed, 2731 insertions(+), 24 deletions(-) create mode 100644 pkg/controllers/clusterresourceplacementeviction/controller.go create mode 100644 pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go create mode 100644 pkg/controllers/clusterresourceplacementeviction/controller_test.go create mode 100644 pkg/controllers/clusterresourceplacementeviction/suite_test.go create mode 100644 pkg/utils/binding/binding.go create mode 100644 pkg/utils/binding/binding_test.go diff --git a/apis/placement/v1alpha1/disruptionbudget_types.go b/apis/placement/v1alpha1/disruptionbudget_types.go index e14f78e4d..de38608b8 100644 --- a/apis/placement/v1alpha1/disruptionbudget_types.go +++ b/apis/placement/v1alpha1/disruptionbudget_types.go @@ -46,9 +46,8 @@ type PlacementDisruptionBudgetSpec struct { // * if the linked Placement object is of the PickFixed placement type, // the percentage is against the number of clusters specified in the placement (i.e., the // length of ClusterNames field in the placement policy); - // * if the linked Placement object is of the PickAll placement type, - // the percentage is against the total number of clusters being selected by the scheduler - // at the time of the evaluation of the disruption budget; + // * if the linked Placement object is of the PickAll placement type, MaxUnavailable cannot + // be specified since we cannot derive the total number of clusters selected. // * if the linked Placement object is of the PickN placement type, // the percentage is against the number of clusters specified in the placement (i.e., the // value of the NumberOfClusters fields in the placement policy). @@ -77,9 +76,8 @@ type PlacementDisruptionBudgetSpec struct { // * if the linked Placement object is of the PickFixed placement type, // the percentage is against the number of clusters specified in the placement (i.e., the // length of ClusterNames field in the placement policy); - // * if the linked Placement object is of the PickAll placement type, - // the percentage is against the total number of clusters being selected by the scheduler - // at the time of the evaluation of the disruption budget; + // * if the linked Placement object is of the PickAll placement type, MinAvailable can be + // specified but only as an integer since we cannot derive the total number of clusters selected. // * if the linked Placement object is of the PickN placement type, // the percentage is against the number of clusters specified in the placement (i.e., the // value of the NumberOfClusters fields in the placement policy). diff --git a/apis/placement/v1alpha1/eviction_types.go b/apis/placement/v1alpha1/eviction_types.go index d91769d55..1525223a4 100644 --- a/apis/placement/v1alpha1/eviction_types.go +++ b/apis/placement/v1alpha1/eviction_types.go @@ -6,6 +6,7 @@ Licensed under the MIT license. package v1alpha1 import ( + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -118,6 +119,18 @@ type ClusterResourcePlacementEvictionList struct { Items []ClusterResourcePlacementEviction `json:"items"` } +// SetConditions set the given conditions on the ClusterResourcePlacementEviction. +func (e *ClusterResourcePlacementEviction) SetConditions(conditions ...metav1.Condition) { + for _, c := range conditions { + meta.SetStatusCondition(&e.Status.Conditions, c) + } +} + +// GetCondition returns the condition of the given ClusterResourcePlacementEviction. +func (e *ClusterResourcePlacementEviction) GetCondition(conditionType string) *metav1.Condition { + return meta.FindStatusCondition(e.Status.Conditions, conditionType) +} + func init() { SchemeBuilder.Register( &ClusterResourcePlacementEviction{}, diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml index 20466e19e..c5a651e38 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementdisruptionbudgets.yaml @@ -72,9 +72,8 @@ spec: * if the linked Placement object is of the PickFixed placement type, the percentage is against the number of clusters specified in the placement (i.e., the length of ClusterNames field in the placement policy); - * if the linked Placement object is of the PickAll placement type, - the percentage is against the total number of clusters being selected by the scheduler - at the time of the evaluation of the disruption budget; + * if the linked Placement object is of the PickAll placement type, MaxUnavailable cannot + be specified since we cannot derive the total number of clusters selected. * if the linked Placement object is of the PickN placement type, the percentage is against the number of clusters specified in the placement (i.e., the value of the NumberOfClusters fields in the placement policy). @@ -113,9 +112,8 @@ spec: * if the linked Placement object is of the PickFixed placement type, the percentage is against the number of clusters specified in the placement (i.e., the length of ClusterNames field in the placement policy); - * if the linked Placement object is of the PickAll placement type, - the percentage is against the total number of clusters being selected by the scheduler - at the time of the evaluation of the disruption budget; + * if the linked Placement object is of the PickAll placement type, MinAvailable can be + specified but only as an integer since we cannot derive the total number of clusters selected. * if the linked Placement object is of the PickN placement type, the percentage is against the number of clusters specified in the placement (i.e., the value of the NumberOfClusters fields in the placement policy). diff --git a/pkg/controllers/clusterresourceplacementeviction/controller.go b/pkg/controllers/clusterresourceplacementeviction/controller.go new file mode 100644 index 000000000..0229f8f69 --- /dev/null +++ b/pkg/controllers/clusterresourceplacementeviction/controller.go @@ -0,0 +1,373 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package clusterresourceplacementeviction + +import ( + "context" + "fmt" + "time" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + runtime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrl "sigs.k8s.io/controller-runtime/pkg/controller" + + placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + bindingutils "go.goms.io/fleet/pkg/utils/binding" + "go.goms.io/fleet/pkg/utils/condition" + "go.goms.io/fleet/pkg/utils/controller" +) + +const ( + clusterResourcePlacementEvictionValidReason = "ClusterResourcePlacementEvictionValid" + clusterResourcePlacementEvictionInvalidReason = "ClusterResourcePlacementEvictionInvalid" + clusterResourcePlacementEvictionExecutedReason = "ClusterResourcePlacementEvictionExecuted" + clusterResourcePlacementEvictionNotExecutedReason = "ClusterResourcePlacementEvictionNotExecuted" + + evictionInvalidMissingCRPMessage = "Failed to find ClusterResourcePlacement targeted by eviction" + evictionInvalidDeletingCRPMessage = "Found deleting ClusterResourcePlacement targeted by eviction" + evictionInvalidMissingCRBMessage = "Failed to find scheduler decision for placement in cluster targeted by eviction" + evictionInvalidMultipleCRBMessage = "Found more than one scheduler decision for placement in cluster targeted by eviction" + evictionValidMessage = "Eviction is valid" + evictionAllowedNoPDBMessage = "Eviction is allowed, no ClusterResourcePlacementDisruptionBudget specified" + evictionAllowedPlacementRemovedMessage = "Eviction is allowed, resources propagated by placement is currently being removed from cluster targeted by eviction" + evictionAllowedPlacementFailedMessage = "Eviction is allowed, placement has failed" + evictionBlockedMisconfiguredPDBSpecifiedMessage = "Eviction is blocked by misconfigured ClusterResourcePlacementDisruptionBudget, either MaxUnavailable is specified or MinAvailable is specified as a percentage for PickAll ClusterResourcePlacement" + evictionBlockedMissingPlacementMessage = "Eviction is blocked, placement has not propagated resources to target cluster yet" + + evictionAllowedPDBSpecifiedFmt = "Eviction is allowed by specified ClusterResourcePlacementDisruptionBudget, availablePlacements: %d, totalPlacements: %d" + evictionBlockedPDBSpecifiedFmt = "Eviction is blocked by specified ClusterResourcePlacementDisruptionBudget, availablePlacements: %d, totalPlacements: %d" +) + +// Reconciler reconciles a ClusterResourcePlacementEviction object. +type Reconciler struct { + client.Client +} + +// Reconcile triggers a single eviction reconcile round. +func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtime.Result, error) { + startTime := time.Now() + evictionName := req.NamespacedName.Name + klog.V(2).InfoS("ClusterResourcePlacementEviction reconciliation starts", "clusterResourcePlacementEviction", evictionName) + defer func() { + latency := time.Since(startTime).Milliseconds() + klog.V(2).InfoS("ClusterResourcePlacementEviction reconciliation ends", "clusterResourcePlacementEviction", evictionName, "latency", latency) + }() + + var eviction placementv1alpha1.ClusterResourcePlacementEviction + if err := r.Client.Get(ctx, req.NamespacedName, &eviction); err != nil { + klog.ErrorS(err, "Failed to get cluster resource placement eviction", "clusterResourcePlacementEviction", evictionName) + return runtime.Result{}, client.IgnoreNotFound(err) + } + + if isEvictionInTerminalState(&eviction) { + return runtime.Result{}, nil + } + + validationResult, err := r.validateEviction(ctx, &eviction) + if err != nil { + return runtime.Result{}, err + } + if !validationResult.isValid { + return runtime.Result{}, r.updateEvictionStatus(ctx, &eviction) + } + + markEvictionValid(&eviction) + + if err = r.executeEviction(ctx, validationResult, &eviction); err != nil { + return runtime.Result{}, err + } + + return runtime.Result{}, r.updateEvictionStatus(ctx, &eviction) +} + +// validateEviction performs validation for eviction object's spec and returns a wrapped validation result. +func (r *Reconciler) validateEviction(ctx context.Context, eviction *placementv1alpha1.ClusterResourcePlacementEviction) (*evictionValidationResult, error) { + validationResult := &evictionValidationResult{isValid: false} + var crp placementv1beta1.ClusterResourcePlacement + if err := r.Client.Get(ctx, types.NamespacedName{Name: eviction.Spec.PlacementName}, &crp); err != nil { + if k8serrors.IsNotFound(err) { + klog.V(2).InfoS(evictionInvalidMissingCRPMessage, "clusterResourcePlacementEviction", eviction.Name, "clusterResourcePlacement", eviction.Spec.PlacementName) + markEvictionInvalid(eviction, evictionInvalidMissingCRPMessage) + return validationResult, nil + } + return nil, controller.NewAPIServerError(true, err) + } + if crp.DeletionTimestamp != nil { + klog.V(2).InfoS(evictionInvalidDeletingCRPMessage, "clusterResourcePlacementEviction", eviction.Name, "clusterResourcePlacement", eviction.Spec.PlacementName) + markEvictionInvalid(eviction, evictionInvalidDeletingCRPMessage) + return validationResult, nil + } + validationResult.crp = &crp + + var crbList placementv1beta1.ClusterResourceBindingList + if err := r.Client.List(ctx, &crbList, client.MatchingLabels{placementv1beta1.CRPTrackingLabel: crp.Name}); err != nil { + return nil, controller.NewAPIServerError(true, err) + } + validationResult.bindings = crbList.Items + + var evictionTargetBinding *placementv1beta1.ClusterResourceBinding + for i := range crbList.Items { + if crbList.Items[i].Spec.TargetCluster == eviction.Spec.ClusterName { + if evictionTargetBinding == nil { + evictionTargetBinding = &crbList.Items[i] + } else { + klog.V(2).InfoS(evictionInvalidMultipleCRBMessage, "clusterResourcePlacementEviction", eviction.Name, "clusterResourcePlacement", eviction.Spec.PlacementName) + markEvictionInvalid(eviction, evictionInvalidMultipleCRBMessage) + return validationResult, nil + } + } + } + if evictionTargetBinding == nil { + klog.V(2).InfoS("Failed to find cluster resource binding for cluster targeted by eviction", "clusterResourcePlacementEviction", eviction.Name, "targetCluster", eviction.Spec.ClusterName) + markEvictionInvalid(eviction, evictionInvalidMissingCRBMessage) + return validationResult, nil + } + validationResult.crb = evictionTargetBinding + + validationResult.isValid = true + return validationResult, nil +} + +// updateEvictionStatus updates eviction status. +func (r *Reconciler) updateEvictionStatus(ctx context.Context, eviction *placementv1alpha1.ClusterResourcePlacementEviction) error { + evictionRef := klog.KObj(eviction) + if err := r.Client.Status().Update(ctx, eviction); err != nil { + klog.ErrorS(err, "Failed to update eviction status", "clusterResourcePlacementEviction", evictionRef) + return controller.NewUpdateIgnoreConflictError(err) + } + klog.V(2).InfoS("Updated the status of a eviction", "clusterResourcePlacementEviction", evictionRef) + return nil +} + +// deleteClusterResourceBinding deletes the specified cluster resource binding. +func (r *Reconciler) deleteClusterResourceBinding(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding) error { + bindingRef := klog.KObj(binding) + deleteOptions := &client.DeleteOptions{ + Preconditions: &metav1.Preconditions{ + ResourceVersion: ptr.To(binding.ResourceVersion), + }, + } + if err := r.Client.Delete(ctx, binding, deleteOptions); err != nil { + klog.ErrorS(err, "Failed to delete cluster resource binding", "clusterResourceBinding", bindingRef) + return controller.NewDeleteIgnoreNotFoundError(err) + } + klog.V(2).InfoS("Issued delete on cluster resource binding, eviction succeeded", "clusterResourceBinding", bindingRef) + return nil +} + +// executeEviction tries to remove resources from target cluster placed by placement targeted by eviction. +func (r *Reconciler) executeEviction(ctx context.Context, validationResult *evictionValidationResult, eviction *placementv1alpha1.ClusterResourcePlacementEviction) error { + // Unwrap validation result for processing. + crp, evictionTargetBinding, bindingList := validationResult.crp, validationResult.crb, validationResult.bindings + + // Check to see if binding is being deleted. + if evictionTargetBinding.GetDeletionTimestamp() != nil { + klog.V(2).InfoS("ClusterResourceBinding targeted by eviction is being deleted", + "clusterResourcePlacementEviction", eviction.Name, "clusterResourceBinding", evictionTargetBinding.Name, "targetCluster", eviction.Spec.ClusterName) + markEvictionExecuted(eviction, evictionAllowedPlacementRemovedMessage) + return nil + } + + if !isPlacementPresent(evictionTargetBinding) { + klog.V(2).InfoS("No resources have been placed for ClusterResourceBinding in target cluster", + "clusterResourcePlacementEviction", eviction.Name, "clusterResourceBinding", evictionTargetBinding.Name, "targetCluster", eviction.Spec.ClusterName) + markEvictionNotExecuted(eviction, evictionBlockedMissingPlacementMessage) + return nil + } + + // Check to see if binding has failed. If so no need to check disruption budget we can evict. + if bindingutils.HasBindingFailed(evictionTargetBinding) { + klog.V(2).InfoS("ClusterResourceBinding targeted by eviction is in failed state", + "clusterResourcePlacementEviction", eviction.Name, "clusterResourceBinding", evictionTargetBinding.Name, "targetCluster", eviction.Spec.ClusterName) + if err := r.deleteClusterResourceBinding(ctx, evictionTargetBinding); err != nil { + return err + } + markEvictionExecuted(eviction, evictionAllowedPlacementFailedMessage) + return nil + } + + var db placementv1alpha1.ClusterResourcePlacementDisruptionBudget + if err := r.Client.Get(ctx, types.NamespacedName{Name: crp.Name}, &db); err != nil { + if k8serrors.IsNotFound(err) { + if err = r.deleteClusterResourceBinding(ctx, evictionTargetBinding); err != nil { + return err + } + markEvictionExecuted(eviction, evictionAllowedNoPDBMessage) + return nil + } + return controller.NewAPIServerError(true, err) + } + + // handle special case for PickAll CRP. + if crp.Spec.Policy.PlacementType == placementv1beta1.PickAllPlacementType { + if db.Spec.MaxUnavailable != nil || (db.Spec.MinAvailable != nil && db.Spec.MinAvailable.Type == intstr.String) { + markEvictionNotExecuted(eviction, evictionBlockedMisconfiguredPDBSpecifiedMessage) + return nil + } + } + + totalBindings := len(bindingList) + allowed, availableBindings := isEvictionAllowed(bindingList, *crp, db) + if allowed { + if err := r.deleteClusterResourceBinding(ctx, evictionTargetBinding); err != nil { + return err + } + markEvictionExecuted(eviction, fmt.Sprintf(evictionAllowedPDBSpecifiedFmt, availableBindings, totalBindings)) + } else { + markEvictionNotExecuted(eviction, fmt.Sprintf(evictionBlockedPDBSpecifiedFmt, availableBindings, totalBindings)) + } + return nil +} + +// isEvictionInTerminalState checks to see if eviction is in a terminal state. +func isEvictionInTerminalState(eviction *placementv1alpha1.ClusterResourcePlacementEviction) bool { + validCondition := eviction.GetCondition(string(placementv1alpha1.PlacementEvictionConditionTypeValid)) + if condition.IsConditionStatusFalse(validCondition, eviction.GetGeneration()) { + klog.V(2).InfoS("Invalid eviction, no need to reconcile", "clusterResourcePlacementEviction", eviction.Name) + return true + } + + executedCondition := eviction.GetCondition(string(placementv1alpha1.PlacementEvictionConditionTypeExecuted)) + if executedCondition != nil { + klog.V(2).InfoS("Eviction has executed condition specified, no need to reconcile", "clusterResourcePlacementEviction", eviction.Name) + return true + } + return false +} + +// isPlacementPresent checks to see if placement on target cluster could be present. +func isPlacementPresent(binding *placementv1beta1.ClusterResourceBinding) bool { + if binding.Spec.State == placementv1beta1.BindingStateBound { + return true + } + if binding.Spec.State == placementv1beta1.BindingStateUnscheduled { + currentAnnotation := binding.GetAnnotations() + previousState, exist := currentAnnotation[placementv1beta1.PreviousBindingStateAnnotation] + if exist && placementv1beta1.BindingState(previousState) == placementv1beta1.BindingStateBound { + return true + } + } + return false +} + +// isEvictionAllowed calculates if eviction allowed based on available bindings and spec specified in placement disruption budget. +func isEvictionAllowed(bindings []placementv1beta1.ClusterResourceBinding, crp placementv1beta1.ClusterResourcePlacement, db placementv1alpha1.ClusterResourcePlacementDisruptionBudget) (bool, int) { + availableBindings := 0 + for i := range bindings { + availableCondition := bindings[i].GetCondition(string(placementv1beta1.ResourceBindingAvailable)) + if condition.IsConditionStatusTrue(availableCondition, bindings[i].GetGeneration()) { + availableBindings++ + } + } + + var desiredBindings int + placementType := crp.Spec.Policy.PlacementType + switch placementType { + case placementv1beta1.PickNPlacementType: + desiredBindings = int(*crp.Spec.Policy.NumberOfClusters) + case placementv1beta1.PickFixedPlacementType: + desiredBindings = len(crp.Spec.Policy.ClusterNames) + case placementv1beta1.PickAllPlacementType: + // we don't know the desired bindings for PickAll. + } + + var disruptionsAllowed int + switch { + // For PickAll CRPs, MaxUnavailable won't be specified in DB. + case db.Spec.MaxUnavailable != nil: + maxUnavailable, _ := intstr.GetScaledValueFromIntOrPercent(db.Spec.MaxUnavailable, desiredBindings, true) + unavailableBindings := len(bindings) - availableBindings + disruptionsAllowed = maxUnavailable - unavailableBindings + case db.Spec.MinAvailable != nil: + var minAvailable int + if placementType == placementv1beta1.PickAllPlacementType { + // MinAvailable will be an Integer value for PickAll CRP. + minAvailable = db.Spec.MinAvailable.IntValue() + } else { + minAvailable, _ = intstr.GetScaledValueFromIntOrPercent(db.Spec.MinAvailable, desiredBindings, true) + } + disruptionsAllowed = availableBindings - minAvailable + } + if disruptionsAllowed < 0 { + disruptionsAllowed = 0 + } + return disruptionsAllowed > 0, availableBindings +} + +// markEvictionValid sets the valid condition as true in eviction status. +func markEvictionValid(eviction *placementv1alpha1.ClusterResourcePlacementEviction) { + cond := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionTrue, + ObservedGeneration: eviction.Generation, + Reason: clusterResourcePlacementEvictionValidReason, + Message: evictionValidMessage, + } + eviction.SetConditions(cond) + + klog.V(2).InfoS("Marked eviction as valid", "clusterResourcePlacementEviction", klog.KObj(eviction)) +} + +// markEvictionInvalid sets the valid condition as false in eviction status. +func markEvictionInvalid(eviction *placementv1alpha1.ClusterResourcePlacementEviction, message string) { + cond := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionFalse, + ObservedGeneration: eviction.Generation, + Reason: clusterResourcePlacementEvictionInvalidReason, + Message: message, + } + eviction.SetConditions(cond) + klog.V(2).InfoS("Marked eviction as invalid", "clusterResourcePlacementEviction", klog.KObj(eviction)) +} + +// markEvictionExecuted sets the executed condition as true in eviction status. +func markEvictionExecuted(eviction *placementv1alpha1.ClusterResourcePlacementEviction, message string) { + cond := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionTrue, + ObservedGeneration: eviction.Generation, + Reason: clusterResourcePlacementEvictionExecutedReason, + Message: message, + } + eviction.SetConditions(cond) + klog.V(2).InfoS("Marked eviction as executed", "clusterResourcePlacementEviction", klog.KObj(eviction)) +} + +// markEvictionNotExecuted sets the executed condition as false in eviction status. +func markEvictionNotExecuted(eviction *placementv1alpha1.ClusterResourcePlacementEviction, message string) { + cond := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionFalse, + ObservedGeneration: eviction.Generation, + Reason: clusterResourcePlacementEvictionNotExecutedReason, + Message: message, + } + eviction.SetConditions(cond) + klog.V(2).InfoS("Marked eviction as not executed", "clusterResourcePlacementEviction", klog.KObj(eviction)) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *Reconciler) SetupWithManager(mgr runtime.Manager) error { + return runtime.NewControllerManagedBy(mgr). + WithOptions(ctrl.Options{MaxConcurrentReconciles: 1}). // max concurrent reconciles is currently set to 1 for concurrency control. + For(&placementv1alpha1.ClusterResourcePlacementEviction{}). + Complete(r) +} + +type evictionValidationResult struct { + crp *placementv1beta1.ClusterResourcePlacement + crb *placementv1beta1.ClusterResourceBinding + bindings []placementv1beta1.ClusterResourceBinding + isValid bool +} diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go new file mode 100644 index 000000000..d576a82b9 --- /dev/null +++ b/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go @@ -0,0 +1,609 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package clusterresourceplacementeviction + +import ( + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +const ( + crbNameTemplate = "crb-%d" + anotherCRBNameTemplate = "another-crb-%d" + crpNameTemplate = "crp-%d" + evictionNameTemplate = "eviction-%d" +) + +var ( + lessFuncCondition = func(a, b metav1.Condition) bool { + return a.Type < b.Type + } + + evictionStatusCmpOptions = cmp.Options{ + cmpopts.SortSlices(lessFuncCondition), + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime"), + cmpopts.EquateEmpty(), + } +) + +const ( + eventuallyDuration = time.Minute * 2 + eventuallyInterval = time.Millisecond * 250 + consistentlyDuration = time.Second * 10 + consistentlyInterval = time.Millisecond * 500 +) + +var _ = Describe("Test ClusterResourcePlacementEviction Controller", func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + evictionName := fmt.Sprintf(evictionNameTemplate, GinkgoParallelProcess()) + + AfterEach(func() { + ensureCRPDBRemoved(crpName) + ensureAllBindingsAreRemoved(crpName) + ensureEvictionRemoved(evictionName) + ensureCRPRemoved(crpName) + }) + + It("Eviction Blocked - ClusterResourcePlacementDisruptionBudget's maxUnavailable blocks eviction", func() { + crbName := fmt.Sprintf(crbNameTemplate, GinkgoParallelProcess()) + + By("Create ClusterResourcePlacement", func() { + // Create ClusterResourcePlacement. + crp := buildTestPickNCRP(crpName, 1) + Expect(k8sClient.Create(ctx, &crp)).Should(Succeed()) + // ensure CRP exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, &crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourceBinding", func() { + // Create CRB. + crb := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: crbName, + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: crpName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + ResourceSnapshotName: "test-resource-snapshot", + SchedulingPolicySnapshotName: "test-scheduling-policy-snapshot", + TargetCluster: "test-cluster", + }, + } + Expect(k8sClient.Create(ctx, &crb)).Should(Succeed()) + // ensure CRB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crb.Name}, &crb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementDisruptionBudget", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + } + Expect(k8sClient.Create(ctx, &crpdb)).Should(Succeed()) + // ensure CRPDB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crpdb.Name}, &crpdb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementEviction", func() { + eviction := buildTestEviction(evictionName, crpName, "test-cluster") + Expect(k8sClient.Create(ctx, eviction)).Should(Succeed()) + }) + + By("Check eviction status", func() { + evictionStatusUpdatedActual := evictionStatusUpdatedActual(&isValidEviction{bool: true, msg: evictionValidMessage}, &isExecutedEviction{bool: false, msg: fmt.Sprintf(evictionBlockedPDBSpecifiedFmt, 0, 1)}) + Eventually(evictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Ensure eviction was not successful", func() { + var crb placementv1beta1.ClusterResourceBinding + // check to see CRB was not deleted. + Consistently(func() bool { + return !k8serrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbName}, &crb)) + }, consistentlyDuration, consistentlyInterval).Should(BeTrue()) + }) + }) + + It("Eviction Allowed - ClusterResourcePlacementDisruptionBudget's maxUnavailable allows eviction", func() { + crbName := fmt.Sprintf(crbNameTemplate, GinkgoParallelProcess()) + + By("Create ClusterResourcePlacement", func() { + // Create ClusterResourcePlacement. + crp := buildTestPickNCRP(crpName, 1) + Expect(k8sClient.Create(ctx, &crp)).Should(Succeed()) + // ensure CRP exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, &crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourceBinding and update status with available condition", func() { + // Create CRB. + crb := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: crbName, + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: crpName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + ResourceSnapshotName: "test-resource-snapshot", + SchedulingPolicySnapshotName: "test-scheduling-policy-snapshot", + TargetCluster: "test-cluster", + }, + } + Expect(k8sClient.Create(ctx, &crb)).Should(Succeed()) + // ensure CRB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crb.Name}, &crb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + + // Update CRB status to have applied, available condition. + // Ideally binding would contain more condition before applied, available. + // But for the sake testing we only specify applied, available condition. + appliedCondition := metav1.Condition{ + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionTrue, + Reason: "applied", + ObservedGeneration: crb.GetGeneration(), + } + availableCondition := metav1.Condition{ + Type: string(placementv1beta1.ResourceBindingAvailable), + Status: metav1.ConditionTrue, + Reason: "available", + ObservedGeneration: crb.GetGeneration(), + } + crb.SetConditions(appliedCondition, availableCondition) + Expect(k8sClient.Status().Update(ctx, &crb)).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementDisruptionBudget", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + } + Expect(k8sClient.Create(ctx, &crpdb)).Should(Succeed()) + // ensure CRPDB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crpdb.Name}, &crpdb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementEviction", func() { + eviction := buildTestEviction(evictionName, crpName, "test-cluster") + Expect(k8sClient.Create(ctx, eviction)).Should(Succeed()) + }) + + By("Check eviction status", func() { + evictionStatusUpdatedActual := evictionStatusUpdatedActual(&isValidEviction{bool: true, msg: evictionValidMessage}, &isExecutedEviction{bool: true, msg: fmt.Sprintf(evictionAllowedPDBSpecifiedFmt, 1, 1)}) + Eventually(evictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Ensure eviction was successful", func() { + var crb placementv1beta1.ClusterResourceBinding + // Ensure CRB was deleted. + Eventually(func() bool { + return k8serrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbName}, &crb)) + }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) + Consistently(func() bool { + return k8serrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbName}, &crb)) + }, consistentlyDuration, consistentlyInterval).Should(BeTrue()) + }) + }) + + It("Eviction Blocked - ClusterResourcePlacementDisruptionBudget's minAvailable blocks eviction", func() { + crbName := fmt.Sprintf(crbNameTemplate, GinkgoParallelProcess()) + + By("Create ClusterResourcePlacement", func() { + // Create ClusterResourcePlacement. + crp := buildTestPickNCRP(crpName, 1) + Expect(k8sClient.Create(ctx, &crp)).Should(Succeed()) + // ensure CRP exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, &crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourceBinding", func() { + // Create CRB. + crb := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: crbName, + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: crpName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + ResourceSnapshotName: "test-resource-snapshot", + SchedulingPolicySnapshotName: "test-scheduling-policy-snapshot", + TargetCluster: "test-cluster", + }, + } + Expect(k8sClient.Create(ctx, &crb)).Should(Succeed()) + // ensure CRB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crb.Name}, &crb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementDisruptionBudget", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + } + Expect(k8sClient.Create(ctx, &crpdb)).Should(Succeed()) + // ensure CRPDB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crpdb.Name}, &crpdb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementEviction", func() { + eviction := buildTestEviction(evictionName, crpName, "test-cluster") + Expect(k8sClient.Create(ctx, eviction)).Should(Succeed()) + }) + + By("Check eviction status", func() { + evictionStatusUpdatedActual := evictionStatusUpdatedActual(&isValidEviction{bool: true, msg: evictionValidMessage}, &isExecutedEviction{bool: false, msg: fmt.Sprintf(evictionBlockedPDBSpecifiedFmt, 0, 1)}) + Eventually(evictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Ensure eviction was not successful", func() { + var crb placementv1beta1.ClusterResourceBinding + // check to see CRB was not deleted. + Consistently(func() bool { + return !k8serrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbName}, &crb)) + }, consistentlyDuration, consistentlyInterval).Should(BeTrue()) + }) + }) + + It("Eviction Allowed - ClusterResourcePlacementDisruptionBudget's minUnavailable allows eviction", func() { + crbName := fmt.Sprintf(crbNameTemplate, GinkgoParallelProcess()) + anotherCRBName := fmt.Sprintf(anotherCRBNameTemplate, GinkgoParallelProcess()) + + By("Create ClusterResourcePlacement", func() { + // Create ClusterResourcePlacement. + crp := buildTestPickNCRP(crpName, 2) + Expect(k8sClient.Create(ctx, &crp)).Should(Succeed()) + // ensure CRP exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, &crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create two ClusterResourceBindings with available condition specified", func() { + // Create CRB. + crb := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: crbName, + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: crpName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + ResourceSnapshotName: "test-resource-snapshot", + SchedulingPolicySnapshotName: "test-scheduling-policy-snapshot", + TargetCluster: "test-cluster", + }, + } + Expect(k8sClient.Create(ctx, &crb)).Should(Succeed()) + // ensure CRB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crb.Name}, &crb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + + // Create another CRB. + anotherCRB := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: anotherCRBName, + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: crpName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + ResourceSnapshotName: "test-resource-snapshot", + SchedulingPolicySnapshotName: "test-scheduling-policy-snapshot", + TargetCluster: "another-test-cluster", + }, + } + Expect(k8sClient.Create(ctx, &anotherCRB)).Should(Succeed()) + // ensure CRB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crb.Name}, &crb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + + // Update CRB status to have applied, available condition. + // Ideally binding would contain more condition before applied, available. + // But for the sake testing we only specify applied, available condition. + appliedCondition := metav1.Condition{ + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionTrue, + Reason: "applied", + ObservedGeneration: crb.GetGeneration(), + } + availableCondition := metav1.Condition{ + Type: string(placementv1beta1.ResourceBindingAvailable), + Status: metav1.ConditionTrue, + Reason: "available", + ObservedGeneration: crb.GetGeneration(), + } + crb.SetConditions(appliedCondition, availableCondition) + Expect(k8sClient.Status().Update(ctx, &crb)).Should(Succeed()) + availableCondition.ObservedGeneration = anotherCRB.GetGeneration() + anotherCRB.SetConditions(availableCondition) + Expect(k8sClient.Status().Update(ctx, &anotherCRB)).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementDisruptionBudget", func() { + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + } + Expect(k8sClient.Create(ctx, &crpdb)).Should(Succeed()) + // ensure CRPDB exists. + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: crpdb.Name}, &crpdb) + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Create ClusterResourcePlacementEviction", func() { + eviction := buildTestEviction(evictionName, crpName, "test-cluster") + Expect(k8sClient.Create(ctx, eviction)).Should(Succeed()) + }) + + By("Check eviction status", func() { + evictionStatusUpdatedActual := evictionStatusUpdatedActual(&isValidEviction{bool: true, msg: evictionValidMessage}, &isExecutedEviction{bool: true, msg: fmt.Sprintf(evictionAllowedPDBSpecifiedFmt, 2, 2)}) + Eventually(evictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + By("Ensure eviction was successful for one ClusterResourceBinding", func() { + var crb placementv1beta1.ClusterResourceBinding + Eventually(func() bool { + return k8serrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbName}, &crb)) + }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) + Consistently(func() bool { + return k8serrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbName}, &crb)) + }, consistentlyDuration, consistentlyInterval).Should(BeTrue()) + }) + + By("Ensure other ClusterResourceBinding was not delete", func() { + var anotherCRB placementv1beta1.ClusterResourceBinding + // Ensure another CRB was not deleted. + Consistently(func() bool { + return !k8serrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: anotherCRBName}, &anotherCRB)) + }, consistentlyDuration, consistentlyInterval).Should(BeTrue()) + }) + }) +}) + +func evictionStatusUpdatedActual(isValid *isValidEviction, isExecuted *isExecutedEviction) func() error { + evictionName := fmt.Sprintf(evictionNameTemplate, GinkgoParallelProcess()) + return func() error { + var eviction placementv1alpha1.ClusterResourcePlacementEviction + if err := k8sClient.Get(ctx, types.NamespacedName{Name: evictionName}, &eviction); err != nil { + return err + } + var conditions []metav1.Condition + if isValid != nil { + if isValid.bool { + validCondition := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionTrue, + ObservedGeneration: eviction.GetGeneration(), + Reason: clusterResourcePlacementEvictionValidReason, + Message: isValid.msg, + } + conditions = append(conditions, validCondition) + } else { + invalidCondition := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionFalse, + ObservedGeneration: eviction.GetGeneration(), + Reason: clusterResourcePlacementEvictionInvalidReason, + Message: isValid.msg, + } + conditions = append(conditions, invalidCondition) + } + } + if isExecuted != nil { + if isExecuted.bool { + executedCondition := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionTrue, + ObservedGeneration: eviction.GetGeneration(), + Reason: clusterResourcePlacementEvictionExecutedReason, + Message: isExecuted.msg, + } + conditions = append(conditions, executedCondition) + } else { + notExecutedCondition := metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionFalse, + ObservedGeneration: eviction.GetGeneration(), + Reason: clusterResourcePlacementEvictionNotExecutedReason, + Message: isExecuted.msg, + } + conditions = append(conditions, notExecutedCondition) + } + } + wantStatus := placementv1alpha1.PlacementEvictionStatus{ + Conditions: conditions, + } + if diff := cmp.Diff(eviction.Status, wantStatus, evictionStatusCmpOptions...); diff != "" { + return fmt.Errorf("CRP status diff (-got, +want): %s", diff) + } + return nil + } +} + +func buildTestPickNCRP(crpName string, clusterCount int32) placementv1beta1.ClusterResourcePlacement { + return placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(clusterCount), + }, + ResourceSelectors: []placementv1beta1.ClusterResourceSelector{ + { + Group: "", + Kind: "Namespace", + Version: "v1", + Name: "test-ns", + }, + }, + }, + } +} + +func buildTestEviction(evictionName, placementName, clusterName string) *placementv1alpha1.ClusterResourcePlacementEviction { + return &placementv1alpha1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: evictionName, + Generation: 1, + }, + Spec: placementv1alpha1.PlacementEvictionSpec{ + PlacementName: placementName, + ClusterName: clusterName, + }, + } +} + +func ensureEvictionRemoved(name string) { + // Delete eviction. + eviction := placementv1alpha1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + Expect(k8sClient.Delete(ctx, &eviction)).Should(Succeed()) + // Ensure eviction doesn't exist. + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, &eviction); !k8serrors.IsNotFound(err) { + return fmt.Errorf("eviction still exists or an unexpected error occurred: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInterval) +} + +func ensureCRPRemoved(name string) { + // Delete CRP. + crp := placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, &crp))).Should(Succeed()) + // Ensure CRP doesn't exist. + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, &crp); !k8serrors.IsNotFound(err) { + return fmt.Errorf("CRP still exists or an unexpected error occurred: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInterval) +} + +func ensureCRPDBRemoved(name string) { + // Delete CRPDB. + crpdb := placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, &crpdb))).Should(Succeed()) + // Ensure CRPDB doesn't exist. + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, &crpdb); !k8serrors.IsNotFound(err) { + return fmt.Errorf("CRPDB still exists or an unexpected error occurred: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInterval) +} + +func ensureCRBRemoved(name string) { + // Delete CRB. + crb := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, &crb))).Should(Succeed()) + // Ensure CRB doesn't exist. + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, &crb); !k8serrors.IsNotFound(err) { + return fmt.Errorf("CRB still exists or an unexpected error occurred: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInterval) +} + +func ensureAllBindingsAreRemoved(crpName string) { + // List all bindings associated with the given CRP. + bindingList := &placementv1beta1.ClusterResourceBindingList{} + labelSelector := labels.SelectorFromSet(labels.Set{placementv1beta1.CRPTrackingLabel: crpName}) + listOptions := &client.ListOptions{LabelSelector: labelSelector} + Expect(k8sClient.List(ctx, bindingList, listOptions)).Should(Succeed()) + + for i := range bindingList.Items { + ensureCRBRemoved(bindingList.Items[i].Name) + } +} + +type isValidEviction struct { + bool + msg string +} + +type isExecutedEviction struct { + bool + msg string +} diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_test.go new file mode 100644 index 000000000..4a69de9c4 --- /dev/null +++ b/pkg/controllers/clusterresourceplacementeviction/controller_test.go @@ -0,0 +1,1453 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package clusterresourceplacementeviction + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +const ( + testBindingName = "test-binding" + testClusterName = "test-cluster" + testCRPName = "test-crp" + testDisruptionBudgetName = "test-disruption-budget" + testEvictionName = "test-eviction" +) + +func TestValidateEviction(t *testing.T) { + testCRP := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + }, + } + testBinding1 := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding-1", + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: testCRPName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateUnscheduled, + TargetCluster: "test-cluster", + }, + } + testBinding2 := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding-2", + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: testCRPName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateScheduled, + TargetCluster: "test-cluster", + }, + } + tests := []struct { + name string + eviction *placementv1alpha1.ClusterResourcePlacementEviction + crp *placementv1beta1.ClusterResourcePlacement + bindings []placementv1beta1.ClusterResourceBinding + wantValidationResult *evictionValidationResult + wantEvictionInvalidCondition *metav1.Condition + wantErr error + }{ + { + name: "invalid eviction - CRP not found", + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantValidationResult: &evictionValidationResult{ + isValid: false, + }, + wantEvictionInvalidCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionInvalidReason, + Message: evictionInvalidMissingCRPMessage, + }, + wantErr: nil, + }, + { + name: "invalid eviction - deleting CRP", + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + crp: &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{"test-finalizer"}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + }, + }, + wantValidationResult: &evictionValidationResult{ + isValid: false, + }, + wantEvictionInvalidCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionInvalidReason, + Message: evictionInvalidDeletingCRPMessage, + }, + wantErr: nil, + }, + { + name: "invalid eviction - multiple CRBs for same cluster", + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + crp: testCRP, + bindings: []placementv1beta1.ClusterResourceBinding{ + testBinding1, testBinding2, + }, + wantValidationResult: &evictionValidationResult{ + isValid: false, + crp: testCRP, + bindings: []placementv1beta1.ClusterResourceBinding{testBinding1, testBinding2}, + }, + wantEvictionInvalidCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionInvalidReason, + Message: evictionInvalidMultipleCRBMessage, + }, + wantErr: nil, + }, + { + name: "invalid eviction - CRB not found", + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + crp: testCRP, + wantValidationResult: &evictionValidationResult{ + isValid: false, + crp: testCRP, + bindings: []placementv1beta1.ClusterResourceBinding{}, + }, + wantEvictionInvalidCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeValid), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionInvalidReason, + Message: evictionInvalidMissingCRBMessage, + }, + wantErr: nil, + }, + { + name: "valid eviction", + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + crp: testCRP, + bindings: []placementv1beta1.ClusterResourceBinding{testBinding2}, + wantValidationResult: &evictionValidationResult{ + isValid: true, + crp: testCRP, + crb: &testBinding2, + bindings: []placementv1beta1.ClusterResourceBinding{testBinding2}, + }, + wantErr: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var objects []client.Object + if tc.crp != nil { + objects = append(objects, tc.crp) + } + for i := range tc.bindings { + objects = append(objects, &tc.bindings[i]) + } + scheme := serviceScheme(t) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + Build() + r := Reconciler{ + Client: fakeClient, + } + gotValidationResult, gotErr := r.validateEviction(ctx, tc.eviction) + if diff := cmp.Diff(tc.wantValidationResult, gotValidationResult, cmp.AllowUnexported(evictionValidationResult{}), cmpopts.IgnoreFields(placementv1beta1.ClusterResourceBinding{}, "ResourceVersion")); diff != "" { + t.Errorf("validateEviction() validation result mismatch (-want, +got):\n%s", diff) + } + gotInvalidCondition := tc.eviction.GetCondition(string(placementv1alpha1.PlacementEvictionConditionTypeValid)) + if diff := cmp.Diff(tc.wantEvictionInvalidCondition, gotInvalidCondition, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")); diff != "" { + t.Errorf("validateEviction() eviction invalid condition mismatch (-want, +got):\n%s", diff) + } + if tc.wantErr == nil { + if gotErr != nil { + t.Errorf("test case `%s` didn't return the expected error, want no error, got error = %+v ", tc.name, gotErr) + } + } else if gotErr == nil || gotErr.Error() != tc.wantErr.Error() { + t.Errorf("test case `%s` didn't return the expected error, want error = %+v, got error = %+v", tc.name, tc.wantErr, gotErr) + } + }) + } +} + +func TestDeleteClusterResourceBinding(t *testing.T) { + tests := []struct { + name string + inputBinding *placementv1beta1.ClusterResourceBinding + storedBinding *placementv1beta1.ClusterResourceBinding + wantErr error + }{ + { + name: "conflict on delete - pre-conditions don't match", + inputBinding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + ResourceVersion: "2", + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + }, + storedBinding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + ResourceVersion: "1", + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + }, + wantErr: errors.New("object might have been modified"), + }, + { + name: "successful delete", + inputBinding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + ResourceVersion: "1", + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + }, + storedBinding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + ResourceVersion: "1", + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + }, + wantErr: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var objects []client.Object + if tc.storedBinding != nil { + objects = append(objects, tc.storedBinding) + } + scheme := serviceScheme(t) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + Build() + r := Reconciler{ + Client: fakeClient, + } + gotErr := r.deleteClusterResourceBinding(ctx, tc.inputBinding) + if tc.wantErr == nil { + if gotErr != nil { + t.Errorf("test case `%s` didn't return the expected error, want no error, got error = %+v ", tc.name, gotErr) + } + } else if gotErr == nil || !strings.Contains(gotErr.Error(), tc.wantErr.Error()) { + t.Errorf("test case `%s` didn't return the expected error, want error = %+v, got error = %+v", tc.name, tc.wantErr, gotErr) + } + }) + } +} + +func TestExecuteEviction(t *testing.T) { + availableBinding := &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + Generation: 1, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionTrue, + Reason: "applied", + ObservedGeneration: 1, + }, + { + Type: string(placementv1beta1.ResourceBindingAvailable), + Status: metav1.ConditionTrue, + Reason: "available", + ObservedGeneration: 1, + }, + }, + }, + } + tests := []struct { + name string + validationResult *evictionValidationResult + eviction *placementv1alpha1.ClusterResourcePlacementEviction + pdb *placementv1alpha1.ClusterResourcePlacementDisruptionBudget + wantEvictionExecutedCondition *metav1.Condition + wantErr error + }{ + { + name: "scheduled binding - eviction not executed", + validationResult: &evictionValidationResult{ + crb: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateScheduled, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionNotExecutedReason, + Message: evictionBlockedMissingPlacementMessage, + }, + wantErr: nil, + }, + { + name: "unscheduled binding with previous state annotation doesn't exist - eviction not executed", + validationResult: &evictionValidationResult{ + crb: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateUnscheduled, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionNotExecutedReason, + Message: evictionBlockedMissingPlacementMessage, + }, + wantErr: nil, + }, + { + name: "unscheduled binding with previous state as scheduled - eviction not executed", + validationResult: &evictionValidationResult{ + crb: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + Annotations: map[string]string{placementv1beta1.PreviousBindingStateAnnotation: string(placementv1beta1.BindingStateScheduled)}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateUnscheduled, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionNotExecutedReason, + Message: evictionBlockedMissingPlacementMessage, + }, + wantErr: nil, + }, + { + name: "deleting binding - eviction executed", + validationResult: &evictionValidationResult{ + crb: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + Annotations: map[string]string{placementv1beta1.PreviousBindingStateAnnotation: string(placementv1beta1.BindingStateBound)}, + Finalizers: []string{"test-finalizer"}, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateUnscheduled, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionExecutedReason, + Message: evictionAllowedPlacementRemovedMessage, + }, + wantErr: nil, + }, + { + name: "failed to apply binding - eviction executed", + validationResult: &evictionValidationResult{ + crb: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + Generation: 1, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionFalse, + Reason: "applied", + ObservedGeneration: 1, + }, + }, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionExecutedReason, + Message: evictionAllowedPlacementFailedMessage, + }, + wantErr: nil, + }, + { + name: "failed to be available binding - eviction executed", + validationResult: &evictionValidationResult{ + crb: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBindingName, + Generation: 1, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionTrue, + Reason: "applied", + ObservedGeneration: 1, + }, + { + Type: string(placementv1beta1.ResourceBindingAvailable), + Status: metav1.ConditionFalse, + Reason: "available", + ObservedGeneration: 1, + }, + }, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionExecutedReason, + Message: evictionAllowedPlacementFailedMessage, + }, + wantErr: nil, + }, + { + name: "pdb not found - eviction executed", + validationResult: &evictionValidationResult{ + crb: availableBinding, + crp: &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionExecutedReason, + Message: evictionAllowedNoPDBMessage, + }, + wantErr: nil, + }, + { + name: "PickAll CRP, Misconfigured PDB MaxUnavailable specified - eviction not executed", + validationResult: &evictionValidationResult{ + crb: availableBinding, + crp: &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + pdb: &placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionNotExecutedReason, + Message: evictionBlockedMisconfiguredPDBSpecifiedMessage, + }, + wantErr: nil, + }, + { + name: "PickAll CRP, Misconfigured PDB MinAvailable specified as percentage - eviction not executed", + validationResult: &evictionValidationResult{ + crb: availableBinding, + crp: &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + }, + }, + }, + eviction: buildTestEviction(testEvictionName, testCRPName, testClusterName), + pdb: &placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + }, + wantEvictionExecutedCondition: &metav1.Condition{ + Type: string(placementv1alpha1.PlacementEvictionConditionTypeExecuted), + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: clusterResourcePlacementEvictionNotExecutedReason, + Message: evictionBlockedMisconfiguredPDBSpecifiedMessage, + }, + wantErr: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var objects []client.Object + if tc.pdb != nil { + objects = append(objects, tc.pdb) + } + scheme := serviceScheme(t) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + Build() + r := Reconciler{ + Client: fakeClient, + } + gotErr := r.executeEviction(ctx, tc.validationResult, tc.eviction) + gotExecutedCondition := tc.eviction.GetCondition(string(placementv1alpha1.PlacementEvictionConditionTypeExecuted)) + if diff := cmp.Diff(tc.wantEvictionExecutedCondition, gotExecutedCondition, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")); diff != "" { + t.Errorf("executeEviction() eviction executed condition mismatch (-want, +got):\n%s", diff) + } + if tc.wantErr == nil { + if gotErr != nil { + t.Errorf("test case `%s` didn't return the expected error, want no error, got error = %+v ", tc.name, gotErr) + } + } else if gotErr == nil || gotErr.Error() != tc.wantErr.Error() { + t.Errorf("test case `%s` didn't return the expected error, want error = %+v, got error = %+v", tc.name, tc.wantErr, gotErr) + } + }) + } +} + +func TestIsEvictionAllowed(t *testing.T) { + availableCondition := metav1.Condition{ + Type: string(placementv1beta1.ResourceBindingAvailable), + Status: metav1.ConditionTrue, + Reason: "available", + ObservedGeneration: 0, + } + scheduledUnavailableBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "scheduled-binding", + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: testCRPName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateScheduled, + TargetCluster: "test-cluster-1", + }, + } + boundAvailableBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bound-available-binding", + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: testCRPName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + TargetCluster: "test-cluster-2", + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{availableCondition}, + }, + } + anotherBoundAvailableBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "another-bound-available-binding", + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: testCRPName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + TargetCluster: "test-cluster-3", + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{availableCondition}, + }, + } + boundUnavailableBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bound-unavailable-binding", + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: testCRPName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + TargetCluster: "test-cluster-4", + }, + } + unScheduledAvailableBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unscheduled-available-binding", + Labels: map[string]string{placementv1beta1.CRPTrackingLabel: testCRPName}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateUnscheduled, + TargetCluster: "test-cluster-5", + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{availableCondition}, + }, + } + tests := []struct { + name string + crp placementv1beta1.ClusterResourcePlacement + bindings []placementv1beta1.ClusterResourceBinding + disruptionBudget placementv1alpha1.ClusterResourcePlacementDisruptionBudget + wantAllowed bool + wantAvailableBindings int + }{ + { + name: "MaxUnavailable specified as Integer zero, one available binding - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 0, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 1, + }, + { + name: "MaxUnavailable specified as Integer zero, one unavailable bindings - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 0, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MaxUnavailable specified as Integer one, one unavailable binding - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MaxUnavailable specified as Integer one, one available binding, upscaling - allow eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 1, + }, + { + name: "MaxUnavailable specified as Integer one, one available, one unavailable binding - block eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, boundUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 1, + }, + { + name: "MaxUnavailable specified as Integer one, two available binding - allow eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 2, + }, + { + name: "MaxUnavailable specified as Integer one, available bindings greater than target, downscaling - allow eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + { + name: "MaxUnavailable specified as Integer greater than one - block eviction", + crp: buildTestPickNCRP(testCRPName, 4), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, anotherBoundAvailableBinding, boundUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 2, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 2, + }, + { + name: "MaxUnavailable specified as Integer greater than one - allow eviction", + crp: buildTestPickNCRP(testCRPName, 3), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 2, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 2, + }, + { + name: "MaxUnavailable specified as Integer large number greater than target number - allows eviction", + crp: buildTestPickNCRP(testCRPName, 4), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, boundUnavailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 10, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 2, + }, + { + name: "MaxUnavailable specified as percentage zero - block eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "0%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 2, + }, + { + name: "MaxUnavailable specified as percentage greater than zero, rounds up to 1 - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MaxUnavailable specified as percentage greater than zero, rounds up to 1 - allow eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 1, + }, + { + name: "MaxUnavailable specified as percentage greater than zero, rounds up to greater than 1 - block eviction", + crp: buildTestPickNCRP(testCRPName, 4), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, boundUnavailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ // equates to 2. + Type: intstr.String, + StrVal: "40%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 2, + }, + { + name: "MaxUnavailable specified as percentage greater than zero, rounds up to greater than 1 - allow eviction", + crp: buildTestPickNCRP(testCRPName, 3), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ // equates to 2. + Type: intstr.String, + StrVal: "50%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 2, + }, + { + name: "MaxUnavailable specified as percentage hundred, target number greater than bindings - allow eviction", + crp: buildTestPickNCRP(testCRPName, 10), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, boundUnavailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ // equates to 10. + Type: intstr.String, + StrVal: "100%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + { + name: "MaxUnavailable specified as percentage hundred, target number equal to bindings - block eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ // equates to 2. + Type: intstr.String, + StrVal: "100%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MaxUnavailable specified as percentage hundred, target number equal to bindings - allow eviction", + crp: buildTestPickNCRP(testCRPName, 4), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, boundUnavailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ // equates to 4. + Type: intstr.String, + StrVal: "100%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 2, + }, + { + name: "MinAvailable specified as Integer zero, unavailable binding - block eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 0, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MinAvailable specified as Integer zero, available binding - allow eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 0, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 1, + }, + { + name: "MinAvailable specified as Integer one, unavailable binding - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MinAvailable specified as Integer one, available binding, upscaling - block eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 1, + }, + { + name: "MinAvailable specified as Integer one, one available, one unavailable binding - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, boundUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 1, + }, + { + name: "MinAvailable specified as Integer one, two available bindings - allow eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 2, + }, + { + name: "MinAvailable specified as Integer one, available bindings greater than target number, downscaling - allow eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as Integer greater than one - block eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 2, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 2, + }, + { + name: "MinAvailable specified as Integer greater than one - allow eviction", + crp: buildTestPickNCRP(testCRPName, 4), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 2, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as Integer greater than one, available bindings greater than target number, downscaling - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 3, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as Integer large number greater than target number - blocks eviction", + crp: buildTestPickNCRP(testCRPName, 5), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, anotherBoundAvailableBinding, boundUnavailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 10, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as percentage zero, all bindings are unavailable - block eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "0%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MinAvailable specified as percentage zero, all bindings are available - allow eviction", + crp: buildTestPickNCRP(testCRPName, 3), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "0%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as percentage rounds upto one - block eviction", + crp: buildTestPickNCRP(testCRPName, 1), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 0, + }, + { + name: "MinAvailable specified as percentage rounds upto one - allow eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 2, + }, + { + name: "MinAvailable specified as percentage greater than zero, rounds up to greater than 1 - block eviction", + crp: buildTestPickNCRP(testCRPName, 3), + bindings: []placementv1beta1.ClusterResourceBinding{scheduledUnavailableBinding, boundAvailableBinding, anotherBoundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ // equates to 2. + Type: intstr.String, + StrVal: "40%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 2, + }, + { + name: "MinAvailable specified as percentage greater than zero, rounds up to greater than 1 - allow eviction", + crp: buildTestPickNCRP(testCRPName, 3), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ // equates to 2. + Type: intstr.String, + StrVal: "40%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as percentage hundred, bindings less than target number - block eviction", + crp: buildTestPickNCRP(testCRPName, 10), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ // equates to 10. + Type: intstr.String, + StrVal: "100%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 2, + }, + { + name: "MinAvailable specified as percentage hundred, bindings equal to target number - block eviction", + crp: buildTestPickNCRP(testCRPName, 3), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ // equates to 3. + Type: intstr.String, + StrVal: "100%", + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as percentage hundred, bindings greater than target number - allow eviction", + crp: buildTestPickNCRP(testCRPName, 2), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ // equates to 2. + Type: intstr.String, + StrVal: "100%", + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + { + name: "MinAvailable specified as Integer zero, available binding, PickAll CRP - allow eviction", + crp: buildTestPickAllCRP(testCRPName), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 0, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 1, + }, + { + name: "MinAvailable specified as Integer one, available binding, PickAll CRP - block eviction", + crp: buildTestPickAllCRP(testCRPName), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + wantAllowed: false, + wantAvailableBindings: 1, + }, + { + name: "MinAvailable specified as Integer greater than one, available binding, PickAll CRP - allow eviction", + crp: buildTestPickAllCRP(testCRPName), + bindings: []placementv1beta1.ClusterResourceBinding{boundAvailableBinding, anotherBoundAvailableBinding, unScheduledAvailableBinding}, + disruptionBudget: placementv1alpha1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDisruptionBudgetName, + }, + Spec: placementv1alpha1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 2, + }, + }, + }, + wantAllowed: true, + wantAvailableBindings: 3, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotAllowed, gotAvailableBindings := isEvictionAllowed(tc.bindings, tc.crp, tc.disruptionBudget) + if gotAllowed != tc.wantAllowed { + t.Errorf("isEvictionAllowed test `%s` failed gotAllowed: %v, wantAllowed: %v", tc.name, gotAllowed, tc.wantAllowed) + } + if gotAvailableBindings != tc.wantAvailableBindings { + t.Errorf("isEvictionAllowed test `%s` failed gotAvailableBindings: %v, wantAvailableBindings: %v", tc.name, gotAvailableBindings, tc.wantAvailableBindings) + } + }) + } +} + +func buildTestPickAllCRP(crpName string) placementv1beta1.ClusterResourcePlacement { + return placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + }, + } +} + +func serviceScheme(t *testing.T) *runtime.Scheme { + scheme := runtime.NewScheme() + if err := placementv1beta1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add placement v1beta1 scheme: %v", err) + } + if err := placementv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add v1alpha1 scheme: %v", err) + } + return scheme +} diff --git a/pkg/controllers/clusterresourceplacementeviction/suite_test.go b/pkg/controllers/clusterresourceplacementeviction/suite_test.go new file mode 100644 index 000000000..5ed603eae --- /dev/null +++ b/pkg/controllers/clusterresourceplacementeviction/suite_test.go @@ -0,0 +1,105 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package clusterresourceplacementeviction + +import ( + "context" + "flag" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "k8s.io/klog/v2/textlogger" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +var ( + cfg *rest.Config + mgr manager.Manager + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "ClusterResourcePlacementEviction Controller Suite") +} + +var _ = BeforeSuite(func() { + klog.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("../../../", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).Should(Succeed()) + Expect(cfg).NotTo(BeNil()) + + err = placementv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = placementv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + By("construct the k8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).Should(Succeed()) + Expect(k8sClient).NotTo(BeNil()) + + By("starting the controller manager") + klog.InitFlags(flag.CommandLine) + flag.Parse() + + mgr, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4))), + }) + Expect(err).Should(Succeed()) + + err = (&Reconciler{ + Client: k8sClient, + }).SetupWithManager(mgr) + Expect(err).Should(Succeed()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).Should(Succeed(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).Should(Succeed()) +}) diff --git a/pkg/controllers/rollout/controller.go b/pkg/controllers/rollout/controller.go index d2b1674d0..bc117b957 100644 --- a/pkg/controllers/rollout/controller.go +++ b/pkg/controllers/rollout/controller.go @@ -33,6 +33,7 @@ import ( fleetv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/controllers/work" + bindingutils "go.goms.io/fleet/pkg/utils/binding" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" "go.goms.io/fleet/pkg/utils/defaulter" @@ -342,7 +343,7 @@ func (r *Reconciler) pickBindingsToRoll(ctx context.Context, allBindings []*flee bindingKObj := klog.KObj(binding) switch binding.Spec.State { case fleetv1beta1.BindingStateUnscheduled: - if hasBindingFailed(binding) { + if bindingutils.HasBindingFailed(binding) { klog.V(3).InfoS("Found a failed to be ready unscheduled binding", "clusterResourcePlacement", crpKObj, "binding", bindingKObj) } else { canBeReadyBindings = append(canBeReadyBindings, binding) @@ -389,7 +390,7 @@ func (r *Reconciler) pickBindingsToRoll(ctx context.Context, allBindings []*flee } } // check if the binding is failed or still on going - if hasBindingFailed(binding) { + if bindingutils.HasBindingFailed(binding) { klog.V(3).InfoS("Found a failed to be ready bound binding", "clusterResourcePlacement", crpKObj, "binding", bindingKObj) bindingFailed = true } else { @@ -435,16 +436,6 @@ func (r *Reconciler) pickBindingsToRoll(ctx context.Context, allBindings []*flee return toBeUpdatedBindingList, staleUnselectedBinding, true, minWaitTime, nil } -// hasBindingFailed checks if ClusterResourceBinding has failed based on its applied and available conditions. -func hasBindingFailed(binding *fleetv1beta1.ClusterResourceBinding) bool { - appliedCondition := binding.GetCondition(string(fleetv1beta1.ResourceBindingApplied)) - availableCondition := binding.GetCondition(string(fleetv1beta1.ResourceBindingAvailable)) - if condition.IsConditionStatusFalse(appliedCondition, binding.Generation) || condition.IsConditionStatusFalse(availableCondition, binding.Generation) { - return true - } - return false -} - // determineBindingsToUpdate determines which bindings to update func determineBindingsToUpdate( crp *fleetv1beta1.ClusterResourcePlacement, diff --git a/pkg/utils/binding/binding.go b/pkg/utils/binding/binding.go new file mode 100644 index 000000000..0030405ec --- /dev/null +++ b/pkg/utils/binding/binding.go @@ -0,0 +1,21 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package binding + +import ( + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/condition" +) + +// HasBindingFailed checks if ClusterResourceBinding has failed based on its applied and available conditions. +func HasBindingFailed(binding *placementv1beta1.ClusterResourceBinding) bool { + appliedCondition := binding.GetCondition(string(placementv1beta1.ResourceBindingApplied)) + availableCondition := binding.GetCondition(string(placementv1beta1.ResourceBindingAvailable)) + if condition.IsConditionStatusFalse(appliedCondition, binding.Generation) || condition.IsConditionStatusFalse(availableCondition, binding.Generation) { + return true + } + return false +} diff --git a/pkg/utils/binding/binding_test.go b/pkg/utils/binding/binding_test.go new file mode 100644 index 000000000..68e77c459 --- /dev/null +++ b/pkg/utils/binding/binding_test.go @@ -0,0 +1,136 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package binding + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +func TestHasBindingFailed(t *testing.T) { + tests := []struct { + name string + binding *placementv1beta1.ClusterResourceBinding + want bool + }{ + { + name: "apply, available conditions not set for binding", + binding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Generation: 1, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + }, + want: false, + }, + { + name: "apply failed binding", + binding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Generation: 1, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.Time{}, + ObservedGeneration: 1, + Reason: "applyFailed", + Message: "test message", + }, + }, + }, + }, + want: true, + }, + { + name: "non available binding", + binding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Generation: 1, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Time{}, + ObservedGeneration: 1, + Reason: "applySucceeded", + Message: "test message", + }, + { + Type: string(placementv1beta1.ResourceBindingAvailable), + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.Time{}, + ObservedGeneration: 1, + Reason: "availableFailed", + Message: "test message", + }, + }, + }, + }, + want: true, + }, + { + name: "available binding", + binding: &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Generation: 1, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + Status: placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ResourceBindingApplied), + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Time{}, + ObservedGeneration: 1, + Reason: "applySucceeded", + Message: "test message", + }, + { + Type: string(placementv1beta1.ResourceBindingAvailable), + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Time{}, + ObservedGeneration: 1, + Reason: "available", + Message: "test message", + }, + }, + }, + }, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := HasBindingFailed(tc.binding) + if got != tc.want { + t.Errorf("HasBindingFailed test `%s` failed got: %v, want: %v", tc.name, got, tc.want) + } + }) + } +} diff --git a/test/apis/placement/v1alpha1/api_validation_integration_test.go b/test/apis/placement/v1alpha1/api_validation_integration_test.go index 7a5f17d10..e437827d5 100644 --- a/test/apis/placement/v1alpha1/api_validation_integration_test.go +++ b/test/apis/placement/v1alpha1/api_validation_integration_test.go @@ -1,3 +1,8 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + package v1alpha1 import ( diff --git a/test/apis/placement/v1alpha1/suite_test.go b/test/apis/placement/v1alpha1/suite_test.go index 45af00112..6b81b00d8 100644 --- a/test/apis/placement/v1alpha1/suite_test.go +++ b/test/apis/placement/v1alpha1/suite_test.go @@ -1,3 +1,8 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + package v1alpha1 import (