From 0362822357e3814716ad5d04e61e255e3b0d96d7 Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Tue, 14 May 2024 17:05:40 -0700 Subject: [PATCH] fix: fix the perf test (#796) * fix perf test * rename --------- Co-authored-by: Ryan Zhang --- hack/loadtest/util/placement.go | 66 +++++++++++++-------------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/hack/loadtest/util/placement.go b/hack/loadtest/util/placement.go index 478e05d37..deb5de932 100644 --- a/hack/loadtest/util/placement.go +++ b/hack/loadtest/util/placement.go @@ -13,7 +13,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/atomic" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/klog/v2" @@ -112,7 +111,7 @@ func MeasureOnePlacement(ctx context.Context, hubClient client.Client, deadline, crpCount.Inc() klog.Infof("verify that the cluster resource placement `%s` is applied", crpName) - fleetSize, clusterNames = collectApplyMetrics(ctx, hubClient, deadline, interval, crpName, currency, fleetSize, clusterNames) + fleetSize, clusterNames = waitForCRPAvailable(ctx, hubClient, deadline, interval, crpName, currency, fleetSize, clusterNames) if fleetSize == "0" { return nil } @@ -130,13 +129,13 @@ func MeasureOnePlacement(ctx context.Context, hubClient client.Client, deadline, // wait for the status of the CRP and make sure all conditions are all true klog.Infof("verify cluster resource placement `%s` is updated", crpName) - waitForCrpToComplete(ctx, hubClient, deadline, interval, deletionStartTime, crpName, currency, fleetSize) + waitForCrpToCompleteUpdate(ctx, hubClient, deadline, interval, deletionStartTime, crpName, currency, fleetSize) } return hubClient.Delete(ctx, crp) } -// collect the crp apply metrics -func collectApplyMetrics(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, crpName string, currency string, fleetSize string, clusterNames ClusterNames) (string, ClusterNames) { +// waitForCRPAvailable waits for the CRP to be available +func waitForCRPAvailable(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, crpName string, currency string, fleetSize string, clusterNames ClusterNames) (string, ClusterNames) { startTime := time.Now() var crp v1beta1.ClusterResourcePlacement var err error @@ -146,28 +145,21 @@ func collectApplyMetrics(ctx context.Context, hubClient client.Client, deadline, defer timer.Stop() for { - if err = hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil { - klog.ErrorS(err, "failed to get crp", "crp", crpName) - } - cond := crp.GetCondition(string(v1beta1.ClusterResourcePlacementAppliedConditionType)) select { case <-timer.C: - // Deadline has been reached - if condition.IsConditionStatusFalse(cond, 1) { - // failed - klog.Infof("the cluster resource placement `%s` failed", crpName) - LoadTestApplyCountMetric.WithLabelValues(currency, fleetSize, "failed").Inc() - applyFailCount.Inc() - return fleetSize, clusterNames - } - // timeout + // Deadline has been reached, timeout klog.Infof("the cluster resource placement `%s` timeout", crpName) LoadTestApplyCountMetric.WithLabelValues(currency, fleetSize, "timeout").Inc() applyTimeoutCount.Inc() return fleetSize, clusterNames case <-ticker.C: // Interval for CRP status check - if condition.IsConditionStatusTrue(cond, 1) { + if err = hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil { + klog.ErrorS(err, "failed to get crp", "crp", crpName) + continue + } + cond := crp.GetCondition(string(v1beta1.ClusterResourcePlacementAvailableConditionType)) + if condition.IsConditionStatusTrue(cond, crp.Generation) { // succeeded klog.Infof("the cluster resource placement `%s` succeeded", crpName) endTime := time.Since(startTime) @@ -180,11 +172,10 @@ func collectApplyMetrics(ctx context.Context, hubClient client.Client, deadline, LoadTestApplyCountMetric.WithLabelValues(currency, fleetSize, "succeed").Inc() applySuccessCount.Inc() return fleetSize, clusterNames - } else if cond == nil || cond.Status == metav1.ConditionUnknown { + } else if condition.IsConditionStatusFalse(cond, crp.Generation) { + klog.Infof("the cluster resource placement `%s` failed with condition %+v. trying again.", crpName, cond) + } else { klog.V(2).Infof("the cluster resource placement `%s` is pending", crpName) - } else if condition.IsConditionStatusFalse(cond, 1) { - klog.Infof("the cluster resource placement `%s` failed. trying again.", crpName) - continue } } } @@ -206,12 +197,13 @@ func resourcesDeletedCheck(ctx context.Context, hubClient client.Client, deadlin case <-timer.C: // Deadline has been reached // timeout - klog.V(3).Infof("the cluster resource placement `%s` delete timeout", crpName) + klog.Infof("the cluster resource placement `%s` delete timeout", crpName) return case <-ticker.C: // Interval for CRP status check if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil { klog.ErrorS(err, "failed to get crp", "crp", crpName) + continue } // the only thing it still selects are namespace and crd if len(crp.Status.SelectedResources) != 2 { @@ -239,7 +231,7 @@ func resourcesDeletedCheck(ctx context.Context, hubClient client.Client, deadlin } // check crp updated/completed before deletion -func waitForCrpToComplete(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, deletionStartTime time.Time, crpName string, currency string, fleetSize string) { +func waitForCrpToCompleteUpdate(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, deletionStartTime time.Time, crpName string, currency string, fleetSize string) { var crp v1beta1.ClusterResourcePlacement var err error timer := time.NewTimer(deadline) @@ -257,21 +249,17 @@ func waitForCrpToComplete(ctx context.Context, hubClient client.Client, deadline scheduledCond := crp.GetCondition(string(v1beta1.ClusterResourcePlacementScheduledConditionType)) select { case <-timer.C: - // Deadline has been reached - if condition.IsConditionStatusFalse(appliedCond, 1) { - // failed - klog.V(3).Infof("the cluster resource placement `%s` failed", crpName) - LoadTestUpdateCountMetric.WithLabelValues(currency, fleetSize, "failed").Inc() - updateFailCount.Inc() - return - } klog.V(3).Infof("the cluster resource placement `%s` timeout", crpName) LoadTestUpdateCountMetric.WithLabelValues(currency, fleetSize, "timeout").Inc() updateTimeoutCount.Inc() return case <-ticker.C: // Interval for CRP status check - if condition.IsConditionStatusTrue(appliedCond, 1) && condition.IsConditionStatusTrue(synchronizedCond, 1) && condition.IsConditionStatusTrue(scheduledCond, 1) { + if err = hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil { + klog.ErrorS(err, "failed to get crp", "crp", crpName) + continue + } + if condition.IsConditionStatusTrue(appliedCond, crp.Generation) && condition.IsConditionStatusTrue(synchronizedCond, crp.Generation) && condition.IsConditionStatusTrue(scheduledCond, crp.Generation) { // succeeded klog.V(3).Infof("the cluster resource placement `%s` succeeded", crpName) var endTime = time.Since(deletionStartTime).Seconds() @@ -279,13 +267,12 @@ func waitForCrpToComplete(ctx context.Context, hubClient client.Client, deadline UpdateLatencyCountMetric.WithLabelValues(currency, fleetSize, strconv.FormatFloat(endTime, 'f', 3, 64)).Inc() LoadTestUpdateCountMetric.WithLabelValues(currency, fleetSize, "succeed").Inc() updateSuccessCount.Inc() - return - } else if appliedCond == nil || appliedCond.Status == metav1.ConditionUnknown || synchronizedCond == nil || synchronizedCond.Status == metav1.ConditionUnknown || scheduledCond == nil || scheduledCond.Status == metav1.ConditionUnknown { - klog.V(3).Infof("the cluster resource placement `%s` is pending", crpName) - } else if condition.IsConditionStatusFalse(appliedCond, 1) { + } else if condition.IsConditionStatusFalse(appliedCond, crp.Generation) { // failed - klog.V(2).Infof("the cluster resource placement `%s` failed. try again", crpName) + klog.Infof("the cluster resource placement `%s` failed. try again", crpName) + } else { + klog.V(2).Infof("the cluster resource placement `%s` is pending", crpName) } } } @@ -294,6 +281,5 @@ func waitForCrpToComplete(ctx context.Context, hubClient client.Client, deadline func PrintTestMetrics() { klog.Infof("CRP count %d", crpCount.Load()) klog.InfoS("Placement apply result", "total applySuccessCount", applySuccessCount.Load(), "applyFailCount", applyFailCount.Load(), "applyTimeoutCount", applyTimeoutCount.Load()) - klog.InfoS("Placement update result", "total updateSuccessCount", updateSuccessCount.Load(), "updateFailCount", updateFailCount.Load(), "updateTimeoutCount", updateTimeoutCount.Load()) }