chore: Refactor HTTPRoute and HTTPRoutePolicy tests condition assertion

test/e2e/framework/assertion.go

@@ -33,6 +33,52 @@ import (
 	"github.com/apache/apisix-ingress-controller/api/v1alpha1"
 )
 
+func GatewayClassMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, gcNN types.NamespacedName, condition metav1.Condition) {
+	err := PollUntilGatewayClassMustHaveStatus(cli, timeout, gcNN, func(gc *gatewayv1.GatewayClass) bool {
+		if err := kubernetes.ConditionsHaveLatestObservedGeneration(gc, gc.Status.Conditions); err != nil {
+			log.Printf("GatewayClass %s %v", gcNN, err)
+			return false
+		}
+		if findConditionInList(gc.Status.Conditions, condition) {
+			return true
+		}
+		log.Printf("NOT FOUND condition %v in %v", condition, gc.Status.Conditions)
+		return false
+	})
+	require.NoError(t, err, "waiting for GatewayClass to have condition %+v", condition)
+}
+
+func PollUntilGatewayClassMustHaveStatus(cli client.Client, timeout time.Duration, gcNN types.NamespacedName, f func(gc *gatewayv1.GatewayClass) bool) error {
+	if err := gatewayv1.Install(cli.Scheme()); err != nil {
+		return err
+	}
+	return genericPollResource(new(gatewayv1.GatewayClass), cli, timeout, gcNN, f)
+}
+
+func GatewayMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, gwNN types.NamespacedName, condition metav1.Condition) {
+	err := PollUntilGatewayHaveStatus(cli, timeout, gwNN, func(gw *gatewayv1.Gateway) bool {
+		if err := kubernetes.ConditionsHaveLatestObservedGeneration(gw, gw.Status.Conditions); err != nil {
+			log.Printf("Gateway %s %v", gwNN, err)
+			return false
+		}
+		if findConditionInList(gw.Status.Conditions, condition) {
+			log.Printf("found condition %v in list %v", condition, gw.Status.Conditions)
+			return true
+		} else {
+			log.Printf("NOT FOUND condition %v in %v", condition, gw.Status.Conditions)
+			return false
+		}
+	})
+	require.NoError(t, err, "waiting for Gateway to have condition %+v", condition)
+}
+
+func PollUntilGatewayHaveStatus(cli client.Client, timeout time.Duration, gwNN types.NamespacedName, f func(gateway *gatewayv1.Gateway) bool) error {
+	if err := gatewayv1.Install(cli.Scheme()); err != nil {
+		return err
+	}
+	return genericPollResource(new(gatewayv1.Gateway), cli, timeout, gwNN, f)
+}
+
 func HTTPRouteMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, refNN, hrNN types.NamespacedName, condition metav1.Condition) {
 	err := PollUntilHTTPRouteHaveStatus(cli, timeout, hrNN, func(hr *gatewayv1.HTTPRoute) bool {
 		for _, parent := range hr.Status.Parents {

test/e2e/gatewayapi/httproute.go

@@ -19,12 +19,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gavv/httpexpect/v2"
 	"github.com/gruntwork-io/terratest/modules/retry"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 	"sigs.k8s.io/gateway-api/apis/v1alpha2"
 
 	"github.com/apache/apisix-ingress-controller/api/v1alpha1"
@@ -101,26 +104,6 @@ spec:
       kind: GatewayProxy
       name: apisix-proxy-config
 `
-
-	var ResourceApplied = func(resourType, resourceName, resourceRaw string, observedGeneration int) {
-		Expect(s.CreateResourceFromString(resourceRaw)).
-			NotTo(HaveOccurred(), fmt.Sprintf("creating %s", resourType))
-
-		Eventually(func() string {
-			hryaml, err := s.GetResourceYaml(resourType, resourceName)
-			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting %s yaml", resourType))
-			return hryaml
-		}, "8s", "2s").
-			Should(
-				SatisfyAll(
-					ContainSubstring(`status: "True"`),
-					ContainSubstring(fmt.Sprintf("observedGeneration: %d", observedGeneration)),
-				),
-				fmt.Sprintf("checking %s condition status", resourType),
-			)
-		time.Sleep(1 * time.Second)
-	}
-
 	var beforeEachHTTP = func() {
 		By("create GatewayProxy")
 		gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Deployer.GetAdminEndpoint(), s.AdminKey())
@@ -132,24 +115,30 @@ spec:
 		gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Unix())
 		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "")
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
-		time.Sleep(5 * time.Second)
 
 		By("check GatewayClass condition")
-		gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName)
-		Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml")
-		Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status")
-		Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message")
+		framework.GatewayClassMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second,
+			types.NamespacedName{Namespace: s.Namespace(), Name: gatewayClassName},
+			metav1.Condition{
+				Type:    string(gatewayv1.GatewayClassConditionStatusAccepted),
+				Status:  metav1.ConditionTrue,
+				Message: "the gatewayclass has been accepted by the apisix-ingress-controller",
+			},
+		)
 
 		By("create Gateway")
 		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayClassName), s.Namespace())
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
-		time.Sleep(5 * time.Second)
 
 		By("check Gateway condition")
-		gwyaml, err := s.GetResourceYaml("Gateway", "apisix")
-		Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml")
-		Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status")
-		Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message")
+		framework.GatewayMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second,
+			types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"},
+			metav1.Condition{
+				Type:    string(gatewayv1.GatewayConditionAccepted),
+				Status:  metav1.ConditionTrue,
+				Message: "the gateway has been accepted by the apisix-ingress-controller",
+			},
+		)
 	}
 
 	var beforeEachHTTPS = func() {
@@ -161,28 +150,35 @@ spec:
 
 		secretName := _secretName
 		createSecret(s, secretName)
+
 		By("create GatewayClass")
 		gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Unix())
 		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "")
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
-		time.Sleep(5 * time.Second)
 
 		By("check GatewayClass condition")
-		gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName)
-		Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml")
-		Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status")
-		Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message")
+		framework.GatewayClassMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second,
+			types.NamespacedName{Namespace: s.Namespace(), Name: gatewayClassName},
+			metav1.Condition{
+				Type:    string(gatewayv1.GatewayClassConditionStatusAccepted),
+				Status:  metav1.ConditionTrue,
+				Message: "the gatewayclass has been accepted by the apisix-ingress-controller",
+			},
+		)
 
 		By("create Gateway")
 		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName), s.Namespace())
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
-		time.Sleep(5 * time.Second)
 
 		By("check Gateway condition")
-		gwyaml, err := s.GetResourceYaml("Gateway", "apisix")
-		Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml")
-		Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status")
-		Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message")
+		framework.GatewayMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second,
+			types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"},
+			metav1.Condition{
+				Type:    string(gatewayv1.GatewayConditionAccepted),
+				Status:  metav1.ConditionTrue,
+				Message: "the gateway has been accepted by the apisix-ingress-controller",
+			},
+		)
 	}
 	Context("HTTPRoute with HTTPS Gateway", func() {
 		var exactRouteByGet = `
@@ -207,26 +203,24 @@ spec:
 
 		BeforeEach(beforeEachHTTPS)
 
-		It("Create/Updtea/Delete HTTPRoute", func() {
+		It("Create/Update/Delete HTTPRoute", func() {
+			request := func() int {
+				return s.NewAPISIXHttpsClient("api6.com").
+					GET("/get").
+					WithHost("api6.com").
+					Expect().
+					Raw().StatusCode
+			}
+
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet, func(_ context.Context) (ok bool, err error) {
+				return request() == http.StatusOK, nil
+			})
 
-			By("access dataplane to check the HTTPRoute")
-			s.NewAPISIXHttpsClient("api6.com").
-				GET("/get").
-				WithHost("api6.com").
-				Expect().
-				Status(200)
 			By("delete HTTPRoute")
 			err := s.DeleteResourceFromString(exactRouteByGet)
 			Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute")
-			time.Sleep(5 * time.Second)
-
-			s.NewAPISIXHttpsClient("api6.com").
-				GET("/get").
-				WithHost("api6.com").
-				Expect().
-				Status(404)
+			Eventually(request).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound))
 		})
 	})
 
@@ -315,12 +309,16 @@ spec:
 			additionalGatewayClassName = fmt.Sprintf("apisix-%d", time.Now().Unix())
 			err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, additionalGatewayClassName, s.GetControllerName()), "")
 			Expect(err).NotTo(HaveOccurred(), "creating additional GatewayClass")
-			time.Sleep(5 * time.Second)
+
 			By("Check additional GatewayClass condition")
-			gcyaml, err := s.GetResourceYaml("GatewayClass", additionalGatewayClassName)
-			Expect(err).NotTo(HaveOccurred(), "getting additional GatewayClass yaml")
-			Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking additional GatewayClass condition status")
-			Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking additional GatewayClass condition message")
+			framework.GatewayClassMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second,
+				types.NamespacedName{Namespace: s.Namespace(), Name: additionalGatewayClassName},
+				metav1.Condition{
+					Type:    string(gatewayv1.GatewayClassConditionStatusAccepted),
+					Status:  metav1.ConditionTrue,
+					Message: "the gatewayclass has been accepted by the apisix-ingress-controller",
+				},
+			)
 
 			additionalGatewayProxy := fmt.Sprintf(additionalGatewayProxyYaml, s.Deployer.GetAdminEndpoint(resources.DataplaneService), resources.AdminAPIKey)
 			err = s.CreateResourceFromStringWithNamespace(additionalGatewayProxy, additionalNamespace)
@@ -336,31 +334,27 @@ spec:
 		})
 
 		It("HTTPRoute should be accessible through both gateways", func() {
+			request := func(client *httpexpect.Expect, host string) int {
+				return client.GET("/get").WithHost(host).Expect().Raw().StatusCode
+			}
+
 			By("Create HTTPRoute referencing both gateways")
 			multiGatewayRoute := fmt.Sprintf(multiGatewayHTTPRoute, s.Namespace(), additionalNamespace)
-			ResourceApplied("HTTPRoute", "multi-gateway-route", multiGatewayRoute, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Name: "multi-gateway-route"}, multiGatewayRoute)
 
 			By("Access through default gateway")
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(http.StatusOK)
+			Eventually(request).WithArguments(s.NewAPISIXClient(), "httpbin.example").
+				WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
 
 			By("Access through additional gateway")
 			client, err := s.NewAPISIXClientForGateway(additionalGatewayGroupID)
 			Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway")
-
-			client.
-				GET("/get").
-				WithHost("httpbin-additional.example").
-				Expect().
-				Status(http.StatusOK)
+			Eventually(request).WithArguments(client, "httpbin-additional.example").
+				WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
 
 			By("Delete Additional Gateway")
 			err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, additionalGatewayClassName), additionalNamespace)
 			Expect(err).NotTo(HaveOccurred(), "deleting additional Gateway")
-			time.Sleep(5 * time.Second)
 
 			By("HTTPRoute should still be accessible through default gateway")
 			s.NewAPISIXClient().
@@ -372,12 +366,8 @@ spec:
 			By("HTTPRoute should not be accessible through additional gateway")
 			client, err = s.NewAPISIXClientForGateway(additionalGatewayGroupID)
 			Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway")
-
-			client.
-				GET("/get").
-				WithHost("httpbin-additional.example").
-				Expect().
-				Status(http.StatusNotFound)
+			Eventually(request).WithArguments(client, "httpbin-additional.example").
+				WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound))
 		})
 	})
 
@@ -493,70 +483,52 @@ spec:
 
 		It("Create/Update/Delete HTTPRoute", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet)
 
+			request := func(host string) int {
+				if host == "" {
+					return s.NewAPISIXClient().GET("/get").Expect().Raw().StatusCode
+				}
+				return s.NewAPISIXClient().GET("/get").WithHost(host).Expect().Raw().StatusCode
+			}
 			By("access dataplane to check the HTTPRoute")
+			Eventually(request).WithArguments("httpbin.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
 			s.NewAPISIXClient().
 				GET("/get").
 				Expect().
 				Status(404)
 
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
-
 			By("delete HTTPRoute")
 			err := s.DeleteResourceFromString(exactRouteByGet)
 			Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute")
-			time.Sleep(5 * time.Second)
-
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(404)
+			Eventually(request).WithArguments("httpbin.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound))
 		})
 
 		It("Delete Gateway after apply HTTPRoute", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet)
 
-			By("access daataplane to check the HTTPRoute")
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
+			request := func() int {
+				return s.NewAPISIXClient().GET("/get").WithHost("httpbin.example").Expect().Raw().StatusCode
+			}
+			By("access dataplane to check the HTTPRoute")
+			Eventually(request).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
 
 			By("delete Gateway")
 			err := s.DeleteResource("Gateway", "apisix")
 			Expect(err).NotTo(HaveOccurred(), "deleting Gateway")
-			time.Sleep(5 * time.Second)
-
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(404)
+			Eventually(request).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound))
 		})
 
-		It("Proxy External Service", func() {
-			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", httprouteWithExternalName, 1)
-
-			By("checking the external service response")
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.external").
-				Expect().
-				Status(200)
+		PIt("Proxy External Service", func() {
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, httprouteWithExternalName, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().GET("/get").WithHost("httpbin.external").Expect().Raw().StatusCode == http.StatusOK, nil
+			})
 		})
 
 		It("Match Port", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", invalidBackendPort, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, invalidBackendPort)
 
 			serviceResources, err := s.DefaultDataplaneResource().Service().List(context.Background())
 			Expect(err).NotTo(HaveOccurred(), "listing services")
@@ -569,23 +541,23 @@ spec:
 		})
 
 		It("Delete HTTPRoute during restart", func() {
-			By("create HTTPRoute httpbin")
-			ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1)
-
-			By("create HTTPRoute httpbin2")
-			ResourceApplied("HTTPRoute", "httpbin2", exactRouteByGet2, 1)
-
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
+			By("create HTTPRoute httpbin and httpbin2")
+			var (
+				httpbinNN  = types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}
+				httpbin2NN = types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin2"}
+			)
+			for nn, spec := range map[types.NamespacedName]string{
+				httpbinNN:  exactRouteByGet,
+				httpbin2NN: exactRouteByGet2,
+			} {
+				s.ApplyHTTPRoute(nn, spec)
+			}
 
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin2.example").
-				Expect().
-				Status(200)
+			request := func(host string) int {
+				return s.NewAPISIXClient().GET("/get").WithHost(host).Expect().Raw().StatusCode
+			}
+			Eventually(request).WithArguments("httpbin.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
+			Eventually(request).WithArguments("httpbin2.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
 
 			s.Deployer.ScaleIngress(0)
 
@@ -727,81 +699,57 @@ spec:
 
 		It("HTTPRoute Exact Match", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1)
-
-			By("access daataplane to check the HTTPRoute")
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet)
 
-			s.NewAPISIXClient().
-				GET("/get/xxx").
-				WithHost("httpbin.example").
-				Expect().
-				Status(404)
+			request := func(uri string) int {
+				return s.NewAPISIXClient().GET(uri).WithHost("httpbin.example").Expect().Raw().StatusCode
+			}
+			By("access dataplane to check the HTTPRoute")
+			Eventually(request).WithArguments("/get").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
+			Eventually(request).WithArguments("/get/xxx").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound))
 		})
 
 		It("HTTPRoute Prefix Match", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", prefixRouteByStatus, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, prefixRouteByStatus)
 
-			By("access daataplane to check the HTTPRoute")
-			s.NewAPISIXClient().
-				GET("/status/200").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
-
-			s.NewAPISIXClient().
-				GET("/status/201").
-				WithHost("httpbin.example").
-				Expect().
-				Status(201)
+			request := func(uri string) int {
+				return s.NewAPISIXClient().GET(uri).WithHost("httpbin.example").Expect().Raw().StatusCode
+			}
+			By("access dataplane to check the HTTPRoute")
+			Eventually(request).WithArguments("/status/200").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
+			Eventually(request).WithArguments("/status/201").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusCreated))
 		})
 
 		It("HTTPRoute Method Match", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", methodRouteGETAndDELETEByAnything, 1)
-
-			By("access daataplane to check the HTTPRoute")
-			s.NewAPISIXClient().
-				GET("/anything").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, methodRouteGETAndDELETEByAnything)
 
-			s.NewAPISIXClient().
-				DELETE("/anything").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
-
-			s.NewAPISIXClient().
-				POST("/anything").
-				WithHost("httpbin.example").
-				Expect().
-				Status(404)
+			request := func(method string) int {
+				return s.NewAPISIXClient().Request(method, "/anything").WithHost("httpbin.example").Expect().Raw().StatusCode
+			}
+			By("access dataplane to check the HTTPRoute")
+			Eventually(request).WithArguments(http.MethodGet).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
+			Eventually(request).WithArguments(http.MethodDelete).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
+			Eventually(request).WithArguments(http.MethodPost).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound))
 		})
 
 		It("HTTPRoute Vars Match", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", varsRoute, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, varsRoute)
 
+			request := func(headers map[string]string) int {
+				cli := s.NewAPISIXClient().GET("/get").WithHost("httpbin.example")
+				for k, v := range headers {
+					cli = cli.WithHeader(k, v)
+				}
+				return cli.Expect().Raw().StatusCode
+			}
 			By("access dataplane to check the HTTPRoute")
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(http.StatusNotFound)
-
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("httpbin.example").
-				WithHeader("X-Route-Name", "httpbin").
-				Expect().
-				Status(http.StatusOK)
+			Eventually(request).WithArguments(map[string]string{}).
+				WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound))
+			Eventually(request).WithArguments(map[string]string{"X-Route-Name": "httpbin"}).
+				WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
 		})
 
 		It("HTTPRoutePolicy in effect", func() {
@@ -1326,9 +1274,17 @@ spec:
 
 		It("HTTPRoute RequestHeaderModifier", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", reqHeaderModifyByHeaders, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, reqHeaderModifyByHeaders, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().
+					GET("/headers").
+					WithHost("httpbin.example").
+					WithHeader("X-Req-Add", "test").
+					WithHeader("X-Req-Removed", "test").
+					WithHeader("X-Req-Set", "test").
+					Expect().Raw().StatusCode == http.StatusOK, nil
+			})
 
-			By("access daataplane to check the HTTPRoute")
+			By("access dataplane to check the HTTPRoute")
 			respExp := s.NewAPISIXClient().
 				GET("/headers").
 				WithHost("httpbin.example").
@@ -1347,9 +1303,14 @@ spec:
 
 		It("HTTPRoute ResponseHeaderModifier", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", respHeaderModifyByHeaders, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, respHeaderModifyByHeaders, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().
+					GET("/headers").
+					WithHost("httpbin.example").
+					Expect().Raw().StatusCode == http.StatusOK, nil
+			})
 
-			By("access daataplane to check the HTTPRoute")
+			By("access dataplane to check the HTTPRoute")
 			respExp := s.NewAPISIXClient().
 				GET("/headers").
 				WithHost("httpbin.example").
@@ -1367,7 +1328,11 @@ spec:
 
 		It("HTTPRoute RequestRedirect", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", httpsRedirectByHeaders, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, httpsRedirectByHeaders, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().GET("/headers").
+					WithHeader("Host", "httpbin.example").
+					Expect().Raw().Header.Get("Location") == "https://httpbin.example:9443/headers", nil
+			})
 
 			s.NewAPISIXClient().GET("/headers").
 				WithHeader("Host", "httpbin.example").
@@ -1376,7 +1341,11 @@ spec:
 				Header("Location").IsEqual("https://httpbin.example:9443/headers")
 
 			By("update HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", hostnameRedirectByHeaders, 2)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, hostnameRedirectByHeaders, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().GET("/headers").
+					WithHeader("Host", "httpbin.example").
+					Expect().Raw().Header.Get("Location") == "http://httpbin.org/headers", nil
+			})
 
 			s.NewAPISIXClient().GET("/headers").
 				WithHeader("Host", "httpbin.example").
@@ -1385,8 +1354,8 @@ spec:
 				Header("Location").IsEqual("http://httpbin.org/headers")
 		})
 
-		It("HTTPRoute RequestMirror", func() {
-			echoRoute := `
+		PIt("HTTPRoute RequestMirror", func() {
+			const echoService = `
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -1419,7 +1388,9 @@ spec:
     port: 80
     protocol: TCP
     targetPort: 8080
----
+`
+
+			const echoRoute = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
@@ -1444,22 +1415,41 @@ spec:
     - name: httpbin-service-e2e-test
       port: 80
 `
-			ResourceApplied("HTTPRoute", "httpbin", echoRoute, 1)
+			// apply echo server Deployment and Service
+			err := s.CreateResourceFromString(echoService)
+			Expect(err).NotTo(HaveOccurred(), "create echo service")
+			err = framework.WaitPodsAvailable(s.GinkgoT, s.KubectlOpts(), metav1.ListOptions{
+				LabelSelector:  "app=echo",
+				TimeoutSeconds: ptr.To(int64(10)),
+			})
+			Expect(err).NotTo(HaveOccurred(), "wait for echo available")
 
-			time.Sleep(time.Second * 6)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, echoRoute, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().GET("/headers").
+					WithHeader("Host", "httpbin.example").
+					Expect().Raw().StatusCode == http.StatusOK, nil
+			})
 
-			_ = s.NewAPISIXClient().GET("/headers").
-				WithHeader("Host", "httpbin.example").
-				Expect().
-				Status(http.StatusOK)
+			time.Sleep(time.Second)
 
 			echoLogs := s.GetDeploymentLogs("echo")
 			Expect(echoLogs).To(ContainSubstring("GET /headers"))
 		})
 
 		It("HTTPRoute URLRewrite with ReplaceFullPath And Hostname", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", replaceFullPathAndHost, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, replaceFullPathAndHost,
+				func(_ context.Context) (done bool, err error) {
+					return s.NewAPISIXClient().GET("/replace/201").
+						WithHeader("Host", "httpbin.example").
+						Expect().Raw().StatusCode == http.StatusOK, nil
+				},
+				func(_ context.Context) (done bool, err error) {
+					return s.NewAPISIXClient().GET("/replace/500").
+						WithHeader("Host", "httpbin.example").
+						Expect().Raw().StatusCode == http.StatusOK, nil
+				},
+			)
 
 			By("/replace/201 should be rewritten to /headers")
 			s.NewAPISIXClient().GET("/replace/201").
@@ -1480,26 +1470,29 @@ spec:
 
 		It("HTTPRoute URLRewrite with ReplacePrefixMatch", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", replacePrefixMatch, 1)
-
-			By("/replace/201 should be rewritten to /status/201")
-			s.NewAPISIXClient().GET("/replace/201").
-				WithHeader("Host", "httpbin.example").
-				Expect().
-				Status(http.StatusCreated)
-
-			By("/replace/500 should be rewritten to /status/500")
-			s.NewAPISIXClient().GET("/replace/500").
-				WithHeader("Host", "httpbin.example").
-				Expect().
-				Status(http.StatusInternalServerError)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, replacePrefixMatch,
+				func(_ context.Context) (done bool, err error) {
+					return s.NewAPISIXClient().GET("/replace/201").
+						WithHeader("Host", "httpbin.example").
+						Expect().Raw().StatusCode == http.StatusCreated, nil
+				},
+				func(_ context.Context) (done bool, err error) {
+					return s.NewAPISIXClient().GET("/replace/500").
+						WithHeader("Host", "httpbin.example").
+						Expect().Raw().StatusCode == http.StatusInternalServerError, nil
+				},
+			)
 		})
 
 		It("HTTPRoute ExtensionRef", func() {
 			By("create HTTPRoute")
 			err := s.CreateResourceFromString(echoPlugin)
 			Expect(err).NotTo(HaveOccurred(), "creating PluginConfig")
-			ResourceApplied("HTTPRoute", "httpbin", extensionRefEchoPlugin, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, extensionRefEchoPlugin, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().GET("/get").
+					WithHeader("Host", "httpbin.example").
+					Expect().Raw().StatusCode == http.StatusOK, nil
+			})
 
 			s.NewAPISIXClient().GET("/get").
 				WithHeader("Host", "httpbin.example").
@@ -1509,18 +1502,16 @@ spec:
 
 			err = s.CreateResourceFromString(echoPluginUpdated)
 			Expect(err).NotTo(HaveOccurred(), "updating PluginConfig")
-			time.Sleep(5 * time.Second)
-
-			s.NewAPISIXClient().GET("/get").
-				WithHeader("Host", "httpbin.example").
-				Expect().
-				Body().
-				Contains("Updated")
+			Eventually(func() string {
+				return s.NewAPISIXClient().GET("/get").
+					WithHeader("Host", "httpbin.example").
+					Expect().Body().Raw()
+			}).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(ContainSubstring("Updated"))
 		})
 	})
 
 	Context("HTTPRoute Multiple Backend", func() {
-		var sameWeiht = `
+		var sameWeight = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
@@ -1543,7 +1534,7 @@ spec:
       port: 80
       weight: 50
  `
-		var oneWeiht = `
+		var oneWeight = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
@@ -1574,7 +1565,11 @@ spec:
 			})
 		})
 		It("HTTPRoute Canary", func() {
-			ResourceApplied("HTTPRoute", "httpbin", sameWeiht, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, sameWeight, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().GET("/get").
+					WithHeader("Host", "httpbin.example").
+					Expect().Raw().StatusCode == http.StatusOK, nil
+			})
 
 			var (
 				hitNginxCnt   = 0
@@ -1595,7 +1590,21 @@ spec:
 			}
 			Expect(hitNginxCnt - hitHttpbinCnt).To(BeNumerically("~", 0, 2))
 
-			ResourceApplied("HTTPRoute", "httpbin", oneWeiht, 2)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, oneWeight)
+
+			var cnt int
+			for cnt < 5 {
+				body := s.NewAPISIXClient().GET("/get").
+					WithHeader("Host", "httpbin.example").
+					Expect().
+					Status(http.StatusOK).
+					Body().Raw()
+				if strings.Contains(body, "Hello") {
+					cnt = 0
+				} else {
+					cnt++
+				}
+			}
 
 			hitNginxCnt = 0
 			hitHttpbinCnt = 0
@@ -1660,7 +1669,12 @@ spec:
 
 		It("Should sync HTTPRoute when GatewayProxy is updated", func() {
 			By("create HTTPRoute")
-			ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1)
+			s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet, func(_ context.Context) (done bool, err error) {
+				return s.NewAPISIXClient().
+					GET("/get").
+					WithHost("httpbin.example").
+					Expect().Raw().StatusCode == http.StatusOK, nil
+			})
 
 			By("verify HTTPRoute works")
 			s.NewAPISIXClient().
@@ -1691,14 +1705,14 @@ spec:
 			updatedProxy := fmt.Sprintf(updatedGatewayProxy, s.Deployer.GetAdminEndpoint(resources.DataplaneService), resources.AdminAPIKey)
 			err = s.CreateResourceFromString(updatedProxy)
 			Expect(err).NotTo(HaveOccurred(), "updating GatewayProxy")
-			time.Sleep(5 * time.Second)
 
 			By("verify HTTPRoute works for additional gateway group")
-			client.
-				GET("/get").
-				WithHost("httpbin.example").
-				Expect().
-				Status(200)
+			Eventually(func() int {
+				return client.
+					GET("/get").
+					WithHost("httpbin.example").
+					Expect().Raw().StatusCode
+			}).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK))
 		})
 	})
 

test/e2e/scaffold/scaffold.go

@@ -395,3 +395,11 @@
 
 	return resources.HttpsTunnel.Endpoint(), nil
 }
+
+func (s *Scaffold) CurrentGatewayGroupID() string {
+	return s.gatewaygroupid
+}
+
+func (s *Scaffold) KubectlOpts() *k8s.KubectlOptions {
+	return s.kubectlOptions
+}