wait for cache sync before ready

Signed-off-by: jukie <[email protected]>

Author: jukie

internal/provider/kubernetes/kubernetes.go

@@ -8,6 +8,7 @@ package kubernetes
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -65,6 +66,28 @@ var (
 	webhookTLSPort = 9443
 )
 
+// cacheReadyCheck returns a healthz.Checker that verifies the manager's cache has synced.
+// This ensures the control plane has populated its cache with all resources from the API server
+// before reporting ready. This prevents serving inconsistent xDS configuration to Envoy proxies
+// when running multiple control plane replicas during periods of resource churn.
+func cacheReadyCheck(mgr manager.Manager) healthz.Checker {
+	return func(req *http.Request) error {
+		// Use a short timeout to avoid blocking the health check indefinitely.
+		// The readiness probe will retry periodically until the cache syncs.
+		//
+		// TODO: For v1.7.0 Make configurable via API and align with helm container readiness probe timeout.
+		ctx, cancel := context.WithTimeout(req.Context(), 1*time.Second)
+		defer cancel()
+
+		// WaitForCacheSync returns true if the cache has synced, false if the context is cancelled.
+		if !mgr.GetCache().WaitForCacheSync(ctx) {
+			return fmt.Errorf("cache not synced yet")
+		}
+
+		return nil
+	}
+}
+
 // New creates a new Provider from the provided EnvoyGateway.
 func New(ctx context.Context, restCfg *rest.Config, svrCfg *ec.Server, resources *message.ProviderResources) (*Provider, error) {
 	// TODO: Decide which mgr opts should be exposed through envoygateway.provider.kubernetes API.
@@ -199,7 +222,10 @@ func New(ctx context.Context, restCfg *rest.Config, svrCfg *ec.Server, resources
 	}
 
 	// Add ready check health probes.
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+	// Use a custom readiness check that waits for the cache to sync before reporting ready.
+	// This ensures the control plane has a consistent view of all resources before serving
+	// xDS configuration to proxies, preventing inconsistent state when multiple replicas exist.
+	if err := mgr.AddReadyzCheck("cache-sync", cacheReadyCheck(mgr)); err != nil {
 		return nil, fmt.Errorf("unable to set up ready check: %w", err)
 	}