Skip to content

Commit 59667cd

Browse files
shuqzshuqz
committed
[feat gw-api]implement finalizer
1 parent 6364771 commit 59667cd

File tree

7 files changed

+173
-12
lines changed

7 files changed

+173
-12
lines changed

‎controllers/gateway/eventhandlers/gateway_events.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func (h *enqueueRequestsForGatewayEvent) Delete(ctx context.Context, e event.Typ
5454

5555
func (h *enqueueRequestsForGatewayEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gwv1.Gateway], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) {
5656
gw := e.Object
57-
h.logger.V(1).Info("enqueue gateway delete event", "gateway", k8s.NamespacedName(gw))
57+
h.logger.V(1).Info("enqueue gateway generic event", "gateway", k8s.NamespacedName(gw))
5858
h.enqueueImpactedGateway(ctx, gw, queue)
5959
}
6060

‎controllers/gateway/eventhandlers/service_events.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,15 @@ package eventhandlers
22

33
import (
44
"context"
5+
"fmt"
56
"github.com/go-logr/logr"
67
corev1 "k8s.io/api/core/v1"
78
"k8s.io/client-go/tools/record"
89
"k8s.io/client-go/util/workqueue"
910
"sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants"
1011
"sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils"
12+
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
13+
"sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants"
1114
"sigs.k8s.io/controller-runtime/pkg/client"
1215
"sigs.k8s.io/controller-runtime/pkg/event"
1316
"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -65,9 +68,33 @@ func (h *enqueueRequestsForServiceEvent) Update(ctx context.Context, e event.Typ
6568
func (h *enqueueRequestsForServiceEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*corev1.Service], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) {
6669
svc := e.Object
6770
h.logger.V(1).Info("enqueue service delete event", "service", svc.Name)
71+
// remove target group configuration finalizer when service is deleted
72+
h.removeTargetGroupConfigurationFinalizer(ctx, svc)
73+
6874
h.enqueueImpactedRoutes(ctx, svc)
6975
}
7076

77+
func (h *enqueueRequestsForServiceEvent) removeTargetGroupConfigurationFinalizer(ctx context.Context, svc *corev1.Service) {
78+
tgConfig, err := routeutils.LookUpTargetGroupConfiguration(ctx, h.k8sClient, k8s.NamespacedName(svc))
79+
if err != nil {
80+
h.logger.Error(err, "failed to look up target group configuration", "service", svc.Name)
81+
return
82+
}
83+
if tgConfig == nil {
84+
h.logger.V(1).Info("TargetGroupConfigurationNotFound, ignoring remove finalizer.", "TargetGroupConfiguration", svc.Name)
85+
return
86+
}
87+
88+
tgFinalizer := shared_constants.TargetGroupConfigurationFinalizer
89+
if k8s.HasFinalizer(tgConfig, tgFinalizer) {
90+
finalizerManager := k8s.NewDefaultFinalizerManager(h.k8sClient, logr.Discard())
91+
if err := finalizerManager.RemoveFinalizers(ctx, tgConfig, tgFinalizer); err != nil {
92+
h.eventRecorder.Event(tgConfig, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed to remove target group configuration finalizer due to %v", err))
93+
}
94+
h.logger.V(1).Info("TargetGroupConfigurationFinalizerRemoved", "TargetGroupConfiguration", tgConfig.Name)
95+
}
96+
}
97+
7198
func (h *enqueueRequestsForServiceEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*corev1.Service], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) {
7299
svc := e.Object
73100
h.logger.V(1).Info("enqueue service generic event", "service", svc.Name)

‎controllers/gateway/gateway_controller.go

Lines changed: 107 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -223,17 +223,17 @@ func (r *gatewayReconciler) reconcileHelper(ctx context.Context, req reconcile.R
223223
}
224224

225225
if lb == nil {
226-
err = r.reconcileDelete(ctx, gw, stack, allRoutes)
226+
err = r.reconcileDelete(ctx, gw, gwClass, stack, allRoutes)
227227
if err != nil {
228228
r.logger.Error(err, "Failed to process gateway delete")
229229
}
230230
return err
231231
}
232232

233-
return r.reconcileUpdate(ctx, gw, stack, lb, backendSGRequired)
233+
return r.reconcileUpdate(ctx, gw, gwClass, stack, lb, backendSGRequired)
234234
}
235235

236-
func (r *gatewayReconciler) reconcileDelete(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, routes map[int32][]routeutils.RouteDescriptor) error {
236+
func (r *gatewayReconciler) reconcileDelete(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, stack core.Stack, routes map[int32][]routeutils.RouteDescriptor) error {
237237
for _, routeList := range routes {
238238
if len(routeList) != 0 {
239239
err := errors.Errorf("Gateway deletion invoked with routes attached [%s]", generateRouteList(routes))
@@ -250,21 +250,34 @@ func (r *gatewayReconciler) reconcileDelete(ctx context.Context, gw *gwv1.Gatewa
250250
if err := r.backendSGProvider.Release(ctx, networking.ResourceTypeGateway, []types.NamespacedName{k8s.NamespacedName(gw)}); err != nil {
251251
return err
252252
}
253+
// remove load balancer configuration finalizer
254+
if err := r.removeLoadBalancerConfigurationFinalizers(ctx, gw, gwClass); err != nil {
255+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.LoadBalancerConfigurationEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove load balancer configuration finalizer due to %v", err))
256+
return err
257+
}
258+
// remove gateway finalizer
253259
if err := r.finalizerManager.RemoveFinalizers(ctx, gw, r.finalizer); err != nil {
254-
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.GatewayEventReasonFailedAddFinalizer, fmt.Sprintf("Failed remove finalizer due to %v", err))
260+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.GatewayEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove gateway finalizer due to %v", err))
255261
return err
256262
}
257263
}
258264
return nil
259265
}
260266

261-
func (r *gatewayReconciler) reconcileUpdate(ctx context.Context, gw *gwv1.Gateway, stack core.Stack,
267+
func (r *gatewayReconciler) reconcileUpdate(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, stack core.Stack,
262268
lb *elbv2model.LoadBalancer, backendSGRequired bool) error {
263-
269+
// add gateway finalizer
264270
if err := r.finalizerManager.AddFinalizers(ctx, gw, r.finalizer); err != nil {
265-
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.GatewayEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
271+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.GatewayEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add gateway finalizer due to %v", err))
266272
return err
267273
}
274+
275+
// add load balancer configuration finalizer
276+
if err := r.addLoadBalancerConfigurationFinalizers(ctx, gw, gwClass); err != nil {
277+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.LoadBalancerConfigurationEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add load balancer configuration finalizer due to %v", err))
278+
return err
279+
}
280+
268281
err := r.deployModel(ctx, gw, stack)
269282
if err != nil {
270283
return err
@@ -284,6 +297,93 @@ func (r *gatewayReconciler) reconcileUpdate(ctx context.Context, gw *gwv1.Gatewa
284297
return nil
285298
}
286299

300+
// add finalizer to load balancer configuration when it is in use by gateway or gatewayClass
301+
func (r *gatewayReconciler) addLoadBalancerConfigurationFinalizers(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass) error {
302+
// add finalizer to lbConfig referred by gatewayClass
303+
if gwClass.Spec.ParametersRef != nil && string(gwClass.Spec.ParametersRef.Kind) == constants.LoadBalancerConfiguration {
304+
lbConfig := &elbv2gw.LoadBalancerConfiguration{}
305+
if err := r.k8sClient.Get(ctx, types.NamespacedName{
306+
Namespace: string(*gwClass.Spec.ParametersRef.Namespace),
307+
Name: gwClass.Spec.ParametersRef.Name,
308+
}, lbConfig); err != nil {
309+
return client.IgnoreNotFound(err)
310+
}
311+
if err := r.finalizerManager.AddFinalizers(ctx, lbConfig, shared_constants.LoadBalancerConfigurationFinalizer); err != nil {
312+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.LoadBalancerConfigurationEventReasonFailedAddFinalizer, fmt.Sprintf("Failed to add load balancer configuration finalizer due to %v", err))
313+
return err
314+
}
315+
}
316+
317+
// add finalizer to lbConfig referred by gateway
318+
if gw.Spec.Infrastructure != nil && gw.Spec.Infrastructure.ParametersRef != nil && string(gw.Spec.Infrastructure.ParametersRef.Kind) == constants.LoadBalancerConfiguration {
319+
lbConfig := &elbv2gw.LoadBalancerConfiguration{}
320+
if err := r.k8sClient.Get(ctx, types.NamespacedName{
321+
Namespace: gw.Namespace,
322+
Name: gw.Spec.Infrastructure.ParametersRef.Name,
323+
}, lbConfig); err != nil {
324+
return client.IgnoreNotFound(err)
325+
}
326+
if err := r.finalizerManager.AddFinalizers(ctx, lbConfig, shared_constants.LoadBalancerConfigurationFinalizer); err != nil {
327+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.LoadBalancerConfigurationEventReasonFailedAddFinalizer, fmt.Sprintf("Failed to add load balancer configuration finalizer due to %v", err))
328+
return err
329+
}
330+
}
331+
return nil
332+
}
333+
334+
func (r *gatewayReconciler) removeLoadBalancerConfigurationFinalizers(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass) error {
335+
// remove finalizer from lbConfig - gatewayClass
336+
if gwClass.Spec.ParametersRef != nil && string(gwClass.Spec.ParametersRef.Kind) == constants.LoadBalancerConfiguration {
337+
lbConfig := &elbv2gw.LoadBalancerConfiguration{}
338+
if err := r.k8sClient.Get(ctx, types.NamespacedName{
339+
Namespace: string(*gwClass.Spec.ParametersRef.Namespace),
340+
Name: gwClass.Spec.ParametersRef.Name,
341+
}, lbConfig); err != nil {
342+
return client.IgnoreNotFound(err)
343+
}
344+
// remove finalizer if it exists and it not in use
345+
if k8s.HasFinalizer(lbConfig, shared_constants.LoadBalancerConfigurationFinalizer) && !r.isLBConfigInUse(ctx, lbConfig, gw) {
346+
if err := r.finalizerManager.RemoveFinalizers(ctx, lbConfig, shared_constants.LoadBalancerConfigurationFinalizer); err != nil {
347+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.LoadBalancerConfigurationEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed to remove load balancer configuration finalizer due to %v", err))
348+
return err
349+
}
350+
}
351+
}
352+
353+
// remove finalizer from lbConfig - gateway
354+
if gw.Spec.Infrastructure != nil && gw.Spec.Infrastructure.ParametersRef != nil && string(gw.Spec.Infrastructure.ParametersRef.Kind) == constants.LoadBalancerConfiguration {
355+
lbConfig := &elbv2gw.LoadBalancerConfiguration{}
356+
if err := r.k8sClient.Get(ctx, types.NamespacedName{
357+
Namespace: gw.Namespace,
358+
Name: gw.Spec.Infrastructure.ParametersRef.Name,
359+
}, lbConfig); err != nil {
360+
return client.IgnoreNotFound(err)
361+
}
362+
// remove finalizer if it exists and it is not in use
363+
if k8s.HasFinalizer(lbConfig, shared_constants.LoadBalancerConfigurationFinalizer) && !r.isLBConfigInUse(ctx, lbConfig, gw) {
364+
if err := r.finalizerManager.RemoveFinalizers(ctx, lbConfig, shared_constants.LoadBalancerConfigurationFinalizer); err != nil {
365+
r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.LoadBalancerConfigurationEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed to remove load balancer configuration finalizer due to %v", err))
366+
return err
367+
}
368+
}
369+
}
370+
return nil
371+
}
372+
373+
func (r *gatewayReconciler) isLBConfigInUse(ctx context.Context, lbConfig *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway) bool {
374+
// check if lbConfig is referred by other gateway
375+
gwsUsingLBConfig := eventhandlers.GetImpactedGatewaysFromLbConfig(ctx, r.k8sClient, lbConfig, r.controllerName)
376+
for _, gwUsingLBConfig := range gwsUsingLBConfig {
377+
if gwUsingLBConfig.Name != gw.Name || gwUsingLBConfig.Namespace != gw.Namespace {
378+
return true
379+
}
380+
}
381+
382+
// check if lbConfig is referred by other gatewayClass
383+
gwClassesUsingLBConfig := eventhandlers.GetImpactedGatewayClassesFromLbConfig(ctx, r.k8sClient, lbConfig, sets.New(r.controllerName))
384+
return len(gwClassesUsingLBConfig) > 0
385+
}
386+
287387
func (r *gatewayReconciler) deployModel(ctx context.Context, gw *gwv1.Gateway, stack core.Stack) error {
288388
if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName, nil); err != nil {
289389
var requeueNeededAfter *runtime.RequeueNeededAfter

‎pkg/gateway/routeutils/backend.go

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ package routeutils
33
import (
44
"context"
55
"fmt"
6+
"github.com/go-logr/logr"
67
"github.com/pkg/errors"
78
corev1 "k8s.io/api/core/v1"
89
"k8s.io/apimachinery/pkg/types"
910
elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
1011
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
12+
"sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants"
1113
"sigs.k8s.io/controller-runtime/pkg/client"
1214
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
1315
gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
@@ -102,12 +104,18 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci
102104
}
103105
}
104106

105-
tgConfig, err := lookUpTargetGroupConfiguration(ctx, k8sClient, k8s.NamespacedName(svc))
107+
tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, k8s.NamespacedName(svc))
106108

107109
if err != nil {
108110
// As of right now, this error can only be thrown because of a k8s api error hence no status update.
109111
return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object"))
110112
}
113+
// add TGConfig finalizer
114+
if tgConfig != nil {
115+
if err := addTargetGroupConfigurationFinalizer(ctx, k8sClient, tgConfig); err != nil {
116+
return nil, errors.Wrap(err, fmt.Sprintf("Unable to add finalizer to tg config object"))
117+
}
118+
}
111119

112120
if servicePort == nil {
113121
initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port)
@@ -164,9 +172,9 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci
164172
}, nil
165173
}
166174

167-
// lookUpTargetGroupConfiguration given a service, lookup the target group configuration associated with the service.
175+
// LookUpTargetGroupConfiguration given a service, lookup the target group configuration associated with the service.
168176
// recall that target group configuration always lives within the same namespace as the service.
169-
func lookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client, serviceMetadata types.NamespacedName) (*elbv2gw.TargetGroupConfiguration, error) {
177+
func LookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client, serviceMetadata types.NamespacedName) (*elbv2gw.TargetGroupConfiguration, error) {
170178
tgConfigList := &elbv2gw.TargetGroupConfigurationList{}
171179

172180
// TODO - Add index
@@ -227,3 +235,15 @@ func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdenti
227235

228236
return false, nil
229237
}
238+
239+
// Implements helper function to add finalizer for target group configuration
240+
func addTargetGroupConfigurationFinalizer(ctx context.Context, k8sClient client.Client, tgConfig *elbv2gw.TargetGroupConfiguration) error {
241+
finalizer := shared_constants.TargetGroupConfigurationFinalizer
242+
// check if finalizer already exist
243+
if k8s.HasFinalizer(tgConfig, finalizer) {
244+
return nil
245+
}
246+
finalizerManager := k8s.NewDefaultFinalizerManager(k8sClient, logr.Discard())
247+
248+
return finalizerManager.AddFinalizers(ctx, tgConfig, finalizer)
249+
}

‎pkg/gateway/routeutils/backend_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -528,7 +528,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) {
528528
assert.NoError(t, err)
529529
}
530530

531-
result, err := lookUpTargetGroupConfiguration(context.Background(), k8sClient, tc.serviceMetadata)
531+
result, err := LookUpTargetGroupConfiguration(context.Background(), k8sClient, tc.serviceMetadata)
532532

533533
if tc.expectErr {
534534
assert.Error(t, err)

‎pkg/k8s/events.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,12 @@ const (
3737
GatewayEventReasonSuccessfullyReconciled = "SuccessfullyReconciled"
3838
GatewayEventReasonFailedDeployModel = "FailedDeployModel"
3939
GatewayEventReasonFailedBuildModel = "FailedBuildModel"
40+
41+
// Target Group Configuration events
42+
TargetGroupConfigurationEventReasonFailedAddFinalizer = "FailedAddFinalizer"
43+
TargetGroupConfigurationEventReasonFailedRemoveFinalizer = "FailedRemoveFinalizer"
44+
45+
// Load Balancer Configuration events
46+
LoadBalancerConfigurationEventReasonFailedAddFinalizer = "FailedAddFinalizer"
47+
LoadBalancerConfigurationEventReasonFailedRemoveFinalizer = "FailedRemoveFinalizer"
4048
)

‎pkg/shared_constants/finalizers.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,10 @@ const (
1515

1616
// ALBGatewayFinalizer the finalizer we attach to an ALB Gateway resource
1717
ALBGatewayFinalizer = "gateway.k8s.aws/alb"
18+
19+
// TargetGroupConfigurationFinalizer the finalizer we attach to a target group configuration resource
20+
TargetGroupConfigurationFinalizer = "gateway.k8s.aws/targetgroupconfigurations"
21+
22+
// LoadBalancerConfigurationFinalizer the finalizer we attach to a load balancer configuration resource
23+
LoadBalancerConfigurationFinalizer = "gateway.k8s.aws/loadbalancerconfigurations"
1824
)

0 commit comments

Comments
 (0)