[Feat] Support logon session affinity

internal/controller/reconcile-capapplication.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/sap/cap-operator/internal/util"
 	"github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1"
+	"golang.org/x/sync/errgroup"
 	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -134,7 +135,7 @@ func (c *Controller) handleCAPApplicationDependentResources(ctx context.Context,
 		return
 	}
 
-	// step 6 - check and set consistent status
+	// step 6 - check and set consistent status; check for newer versions and trigger tenant networking updates
 	return c.verifyApplicationConsistent(ctx, ca)
 }
 
@@ -150,11 +151,12 @@ func (c *Controller) verifyApplicationConsistent(ctx context.Context, ca *v1alph
 		}
 	}
 
-	// Check for newer CAPApplicationVersion
-	return nil, c.checkNewCAPApplicationVersion(ctx, ca)
+	// Check for newer CAPApplicationVersion and trigger tenant networking updates
+	return nil, c.checkNewCavAndTenantNetworking(ctx, ca)
 }
 
-func (c *Controller) checkNewCAPApplicationVersion(ctx context.Context, ca *v1alpha1.CAPApplication) error {
+func (c *Controller) checkNewCavAndTenantNetworking(ctx context.Context, ca *v1alpha1.CAPApplication) error {
+	// Get the latest CAV for the tenant
 	cav, err := c.getLatestReadyCAPApplicationVersion(ca, false)
 	if err != nil {
 		return err
@@ -165,31 +167,28 @@ func (c *Controller) checkNewCAPApplicationVersion(ctx context.Context, ca *v1al
 	if err != nil || len(tenants) == 0 {
 		return err
 	}
+
+	netUpdGrp := errgroup.Group{}
 	updated := false
 	for _, tenant := range tenants {
-		if tenant.Spec.VersionUpgradeStrategy == v1alpha1.VersionUpgradeStrategyTypeNever {
-			// Skip non relevant tenants
-			continue
-		}
-		if tenant.Status.State == v1alpha1.CAPTenantStateProvisioning || tenant.Status.State == v1alpha1.CAPTenantStateUpgrading || tenant.Status.State == v1alpha1.CAPTenantStateDeleting {
-			// Skip tenants that are not ready or not in processing or not in error
-			continue
+		if tenant.Status.CurrentCAPApplicationVersionInstance != "" {
+			t := tenant
+			netUpdGrp.Go(func() error {
+				return c.reconcileTenantNetworking(ctx, t, t.Status.CurrentCAPApplicationVersionInstance, ca)
+			})
 		}
-		// Assume we may have to update the tenant and prepare a copy
-		cat := tenant.DeepCopy()
-
-		// Check version of tenant
-		if cat.Spec.Version != cav.Spec.Version {
-			// update CAPTenant Spec to point to the latest version
-			cat.Spec.Version = cav.Spec.Version
-			// Trigger update on CAPTenant (modifies Generation) --> which would reconcile the tenant
-			if _, err = c.crdClient.SmeV1alpha1().CAPTenants(ca.Namespace).Update(ctx, cat, metav1.UpdateOptions{}); err != nil {
-				return fmt.Errorf("could not update %s %s.%s: %w", v1alpha1.CAPTenantKind, cat.Namespace, cat.Name, err)
-			}
-			c.Event(tenant, ca, corev1.EventTypeNormal, CAPTenantEventAutoVersionUpdate, EventActionUpgrade, fmt.Sprintf("version updated to %s for initiating tenant upgrade", cav.Spec.Version))
+
+		if upd, err := c.checkForTenantVersionUpgrade(ctx, ca, cav, tenant); err != nil {
+			return err
+		} else if upd {
 			updated = true
 		}
 	}
+
+	if err = netUpdGrp.Wait(); err != nil {
+		return fmt.Errorf("failed to reconcile tenant networking: %w", err)
+	}
+
 	if updated {
 		msg := fmt.Sprintf("new version %s.%s was used to trigger tenant upgrades", cav.Namespace, cav.Name)
 		ca.SetStatusWithReadyCondition(v1alpha1.CAPApplicationStateProcessing, metav1.ConditionFalse, CAPApplicationEventNewCAVTriggeredTenantUpgrade, msg)
@@ -200,6 +199,33 @@ func (c *Controller) checkNewCAPApplicationVersion(ctx context.Context, ca *v1al
 	return nil
 }
 
+func (c *Controller) checkForTenantVersionUpgrade(ctx context.Context, ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, tenant *v1alpha1.CAPTenant) (bool, error) {
+	if tenant.Spec.VersionUpgradeStrategy == v1alpha1.VersionUpgradeStrategyTypeNever {
+		// Skip non relevant tenants
+		return false, nil
+	}
+	if tenant.Status.State == v1alpha1.CAPTenantStateProvisioning || tenant.Status.State == v1alpha1.CAPTenantStateUpgrading || tenant.Status.State == v1alpha1.CAPTenantStateDeleting {
+		// Skip tenants that are not ready or not in processing or not in error
+		return false, nil
+	}
+
+	// Assume we may have to update the tenant and prepare a copy
+	cat := tenant.DeepCopy()
+
+	// Check version of tenant
+	if cat.Spec.Version != cav.Spec.Version {
+		// update CAPTenant Spec to point to the latest version
+		cat.Spec.Version = cav.Spec.Version
+		// Trigger update on CAPTenant (modifies Generation) --> which would reconcile the tenant
+		if _, err := c.crdClient.SmeV1alpha1().CAPTenants(ca.Namespace).Update(ctx, cat, metav1.UpdateOptions{}); err != nil {
+			return false, fmt.Errorf("could not update %s %s.%s: %w", v1alpha1.CAPTenantKind, cat.Namespace, cat.Name, err)
+		}
+		c.Event(tenant, ca, corev1.EventTypeNormal, CAPTenantEventAutoVersionUpdate, EventActionUpgrade, fmt.Sprintf("version updated to %s for initiating tenant upgrade", cav.Spec.Version))
+		return true, nil
+	}
+	return false, nil
+}
+
 func (c *Controller) checkAdditionalConditions(ca *v1alpha1.CAPApplication, result *ReconcileResult, err error) (*ReconcileResult, error) {
 	// In case of explicit Reconcile or errors return back with the original result
 	if result != nil || err != nil {

internal/controller/reconcile-capapplication_test.go

@@ -375,7 +375,7 @@ func TestController_handleCAPApplicationConsistent_Case2(t *testing.T) {
 }
 
 func TestController_handleCAPApplicationConsistent_Case3(t *testing.T) {
-	reconcileTestItem(
+	err := reconcileTestItem(
 		context.TODO(), t,
 		QueueItem{Key: ResourceCAPApplication, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01"}},
 		TestData{
@@ -389,35 +389,13 @@ func TestController_handleCAPApplicationConsistent_Case3(t *testing.T) {
 				"testdata/capapplication/cat-consumer-upg-never-ready.yaml",
 				"testdata/common/credential-secrets.yaml",
 			},
-			expectedResources: "testdata/capapplication/ca-31.expected.yaml",
-			backlogItems: []string{
-				"ERP4SMEPREPWORKAPPPLAT-2881",
-			},
+			expectError: true,
 		},
 	)
-}
 
-func TestController_handleCAPApplicationConsistent_Case4(t *testing.T) {
-	reconcileTestItem(
-		context.TODO(), t,
-		QueueItem{Key: ResourceCAPApplication, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01"}},
-		TestData{
-			description: "Consistent state with a CAV name update",
-			initialResources: []string{
-				"testdata/common/domain-ready.yaml",
-				"testdata/common/cluster-domain-ready.yaml",
-				"testdata/capapplication/ca-32.initial.yaml",
-				"testdata/capapplication/cav-name-modified-ready.yaml",
-				"testdata/capapplication/cat-provider-no-finalizers-ready.yaml",
-				"testdata/capapplication/cat-consumer-no-finalizers-ready.yaml",
-				"testdata/common/credential-secrets.yaml",
-			},
-			expectedResources: "testdata/capapplication/ca-32.expected.yaml",
-			backlogItems: []string{
-				"ERP4SMEPREPWORKAPPPLAT-2881",
-			},
-		},
-	)
+	if err.Error() != "failed to reconcile tenant networking: capapplicationversion.sme.sap.com \"test-cap-01-cav-v1\" not found" {
+		t.Error("Wrong error message")
+	}
 }
 
 func TestController_handleCAPApplicationConsistent_Case5(t *testing.T) {
@@ -481,12 +459,11 @@ func TestCAPApplicationConsistentWithNewCAPApplicationVersionTenantUpdateError(t
 				"testdata/capapplication/cat-provider-no-finalizers-ready.yaml",
 				"testdata/common/credential-secrets.yaml",
 			},
-			expectError:           true,
-			mockErrorForResources: []ResourceAction{{Verb: "update", Group: "sme.sap.com", Version: "v1alpha1", Resource: "captenants", Namespace: "*", Name: "*"}},
+			expectError: true,
 		},
 	)
-	if err.Error() != "could not update CAPTenant default.test-cap-01-provider: mocked api error (captenants.sme.sap.com/v1alpha1)" {
-		t.Error("error message is different from expected")
+	if err.Error() != "failed to reconcile tenant networking: capapplicationversion.sme.sap.com \"test-cap-01-cav-v1\" not found" {
+		t.Error("Wrong error message")
 	}
 }
 
@@ -535,7 +512,7 @@ func TestAdditionalConditionsWithTenantDeletingUpgradeStrategyNever(t *testing.T
 }
 
 func TestController_handleCAPApplicationConsistent_versionUpgrade(t *testing.T) {
-	reconcileTestItem(
+	err := reconcileTestItem(
 		context.TODO(), t,
 		QueueItem{Key: ResourceCAPApplication, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01"}},
 		TestData{
@@ -549,9 +526,13 @@ func TestController_handleCAPApplicationConsistent_versionUpgrade(t *testing.T)
 				"testdata/capapplication/cat-consumer-upgrading.yaml",
 				"testdata/common/credential-secrets.yaml",
 			},
-			expectedResources: "testdata/capapplication/ca-31.expected.yaml",
+			expectError: true,
 		},
 	)
+
+	if err.Error() != "failed to reconcile tenant networking: capapplicationversion.sme.sap.com \"test-cap-01-cav-v1\" not found" {
+		t.Error("Wrong error message")
+	}
 }
 
 func TestCA_ServicesOnly_Consistent(t *testing.T) {

internal/controller/reconcile-captenant.go

@@ -163,6 +163,10 @@ var handleCompletedProvisioningUpgradeOperation = func(ctx context.Context, c *C
 		return NewReconcileResultWithResource(ResourceCAPTenant, cat.Name, cat.Namespace, 10*time.Second), nil
 	}
 
+	// set current CAPApplicationVersionInstance and update previous versions
+	// required for tenant networking reconciliation as it relies on the current and previous version in the status of the tenant
+	cat.SetStatusCAPApplicationVersion(ctop.Spec.CAPApplicationVersionInstance)
+
 	// check and reconcile tenant virtual service
 	// adjust virtual service only when tenant is finalizing (after provisioning or upgrade)
 	err = c.reconcileTenantNetworking(ctx, cat, ctop.Spec.CAPApplicationVersionInstance, ca)
@@ -172,7 +176,6 @@ var handleCompletedProvisioningUpgradeOperation = func(ctx context.Context, c *C
 
 	// the ObservedGeneration of the tenant should be updated here (when Ready)
 	cat.SetStatusWithReadyCondition(target.state, target.conditionStatus, target.conditionReason, message)
-	cat.SetStatusCAPApplicationVersion(ctop.Spec.CAPApplicationVersionInstance)
 	return getTenantReconcileResultConsideringDeletion(cat, nil), nil
 }
 
@@ -242,7 +245,11 @@ func (c *Controller) reconcileCAPTenant(ctx context.Context, item QueueItem, _ i
 	}
 
 	if cat.DeletionTimestamp == nil && cat.Status.CurrentCAPApplicationVersionInstance != "" {
-		err = c.reconcileTenantNetworking(ctx, cat, cat.Status.CurrentCAPApplicationVersionInstance, nil)
+		ca, caGetErr := c.getCachedCAPApplication(cat.Namespace, cat.Spec.CAPApplicationInstance)
+		if caGetErr != nil {
+			return nil, caGetErr
+		}
+		err = c.reconcileTenantNetworking(ctx, cat, cat.Status.CurrentCAPApplicationVersionInstance, ca)
 		if err == nil {
 			util.LogInfo("Tenant processing completed", string(Ready), cat, nil, "tenantId", cat.Spec.TenantId, "version", cat.Spec.Version)
 		}

internal/controller/reconcile-captenant_test.go

@@ -576,3 +576,158 @@ func TestCAPTenantDeprovisioningVersionNotReady(t *testing.T) {
 		},
 	)
 }
+
+func TestCAPTenantProvisioningCompletedWithSessionAffinityEnabled(t *testing.T) {
+	reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant provisioning operation completed (creates virtual service and destination rule) with session affinity enabled",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication-session-affinity.yaml",
+				"testdata/common/capapplicationversion-v1.yaml",
+				"testdata/captenant/cat-04.initial.yaml",
+			},
+			expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs.yaml",
+		},
+	)
+}
+
+func TestCAPTenantProvisioningCompletedWithSessionAffinityEnabledAndVsheaders(t *testing.T) {
+	reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant provisioning operation completed (creates virtual service and destination rule) with session affinity enabled and virtual service headers",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication-session-affinity-vs-headers.yaml",
+				"testdata/common/capapplicationversion-v1.yaml",
+				"testdata/captenant/cat-04.initial.yaml",
+			},
+			expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-headers.yaml",
+		},
+	)
+}
+
+func TestCAPTenantProvisioningCompletedWithSessionAffinityEnabledCustomLogout(t *testing.T) {
+	reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant provisioning operation completed (creates virtual service and destination rule) with session affinity enabled using custom logout routes",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication-session-affinity.yaml",
+				"testdata/common/capapplicationversion-v1-custom-logout-endpoint.yaml",
+				"testdata/captenant/cat-04.initial.yaml",
+			},
+			expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-logout-endpoint.yaml",
+		},
+	)
+}
+
+func TestCAPTenantUpgradeOperationCompletedWithSessionAffinityEnabled(t *testing.T) {
+	reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant upgrade operation completed expecting virtual service, destination rule adjustments with session affinity enabled",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication-session-affinity.yaml",
+				"testdata/common/capapplicationversion-v1.yaml",
+				"testdata/common/capapplicationversion-v2.yaml",
+				"testdata/captenant/provider-tenant-vs-v1.yaml",
+				"testdata/captenant/provider-tenant-dr-v1.yaml",
+				"testdata/captenant/cat-13.initial.yaml",
+			},
+			expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml",
+		},
+	)
+}
+
+func TestCAPTenantUpgradedThirdTimeWithSessionAffinityEnabled(t *testing.T) {
+	reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant upgraded third time - expecting virtual service, destination rule adjustments by removing config corresponding to v1 and by adding config for v3",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication-session-affinity.yaml",
+				"testdata/common/capapplicationversion-v1.yaml",
+				"testdata/common/capapplicationversion-v2.yaml",
+				"testdata/common/capapplicationversion-v3.yaml",
+				"testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.yaml",
+			},
+			expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.expected.yaml",
+		},
+	)
+}
+
+func TestCAPTenantUpgradeOperationCompletedWithSessionAffinityEnabledAndPreviousCAVRemoved(t *testing.T) {
+	reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant upgraded - expecting virtual service, destination rule adjustments after removing previous cav v1",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication-session-affinity.yaml",
+				"testdata/common/capapplicationversion-v2.yaml",
+				"testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml",
+			},
+			expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-prev-cav-removed.yaml",
+		},
+	)
+}
+
+func TestCAPTenantUpgradeOperationCompletedWithSessionAffinitySwitchedFromEnabledToDisabled(t *testing.T) {
+	reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant upgraded - expecting virtual service, destination rule adjustments after switching session affinity from enabled to disabled",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication.yaml",
+				"testdata/common/capapplicationversion-v1.yaml",
+				"testdata/common/capapplicationversion-v2.yaml",
+				"testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml",
+			},
+			expectedResources: "testdata/captenant/cat-with-session-affinity-disabled-dr-vs.yaml",
+		},
+	)
+}
+
+func TestCAPTenantUpgradeOperationCompletedWithSessionAffinityEnabledAndPreviousCAVRemovedButDRDeletionFailed(t *testing.T) {
+	err := reconcileTestItem(
+		context.TODO(), t,
+		QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}},
+		TestData{
+			description: "captenant upgraded - expecting virtual service, destination rule adjustments after removing previous cav v1 but DR deletion fails for some reason",
+			initialResources: []string{
+				"testdata/common/domain-ready.yaml",
+				"testdata/common/cluster-domain-ready.yaml",
+				"testdata/common/capapplication-session-affinity.yaml",
+				"testdata/common/capapplicationversion-v2.yaml",
+				"testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml",
+			},
+			mockErrorForResources: []ResourceAction{{Verb: "delete", Group: "networking.istio.io", Version: "v1", Resource: "destinationrules", Namespace: "*", Name: "test-cap-01-provider-test-cap-01-cav-v1"}},
+			expectError:           true,
+		},
+	)
+
+	if err.Error() != "mocked api error (destinationrules.networking.istio.io/v1)" {
+		t.Error("error message is different from expected, actual:", err.Error())
+	}
+}