Skip to content

Commit

Permalink
🌱 test: shutdown clustercache when ctx is done (#3344)
Browse files Browse the repository at this point in the history 
* test: shutdown clustercache when ctx is done

* use separate testenv for tests using clustercache

* fixup naming

* Properly turn off testenvs

* fixup
  • Loading branch information
@chrischdi
chrischdi authored Feb 7, 2025
1 parent 411ccc3 commit 4d91440
Show file tree
Hide file tree
Showing 8 changed files with 93 additions and 45 deletions.
4 changes: 4 additions & 0 deletions controllers/controllers_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ func setup() {
if err != nil {
panic(fmt.Sprintf("Unable to setup ClusterCache: %v", err))
}
go func() {
<-ctx.Done()
clusterCache.(interface{ Shutdown() }).Shutdown()
}()

controllerOpts := controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: ptr.To(true)}

Expand Down
15 changes: 11 additions & 4 deletions controllers/vmware/controllers_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package vmware

import (
"context"
"fmt"
"os"
"path/filepath"
Expand Down Expand Up @@ -60,18 +61,18 @@ var (
)

func TestMain(m *testing.M) {
setup()
testEnv, clusterCache = setup(ctx)
code := m.Run()
teardown()
os.Exit(code)
}

func setup() {
func setup(ctx context.Context) (*helpers.TestEnvironment, clustercache.ClusterCache) {
utilruntime.Must(infrav1.AddToScheme(scheme.Scheme))
utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme))
utilruntime.Must(vmwarev1.AddToScheme(scheme.Scheme))

testEnv = helpers.NewTestEnvironment(ctx)
testEnv := helpers.NewTestEnvironment(ctx)

secretCachingClient, err := client.New(testEnv.Manager.GetConfig(), client.Options{
HTTPClient: testEnv.Manager.GetHTTPClient(),
Expand Down Expand Up @@ -99,6 +100,10 @@ func setup() {
if err != nil {
panic(fmt.Sprintf("Unable to setup ClusterCache: %v", err))
}
go func() {
<-ctx.Done()
clusterCache.(interface{ Shutdown() }).Shutdown()
}()

controllerOpts := controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: ptr.To(true)}

Expand Down Expand Up @@ -127,8 +132,10 @@ func setup() {
},
}
if err := testEnv.Create(ctx, ns); err != nil {
panic("unable to create controller namespace")
panic(fmt.Sprintf("unable to create controller namespace: %v", err))
}

return testEnv, clusterCache
}

func teardown() {
Expand Down
48 changes: 33 additions & 15 deletions controllers/vmware/serviceaccount_controller_intg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package vmware

import (
"context"
"fmt"
"reflect"
"time"
Expand All @@ -30,23 +31,40 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
"sigs.k8s.io/cluster-api/controllers/clustercache"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/controller-runtime/pkg/client"

vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
"sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers"
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
)

var _ = Describe("ProviderServiceAccount controller integration tests", func() {
var intCtx *helpers.IntegrationTestContext
// This test suite requires its own management cluster and controllers including clustercache.
// Otherwise the workload cluster's kube-apiserver would not shutdown due to the global
// test's clustercache still being running.
var (
intCtx *vmwarehelpers.IntegrationTestContext
testEnv *helpers.TestEnvironment
clusterCache clustercache.ClusterCache
clusterCacheCancel context.CancelFunc
)

BeforeEach(func() {
intCtx = helpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())
var clusterCacheCtx context.Context
clusterCacheCtx, clusterCacheCancel = context.WithCancel(ctx)
testEnv, clusterCache = setup(clusterCacheCtx)
intCtx = vmwarehelpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())
})

AfterEach(func() {
// Stop clustercache
clusterCacheCancel()

intCtx.AfterEach()
Expect(testEnv.Stop()).To(Succeed())
})

Describe("When the ProviderServiceAccount is created", func() {
Expand All @@ -56,10 +74,10 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
)
BeforeEach(func() {
By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() {
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
})

By("Verifying that the guest cluster client works")
Expand Down Expand Up @@ -142,8 +160,8 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
Context("With non-existent Cluster object", func() {
It("cannot reconcile the ProviderServiceAccount object", func() {
By("Creating the vSphereCluster and KubeconfigSecret only", func() {
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
})

By("Creating the ProviderServiceAccount", func() {
Expand All @@ -164,8 +182,8 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
Context("With non-existent Cluster credentials secret", func() {
It("cannot reconcile the ProviderServiceAccount object", func() {
By("Creating the Cluster and vSphereCluster only", func() {
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
})

By("Creating the ProviderServiceAccount", func() {
Expand All @@ -189,10 +207,10 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
var roleBinding *rbacv1.RoleBinding
BeforeEach(func() {
By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() {
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
})
pSvcAccount = getTestProviderServiceAccount(intCtx.Namespace, intCtx.VSphereCluster)
pSvcAccount.Spec.TargetNamespace = "default"
Expand Down
8 changes: 4 additions & 4 deletions controllers/vmware/serviceaccount_controller_unit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,23 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"

vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake"
)

var _ = Describe("ServiceAccountReconciler reconcileNormal", unitTestsReconcileNormal)

func unitTestsReconcileNormal() {
var (
controllerCtx *helpers.UnitTestContextForController
controllerCtx *vmwarehelpers.UnitTestContextForController
vsphereCluster *vmwarev1.VSphereCluster
initObjects []client.Object
namespace string
reconciler ServiceAccountReconciler
)

JustBeforeEach(func() {
controllerCtx = helpers.NewUnitTestContextForController(ctx, namespace, vsphereCluster, false, initObjects, nil)
controllerCtx = vmwarehelpers.NewUnitTestContextForController(ctx, namespace, vsphereCluster, false, initObjects, nil)
// Note: The service account provider requires a reference to the vSphereCluster hence the need to create
// a fake vSphereCluster in the test and pass it to during context setup.
reconciler = ServiceAccountReconciler{
Expand Down Expand Up @@ -128,7 +128,7 @@ func unitTestsReconcileNormal() {

// Updates the service account secret similar to how a token controller would act upon a service account
// and then re-invokes reconcileNormal.
func updateServiceAccountSecretAndReconcileNormal(ctx context.Context, controllerCtx *helpers.UnitTestContextForController, reconciler ServiceAccountReconciler, object client.Object) {
func updateServiceAccountSecretAndReconcileNormal(ctx context.Context, controllerCtx *vmwarehelpers.UnitTestContextForController, reconciler ServiceAccountReconciler, object client.Object) {
assertServiceAccountAndUpdateSecret(ctx, controllerCtx.ControllerManagerContext.Client, object.GetNamespace(), object.GetName())
_, err := reconciler.reconcileNormal(ctx, controllerCtx.GuestClusterContext)
Expect(err).NotTo(HaveOccurred())
Expand Down
33 changes: 25 additions & 8 deletions controllers/vmware/servicediscovery_controller_intg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,43 @@ limitations under the License.
package vmware

import (
"context"
"fmt"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/cluster-api/controllers/clustercache"
"sigs.k8s.io/controller-runtime/pkg/client"

helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
"sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers"
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
)

var _ = Describe("Service Discovery controller integration tests", func() {
// This test suite requires its own management cluster and controllers including clustercache.
// Otherwise the workload cluster's kube-apiserver would not shutdown due to the global
// test's clustercache still being running.
var (
intCtx *helpers.IntegrationTestContext
initObjects []client.Object
intCtx *vmwarehelpers.IntegrationTestContext
testEnv *helpers.TestEnvironment
clusterCache clustercache.ClusterCache
clusterCacheCancel context.CancelFunc
initObjects []client.Object
)
BeforeEach(func() {
intCtx = helpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())
var clusterCacheCtx context.Context
clusterCacheCtx, clusterCacheCancel = context.WithCancel(ctx)
testEnv, clusterCache = setup(clusterCacheCtx)
intCtx = vmwarehelpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())

By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() {
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
})

By("Verifying that the guest cluster client works")
Expand All @@ -54,10 +67,14 @@ var _ = Describe("Service Discovery controller integration tests", func() {
Expect(guestClient.List(ctx, &corev1.ServiceList{}, client.InNamespace(metav1.NamespaceDefault))).To(Succeed())
})
AfterEach(func() {
// Stop clustercache
clusterCacheCancel()

deleteTestResource(ctx, intCtx.Client, intCtx.VSphereCluster)
deleteTestResource(ctx, intCtx.Client, intCtx.Cluster)
deleteTestResource(ctx, intCtx.Client, intCtx.KubeconfigSecret)
intCtx.AfterEach()
Expect(testEnv.Stop()).To(Succeed())
})

Context("When VIP is available", func() {
Expand Down
6 changes: 3 additions & 3 deletions controllers/vmware/servicediscovery_controller_unit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,23 +26,23 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"

vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake"
)

var _ = Describe("ServiceDiscoveryReconciler reconcileNormal", serviceDiscoveryUnitTestsReconcileNormal)

func serviceDiscoveryUnitTestsReconcileNormal() {
var (
controllerCtx *helpers.UnitTestContextForController
controllerCtx *vmwarehelpers.UnitTestContextForController
vsphereCluster vmwarev1.VSphereCluster
initObjects []client.Object
reconciler serviceDiscoveryReconciler
)
namespace := capiutil.RandomString(6)
JustBeforeEach(func() {
vsphereCluster = fake.NewVSphereCluster(namespace)
controllerCtx = helpers.NewUnitTestContextForController(ctx, namespace, &vsphereCluster, false, initObjects, nil)
controllerCtx = vmwarehelpers.NewUnitTestContextForController(ctx, namespace, &vsphereCluster, false, initObjects, nil)
reconciler = serviceDiscoveryReconciler{
Client: controllerCtx.ControllerManagerContext.Client,
}
Expand Down
22 changes: 12 additions & 10 deletions internal/test/helpers/envtest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ func init() {
}

var (
scheme = runtime.NewScheme()
env *envtest.Environment
scheme = runtime.NewScheme()
crdPaths []string
)

func init() {
Expand All @@ -97,7 +97,7 @@ func init() {
}
root := path.Join(path.Dir(filename), "..", "..", "..")

crdPaths := []string{
crdPaths = []string{
filepath.Join(root, "config", "default", "crd", "bases"),
filepath.Join(root, "config", "supervisor", "crd", "bases"),
}
Expand All @@ -106,12 +106,6 @@ func init() {
if capiPaths := getFilePathToCAPICRDs(); capiPaths != nil {
crdPaths = append(crdPaths, capiPaths...)
}

// Create the test environment.
env = &envtest.Environment{
ErrorIfCRDPathMissing: true,
CRDDirectoryPaths: crdPaths,
}
}

var (
Expand All @@ -130,13 +124,20 @@ type (
client.Client
Config *rest.Config
Simulator *vcsim.Simulator
env *envtest.Environment

cancel context.CancelFunc
}
)

// NewTestEnvironment creates a new environment spinning up a local api-server.
func NewTestEnvironment(ctx context.Context) *TestEnvironment {
// Create the test environment.
env := &envtest.Environment{
ErrorIfCRDPathMissing: true,
CRDDirectoryPaths: crdPaths,
}

// Get the root of the current file to use in CRD paths.
_, filename, _, ok := goruntime.Caller(0)
if !ok {
Expand Down Expand Up @@ -227,6 +228,7 @@ func NewTestEnvironment(ctx context.Context) *TestEnvironment {
Client: mgr.GetClient(),
Config: mgr.GetConfig(),
Simulator: simr,
env: env,
}
}

Expand All @@ -241,7 +243,7 @@ func (t *TestEnvironment) StartManager(ctx context.Context) error {
func (t *TestEnvironment) Stop() error {
t.cancel()
t.Simulator.Destroy()
return env.Stop()
return t.env.Stop()
}

// Cleanup removes objects from the TestEnvironment.
Expand Down
2 changes: 1 addition & 1 deletion internal/test/helpers/webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func InitializeWebhookInEnvironment(e *envtest.Environment, configPath string) {

// WaitForWebhooks waits for TestEnvironment's webhooks to accept connections.
func (t *TestEnvironment) WaitForWebhooks() {
port := env.WebhookInstallOptions.LocalServingPort
port := t.env.WebhookInstallOptions.LocalServingPort

klog.Infof("Waiting for webhook port %d to be open prior to running tests", port)
timeout := 1 * time.Second
Expand Down

0 comments on commit 4d91440

Please sign in to comment.