diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index e62693479a..c8a5b132b6 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -98,6 +98,10 @@ func setup() { if err != nil { panic(fmt.Sprintf("Unable to setup ClusterCache: %v", err)) } + go func() { + <-ctx.Done() + clusterCache.(interface{ Shutdown() }).Shutdown() + }() controllerOpts := controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: ptr.To(true)} diff --git a/controllers/vmware/controllers_suite_test.go b/controllers/vmware/controllers_suite_test.go index 55bdaa7781..330e1518eb 100644 --- a/controllers/vmware/controllers_suite_test.go +++ b/controllers/vmware/controllers_suite_test.go @@ -17,6 +17,7 @@ limitations under the License. package vmware import ( + "context" "fmt" "os" "path/filepath" @@ -60,18 +61,18 @@ var ( ) func TestMain(m *testing.M) { - setup() + testEnv, clusterCache = setup(ctx) code := m.Run() teardown() os.Exit(code) } -func setup() { +func setup(ctx context.Context) (*helpers.TestEnvironment, clustercache.ClusterCache) { utilruntime.Must(infrav1.AddToScheme(scheme.Scheme)) utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) utilruntime.Must(vmwarev1.AddToScheme(scheme.Scheme)) - testEnv = helpers.NewTestEnvironment(ctx) + testEnv := helpers.NewTestEnvironment(ctx) secretCachingClient, err := client.New(testEnv.Manager.GetConfig(), client.Options{ HTTPClient: testEnv.Manager.GetHTTPClient(), @@ -99,6 +100,10 @@ func setup() { if err != nil { panic(fmt.Sprintf("Unable to setup ClusterCache: %v", err)) } + go func() { + <-ctx.Done() + clusterCache.(interface{ Shutdown() }).Shutdown() + }() controllerOpts := controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: ptr.To(true)} @@ -127,8 +132,10 @@ func setup() { }, } if err := testEnv.Create(ctx, ns); err != nil { - panic("unable to create controller namespace") + panic(fmt.Sprintf("unable to create controller namespace: %v", err)) } + + return testEnv, clusterCache } func teardown() { diff --git a/controllers/vmware/serviceaccount_controller_intg_test.go b/controllers/vmware/serviceaccount_controller_intg_test.go index 7cd87e8518..54ffe915a0 100644 --- a/controllers/vmware/serviceaccount_controller_intg_test.go +++ b/controllers/vmware/serviceaccount_controller_intg_test.go @@ -17,6 +17,7 @@ limitations under the License. package vmware import ( + "context" "fmt" "reflect" "time" @@ -30,23 +31,40 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/controllers/clustercache" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" - helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" + "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers" + vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" ) var _ = Describe("ProviderServiceAccount controller integration tests", func() { - var intCtx *helpers.IntegrationTestContext + // This test suite requires its own management cluster and controllers including clustercache. + // Otherwise the workload cluster's kube-apiserver would not shutdown due to the global + // test's clustercache still being running. + var ( + intCtx *vmwarehelpers.IntegrationTestContext + testEnv *helpers.TestEnvironment + clusterCache clustercache.ClusterCache + clusterCacheCancel context.CancelFunc + ) BeforeEach(func() { - intCtx = helpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient()) + var clusterCacheCtx context.Context + clusterCacheCtx, clusterCacheCancel = context.WithCancel(ctx) + testEnv, clusterCache = setup(clusterCacheCtx) + intCtx = vmwarehelpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient()) }) AfterEach(func() { + // Stop clustercache + clusterCacheCancel() + intCtx.AfterEach() + Expect(testEnv.Stop()).To(Succeed()) }) Describe("When the ProviderServiceAccount is created", func() { @@ -56,10 +74,10 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() { ) BeforeEach(func() { By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() { - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) - helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) + vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster) }) By("Verifying that the guest cluster client works") @@ -142,8 +160,8 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() { Context("With non-existent Cluster object", func() { It("cannot reconcile the ProviderServiceAccount object", func() { By("Creating the vSphereCluster and KubeconfigSecret only", func() { - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) }) By("Creating the ProviderServiceAccount", func() { @@ -164,8 +182,8 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() { Context("With non-existent Cluster credentials secret", func() { It("cannot reconcile the ProviderServiceAccount object", func() { By("Creating the Cluster and vSphereCluster only", func() { - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) }) By("Creating the ProviderServiceAccount", func() { @@ -189,10 +207,10 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() { var roleBinding *rbacv1.RoleBinding BeforeEach(func() { By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() { - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) - helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) + vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster) }) pSvcAccount = getTestProviderServiceAccount(intCtx.Namespace, intCtx.VSphereCluster) pSvcAccount.Spec.TargetNamespace = "default" diff --git a/controllers/vmware/serviceaccount_controller_unit_test.go b/controllers/vmware/serviceaccount_controller_unit_test.go index 531bc838c6..2ff38234f1 100644 --- a/controllers/vmware/serviceaccount_controller_unit_test.go +++ b/controllers/vmware/serviceaccount_controller_unit_test.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" - helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" + vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake" ) @@ -35,7 +35,7 @@ var _ = Describe("ServiceAccountReconciler reconcileNormal", unitTestsReconcileN func unitTestsReconcileNormal() { var ( - controllerCtx *helpers.UnitTestContextForController + controllerCtx *vmwarehelpers.UnitTestContextForController vsphereCluster *vmwarev1.VSphereCluster initObjects []client.Object namespace string @@ -43,7 +43,7 @@ func unitTestsReconcileNormal() { ) JustBeforeEach(func() { - controllerCtx = helpers.NewUnitTestContextForController(ctx, namespace, vsphereCluster, false, initObjects, nil) + controllerCtx = vmwarehelpers.NewUnitTestContextForController(ctx, namespace, vsphereCluster, false, initObjects, nil) // Note: The service account provider requires a reference to the vSphereCluster hence the need to create // a fake vSphereCluster in the test and pass it to during context setup. reconciler = ServiceAccountReconciler{ @@ -128,7 +128,7 @@ func unitTestsReconcileNormal() { // Updates the service account secret similar to how a token controller would act upon a service account // and then re-invokes reconcileNormal. -func updateServiceAccountSecretAndReconcileNormal(ctx context.Context, controllerCtx *helpers.UnitTestContextForController, reconciler ServiceAccountReconciler, object client.Object) { +func updateServiceAccountSecretAndReconcileNormal(ctx context.Context, controllerCtx *vmwarehelpers.UnitTestContextForController, reconciler ServiceAccountReconciler, object client.Object) { assertServiceAccountAndUpdateSecret(ctx, controllerCtx.ControllerManagerContext.Client, object.GetNamespace(), object.GetName()) _, err := reconciler.reconcileNormal(ctx, controllerCtx.GuestClusterContext) Expect(err).NotTo(HaveOccurred()) diff --git a/controllers/vmware/servicediscovery_controller_intg_test.go b/controllers/vmware/servicediscovery_controller_intg_test.go index 170743ec00..50e60e46f7 100644 --- a/controllers/vmware/servicediscovery_controller_intg_test.go +++ b/controllers/vmware/servicediscovery_controller_intg_test.go @@ -17,6 +17,7 @@ limitations under the License. package vmware import ( + "context" "fmt" "time" @@ -24,23 +25,35 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/cluster-api/controllers/clustercache" "sigs.k8s.io/controller-runtime/pkg/client" - helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" + "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers" + vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" ) var _ = Describe("Service Discovery controller integration tests", func() { + // This test suite requires its own management cluster and controllers including clustercache. + // Otherwise the workload cluster's kube-apiserver would not shutdown due to the global + // test's clustercache still being running. var ( - intCtx *helpers.IntegrationTestContext - initObjects []client.Object + intCtx *vmwarehelpers.IntegrationTestContext + testEnv *helpers.TestEnvironment + clusterCache clustercache.ClusterCache + clusterCacheCancel context.CancelFunc + initObjects []client.Object ) BeforeEach(func() { - intCtx = helpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient()) + var clusterCacheCtx context.Context + clusterCacheCtx, clusterCacheCancel = context.WithCancel(ctx) + testEnv, clusterCache = setup(clusterCacheCtx) + intCtx = vmwarehelpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient()) + By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() { - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) - helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) - helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster) + vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret) + vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster) }) By("Verifying that the guest cluster client works") @@ -54,10 +67,14 @@ var _ = Describe("Service Discovery controller integration tests", func() { Expect(guestClient.List(ctx, &corev1.ServiceList{}, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) }) AfterEach(func() { + // Stop clustercache + clusterCacheCancel() + deleteTestResource(ctx, intCtx.Client, intCtx.VSphereCluster) deleteTestResource(ctx, intCtx.Client, intCtx.Cluster) deleteTestResource(ctx, intCtx.Client, intCtx.KubeconfigSecret) intCtx.AfterEach() + Expect(testEnv.Stop()).To(Succeed()) }) Context("When VIP is available", func() { diff --git a/controllers/vmware/servicediscovery_controller_unit_test.go b/controllers/vmware/servicediscovery_controller_unit_test.go index c2f28ac62f..5a2a6dc33a 100644 --- a/controllers/vmware/servicediscovery_controller_unit_test.go +++ b/controllers/vmware/servicediscovery_controller_unit_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" - helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" + vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake" ) @@ -34,7 +34,7 @@ var _ = Describe("ServiceDiscoveryReconciler reconcileNormal", serviceDiscoveryU func serviceDiscoveryUnitTestsReconcileNormal() { var ( - controllerCtx *helpers.UnitTestContextForController + controllerCtx *vmwarehelpers.UnitTestContextForController vsphereCluster vmwarev1.VSphereCluster initObjects []client.Object reconciler serviceDiscoveryReconciler @@ -42,7 +42,7 @@ func serviceDiscoveryUnitTestsReconcileNormal() { namespace := capiutil.RandomString(6) JustBeforeEach(func() { vsphereCluster = fake.NewVSphereCluster(namespace) - controllerCtx = helpers.NewUnitTestContextForController(ctx, namespace, &vsphereCluster, false, initObjects, nil) + controllerCtx = vmwarehelpers.NewUnitTestContextForController(ctx, namespace, &vsphereCluster, false, initObjects, nil) reconciler = serviceDiscoveryReconciler{ Client: controllerCtx.ControllerManagerContext.Client, } diff --git a/internal/test/helpers/envtest.go b/internal/test/helpers/envtest.go index cb91e02fbb..3c8ec04eb4 100644 --- a/internal/test/helpers/envtest.go +++ b/internal/test/helpers/envtest.go @@ -78,8 +78,8 @@ func init() { } var ( - scheme = runtime.NewScheme() - env *envtest.Environment + scheme = runtime.NewScheme() + crdPaths []string ) func init() { @@ -97,7 +97,7 @@ func init() { } root := path.Join(path.Dir(filename), "..", "..", "..") - crdPaths := []string{ + crdPaths = []string{ filepath.Join(root, "config", "default", "crd", "bases"), filepath.Join(root, "config", "supervisor", "crd", "bases"), } @@ -106,12 +106,6 @@ func init() { if capiPaths := getFilePathToCAPICRDs(); capiPaths != nil { crdPaths = append(crdPaths, capiPaths...) } - - // Create the test environment. - env = &envtest.Environment{ - ErrorIfCRDPathMissing: true, - CRDDirectoryPaths: crdPaths, - } } var ( @@ -130,6 +124,7 @@ type ( client.Client Config *rest.Config Simulator *vcsim.Simulator + env *envtest.Environment cancel context.CancelFunc } @@ -137,6 +132,12 @@ type ( // NewTestEnvironment creates a new environment spinning up a local api-server. func NewTestEnvironment(ctx context.Context) *TestEnvironment { + // Create the test environment. + env := &envtest.Environment{ + ErrorIfCRDPathMissing: true, + CRDDirectoryPaths: crdPaths, + } + // Get the root of the current file to use in CRD paths. _, filename, _, ok := goruntime.Caller(0) if !ok { @@ -227,6 +228,7 @@ func NewTestEnvironment(ctx context.Context) *TestEnvironment { Client: mgr.GetClient(), Config: mgr.GetConfig(), Simulator: simr, + env: env, } } @@ -241,7 +243,7 @@ func (t *TestEnvironment) StartManager(ctx context.Context) error { func (t *TestEnvironment) Stop() error { t.cancel() t.Simulator.Destroy() - return env.Stop() + return t.env.Stop() } // Cleanup removes objects from the TestEnvironment. diff --git a/internal/test/helpers/webhook.go b/internal/test/helpers/webhook.go index eac4c338b1..af46b9ddde 100644 --- a/internal/test/helpers/webhook.go +++ b/internal/test/helpers/webhook.go @@ -101,7 +101,7 @@ func InitializeWebhookInEnvironment(e *envtest.Environment, configPath string) { // WaitForWebhooks waits for TestEnvironment's webhooks to accept connections. func (t *TestEnvironment) WaitForWebhooks() { - port := env.WebhookInstallOptions.LocalServingPort + port := t.env.WebhookInstallOptions.LocalServingPort klog.Infof("Waiting for webhook port %d to be open prior to running tests", port) timeout := 1 * time.Second