Skip to content

Commit

Permalink
test: enable vcsim to run clusterctl upgrade e2e tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrischdi
chrischdi committed Jul 18, 2024
1 parent 4a7d0ff commit f9e8ae1
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 93 deletions.
8 changes: 4 additions & 4 deletions test/e2e/clusterctl_upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ var (
capvReleaseMarkerPrefix = "go://sigs.k8s.io/cluster-api-provider-vsphere@v%s"
)

var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10=>current, CAPI 1.7=>1.8) [supervisor] [ClusterClass]", func() {
var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10=>current, CAPI 1.7=>1.8) [vcsim] [supervisor] [ClusterClass]", func() {
const specName = "clusterctl-upgrade-1.10-current" // prefix (clusterctl-upgrade) copied from CAPI
Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
Expand Down Expand Up @@ -71,10 +71,10 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10
KindManagementClusterNewClusterProxyFunc: kindManagementClusterNewClusterProxyFunc,
}
})
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"))
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"), WithUseKindForManagementCluster())
})

var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=>current, CAPI 1.6=>1.8) [supervisor] [ClusterClass]", func() {
var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=>current, CAPI 1.6=>1.8) [vcsim] [supervisor] [ClusterClass]", func() {
const specName = "clusterctl-upgrade-1.9-current" // prefix (clusterctl-upgrade) copied from CAPI
Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
Expand Down Expand Up @@ -109,7 +109,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=
KindManagementClusterNewClusterProxyFunc: kindManagementClusterNewClusterProxyFunc,
}
})
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"))
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"), WithUseKindForManagementCluster())
})

// getStableReleaseOfMinor returns the latest stable version of minorRelease.
Expand Down
179 changes: 108 additions & 71 deletions test/e2e/e2e_setup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,10 @@ import (
)

type setupOptions struct {
additionalIPVariableNames []string
gatewayIPVariableName string
prefixVariableName string
additionalIPVariableNames []string
gatewayIPVariableName string
prefixVariableName string
useKindForManagementCluster bool
}

// SetupOption is a configuration option supplied to Setup.
Expand Down Expand Up @@ -73,6 +74,13 @@ func WithPrefix(variableName string) SetupOption {
}
}

// WithUseKindForManagementCluster instructs Setup to run extra steps for the separate kind management cluster.
func WithUseKindForManagementCluster() SetupOption {
return func(o *setupOptions) {
o.useKindForManagementCluster = true
}
}

type testSettings struct {
ClusterctlConfigPath string
Variables map[string]string
Expand All @@ -91,81 +99,111 @@ func Setup(specName string, f func(testSpecificSettings func() testSettings), op
var (
testSpecificClusterctlConfigPath string
testSpecificIPAddressClaims vsphereip.AddressClaims
testSpecificIPAddressManager vsphereip.AddressManager
testSpecificVariables map[string]string
postNamespaceCreatedFunc func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
runtimeExtensionProviders []string
)
BeforeEach(func() {
Byf("Setting up test env for %s", specName)
switch testTarget {
case VCenterTestTarget:
Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
// get IPs from the in cluster address manager
testSpecificIPAddressClaims, testSpecificVariables = inClusterAddressManager.ClaimIPs(ctx, vsphereip.WithGateway(options.gatewayIPVariableName), vsphereip.WithPrefix(options.prefixVariableName), vsphereip.WithIP(options.additionalIPVariableNames...))
case VCSimTestTarget:
c := bootstrapClusterProxy.GetClient()

// get IPs from the vcsim controller
// NOTE: ControlPlaneEndpointIP is the first claim in the returned list (this assumption is used below).
Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
testSpecificIPAddressClaims, testSpecificVariables = vcsimAddressManager.ClaimIPs(ctx, vsphereip.WithIP(options.additionalIPVariableNames...))

// variables derived from the vCenterSimulator
vCenterSimulator, err := vspherevcsim.Get(ctx, bootstrapClusterProxy.GetClient())
Expect(err).ToNot(HaveOccurred(), "Failed to get VCenterSimulator")

Byf("Creating EnvVar %s", klog.KRef(metav1.NamespaceDefault, specName))
envVar := &vcsimv1.EnvVar{
ObjectMeta: metav1.ObjectMeta{
Name: specName,
Namespace: metav1.NamespaceDefault,
},
Spec: vcsimv1.EnvVarSpec{
VCenterSimulator: &vcsimv1.NamespacedRef{
Namespace: vCenterSimulator.Namespace,
Name: vCenterSimulator.Name,

postNamespaceCreatedFunc = func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) {
additionalTestSpecificVariables := map[string]string{}
switch testTarget {
case VCenterTestTarget:
testSpecificIPAddressManager = inClusterAddressManager

Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
// get IPs from the in cluster address manager
testSpecificIPAddressClaims, additionalTestSpecificVariables = testSpecificIPAddressManager.ClaimIPs(ctx, vsphereip.WithGateway(options.gatewayIPVariableName), vsphereip.WithPrefix(options.prefixVariableName), vsphereip.WithIP(options.additionalIPVariableNames...))

case VCSimTestTarget:
testSpecificIPAddressManager = vcsimAddressManager
if options.useKindForManagementCluster {
Byf("Creating a vcsim server")
Eventually(func() error {
return vspherevcsim.Create(ctx, managementClusterProxy.GetClient())
}, time.Minute, 3*time.Second).ShouldNot(HaveOccurred(), "Failed to create VCenterSimulator")

var err error
testSpecificIPAddressManager, err = vsphereip.VCSIMAddressManager(managementClusterProxy.GetClient(), map[string]string{}, skipCleanup)
Expect(err).ToNot(HaveOccurred())
}

// get IPs from the vcsim controller
// NOTE: ControlPlaneEndpointIP is the first claim in the returned list (this assumption is used below).
Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
testSpecificIPAddressClaims, additionalTestSpecificVariables = testSpecificIPAddressManager.ClaimIPs(ctx, vsphereip.WithIP(options.additionalIPVariableNames...))

// variables derived from the vCenterSimulator
vCenterSimulator, err := vspherevcsim.Get(ctx, managementClusterProxy.GetClient())
Expect(err).ToNot(HaveOccurred(), "Failed to get VCenterSimulator")

Byf("Creating EnvVar %s", klog.KRef(metav1.NamespaceDefault, specName))
envVar := &vcsimv1.EnvVar{
ObjectMeta: metav1.ObjectMeta{
Name: specName,
Namespace: metav1.NamespaceDefault,
},
ControlPlaneEndpoint: vcsimv1.NamespacedRef{
Namespace: testSpecificIPAddressClaims[0].Namespace,
Name: testSpecificIPAddressClaims[0].Name,
Spec: vcsimv1.EnvVarSpec{
VCenterSimulator: &vcsimv1.NamespacedRef{
Namespace: vCenterSimulator.Namespace,
Name: vCenterSimulator.Name,
},
ControlPlaneEndpoint: vcsimv1.NamespacedRef{
Namespace: testSpecificIPAddressClaims[0].Namespace,
Name: testSpecificIPAddressClaims[0].Name,
},
// NOTE: we are omitting VMOperatorDependencies because it is not created yet (it will be created by the PostNamespaceCreated hook)
// But this is not a issue because a default dependenciesConfig that works for vcsim will be automatically used.
},
// NOTE: we are omitting VMOperatorDependencies because it is not created yet (it will be created by the PostNamespaceCreated hook)
// But this is not a issue because a default dependenciesConfig that works for vcsim will be automatically used.
},
}
}

err = c.Create(ctx, envVar)
Expect(err).ToNot(HaveOccurred(), "Failed to create EnvVar")
err = managementClusterProxy.GetClient().Create(ctx, envVar)
Expect(err).ToNot(HaveOccurred(), "Failed to create EnvVar")

Eventually(func() bool {
if err := c.Get(ctx, crclient.ObjectKeyFromObject(envVar), envVar); err != nil {
return false
}
return len(envVar.Status.Variables) > 0
}, 30*time.Second, 5*time.Second).Should(BeTrue(), "Failed to get EnvVar %s", klog.KObj(envVar))

Byf("Setting test variables for %s", specName)
for k, v := range envVar.Status.Variables {
// ignore variables that will be set later on by the test
if sets.New("NAMESPACE", "CLUSTER_NAME", "KUBERNETES_VERSION", "CONTROL_PLANE_MACHINE_COUNT", "WORKER_MACHINE_COUNT").Has(k) {
continue
}
Eventually(func() bool {
if err := managementClusterProxy.GetClient().Get(ctx, crclient.ObjectKeyFromObject(envVar), envVar); err != nil {
return false
}
return len(envVar.Status.Variables) > 0
}, 30*time.Second, 5*time.Second).Should(BeTrue(), "Failed to get EnvVar %s", klog.KObj(envVar))

Byf("Setting test variables for %s", specName)
for k, v := range envVar.Status.Variables {
// ignore variables that will be set later on by the test
if sets.New("NAMESPACE", "CLUSTER_NAME", "KUBERNETES_VERSION", "CONTROL_PLANE_MACHINE_COUNT", "WORKER_MACHINE_COUNT").Has(k) {
continue
}

// unset corresponding env variable (that in CI contains VMC data), so we are sure we use the value for vcsim
if strings.HasPrefix(k, "VSPHERE_") {
Expect(os.Unsetenv(k)).To(Succeed())
// unset corresponding env variable (that in CI contains VMC data), so we are sure we use the value for vcsim
if strings.HasPrefix(k, "VSPHERE_") {
Expect(os.Unsetenv(k)).To(Succeed())
}

additionalTestSpecificVariables[k] = v
}
}

testSpecificVariables[k] = v
// re-write the clusterctl config file and add the new variables for the IPs.
testSpecificClusterctlConfigPath = fmt.Sprintf("%s-%s.yaml", strings.TrimSuffix(clusterctlConfigPath, ".yaml"), specName)
Byf("Writing a new clusterctl config to %s", testSpecificClusterctlConfigPath)
copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{
ClusterctlConfigPath: testSpecificClusterctlConfigPath,
OutputPath: testSpecificClusterctlConfigPath,
Variables: additionalTestSpecificVariables,
})

if testMode == SupervisorTestMode {
switch testTarget {
case VCenterTestTarget:
setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy, workloadClusterNamespace)
case VCSimTestTarget:
setupNamespaceWithVMOperatorDependenciesVCSim(managementClusterProxy, workloadClusterNamespace)
}
}
}

if testMode == SupervisorTestMode {
postNamespaceCreatedFunc = setupNamespaceWithVMOperatorDependenciesVCenter
if testTarget == VCSimTestTarget {
postNamespaceCreatedFunc = setupNamespaceWithVMOperatorDependenciesVCSim
}

if testSpecificVariables == nil {
testSpecificVariables = map[string]string{}
}
Expand Down Expand Up @@ -194,14 +232,13 @@ func Setup(specName string, f func(testSpecificSettings func() testSettings), op
}
})
defer AfterEach(func() {
Byf("Cleaning up test env for %s", specName)
switch testTarget {
case VCenterTestTarget:
// cleanup IPs/controlPlaneEndpoint created by the in cluster ipam provider.
Expect(inClusterAddressManager.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
case VCSimTestTarget:
// cleanup IPs/controlPlaneEndpoint created by the vcsim controller manager.
Expect(vcsimAddressManager.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
if !skipCleanup {
Byf("Cleaning up test env for %s", specName)
// We can't cleanup when a kind management cluster is used, because it won't exist anymore.
if !options.useKindForManagementCluster {
// cleanup IPs/controlPlaneEndpoint created by the IPAddressManager.
Expect(testSpecificIPAddressManager.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
}
}
})

Expand Down Expand Up @@ -232,7 +269,7 @@ func Setup(specName string, f func(testSpecificSettings func() testSettings), op
func setupNamespaceWithVMOperatorDependenciesVCSim(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) {
c := managementClusterProxy.GetClient()

vCenterSimulator, err := vspherevcsim.Get(ctx, bootstrapClusterProxy.GetClient())
vCenterSimulator, err := vspherevcsim.Get(ctx, c)
Expect(err).ToNot(HaveOccurred(), "Failed to get VCenterSimulator")

Byf("Creating VMOperatorDependencies %s", klog.KRef(workloadClusterNamespace, "vcsim"))
Expand Down
37 changes: 19 additions & 18 deletions test/e2e/ownerrefs_finalizers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,31 +53,32 @@ var _ = Describe("Ensure OwnerReferences and Finalizers are resilient [vcsim] [s
const specName = "owner-reference"
Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput {
if testMode == GovmomiTestMode {
// NOTE: When testing with vcsim VSPHERE_USERNAME and VSPHERE_PASSWORD are provided as a test specific variables,
// when running on CI same variables are provided as env variables.
input := testSpecificSettingsGetter()
username, ok := input.Variables["VSPHERE_USERNAME"]
if !ok {
username = os.Getenv("VSPHERE_USERNAME")
}
password, ok := input.Variables["VSPHERE_PASSWORD"]
if !ok {
password = os.Getenv("VSPHERE_PASSWORD")
}

// Before running the test create the secret used by the VSphereClusterIdentity to connect to the vCenter.
createVsphereIdentitySecret(ctx, bootstrapClusterProxy, username, password)
}

return capi_e2e.QuickStartSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: testSpecificSettingsGetter().ClusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("ownerrefs-finalizers")),
PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc,
PostNamespaceCreated: func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) {
testSpecificSettingsGetter().PostNamespaceCreatedFunc(managementClusterProxy, workloadClusterNamespace)
if testMode == GovmomiTestMode {
// NOTE: When testing with vcsim VSPHERE_USERNAME and VSPHERE_PASSWORD are provided as a test specific variables,
// when running on CI same variables are provided as env variables.
input := testSpecificSettingsGetter()
username, ok := input.Variables["VSPHERE_USERNAME"]
if !ok {
username = os.Getenv("VSPHERE_USERNAME")
}
password, ok := input.Variables["VSPHERE_PASSWORD"]
if !ok {
password = os.Getenv("VSPHERE_PASSWORD")
}

// Before running the test create the secret used by the VSphereClusterIdentity to connect to the vCenter.
createVsphereIdentitySecret(ctx, bootstrapClusterProxy, username, password)
}
},
PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
forceCtx, forceCancelFunc := context.WithCancel(ctx)
if testMode == GovmomiTestMode {
Expand Down

0 comments on commit f9e8ae1

Please sign in to comment.