Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 test: enable vcsim to run clusterctl upgrade e2e tests #3109

Merged
merged 9 commits into from
Jul 23, 2024
Merged

🌱 test: enable vcsim to run clusterctl upgrade e2e tests #3109

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
40c2ffe
test: enable vcsim to run clusterctl upgrade e2e tests
chrischdi Jul 18, 2024
021f855
fix old capv version for clusterctl upgrade
chrischdi Jul 19, 2024
a37eea9
test: adjust templates for v1.9 to allow setting controlPlane port
chrischdi Jul 19, 2024
61e1c23
review fixes
chrischdi Jul 19, 2024
89ef136
test: use postNamespaceCreated func in custom e2e tests
chrischdi Jul 19, 2024
e4d9792
review fixes
chrischdi Jul 19, 2024
0f91fb4
Review fixes
chrischdi Jul 22, 2024
5e73a82
fixup
chrischdi Jul 22, 2024
76d24e2
review fixes
chrischdi Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions test/e2e/clusterctl_upgrade_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ var (
capvReleaseMarkerPrefix = "go://sigs.k8s.io/cluster-api-provider-vsphere@v%s"
)

var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10=>current, CAPI 1.7=>1.8) [supervisor] [ClusterClass]", func() {
var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10=>current, CAPI 1.7=>1.8) [vcsim] [supervisor] [ClusterClass]", func() {
const specName = "clusterctl-upgrade-1.10-current" // prefix (clusterctl-upgrade) copied from CAPI
Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
Expand Down Expand Up @@ -71,10 +71,10 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10
KindManagementClusterNewClusterProxyFunc: kindManagementClusterNewClusterProxyFunc,
}
})
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"))
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"), WithUseKindForManagementCluster())
})

var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=>current, CAPI 1.6=>1.8) [supervisor] [ClusterClass]", func() {
var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=>current, CAPI 1.6=>1.8) [vcsim] [supervisor] [ClusterClass]", func() {
const specName = "clusterctl-upgrade-1.9-current" // prefix (clusterctl-upgrade) copied from CAPI
Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
Expand Down Expand Up @@ -109,7 +109,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=
KindManagementClusterNewClusterProxyFunc: kindManagementClusterNewClusterProxyFunc,
}
})
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"))
}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"), WithUseKindForManagementCluster())
})

// getStableReleaseOfMinor returns the latest stable version of minorRelease.
Expand Down
231 changes: 142 additions & 89 deletions test/e2e/e2e_setup_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,10 @@ import (
)

type setupOptions struct {
additionalIPVariableNames []string
gatewayIPVariableName string
prefixVariableName string
additionalIPVariableNames []string
gatewayIPVariableName string
prefixVariableName string
useKindForManagementCluster bool
}

// SetupOption is a configuration option supplied to Setup.
Expand Down Expand Up @@ -73,6 +74,13 @@ func WithPrefix(variableName string) SetupOption {
}
}

// WithUseKindForManagementCluster instructs Setup to run extra steps for the separate kind management cluster.
func WithUseKindForManagementCluster() SetupOption {
return func(o *setupOptions) {
o.useKindForManagementCluster = true
}
}

type testSettings struct {
ClusterctlConfigPath string
Variables map[string]string
Expand All @@ -91,92 +99,34 @@ func Setup(specName string, f func(testSpecificSettings func() testSettings), op
var (
testSpecificClusterctlConfigPath string
testSpecificIPAddressClaims vsphereip.AddressClaims
testSpecificIPAddressManager vsphereip.AddressManager
testSpecificVariables map[string]string
postNamespaceCreatedFunc func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
runtimeExtensionProviders []string
)
BeforeEach(func() {
Byf("Setting up test env for %s", specName)
switch testTarget {
case VCenterTestTarget:
Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
// get IPs from the in cluster address manager
testSpecificIPAddressClaims, testSpecificVariables = inClusterAddressManager.ClaimIPs(ctx, vsphereip.WithGateway(options.gatewayIPVariableName), vsphereip.WithPrefix(options.prefixVariableName), vsphereip.WithIP(options.additionalIPVariableNames...))
case VCSimTestTarget:
c := bootstrapClusterProxy.GetClient()

// get IPs from the vcsim controller
// NOTE: ControlPlaneEndpointIP is the first claim in the returned list (this assumption is used below).
Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
testSpecificIPAddressClaims, testSpecificVariables = vcsimAddressManager.ClaimIPs(ctx, vsphereip.WithIP(options.additionalIPVariableNames...))

// variables derived from the vCenterSimulator
vCenterSimulator, err := vspherevcsim.Get(ctx, bootstrapClusterProxy.GetClient())
Expect(err).ToNot(HaveOccurred(), "Failed to get VCenterSimulator")

Byf("Creating EnvVar %s", klog.KRef(metav1.NamespaceDefault, specName))
envVar := &vcsimv1.EnvVar{
ObjectMeta: metav1.ObjectMeta{
Name: specName,
Namespace: metav1.NamespaceDefault,
},
Spec: vcsimv1.EnvVarSpec{
VCenterSimulator: &vcsimv1.NamespacedRef{
Namespace: vCenterSimulator.Namespace,
Name: vCenterSimulator.Name,
},
ControlPlaneEndpoint: vcsimv1.NamespacedRef{
Namespace: testSpecificIPAddressClaims[0].Namespace,
Name: testSpecificIPAddressClaims[0].Name,
},
// NOTE: we are omitting VMOperatorDependencies because it is not created yet (it will be created by the PostNamespaceCreated hook)
// But this is not a issue because a default dependenciesConfig that works for vcsim will be automatically used.
},
}

err = c.Create(ctx, envVar)
Expect(err).ToNot(HaveOccurred(), "Failed to create EnvVar")

Eventually(func() bool {
if err := c.Get(ctx, crclient.ObjectKeyFromObject(envVar), envVar); err != nil {
return false
}
return len(envVar.Status.Variables) > 0
}, 30*time.Second, 5*time.Second).Should(BeTrue(), "Failed to get EnvVar %s", klog.KObj(envVar))

Byf("Setting test variables for %s", specName)
for k, v := range envVar.Status.Variables {
// ignore variables that will be set later on by the test
if sets.New("NAMESPACE", "CLUSTER_NAME", "KUBERNETES_VERSION", "CONTROL_PLANE_MACHINE_COUNT", "WORKER_MACHINE_COUNT").Has(k) {
continue
}

// unset corresponding env variable (that in CI contains VMC data), so we are sure we use the value for vcsim
if strings.HasPrefix(k, "VSPHERE_") {
Expect(os.Unsetenv(k)).To(Succeed())
}

testSpecificVariables[k] = v
}
if testSpecificVariables == nil {
testSpecificVariables = map[string]string{}
}

// Update the CLUSTER_CLASS_NAME variable adding the supervisor suffix.
if testMode == SupervisorTestMode {
postNamespaceCreatedFunc = setupNamespaceWithVMOperatorDependenciesVCenter
if testTarget == VCSimTestTarget {
postNamespaceCreatedFunc = setupNamespaceWithVMOperatorDependenciesVCSim
}

if testSpecificVariables == nil {
testSpecificVariables = map[string]string{}
}

// Update the CLUSTER_CLASS_NAME variable adding the supervisor suffix.
if e2eConfig.HasVariable("CLUSTER_CLASS_NAME") {
testSpecificVariables["CLUSTER_CLASS_NAME"] = fmt.Sprintf("%s-supervisor", e2eConfig.GetVariable("CLUSTER_CLASS_NAME"))
}
}

// Create a new clusterctl config file based on the passed file and add the new variables for the IPs.
// Enable additional providers depending on testMode and testTarget.
if testMode == SupervisorTestMode {
runtimeExtensionProviders = append(runtimeExtensionProviders, "vm-operator", "net-operator")
}
if testTarget == VCSimTestTarget {
runtimeExtensionProviders = append(runtimeExtensionProviders, "vcsim")
}

// Create a new clusterctl config file based on the passed file. The postNamespaceCreatedFunc
// may re-write the file to add some variables, but it needs to exist already before that.
testSpecificClusterctlConfigPath = fmt.Sprintf("%s-%s.yaml", strings.TrimSuffix(clusterctlConfigPath, ".yaml"), specName)
Byf("Writing a new clusterctl config to %s", testSpecificClusterctlConfigPath)
copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{
Expand All @@ -185,23 +135,126 @@ func Setup(specName string, f func(testSpecificSettings func() testSettings), op
Variables: testSpecificVariables,
})

if testMode == SupervisorTestMode {
runtimeExtensionProviders = append(runtimeExtensionProviders, "vm-operator", "net-operator")
}
postNamespaceCreatedFunc = func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) {
var ipVariables map[string]string

if testTarget == VCSimTestTarget {
runtimeExtensionProviders = append(runtimeExtensionProviders, "vcsim")
if testTarget == VCSimTestTarget && options.useKindForManagementCluster {
Byf("Creating a vcsim server")
Eventually(func() error {
Copy link
Member

@fabriziopandini fabriziopandini Jul 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

q: why are we creating another VC sim server? is it the one running on the kind cluster?

might be we need to check that it is reconciled as something pops up in status to get sure it is up and running

Copy link
Member Author

@chrischdi chrischdi Jul 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is on the second kind cluster, created by the CAPI part of the clusterctl upgrade test.

return vspherevcsim.Create(ctx, managementClusterProxy.GetClient())
}, time.Minute, 3*time.Second).ShouldNot(HaveOccurred(), "Failed to create VCenterSimulator")
}

// IP Address Management
switch testTarget {
case VCenterTestTarget:
testSpecificIPAddressManager = inClusterAddressManager

Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
// get IPs from the in cluster address manager
testSpecificIPAddressClaims, ipVariables = testSpecificIPAddressManager.ClaimIPs(ctx, vsphereip.WithGateway(options.gatewayIPVariableName), vsphereip.WithPrefix(options.prefixVariableName), vsphereip.WithIP(options.additionalIPVariableNames...))
for k, v := range ipVariables {
testSpecificVariables[k] = v
}

case VCSimTestTarget:
testSpecificIPAddressManager = vcsimAddressManager
// Use a new address manager when using VCSim in a separate kind management cluster.
if options.useKindForManagementCluster {
var err error
testSpecificIPAddressManager, err = vsphereip.VCSIMAddressManager(managementClusterProxy.GetClient(), map[string]string{}, skipCleanup)
Expect(err).ToNot(HaveOccurred())
}

// get IPs from the vcsim controller
// NOTE: ControlPlaneEndpointIP is the first claim in the returned list (this assumption is used below).
Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
testSpecificIPAddressClaims, ipVariables = testSpecificIPAddressManager.ClaimIPs(ctx, vsphereip.WithIP(options.additionalIPVariableNames...))
for k, v := range ipVariables {
testSpecificVariables[k] = v
}
}

// Additional initialization required when running on VCSim.
if testTarget == VCSimTestTarget {
// variables derived from the vCenterSimulator
vCenterSimulator, err := vspherevcsim.Get(ctx, managementClusterProxy.GetClient())
Expect(err).ToNot(HaveOccurred(), "Failed to get VCenterSimulator")

Byf("Creating EnvVar %s", klog.KRef(metav1.NamespaceDefault, specName))
envVar := &vcsimv1.EnvVar{
ObjectMeta: metav1.ObjectMeta{
Name: specName,
Namespace: metav1.NamespaceDefault,
},
Spec: vcsimv1.EnvVarSpec{
VCenterSimulator: &vcsimv1.NamespacedRef{
Namespace: vCenterSimulator.Namespace,
Name: vCenterSimulator.Name,
},
ControlPlaneEndpoint: vcsimv1.NamespacedRef{
Namespace: testSpecificIPAddressClaims[0].Namespace,
Name: testSpecificIPAddressClaims[0].Name,
},
// NOTE: we are omitting VMOperatorDependencies because it is not created yet (it will be created by the PostNamespaceCreated hook)
// But this is not a issue because a default dependenciesConfig that works for vcsim will be automatically used.
},
}

err = managementClusterProxy.GetClient().Create(ctx, envVar)
Expect(err).ToNot(HaveOccurred(), "Failed to create EnvVar")

Eventually(func() bool {
if err := managementClusterProxy.GetClient().Get(ctx, crclient.ObjectKeyFromObject(envVar), envVar); err != nil {
return false
}
return len(envVar.Status.Variables) > 0
}, 30*time.Second, 5*time.Second).Should(BeTrue(), "Failed to get EnvVar %s", klog.KObj(envVar))

Byf("Setting test variables for %s", specName)
for k, v := range envVar.Status.Variables {
// ignore variables that will be set later on by the test
if sets.New("NAMESPACE", "CLUSTER_NAME", "KUBERNETES_VERSION", "CONTROL_PLANE_MACHINE_COUNT", "WORKER_MACHINE_COUNT").Has(k) {
continue
}

// unset corresponding env variable (that in CI contains VMC data), so we are sure we use the value for vcsim
if strings.HasPrefix(k, "VSPHERE_") {
Expect(os.Unsetenv(k)).To(Succeed())
}

testSpecificVariables[k] = v
}
}

// Re-write the clusterctl config file and add the new variables.
testSpecificClusterctlConfigPath = fmt.Sprintf("%s-%s.yaml", strings.TrimSuffix(clusterctlConfigPath, ".yaml"), specName)
Byf("Writing a new clusterctl config to %s", testSpecificClusterctlConfigPath)
copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{
ClusterctlConfigPath: testSpecificClusterctlConfigPath,
OutputPath: testSpecificClusterctlConfigPath,
Variables: testSpecificVariables,
})

// Run additional initialization required for supervisor.
if testMode == SupervisorTestMode {
switch testTarget {
case VCenterTestTarget:
setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy, workloadClusterNamespace)
case VCSimTestTarget:
setupNamespaceWithVMOperatorDependenciesVCSim(managementClusterProxy, workloadClusterNamespace)
}
}
}
})
defer AfterEach(func() {
Byf("Cleaning up test env for %s", specName)
switch testTarget {
case VCenterTestTarget:
// cleanup IPs/controlPlaneEndpoint created by the in cluster ipam provider.
Expect(inClusterAddressManager.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
case VCSimTestTarget:
// cleanup IPs/controlPlaneEndpoint created by the vcsim controller manager.
Expect(vcsimAddressManager.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
if !skipCleanup {
Byf("Cleaning up test env for %s", specName)
// We can't cleanup when a kind management cluster is used, because it won't exist anymore.
if !options.useKindForManagementCluster {
// cleanup IPs/controlPlaneEndpoint created by the IPAddressManager.
Expect(testSpecificIPAddressManager.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
}
}
})

Expand Down Expand Up @@ -232,7 +285,7 @@ func Setup(specName string, f func(testSpecificSettings func() testSettings), op
func setupNamespaceWithVMOperatorDependenciesVCSim(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) {
c := managementClusterProxy.GetClient()

vCenterSimulator, err := vspherevcsim.Get(ctx, bootstrapClusterProxy.GetClient())
vCenterSimulator, err := vspherevcsim.Get(ctx, c)
Expect(err).ToNot(HaveOccurred(), "Failed to get VCenterSimulator")

Byf("Creating VMOperatorDependencies %s", klog.KRef(workloadClusterNamespace, "vcsim"))
Expand Down
37 changes: 19 additions & 18 deletions test/e2e/ownerrefs_finalizers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,31 +53,32 @@ var _ = Describe("Ensure OwnerReferences and Finalizers are resilient [vcsim] [s
const specName = "owner-reference"
Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput {
if testMode == GovmomiTestMode {
// NOTE: When testing with vcsim VSPHERE_USERNAME and VSPHERE_PASSWORD are provided as a test specific variables,
// when running on CI same variables are provided as env variables.
input := testSpecificSettingsGetter()
username, ok := input.Variables["VSPHERE_USERNAME"]
if !ok {
username = os.Getenv("VSPHERE_USERNAME")
}
password, ok := input.Variables["VSPHERE_PASSWORD"]
if !ok {
password = os.Getenv("VSPHERE_PASSWORD")
}

// Before running the test create the secret used by the VSphereClusterIdentity to connect to the vCenter.
createVsphereIdentitySecret(ctx, bootstrapClusterProxy, username, password)
}

return capi_e2e.QuickStartSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: testSpecificSettingsGetter().ClusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("ownerrefs-finalizers")),
PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc,
PostNamespaceCreated: func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) {
testSpecificSettingsGetter().PostNamespaceCreatedFunc(managementClusterProxy, workloadClusterNamespace)
if testMode == GovmomiTestMode {
// NOTE: When testing with vcsim VSPHERE_USERNAME and VSPHERE_PASSWORD are provided as a test specific variables,
// when running on CI same variables are provided as env variables.
input := testSpecificSettingsGetter()
username, ok := input.Variables["VSPHERE_USERNAME"]
if !ok {
username = os.Getenv("VSPHERE_USERNAME")
}
password, ok := input.Variables["VSPHERE_PASSWORD"]
if !ok {
password = os.Getenv("VSPHERE_PASSWORD")
}

// Before running the test create the secret used by the VSphereClusterIdentity to connect to the vCenter.
createVsphereIdentitySecret(ctx, bootstrapClusterProxy, username, password)
}
},
PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
forceCtx, forceCancelFunc := context.WithCancel(ctx)
if testMode == GovmomiTestMode {
Expand Down