🐛 [WIP] supervisor: create ClusterModule per MachineDeployment and re-reconcile VirtualMachineResourceSetPolicy to update VirtualMachineSetResourcePolicy

apis/vmware/v1beta1/vspherecluster_types.go

@@ -38,8 +38,43 @@ const (
 type VSphereClusterSpec struct {
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
+	// placement allows to configure the placement of machines of a VSphereCluster.
+	// +optional
+	Placement *VSphereClusterPlacement `json:"placement,omitempty"`
+}
+
+// VSphereClusterPlacement defines a placement strategy for machines of a VSphereCluster.
+// +kubebuilder:validation:MinProperties=1
+type VSphereClusterPlacement struct {
+	// workerAntiAffinity configures soft anti-affinity for workers.
+	// +optional
+	WorkerAntiAffinity *VSphereClusterWorkerAntiAffinity `json:"workerAntiAffinity,omitempty"`
+}
+
+// VSphereClusterWorkerAntiAffinity defines the anti-affinity configuration for workers.
+// +kubebuilder:validation:MinProperties=1
+type VSphereClusterWorkerAntiAffinity struct {
+	// mode allows to set the grouping of (soft) anti-affinity for worker nodes.
+	// Defaults to `Cluster`.
+	// +kubebuilder:validation:Enum=Cluster;None;MachineDeployment
+	// +optional
+	Mode VSphereClusterWorkerAntiAffinityMode `json:"mode,omitempty"`
 }
 
+// VSphereClusterWorkerAntiAffinityMode describes the soft anti-affinity mode used across a for distributing virtual machines.
+type VSphereClusterWorkerAntiAffinityMode string
+
+const (
+	// VSphereClusterWorkerAntiAffinityModeCluster means to use all workers as a single group for soft anti-affinity.
+	VSphereClusterWorkerAntiAffinityModeCluster VSphereClusterWorkerAntiAffinityMode = "Cluster"
+
+	// VSphereClusterWorkerAntiAffinityModeNone means to not configure any soft anti-affinity for workers.
+	VSphereClusterWorkerAntiAffinityModeNone VSphereClusterWorkerAntiAffinityMode = "None"
+
+	// VSphereClusterWorkerAntiAffinityModeMachineDeployment means to configure soft anti-affinity for all workers per MachineDeployment.
+	VSphereClusterWorkerAntiAffinityModeMachineDeployment VSphereClusterWorkerAntiAffinityMode = "MachineDeployment"
+)
+
 // VSphereClusterStatus defines the observed state of VSphereClusterSpec.
 type VSphereClusterStatus struct {
 	// Ready indicates the infrastructure required to deploy this cluster is

config/manager/manager.yaml

@@ -21,7 +21,7 @@ spec:
         - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
         - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
         - --v=4
-        - "--feature-gates=NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false}"
+        - "--feature-gates=NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},WorkerAntiAffinity=${EXP_WORKER_ANTI_AFFINITY:=false}"
         image: controller:latest
         imagePullPolicy: IfNotPresent
         name: manager

config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vsphereclusters.yaml

@@ -55,6 +55,27 @@ spec:
                 - host
                 - port
                 type: object
+              placement:
+                description: placement allows to configure the placement of machines
+                  of a VSphereCluster.
+                minProperties: 1
+                properties:
+                  workerAntiAffinity:
+                    description: workerAntiAffinity configures soft anti-affinity
+                      for workers.
+                    minProperties: 1
+                    properties:
+                      mode:
+                        description: |-
+                          mode allows to set the grouping of (soft) anti-affinity for worker nodes.
+                          Defaults to `Cluster`.
+                        enum:
+                        - Cluster
+                        - None
+                        - MachineDeployment
+                        type: string
+                    type: object
+                type: object
             type: object
           status:
             description: VSphereClusterStatus defines the observed state of VSphereClusterSpec.

config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vsphereclustertemplates.yaml

@@ -65,6 +65,27 @@ spec:
                         - host
                         - port
                         type: object
+                      placement:
+                        description: placement allows to configure the placement of
+                          machines of a VSphereCluster.
+                        minProperties: 1
+                        properties:
+                          workerAntiAffinity:
+                            description: workerAntiAffinity configures soft anti-affinity
+                              for workers.
+                            minProperties: 1
+                            properties:
+                              mode:
+                                description: |-
+                                  mode allows to set the grouping of (soft) anti-affinity for worker nodes.
+                                  Defaults to `Cluster`.
+                                enum:
+                                - Cluster
+                                - None
+                                - MachineDeployment
+                                type: string
+                            type: object
+                        type: object
                     type: object
                 required:
                 - spec

config/supervisor/webhook/manifests.yaml

@@ -31,6 +31,27 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: validating-webhook-configuration
 webhooks:
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspherecluster
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: validation.vspherecluster.vmware.infrastructure.cluster.x-k8s.io
+  rules:
+  - apiGroups:
+    - vmware.infrastructure.cluster.x-k8s.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - vsphereclusters
+  sideEffects: None
 - admissionReviewVersions:
   - v1beta1
   clientConfig:

controllers/vmware/test/controllers_test.go

@@ -478,7 +478,7 @@ var _ = Describe("Reconciliation tests", func() {
 			Eventually(func() error {
 				return k8sClient.Get(ctx, rpKey, resourcePolicy)
 			}, time.Second*30).Should(Succeed())
-			Expect(len(resourcePolicy.Spec.ClusterModuleGroups)).To(BeEquivalentTo(2))
+			Expect(len(resourcePolicy.Spec.ClusterModuleGroups)).To(BeEquivalentTo(1))
 
 			By("Create the CAPI Machine and wait for it to exist")
 			machineKey, machine := deployCAPIMachine(ns.Name, cluster, k8sClient)

controllers/vmware/vspherecluster_reconciler.go

@@ -69,6 +69,8 @@ type ClusterReconciler struct {
 // +kubebuilder:rbac:groups=netoperator.vmware.com,resources=networks,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update;create;delete
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinedeployments,verbs=get;list;watch
+// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;list;watch
 
 func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) {
 	log := ctrl.LoggerFrom(ctx)
@@ -172,7 +174,7 @@ func (r *ClusterReconciler) reconcileNormal(ctx context.Context, clusterCtx *vmw
 	// Reconcile ResourcePolicy before we create the machines. If the ResourcePolicy is not reconciled before we create the Node VMs,
 	// it will be handled by vm operator by relocating the VMs to the ResourcePool and Folder specified by the ResourcePolicy.
 	// Reconciling the ResourcePolicy early potentially saves us the extra relocate operation.
-	resourcePolicyName, err := r.ResourcePolicyService.ReconcileResourcePolicy(ctx, clusterCtx)
+	resourcePolicyName, err := r.ResourcePolicyService.ReconcileResourcePolicy(ctx, clusterCtx.Cluster, clusterCtx.VSphereCluster)
 	if err != nil {
 		conditions.MarkFalse(clusterCtx.VSphereCluster, vmwarev1.ResourcePolicyReadyCondition, vmwarev1.ResourcePolicyCreationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
 		return errors.Wrapf(err,
@@ -370,6 +372,34 @@ func (r *ClusterReconciler) VSphereMachineToCluster(ctx context.Context, o clien
 	}}
 }
 
+// MachineDeploymentToCluster adds reconcile requests for a Cluster when one of its machineDeployments has an event.
+func (r *ClusterReconciler) MachineDeploymentToCluster(ctx context.Context, o client.Object) []reconcile.Request {
+	log := ctrl.LoggerFrom(ctx)
+
+	machineDeployment, ok := o.(*clusterv1.MachineDeployment)
+	if !ok {
+		log.Error(nil, fmt.Sprintf("Expected a MachineDeployment but got a %T", o))
+		return nil
+	}
+	log = log.WithValues("MachineDeployment", klog.KObj(machineDeployment))
+	ctx = ctrl.LoggerInto(ctx, log)
+
+	vsphereCluster, err := util.GetVMwareVSphereClusterFromMachineDeployment(ctx, r.Client, machineDeployment)
+	if err != nil {
+		log.V(4).Error(err, "Failed to get VSphereCluster from MachineDeployment")
+		return nil
+	}
+
+	// Can add further filters on Cluster state so that we don't keep reconciling Cluster
+	log.V(6).Info("Triggering VSphereCluster reconcile from MachineDeployment")
+	return []ctrl.Request{{
+		NamespacedName: types.NamespacedName{
+			Namespace: vsphereCluster.Namespace,
+			Name:      vsphereCluster.Name,
+		},
+	}}
+}
+
 // ZoneToVSphereClusters adds reconcile requests for VSphereClusters when Zone has an event.
 func (r *ClusterReconciler) ZoneToVSphereClusters(ctx context.Context, o client.Object) []reconcile.Request {
 	log := ctrl.LoggerFrom(ctx)

controllers/vspherecluster_controller.go

@@ -83,6 +83,10 @@ func AddClusterControllerToManager(ctx context.Context, controllerManagerCtx *ca
 				&vmwarev1.VSphereMachine{},
 				handler.EnqueueRequestsFromMapFunc(reconciler.VSphereMachineToCluster),
 			).
+			Watches(
+				&clusterv1.MachineDeployment{},
+				handler.EnqueueRequestsFromMapFunc(reconciler.MachineDeploymentToCluster),
+			).
 			WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(mgr.GetScheme(), predicateLog, controllerManagerCtx.WatchFilterValue))
 
 		// Conditionally add a Watch for topologyv1.Zone when the feature gate is enabled

feature/feature.go

@@ -44,6 +44,15 @@ const (
 	//
 	// alpha: v1.10
 	PriorityQueue featuregate.Feature = "PriorityQueue"
+
+	// WorkerAntiAffinity allows configuring how soft anti-affinity should be done for worker nodes.[]
+	// If disabled it disallows:
+	// * mutating `VSphereCluster.spec.placement.workerAntiAffinity.mode`.
+	// * Setting `MachineDeployment` as value for `VSphereCluster.spec.placement.workerAntiAffinity.mode` on creation.
+	// Note: the feature requires a version of vm-operator which allows mutation of `VirtualMachineSetResourcePolicy's`.
+	//
+	// alpha: v1.13
+	WorkerAntiAffinity featuregate.Feature = "WorkerAntiAffinity"
 )
 
 func init() {
@@ -57,4 +66,7 @@ var defaultCAPVFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	NodeAntiAffinity:     {Default: false, PreRelease: featuregate.Alpha},
 	NamespaceScopedZones: {Default: false, PreRelease: featuregate.Alpha},
 	PriorityQueue:        {Default: false, PreRelease: featuregate.Alpha},
+
+	// Feature gates specific to supervisor mode:
+	WorkerAntiAffinity: {Default: false, PreRelease: featuregate.Alpha},
 }