feat: Add delete override implementation (Azure#977)

apis/placement/v1alpha1/override_types.go

@@ -76,7 +76,7 @@ type OverrideRule struct {
 
 	// OverrideType defines the type of the override rules.
 	// +kubebuilder:validation:Enum=JSONPatch;Delete
-	// +kubebuilder:default:JSONPatch
+	// +kubebuilder:default=JSONPatch
 	// +optional
 	OverrideType OverrideType `json:"overrideType,omitempty"`
 

config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverrides.yaml

@@ -303,8 +303,9 @@ spec:
                           - clusterSelectorTerms
                           type: object
                         jsonPatchOverrides:
-                          description: JSONPatchOverrides defines a list of JSON patch
-                            override rules.
+                          description: |-
+                            JSONPatchOverrides defines a list of JSON patch override rules.
+                            This field is only allowed when OverrideType is JSONPatch.
                           items:
                             description: JSONPatchOverride applies a JSON patch on
                               the selected resources following [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902).
@@ -334,8 +335,14 @@ spec:
                           maxItems: 20
                           minItems: 1
                           type: array
-                      required:
-                      - jsonPatchOverrides
+                        overrideType:
+                          default: JSONPatch
+                          description: OverrideType defines the type of the override
+                            rules.
+                          enum:
+                          - JSONPatch
+                          - Delete
+                          type: string
                       type: object
                     maxItems: 20
                     minItems: 1

config/crd/bases/placement.kubernetes-fleet.io_clusterresourceoverridesnapshots.yaml

@@ -317,8 +317,9 @@ spec:
                               - clusterSelectorTerms
                               type: object
                             jsonPatchOverrides:
-                              description: JSONPatchOverrides defines a list of JSON
-                                patch override rules.
+                              description: |-
+                                JSONPatchOverrides defines a list of JSON patch override rules.
+                                This field is only allowed when OverrideType is JSONPatch.
                               items:
                                 description: JSONPatchOverride applies a JSON patch
                                   on the selected resources following [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902).
@@ -348,8 +349,14 @@ spec:
                               maxItems: 20
                               minItems: 1
                               type: array
-                          required:
-                          - jsonPatchOverrides
+                            overrideType:
+                              default: JSONPatch
+                              description: OverrideType defines the type of the override
+                                rules.
+                              enum:
+                              - JSONPatch
+                              - Delete
+                              type: string
                           type: object
                         maxItems: 20
                         minItems: 1

config/crd/bases/placement.kubernetes-fleet.io_resourceoverrides.yaml

@@ -218,8 +218,9 @@ spec:
                           - clusterSelectorTerms
                           type: object
                         jsonPatchOverrides:
-                          description: JSONPatchOverrides defines a list of JSON patch
-                            override rules.
+                          description: |-
+                            JSONPatchOverrides defines a list of JSON patch override rules.
+                            This field is only allowed when OverrideType is JSONPatch.
                           items:
                             description: JSONPatchOverride applies a JSON patch on
                               the selected resources following [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902).
@@ -249,8 +250,14 @@ spec:
                           maxItems: 20
                           minItems: 1
                           type: array
-                      required:
-                      - jsonPatchOverrides
+                        overrideType:
+                          default: JSONPatch
+                          description: OverrideType defines the type of the override
+                            rules.
+                          enum:
+                          - JSONPatch
+                          - Delete
+                          type: string
                       type: object
                     maxItems: 20
                     minItems: 1

config/crd/bases/placement.kubernetes-fleet.io_resourceoverridesnapshots.yaml

@@ -232,8 +232,9 @@ spec:
                               - clusterSelectorTerms
                               type: object
                             jsonPatchOverrides:
-                              description: JSONPatchOverrides defines a list of JSON
-                                patch override rules.
+                              description: |-
+                                JSONPatchOverrides defines a list of JSON patch override rules.
+                                This field is only allowed when OverrideType is JSONPatch.
                               items:
                                 description: JSONPatchOverride applies a JSON patch
                                   on the selected resources following [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902).
@@ -263,8 +264,14 @@ spec:
                               maxItems: 20
                               minItems: 1
                               type: array
-                          required:
-                          - jsonPatchOverrides
+                            overrideType:
+                              default: JSONPatch
+                              description: OverrideType defines the type of the override
+                                rules.
+                              enum:
+                              - JSONPatch
+                              - Delete
+                              type: string
                           type: object
                         maxItems: 20
                         minItems: 1

pkg/controllers/workgenerator/controller.go

@@ -425,17 +425,22 @@ func (r *Reconciler) syncAllWork(ctx context.Context, resourceBinding *fleetv1be
 		}
 		var simpleManifests []fleetv1beta1.Manifest
 		for j := range snapshot.Spec.SelectedResources {
-			selectedResource := snapshot.Spec.SelectedResources[j]
-			if err := r.applyOverrides(&selectedResource, cluster, croMap, roMap); err != nil {
-				return false, false, err
+			selectedResource := snapshot.Spec.SelectedResources[j].DeepCopy()
+			// TODO: override the content of the wrapped resource instead of the envelope itself
+			resourceDeleted, overrideErr := r.applyOverrides(selectedResource, cluster, croMap, roMap)
+			if overrideErr != nil {
+				return false, false, overrideErr
+			}
+			if resourceDeleted {
+				klog.V(2).InfoS("The resource is deleted by the override rules", "snapshot", klog.KObj(snapshot), "selectedResource", snapshot.Spec.SelectedResources[j])
+				continue
 			}
-
 			// we need to special treat configMap with envelopeConfigMapAnnotation annotation,
 			// so we need to check the GVK and annotation of the selected resource
 			var uResource unstructured.Unstructured
-			if err := uResource.UnmarshalJSON(selectedResource.Raw); err != nil {
-				klog.ErrorS(err, "work has invalid content", "snapshot", klog.KObj(snapshot), "selectedResource", selectedResource.Raw)
-				return true, false, controller.NewUnexpectedBehaviorError(err)
+			if unMarshallErr := uResource.UnmarshalJSON(selectedResource.Raw); unMarshallErr != nil {
+				klog.ErrorS(unMarshallErr, "work has invalid content", "snapshot", klog.KObj(snapshot), "selectedResource", selectedResource.Raw)
+				return true, false, controller.NewUnexpectedBehaviorError(unMarshallErr)
 			}
 			if uResource.GetObjectKind().GroupVersionKind() == utils.ConfigMapGVK &&
 				len(uResource.GetAnnotations()[fleetv1beta1.EnvelopeConfigMapAnnotation]) != 0 {
@@ -447,11 +452,11 @@ func (r *Reconciler) syncAllWork(ctx context.Context, resourceBinding *fleetv1be
 				activeWork[work.Name] = work
 				newWork = append(newWork, work)
 			} else {
-				simpleManifests = append(simpleManifests, fleetv1beta1.Manifest(selectedResource))
+				simpleManifests = append(simpleManifests, fleetv1beta1.Manifest(*selectedResource))
 			}
 		}
 		if len(simpleManifests) == 0 {
-			klog.V(2).InfoS("the snapshot contains enveloped resource only", "snapshot", klog.KObj(snapshot))
+			klog.V(2).InfoS("the snapshot contains no resource to apply either because of override or enveloped resources", "snapshot", klog.KObj(snapshot))
 		}
 		// generate a work object for the manifests even if there is nothing to place
 		// to allow CRP to collect the status of the placement

pkg/controllers/workgenerator/override.go

@@ -92,15 +92,21 @@ func (r *Reconciler) fetchResourceOverrideSnapshots(ctx context.Context, resourc
 	return roMap, nil
 }
 
-func (r *Reconciler) applyOverrides(resource *placementv1beta1.ResourceContent, cluster clusterv1beta1.MemberCluster, croMap map[placementv1beta1.ResourceIdentifier][]*placementv1alpha1.ClusterResourceOverrideSnapshot, roMap map[placementv1beta1.ResourceIdentifier][]*placementv1alpha1.ResourceOverrideSnapshot) error {
+// applyOverrides applies the overrides on the selected resources.
+// The resource could be selected by both ClusterResourceOverride and ResourceOverride.
+// It returns
+//   - true if the resource is deleted by the overrides.
+//   - an error if the override rules are invalid.
+func (r *Reconciler) applyOverrides(resource *placementv1beta1.ResourceContent, cluster clusterv1beta1.MemberCluster,
+	croMap map[placementv1beta1.ResourceIdentifier][]*placementv1alpha1.ClusterResourceOverrideSnapshot, roMap map[placementv1beta1.ResourceIdentifier][]*placementv1alpha1.ResourceOverrideSnapshot) (bool, error) {
 	if len(croMap) == 0 && len(roMap) == 0 {
-		return nil
+		return false, nil
 	}
 
 	var uResource unstructured.Unstructured
 	if err := uResource.UnmarshalJSON(resource.Raw); err != nil {
 		klog.ErrorS(err, "Work has invalid content", "selectedResource", resource.Raw)
-		return controller.NewUnexpectedBehaviorError(err)
+		return false, controller.NewUnexpectedBehaviorError(err)
 	}
 	gvk := uResource.GetObjectKind().GroupVersionKind()
 	key := placementv1beta1.ResourceIdentifier{
@@ -131,13 +137,12 @@ func (r *Reconciler) applyOverrides(resource *placementv1beta1.ResourceContent,
 		}
 		if err := applyOverrideRules(resource, cluster, snapshot.Spec.OverrideSpec.Policy.OverrideRules); err != nil {
 			klog.ErrorS(err, "Failed to apply the override rules", "clusterResourceOverrideSnapshot", klog.KObj(snapshot))
-			return err
+			return false, err
 		}
 	}
 	klog.V(2).InfoS("Applied clusterResourceOverrideSnapshots", "resource", klog.KObj(&uResource), "numberOfOverrides", len(croMap[key]))
 
-	// If the resource is selected by both ClusterResourceOverride and ResourceOverride, ResourceOverride will win when
-	// resolving conflicts.
+	// If the resource is selected by both ClusterResourceOverride and ResourceOverride, ResourceOverride will win when resolving conflicts.
 	// Apply ResourceOverrideSnapshots.
 	if !isClusterScopeResource {
 		key = placementv1beta1.ResourceIdentifier{
@@ -155,12 +160,12 @@ func (r *Reconciler) applyOverrides(resource *placementv1beta1.ResourceContent,
 			}
 			if err := applyOverrideRules(resource, cluster, snapshot.Spec.OverrideSpec.Policy.OverrideRules); err != nil {
 				klog.ErrorS(err, "Failed to apply the override rules", "resourceOverrideSnapshot", klog.KObj(snapshot))
-				return err
+				return false, err
 			}
 		}
 		klog.V(2).InfoS("Applied resourceOverrideSnapshots", "resource", klog.KObj(&uResource), "numberOfOverrides", len(roMap[key]))
 	}
-	return nil
+	return resource.Raw == nil, nil
 }
 
 func applyOverrideRules(resource *placementv1beta1.ResourceContent, cluster clusterv1beta1.MemberCluster, rules []placementv1alpha1.OverrideRule) error {
@@ -173,7 +178,12 @@ func applyOverrideRules(resource *placementv1beta1.ResourceContent, cluster clus
 		if !matched {
 			continue
 		}
-
+		if rule.OverrideType == placementv1alpha1.DeleteOverrideType {
+			// Delete the resource
+			resource.Raw = nil
+			return nil
+		}
+		// Apply JSONPatchOverrides by default
 		if err := applyJSONPatchOverride(resource, rule.JSONPatchOverrides); err != nil {
 			klog.ErrorS(err, "Failed to apply JSON patch override")
 			return controller.NewUserError(err)