diff --git a/apis/placement/v1beta1/work_types.go b/apis/placement/v1beta1/work_types.go index 05ae59a59..1b3b88d49 100644 --- a/apis/placement/v1beta1/work_types.go +++ b/apis/placement/v1beta1/work_types.go @@ -39,6 +39,10 @@ const ( // WorkConditionTypeAvailable represents workload in Work is available on the spoke cluster. WorkConditionTypeAvailable = "Available" + + // WorkConditionTypeDiffReported reports whether Fleet has successfully reported the + // configuration difference between the states in the hub cluster and a member cluster. + WorkConditionTypeDiffReported = "DiffReported" ) // This api is copied from https://github.com/kubernetes-sigs/work-api/blob/master/pkg/apis/v1alpha1/work_types.go. diff --git a/go.mod b/go.mod index 2d13fc9e7..2199552f2 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,12 @@ require ( github.com/onsi/gomega v1.35.1 github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.6.1 + github.com/qri-io/jsonpointer v0.1.1 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - go.goms.io/fleet-networking v0.2.7 + github.com/wI2L/jsondiff v0.6.0 + go.goms.io/fleet-networking v0.2.11 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 @@ -102,6 +104,10 @@ require ( github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/samber/lo v1.38.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/metric v1.31.0 // indirect diff --git a/go.sum b/go.sum index d4b41b5a6..154eb4abb 100644 --- a/go.sum +++ b/go.sum @@ -210,6 +210,8 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA= +github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= @@ -232,11 +234,23 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.goms.io/fleet-networking v0.2.7 h1:lVs2/GiCjo18BRgACib+VPnENUMh+2YbYXoeNtcAvw0= -go.goms.io/fleet-networking v0.2.7/go.mod h1:JoWG82La5nV29mooOnPpIhy6/Pi4oGXQk21CPF1UStg= +go.goms.io/fleet-networking v0.2.11 h1:N5/5TckwEipA8s4DC71lzwdz88P3mtXdvRKGL8Cg/JA= +go.goms.io/fleet-networking v0.2.11/go.mod h1:tenpiBceno4hiCakjCMz/KCAwRlBz0+/kER6lSXy+Mk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_test.go index f67e53fae..7539e597e 100644 --- a/pkg/controllers/clusterresourceplacementeviction/controller_test.go +++ b/pkg/controllers/clusterresourceplacementeviction/controller_test.go @@ -237,6 +237,13 @@ func TestValidateEviction(t *testing.T) { Client: fakeClient, } gotValidationResult, gotErr := r.validateEviction(ctx, tc.eviction) + + // Since default values are applied to the affected CRP in the eviction controller; the + // the same must be done on the expected result as well. + if tc.wantValidationResult.crp != nil { + defaulter.SetDefaultsClusterResourcePlacement(tc.wantValidationResult.crp) + } + if diff := cmp.Diff(tc.wantValidationResult, gotValidationResult, validationResultCmpOptions...); diff != "" { t.Errorf("validateEviction() validation result mismatch (-want, +got):\n%s", diff) } diff --git a/pkg/controllers/work/applier.go b/pkg/controllers/work/applier.go index 9983a685b..1fe3ff15a 100644 --- a/pkg/controllers/work/applier.go +++ b/pkg/controllers/work/applier.go @@ -75,6 +75,9 @@ func findConflictedWork(ctx context.Context, hubClient client.Client, namespace // * the defaulting webhook failure policy is configured as "fail". // * user cannot update/delete the webhook. defaulter.SetDefaultsWork(work) + // Note (chenyu1): set the defaults here to accommodate for new apply strategy fields added in v1beta1 + // API version. + defaulter.SetDefaultsApplyStrategy(strategy) if !equality.Semantic.DeepEqual(strategy, work.Spec.ApplyStrategy) { return work, nil } diff --git a/pkg/controllers/work/apply_controller.go b/pkg/controllers/work/apply_controller.go index cb0ed7c87..2892ce93e 100644 --- a/pkg/controllers/work/apply_controller.go +++ b/pkg/controllers/work/apply_controller.go @@ -441,10 +441,10 @@ func trackResourceAvailability(gvr schema.GroupVersionResource, curObj *unstruct case utils.DeploymentGVR: return trackDeploymentAvailability(curObj) - case utils.StatefulSettGVR: + case utils.StatefulSetGVR: return trackStatefulSetAvailability(curObj) - case utils.DaemonSettGVR: + case utils.DaemonSetGVR: return trackDaemonSetAvailability(curObj) case utils.ServiceGVR: diff --git a/pkg/controllers/work/apply_controller_test.go b/pkg/controllers/work/apply_controller_test.go index 3722982ff..3a50a8439 100644 --- a/pkg/controllers/work/apply_controller_test.go +++ b/pkg/controllers/work/apply_controller_test.go @@ -1025,7 +1025,7 @@ func TestTrackResourceAvailability(t *testing.T) { err: nil, }, "Test StatefulSet available": { - gvr: utils.StatefulSettGVR, + gvr: utils.StatefulSetGVR, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "apps/v1", @@ -1049,7 +1049,7 @@ func TestTrackResourceAvailability(t *testing.T) { err: nil, }, "Test StatefulSet not available": { - gvr: utils.StatefulSettGVR, + gvr: utils.StatefulSetGVR, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "apps/v1", @@ -1073,7 +1073,7 @@ func TestTrackResourceAvailability(t *testing.T) { err: nil, }, "Test StatefulSet observed old generation": { - gvr: utils.StatefulSettGVR, + gvr: utils.StatefulSetGVR, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "apps/v1", @@ -1097,7 +1097,7 @@ func TestTrackResourceAvailability(t *testing.T) { err: nil, }, "Test DaemonSet Available": { - gvr: utils.DaemonSettGVR, + gvr: utils.DaemonSetGVR, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "apps/v1", @@ -1118,7 +1118,7 @@ func TestTrackResourceAvailability(t *testing.T) { err: nil, }, "Test DaemonSet not available": { - gvr: utils.DaemonSettGVR, + gvr: utils.DaemonSetGVR, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "apps/v1", @@ -1139,7 +1139,7 @@ func TestTrackResourceAvailability(t *testing.T) { err: nil, }, "Test DaemonSet not observe current generation": { - gvr: utils.DaemonSettGVR, + gvr: utils.DaemonSetGVR, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "apps/v1", diff --git a/pkg/controllers/workapplier/apply.go b/pkg/controllers/workapplier/apply.go new file mode 100644 index 000000000..a689c71f6 --- /dev/null +++ b/pkg/controllers/workapplier/apply.go @@ -0,0 +1,539 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + "reflect" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/jsonmergepatch" + "k8s.io/apimachinery/pkg/util/mergepatch" + "k8s.io/apimachinery/pkg/util/strategicpatch" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/resource" +) + +var builtInScheme = runtime.NewScheme() + +func init() { + // This is a trick that allows Fleet to check if a resource is a K8s built-in one. + _ = clientgoscheme.AddToScheme(builtInScheme) +} + +// applyInDryRunMode +func (r *Reconciler) applyInDryRunMode( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObj, inMemberClusterObj *unstructured.Unstructured, +) (*unstructured.Unstructured, error) { + // In this method, Fleet will always use forced server-side apply + // w/o optimistic lock for diff calculation. + // + // This is OK as partial comparison concerns only fields that are currently present + // in the manifest object, and Fleet will clear out system managed and read-only fields + // before the comparison. + // + // Note that full comparison can be carried out directly without involving the apply op. + return r.serverSideApply(ctx, gvr, manifestObj, inMemberClusterObj, true, false, true) +} + +func (r *Reconciler) apply( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObj, inMemberClusterObj *unstructured.Unstructured, + applyStrategy *fleetv1beta1.ApplyStrategy, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) (*unstructured.Unstructured, error) { + // Create a sanitized copy of the manifest object. + // + // Note (chenyu1): for objects directly created on the hub, the Fleet hub agent has already + // performed a round of sanitization (i.e., clearing fields such as resource version, + // UID, etc.); this, however, does not apply to enveloped objects, which are currently + // placed as-is. To accommodate for this, the work applier here will perform an additional + // round of sanitization. + // + // TO-DO (chenyu1): this processing step can be removed if the enveloped objects are + // pre-processed by the Fleet hub agent as well, provided that there are no further + // backwards compatibility concerns. + manifestObjCopy := sanitizeManifestObject(manifestObj) + + // Compute the hash of the manifest object. + // + // Originally the manifest hash is kept only if three-way merge patch (client side apply esque + // strategy) is used; with the new drift detection and takeover capabilities, the manifest hash + // will always be kept regardless of the apply strategy in use, as it is needed for + // drift detection purposes. + // + // Note that certain fields have been removed from the manifest object in the hash computation + // process. + if err := setManifestHashAnnotation(manifestObjCopy); err != nil { + return nil, fmt.Errorf("failed to set manifest hash annotation: %w", err) + } + + // Validate owner references. + // + // As previously mentioned, with the new capabilities, at this point of the workflow, + // Fleet has been added as an owner for the object. Still, to guard against cases where + // the allow co-ownership switch is turned on then off or the addition of new owner references + // in the manifest, Fleet will still perform a validation round. + if err := validateOwnerReferences(manifestObj, inMemberClusterObj, applyStrategy, expectedAppliedWorkOwnerRef); err != nil { + return nil, fmt.Errorf("failed to validate owner references: %w", err) + } + + // Add the owner reference information. + setOwnerRef(manifestObjCopy, expectedAppliedWorkOwnerRef) + + // If three-way merge patch is used, set the Fleet-specific last applied annotation. + // Note that this op might not complete due to the last applied annotation being too large; + // this is not recognized as an error and Fleet will switch to server-side apply instead. + isLastAppliedAnnotationSet := false + if applyStrategy.Type == fleetv1beta1.ApplyStrategyTypeClientSideApply { + var err error + isLastAppliedAnnotationSet, err = setFleetLastAppliedAnnotation(manifestObjCopy) + if err != nil { + return nil, fmt.Errorf("failed to set last applied annotation: %w", err) + } + + // Note that Fleet might choose to skip the last applied annotation due to size limits + // even if no error has occurred. + } + + // Create the object if it does not exist in the member cluster. + if inMemberClusterObj == nil { + return r.createManifestObject(ctx, gvr, manifestObjCopy) + } + + // Note: originally Fleet will add its owner reference and + // retrieve existing (previously applied) object during the apply op; with the addition + // of drift detection and takeover capabilities, such steps have been completed before + // the apply op actually runs. + + // Run the apply op. Note that Fleet will always attempt to apply the manifest, even if + // the manifest object hash does not change. + + // Optimistic lock is enabled when the apply strategy dictates that an apply op should + // not be carries through if a drift has been found (i.e., the WhenToApply field is set + // to IfNotDrifted); this helps Fleet guard against + // cases where inadvertent changes are being made in an untimely manner (i.e., changes + // are made when the Fleet agent is preparing an apply op). + // + // Note that if the apply strategy dictates that apply ops are always executed (i.e., + // the WhenToApply field is set to Always), Fleet will not enable optimistic lock. This + // is consistent with the behavior before the drift detection and takeover experience + // is added. + isOptimisticLockEnabled := shouldEnableOptimisticLock(applyStrategy) + + switch { + case applyStrategy.Type == fleetv1beta1.ApplyStrategyTypeClientSideApply && isLastAppliedAnnotationSet: + // The apply strategy dictates that three-way merge patch + // (client-side apply esque patch) should be used, and the last applied annotation + // has been set. + return r.threeWayMergePatch(ctx, gvr, manifestObjCopy, inMemberClusterObj, isOptimisticLockEnabled, false) + case applyStrategy.Type == fleetv1beta1.ApplyStrategyTypeClientSideApply: + // The apply strategy dictates that three-way merge patch + // (client-side apply esque patch) should be used, but the last applied annotation + // cannot be set. Fleet will fall back to server-side apply. + return r.serverSideApply( + ctx, + gvr, manifestObjCopy, inMemberClusterObj, + applyStrategy.ServerSideApplyConfig.ForceConflicts, isOptimisticLockEnabled, false, + ) + case applyStrategy.Type == fleetv1beta1.ApplyStrategyTypeServerSideApply: + // The apply strategy dictates that server-side apply should be used. + return r.serverSideApply( + ctx, + gvr, manifestObjCopy, inMemberClusterObj, + applyStrategy.ServerSideApplyConfig.ForceConflicts, isOptimisticLockEnabled, false, + ) + default: + // An unexpected apply strategy has been set. + return nil, fmt.Errorf("unexpected apply strategy %s is found", applyStrategy.Type) + } +} + +// createManifestObject creates the manifest object in the member cluster. +func (r *Reconciler) createManifestObject( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObject *unstructured.Unstructured, +) (*unstructured.Unstructured, error) { + createOpts := metav1.CreateOptions{ + FieldManager: workFieldManagerName, + } + createdObj, err := r.spokeDynamicClient.Resource(*gvr).Namespace(manifestObject.GetNamespace()).Create(ctx, manifestObject, createOpts) + if err != nil { + return nil, fmt.Errorf("failed to create manifest object: %w", err) + } + return createdObj, nil +} + +// threeWayMergePatch uses three-way merge patch to apply the manifest object. +func (r *Reconciler) threeWayMergePatch( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObj, inMemberClusterObj *unstructured.Unstructured, + optimisticLock, dryRun bool, +) (*unstructured.Unstructured, error) { + // Enable optimistic lock by forcing the resource version field to be added to the + // JSON merge patch. Optimistic lock is always enabled in the dry run mode. + if optimisticLock || dryRun { + curResourceVer := inMemberClusterObj.GetResourceVersion() + if len(curResourceVer) == 0 { + return nil, fmt.Errorf("failed to enable optimistic lock: resource version is empty on the object %s/%s from the member cluster", inMemberClusterObj.GroupVersionKind(), klog.KObj(inMemberClusterObj)) + } + + // Add the resource version to the manifest object. + manifestObj.SetResourceVersion(curResourceVer) + + // Remove the resource version from the object in the member cluster. + inMemberClusterObj.SetResourceVersion("") + } + + // Create a three-way merge patch. + patch, err := buildThreeWayMergePatch(manifestObj, inMemberClusterObj) + if err != nil { + return nil, fmt.Errorf("failed to create three-way merge patch: %w", err) + } + data, err := patch.Data(manifestObj) + if err != nil { + // Fleet uses raw patch; this branch should never run. + return nil, fmt.Errorf("failed to get patch data: %w", err) + } + + // Use three-way merge (similar to kubectl client side apply) to patch the object in the + // member cluster. + // + // This will: + // * Remove fields that are present in the last applied configuration but not in the + // manifest object. + // * Create fields that are present in the manifest object but not in the object from the member cluster. + // * Update fields that are present in both the manifest object and the object from the member cluster. + patchOpts := metav1.PatchOptions{ + FieldManager: workFieldManagerName, + } + if dryRun { + patchOpts.DryRun = []string{metav1.DryRunAll} + } + patchedObj, err := r.spokeDynamicClient. + Resource(*gvr).Namespace(manifestObj.GetNamespace()). + Patch(ctx, manifestObj.GetName(), patch.Type(), data, patchOpts) + if err != nil { + return nil, fmt.Errorf("failed to patch the manifest object: %w", err) + } + return patchedObj, nil +} + +// serverSideApply uses server-side apply to apply the manifest object. +func (r *Reconciler) serverSideApply( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObj, inMemberClusterObj *unstructured.Unstructured, + force, optimisticLock, dryRun bool, +) (*unstructured.Unstructured, error) { + // Enable optimistic lock by forcing the resource version field to be added to the + // JSON merge patch. Optimistic lock is always disabled in the dry run mode. + if optimisticLock && !dryRun { + curResourceVer := inMemberClusterObj.GetResourceVersion() + if len(curResourceVer) == 0 { + return nil, fmt.Errorf("failed to enable optimistic lock: resource version is empty on the object %s/%s from the member cluster", inMemberClusterObj.GroupVersionKind(), klog.KObj(inMemberClusterObj)) + } + + // Add the resource version to the manifest object. + manifestObj.SetResourceVersion(curResourceVer) + + // Remove the resource version from the object in the member cluster. + inMemberClusterObj.SetResourceVersion("") + } + + // Use server-side apply to apply the manifest object. + // + // See the Kubernetes documentation on structured merged diff for the exact behaviors. + applyOpts := metav1.ApplyOptions{ + FieldManager: workFieldManagerName, + Force: force, + } + if dryRun { + applyOpts.DryRun = []string{metav1.DryRunAll} + } + appliedObj, err := r.spokeDynamicClient. + Resource(*gvr).Namespace(manifestObj.GetNamespace()). + Apply(ctx, manifestObj.GetName(), manifestObj, applyOpts) + if err != nil { + return nil, fmt.Errorf("failed to apply the manifest object: %w", err) + } + return appliedObj, nil +} + +// threeWayMergePatch creates a patch by computing a three-way diff based on +// the manifest object, the live object, and the last applied configuration as kept in +// the annotations. +func buildThreeWayMergePatch(manifestObj, liveObj *unstructured.Unstructured) (client.Patch, error) { + // Marshal the manifest object into JSON bytes. + manifestObjJSONBytes, err := manifestObj.MarshalJSON() + if err != nil { + return nil, err + } + // Marshal the live object into JSON bytes. + liveObjJSONBytes, err := liveObj.MarshalJSON() + if err != nil { + return nil, err + } + // Retrieve the last applied configuration from the annotations. This can be an empty string. + lastAppliedObjJSONBytes := getFleetLastAppliedAnnotation(liveObj) + + var patchType types.PatchType + var patchData []byte + var lookupPatchMeta strategicpatch.LookupPatchMeta + + versionedObject, err := builtInScheme.New(liveObj.GetObjectKind().GroupVersionKind()) + switch { + case runtime.IsNotRegisteredError(err): + // use JSONMergePatch for custom resources + // because StrategicMergePatch doesn't support custom resources + patchType = types.MergePatchType + preconditions := []mergepatch.PreconditionFunc{ + mergepatch.RequireKeyUnchanged("apiVersion"), + mergepatch.RequireKeyUnchanged("kind"), + mergepatch.RequireMetadataKeyUnchanged("name"), + } + patchData, err = jsonmergepatch.CreateThreeWayJSONMergePatch( + lastAppliedObjJSONBytes, manifestObjJSONBytes, liveObjJSONBytes, preconditions...) + if err != nil { + return nil, err + } + case err != nil: + return nil, err + default: + // use StrategicMergePatch for K8s built-in resources + patchType = types.StrategicMergePatchType + lookupPatchMeta, err = strategicpatch.NewPatchMetaFromStruct(versionedObject) + if err != nil { + return nil, err + } + patchData, err = strategicpatch.CreateThreeWayMergePatch(lastAppliedObjJSONBytes, manifestObjJSONBytes, liveObjJSONBytes, lookupPatchMeta, true) + if err != nil { + return nil, err + } + } + return client.RawPatch(patchType, patchData), nil +} + +// setFleetLastAppliedAnnotation sets the last applied annotation on the provided manifest object. +func setFleetLastAppliedAnnotation(manifestObj *unstructured.Unstructured) (bool, error) { + annotations := manifestObj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + + // Remove the last applied annotation just in case. + delete(annotations, fleetv1beta1.LastAppliedConfigAnnotation) + manifestObj.SetAnnotations(annotations) + + lastAppliedManifestJSONBytes, err := manifestObj.MarshalJSON() + if err != nil { + return false, fmt.Errorf("failed to marshal the manifest object into JSON: %w", err) + } + annotations[fleetv1beta1.LastAppliedConfigAnnotation] = string(lastAppliedManifestJSONBytes) + isLastAppliedAnnotationSet := true + + if err := validation.ValidateAnnotationsSize(annotations); err != nil { + // If the annotation size exceeds the limit, Fleet will set the annotation to an empty string. + annotations[fleetv1beta1.LastAppliedConfigAnnotation] = "" + isLastAppliedAnnotationSet = false + } + + manifestObj.SetAnnotations(annotations) + return isLastAppliedAnnotationSet, nil +} + +// getFleetLastAppliedAnnotation returns the last applied annotation of a manifest object. +func getFleetLastAppliedAnnotation(inMemberClusterObj *unstructured.Unstructured) []byte { + annotations := inMemberClusterObj.GetAnnotations() + if annotations == nil { + // The last applied annotation is not found in the live object; normally this should not + // happen, but Fleet can still handle this situation. + klog.Warningf("no annotations in the live object %s/%s", inMemberClusterObj.GroupVersionKind(), klog.KObj(inMemberClusterObj)) + return nil + } + + lastAppliedManifestJSONStr, found := annotations[fleetv1beta1.LastAppliedConfigAnnotation] + if !found { + // The last applied annotation is not found in the live object; normally this should not + // happen, but Fleet can still handle this situation. + klog.Warningf("the last applied annotation is not found in the live object %s/%s", inMemberClusterObj.GroupVersionKind(), klog.KObj(inMemberClusterObj)) + return nil + } + + return []byte(lastAppliedManifestJSONStr) +} + +// setManifestHashAnnotation computes the hash of the provided manifest and sets an annotation of the +// hash on the provided unstructured object. +func setManifestHashAnnotation(manifestObj *unstructured.Unstructured) error { + cleanedManifestObj := discardFieldsIrrelevantInComparisonFrom(manifestObj) + manifestObjHash, err := resource.HashOf(cleanedManifestObj.Object) + if err != nil { + return err + } + + annotations := manifestObj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[fleetv1beta1.ManifestHashAnnotation] = manifestObjHash + manifestObj.SetAnnotations(annotations) + return nil +} + +// setOwnerRef sets the expected owner reference (reference to an AppliedWork object) +// on a manifest to be applied. +func setOwnerRef(obj *unstructured.Unstructured, expectedAppliedWorkOwnerRef *metav1.OwnerReference) { + ownerRefs := obj.GetOwnerReferences() + if ownerRefs == nil { + ownerRefs = []metav1.OwnerReference{} + } + // Typically owner references is a system-managed field, and at this moment Fleet will + // clear owner references (if any) set in the manifest object. However, for consistency + // reasons, here Fleet will still assume that there might be some owner references set + // in the manifest object. + ownerRefs = append(ownerRefs, *expectedAppliedWorkOwnerRef) + obj.SetOwnerReferences(ownerRefs) +} + +// validateOwnerReferences validates the owner references of an applied manifest, checking +// if an apply op can be performed on the object. +func validateOwnerReferences( + manifestObj, inMemberClusterObj *unstructured.Unstructured, + applyStrategy *fleetv1beta1.ApplyStrategy, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) error { + manifestObjOwnerRefs := manifestObj.GetOwnerReferences() + + // If the manifest object already features some owner reference(s), but co-ownership is + // disallowed, the validation fails. + // + // This is just a sanity check; normally the branch will never get triggered as Fleet would + // perform sanitization on the manifest object before applying it, which removes all owner + // references. + if len(manifestObjOwnerRefs) > 0 && !applyStrategy.AllowCoOwnership { + return fmt.Errorf("manifest is set to have multiple owner references but co-ownership is disallowed") + } + + // Do a sanity check to verify that no AppliedWork object is directly added as an owner + // in the manifest object. Normally the branch will never get triggered as Fleet would + // perform sanitization on the manifest object before applying it, which removes all owner + // references. + for _, ownerRef := range manifestObjOwnerRefs { + if ownerRef.APIVersion == fleetv1beta1.GroupVersion.String() && ownerRef.Kind == fleetv1beta1.AppliedWorkKind { + return fmt.Errorf("an AppliedWork object is unexpectedly added as an owner in the manifest object") + } + } + + if inMemberClusterObj == nil { + // The manifest object has never been applied yet; no need to do further validation. + return nil + } + inMemberClusterObjOwnerRefs := inMemberClusterObj.GetOwnerReferences() + + // If the live object is co-owned but co-ownership is no longer allowed, the validation fails. + if len(inMemberClusterObjOwnerRefs) > 1 && !applyStrategy.AllowCoOwnership { + return fmt.Errorf("object is co-owned by multiple objects but co-ownership has been disallowed") + } + + // Note that at this point of execution, one of the owner references is guaranteed to be the + // expected AppliedWork object. For safety reasons, Fleet will still do a sanity check. + found := false + for _, ownerRef := range inMemberClusterObjOwnerRefs { + if reflect.DeepEqual(ownerRef, *expectedAppliedWorkOwnerRef) { + found = true + break + } + } + if !found { + return fmt.Errorf("object is not owned by the expected AppliedWork object") + } + + // If the object is already owned by another AppliedWork object, the validation fails. + // + // Normally this branch will never get executed as Fleet would refuse to take over an object + // that has been owned by another AppliedWork object. + if isPlacedByFleetInDuplicate(inMemberClusterObjOwnerRefs, expectedAppliedWorkOwnerRef) { + return fmt.Errorf("object is already owned by another AppliedWork object") + } + + return nil +} + +// sanitizeManifestObject sanitizes the manifest object before applying it. +// +// The sanitization logic here is consistent with that of the CRP controller, sans the API server +// specific parts; see also the generateRawContent function in the respective controller. +// +// Note that this function returns a copy of the manifest object; the original object will be left +// untouched. +func sanitizeManifestObject(manifestObj *unstructured.Unstructured) *unstructured.Unstructured { + // Create a deep copy of the object. + manifestObjCopy := manifestObj.DeepCopy() + + // Remove certain labels and annotations. + if annotations := manifestObjCopy.GetAnnotations(); annotations != nil { + // Remove the two Fleet reserved annotations. This is normally not set by users. + delete(annotations, fleetv1beta1.ManifestHashAnnotation) + delete(annotations, fleetv1beta1.LastAppliedConfigAnnotation) + + // Remove the last applied configuration set by kubectl. + delete(annotations, corev1.LastAppliedConfigAnnotation) + if len(annotations) == 0 { + manifestObjCopy.SetAnnotations(nil) + } else { + manifestObjCopy.SetAnnotations(annotations) + } + } + + // Remove certain system-managed fields. + manifestObjCopy.SetOwnerReferences(nil) + manifestObjCopy.SetManagedFields(nil) + + // Remove the read-only fields. + manifestObjCopy.SetCreationTimestamp(metav1.Time{}) + manifestObjCopy.SetDeletionTimestamp(nil) + manifestObjCopy.SetDeletionGracePeriodSeconds(nil) + manifestObjCopy.SetGeneration(0) + manifestObjCopy.SetResourceVersion("") + manifestObjCopy.SetSelfLink("") + manifestObjCopy.SetUID("") + + // Remove the status field. + unstructured.RemoveNestedField(manifestObjCopy.Object, "status") + + // Note: in the Fleet hub agent logic, the system also handles the Service and Job objects + // in a special way, so as to remove certain fields that are set by the hub cluster API + // server automatically; for the Fleet member agent logic here, however, Fleet assumes + // that if these fields are set, users must have set them on purpose, and they should not + // be removed. The difference comes to the fact that the Fleet member agent sanitization + // logic concerns only the enveloped objects, which are free from any hub cluster API + // server manipulation anyway. + + return manifestObjCopy +} + +// shouldEnableOptimisticLock checks if optimistic lock should be enabled given an apply strategy. +func shouldEnableOptimisticLock(applyStrategy *fleetv1beta1.ApplyStrategy) bool { + // Optimistic lock is enabled if the apply strategy is set to IfNotDrifted. + return applyStrategy.WhenToApply == fleetv1beta1.WhenToApplyTypeIfNotDrifted +} diff --git a/pkg/controllers/workapplier/apply_test.go b/pkg/controllers/workapplier/apply_test.go new file mode 100644 index 000000000..ac963e0a6 --- /dev/null +++ b/pkg/controllers/workapplier/apply_test.go @@ -0,0 +1,450 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "crypto/rand" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +// Note (chenyu1): The fake client Fleet uses for unit tests has trouble processing certain requests +// at the moment; affected test cases will be covered in the integration tests (w/ real clients) instead. + +// TestSanitizeManifestObject tests the sanitizeManifestObject function. +func TestSanitizeManifestObject(t *testing.T) { + // This test spec uses a Deployment object as the target. + dummyManager := "dummy-manager" + now := metav1.Now() + dummyGeneration := int64(1) + dummyResourceVersion := "abc" + dummySelfLink := "self-link" + dummyUID := "123-xyz-abcd" + + deploy := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: deployName, + Namespace: nsName, + Annotations: map[string]string{ + fleetv1beta1.ManifestHashAnnotation: dummyLabelValue1, + fleetv1beta1.LastAppliedConfigAnnotation: dummyLabelValue1, + corev1.LastAppliedConfigAnnotation: dummyLabelValue1, + dummyLabelKey: dummyLabelValue1, + }, + Labels: map[string]string{ + dummyLabelKey: dummyLabelValue2, + }, + Finalizers: []string{ + dummyLabelKey, + }, + ManagedFields: []metav1.ManagedFieldsEntry{ + { + Manager: dummyManager, + Operation: metav1.ManagedFieldsOperationUpdate, + APIVersion: "v1", + Time: &now, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{}, + }, + }, + OwnerReferences: []metav1.OwnerReference{ + dummyOwnerRef, + }, + CreationTimestamp: now, + DeletionTimestamp: &now, + DeletionGracePeriodSeconds: ptr.To(int64(30)), + Generation: dummyGeneration, + ResourceVersion: dummyResourceVersion, + SelfLink: dummySelfLink, + UID: types.UID(dummyUID), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + }, + Status: appsv1.DeploymentStatus{ + Replicas: 1, + }, + } + wantDeploy := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: deployName, + Namespace: nsName, + Annotations: map[string]string{ + dummyLabelKey: dummyLabelValue1, + }, + Labels: map[string]string{ + dummyLabelKey: dummyLabelValue2, + }, + Finalizers: []string{ + dummyLabelKey, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + }, + Status: appsv1.DeploymentStatus{}, + } + + testCases := []struct { + name string + manifestObj *unstructured.Unstructured + wantSanitizedObj *unstructured.Unstructured + }{ + { + name: "deploy", + manifestObj: toUnstructured(t, deploy), + wantSanitizedObj: toUnstructured(t, wantDeploy), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotSanitizedObj := sanitizeManifestObject(tc.manifestObj) + + // There are certain fields that need to be set manually as they might got omitted + // when being cast to an Unstructured object. + tc.wantSanitizedObj.SetCreationTimestamp(metav1.Time{}) + tc.wantSanitizedObj.SetManagedFields(nil) + tc.wantSanitizedObj.SetOwnerReferences(nil) + unstructured.RemoveNestedField(tc.wantSanitizedObj.Object, "status") + + if diff := cmp.Diff(gotSanitizedObj, tc.wantSanitizedObj); diff != "" { + t.Errorf("sanitized obj mismatches (-got +want):\n%s", diff) + } + }) + } +} + +// TestValidateOwnerReferences tests the validateOwnerReferences function. +func TestValidateOwnerReferences(t *testing.T) { + deployManifestObj1 := deploy.DeepCopy() + deployManifestObj1.OwnerReferences = []metav1.OwnerReference{ + dummyOwnerRef, + } + deployInMemberClusterObj1 := deploy.DeepCopy() + + deployManifestObj2 := deploy.DeepCopy() + deployManifestObj2.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + deployInMemberClusterObj2 := deploy.DeepCopy() + + deployManifestObj3 := deploy.DeepCopy() + + deployManifestObj4 := deploy.DeepCopy() + deployInMemberClusterObj4 := deploy.DeepCopy() + + deployManifestObj5 := deploy.DeepCopy() + deployInMemberClusterObj5 := deploy.DeepCopy() + deployInMemberClusterObj5.OwnerReferences = []metav1.OwnerReference{ + dummyOwnerRef, + *appliedWorkOwnerRef, + } + + deployManifestObj6 := deploy.DeepCopy() + deployInMemberClusterObj6 := deploy.DeepCopy() + deployInMemberClusterObj6.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + { + APIVersion: "placement.kubernetes-fleet.io/v1beta1", + Kind: "AppliedWork", + Name: "work-2", + UID: "uid-2", + }, + } + + deployManifestObj7 := deploy.DeepCopy() + deployInMemberClusterObj7 := deploy.DeepCopy() + deployInMemberClusterObj7.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + dummyOwnerRef, + } + + deployManifestObj8 := deploy.DeepCopy() + deployInMemberClusterObj8 := deploy.DeepCopy() + deployInMemberClusterObj8.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + testCases := []struct { + name string + manifestObj *unstructured.Unstructured + inMemberClusterObj *unstructured.Unstructured + applyStrategy *fleetv1beta1.ApplyStrategy + wantErred bool + wantErrMsgSubStr string + }{ + { + name: "multiple owners set on manifest, co-ownership is not allowed", + manifestObj: toUnstructured(t, deployManifestObj1), + inMemberClusterObj: toUnstructured(t, deployInMemberClusterObj1), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: false, + }, + wantErred: true, + wantErrMsgSubStr: "manifest is set to have multiple owner references but co-ownership is disallowed", + }, + { + name: "unexpected AppliedWork owner ref", + manifestObj: toUnstructured(t, deployManifestObj2), + inMemberClusterObj: toUnstructured(t, deployInMemberClusterObj2), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: true, + }, + wantErred: true, + wantErrMsgSubStr: "an AppliedWork object is unexpectedly added as an owner", + }, + { + name: "not created yet", + manifestObj: toUnstructured(t, deployManifestObj3), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: true, + }, + wantErred: false, + }, + { + name: "not taken over yet", + manifestObj: toUnstructured(t, deployManifestObj4), + inMemberClusterObj: toUnstructured(t, deployInMemberClusterObj4), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: false, + }, + wantErred: true, + wantErrMsgSubStr: "object is not owned by the expected AppliedWork object", + }, + { + name: "multiple owners set on applied object, co-ownership is not allowed", + manifestObj: toUnstructured(t, deployManifestObj5), + inMemberClusterObj: toUnstructured(t, deployInMemberClusterObj5), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: false, + }, + wantErred: true, + wantErrMsgSubStr: "object is co-owned by multiple objects but co-ownership has been disallowed", + }, + { + name: "placed by Fleet in duplication", + manifestObj: toUnstructured(t, deployManifestObj6), + inMemberClusterObj: toUnstructured(t, deployInMemberClusterObj6), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: true, + }, + wantErred: true, + wantErrMsgSubStr: "object is already owned by another AppliedWork object", + }, + { + name: "regular (multiple owners, co-ownership is allowed)", + manifestObj: toUnstructured(t, deployManifestObj7), + inMemberClusterObj: toUnstructured(t, deployInMemberClusterObj7), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: true, + }, + wantErred: false, + }, + { + name: "regular (single owner, co-ownership is not allowed)", + manifestObj: toUnstructured(t, deployManifestObj8), + inMemberClusterObj: toUnstructured(t, deployInMemberClusterObj8), + applyStrategy: &fleetv1beta1.ApplyStrategy{ + AllowCoOwnership: false, + }, + wantErred: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateOwnerReferences(tc.manifestObj, tc.inMemberClusterObj, tc.applyStrategy, appliedWorkOwnerRef) + switch { + case tc.wantErred && err == nil: + t.Fatalf("validateOwnerReferences() = nil, want error") + case !tc.wantErred && err != nil: + t.Fatalf("validateOwnerReferences() = %v, want no error", err) + case tc.wantErred && err != nil && !strings.Contains(err.Error(), tc.wantErrMsgSubStr): + t.Fatalf("validateOwnerReferences() = %v, want error message with sub-string %s", err, tc.wantErrMsgSubStr) + } + }) + } +} + +// TestSetManifestHashAnnotation tests the setManifestHashAnnotation function. +func TestSetManifestHashAnnotation(t *testing.T) { + nsManifestObj := ns.DeepCopy() + wantNSManifestObj := ns.DeepCopy() + wantNSManifestObj.Annotations = map[string]string{ + fleetv1beta1.ManifestHashAnnotation: string("08a19eb5a085293fdc5b9e252422e44002e5cfeea1ae3cb303ce1a6537d97c69"), + } + + testCases := []struct { + name string + manifestObj *unstructured.Unstructured + wantManifestObj *unstructured.Unstructured + }{ + { + name: "namespace", + manifestObj: toUnstructured(t, nsManifestObj), + wantManifestObj: toUnstructured(t, wantNSManifestObj), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if err := setManifestHashAnnotation(tc.manifestObj); err != nil { + t.Fatalf("setManifestHashAnnotation() = %v, want no error", err) + } + + if diff := cmp.Diff(tc.manifestObj, tc.wantManifestObj); diff != "" { + t.Errorf("manifest obj mismatches (-got +want):\n%s", diff) + } + }) + } +} + +// TestGetFleetLastAppliedAnnotation tests the getFleetLastAppliedAnnotation function. +func TestGetFleetLastAppliedAnnotation(t *testing.T) { + nsInMemberClusterObj1 := ns.DeepCopy() + + nsInMemberClusterObj2 := ns.DeepCopy() + nsInMemberClusterObj2.SetAnnotations(map[string]string{ + dummyLabelKey: dummyLabelValue1, + }) + + nsInMemberClusterObj3 := ns.DeepCopy() + nsInMemberClusterObj3.SetAnnotations(map[string]string{ + fleetv1beta1.LastAppliedConfigAnnotation: dummyLabelValue1, + }) + + testCases := []struct { + name string + inMemberClusterObj *unstructured.Unstructured + wantLastAppliedManifestJSONBytes []byte + }{ + { + name: "no annotations", + inMemberClusterObj: toUnstructured(t, nsInMemberClusterObj1), + }, + { + name: "no last applied annotation", + inMemberClusterObj: toUnstructured(t, nsInMemberClusterObj2), + }, + { + name: "last applied annotation set", + inMemberClusterObj: toUnstructured(t, nsInMemberClusterObj3), + wantLastAppliedManifestJSONBytes: []byte(dummyLabelValue1), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotBytes := getFleetLastAppliedAnnotation(tc.inMemberClusterObj) + if diff := cmp.Diff(gotBytes, tc.wantLastAppliedManifestJSONBytes); diff != "" { + t.Errorf("last applied manifest JSON bytes mismatches (-got +want):\n%s", diff) + } + }) + } +} + +// TestSetFleetLastAppliedAnnotation tests the setFleetLastAppliedAnnotation function. +func TestSetFleetLastAppliedAnnotation(t *testing.T) { + nsManifestObj1 := ns.DeepCopy() + wantNSManifestObj1 := ns.DeepCopy() + wantNSManifestObj1.SetAnnotations(map[string]string{ + fleetv1beta1.LastAppliedConfigAnnotation: string("{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"creationTimestamp\":null,\"name\":\"ns-1\"},\"spec\":{},\"status\":{}}\n"), + }) + + nsManifestObj2 := ns.DeepCopy() + nsManifestObj2.SetAnnotations(map[string]string{ + fleetv1beta1.LastAppliedConfigAnnotation: dummyLabelValue2, + }) + wantNSManifestObj2 := ns.DeepCopy() + wantNSManifestObj2.SetAnnotations(map[string]string{ + fleetv1beta1.LastAppliedConfigAnnotation: string("{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"creationTimestamp\":null,\"name\":\"ns-1\"},\"spec\":{},\"status\":{}}\n"), + }) + + // Annotation size limit is 262144 bytes. + longDataBytes := make([]byte, 300000) + _, err := rand.Read(longDataBytes) + if err != nil { + t.Fatalf("failed to generate random bytes: %v", err) + } + configMapManifestObj3 := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: nsName, + }, + Data: map[string]string{ + "data": string(longDataBytes), + }, + } + wantConfigMapManifestObj3 := configMapManifestObj3.DeepCopy() + wantConfigMapManifestObj3.SetAnnotations(map[string]string{ + fleetv1beta1.LastAppliedConfigAnnotation: "", + }) + + testCases := []struct { + name string + manifestObj *unstructured.Unstructured + wantManifestObj *unstructured.Unstructured + wantSetFlag bool + }{ + { + name: "last applied annotation set", + manifestObj: toUnstructured(t, nsManifestObj1), + wantManifestObj: toUnstructured(t, wantNSManifestObj1), + wantSetFlag: true, + }, + { + name: "last applied annotation replaced", + manifestObj: toUnstructured(t, nsManifestObj2), + wantManifestObj: toUnstructured(t, wantNSManifestObj2), + wantSetFlag: true, + }, + { + name: "last applied annotation not set (annotation oversized)", + manifestObj: toUnstructured(t, configMapManifestObj3), + wantManifestObj: toUnstructured(t, wantConfigMapManifestObj3), + wantSetFlag: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotSetFlag, err := setFleetLastAppliedAnnotation(tc.manifestObj) + if err != nil { + t.Fatalf("setFleetLastAppliedAnnotation() = %v, want no error", err) + } + + if gotSetFlag != tc.wantSetFlag { + t.Errorf("isLastAppliedAnnotationSet flag mismatches, got %t, want %t", gotSetFlag, tc.wantSetFlag) + } + if diff := cmp.Diff(tc.manifestObj, tc.wantManifestObj); diff != "" { + t.Errorf("manifest obj mismatches (-got +want):\n%s", diff) + } + }) + } +} diff --git a/pkg/controllers/workapplier/availability_tracker.go b/pkg/controllers/workapplier/availability_tracker.go new file mode 100644 index 000000000..0c8fbfef3 --- /dev/null +++ b/pkg/controllers/workapplier/availability_tracker.go @@ -0,0 +1,241 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" + + "go.goms.io/fleet/pkg/utils" + "go.goms.io/fleet/pkg/utils/controller" +) + +// trackInMemberClusterObjAvailability tracks the availability of an applied objects in the member cluster. +func (r *Reconciler) trackInMemberClusterObjAvailability(ctx context.Context, bundles []*manifestProcessingBundle, workRef klog.ObjectRef) { + // Track the availability of all the applied objects in the member cluster in parallel. + // + // This is concurrency-safe as the bundles slice has been pre-allocated. + + // Prepare a child context. + // Cancel the child context anyway to avoid leaks. + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + + doWork := func(pieces int) { + bundle := bundles[pieces] + if !isManifestObjectApplied(bundle.applyResTyp) { + // The manifest object in the bundle has not been applied yet. No availability check + // is needed. + bundle.availabilityResTyp = ManifestProcessingAvailabilityResultTypeSkipped + return + } + + availabilityResTyp, err := trackInMemberClusterObjAvailabilityByGVR(bundle.gvr, bundle.inMemberClusterObj) + if err != nil { + // An unexpected error has occurred during the availability check. + bundle.availabilityErr = err + bundle.availabilityResTyp = ManifestProcessingAvailabilityResultTypeFailed + klog.ErrorS(err, + "Failed to track the availability of the applied object in the member cluster", + "work", workRef, "GVR", *bundle.gvr, "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj)) + return + } + bundle.availabilityResTyp = availabilityResTyp + klog.V(2).InfoS("Tracked availability of a resource", + "work", workRef, "GVR", *bundle.gvr, "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), + "availabilityResTyp", availabilityResTyp) + } + + // Run the availability check in parallel. + r.parallelizer.ParallelizeUntil(childCtx, len(bundles), doWork, "trackInMemberClusterObjAvailability") +} + +// trackInMemberClusterObjAvailabilityByGVR tracks the availability of an object in the member cluster based +// on its GVR. +func trackInMemberClusterObjAvailabilityByGVR( + gvr *schema.GroupVersionResource, + inMemberClusterObj *unstructured.Unstructured, +) (ManifestProcessingAvailabilityResultType, error) { + switch *gvr { + case utils.DeploymentGVR: + return trackDeploymentAvailability(inMemberClusterObj) + case utils.StatefulSetGVR: + return trackStatefulSetAvailability(inMemberClusterObj) + case utils.DaemonSetGVR: + return trackDaemonSetAvailability(inMemberClusterObj) + case utils.ServiceGVR: + return trackServiceAvailability(inMemberClusterObj) + case utils.CustomResourceDefinitionGVR: + return trackCRDAvailability(inMemberClusterObj) + default: + if isDataResource(*gvr) { + klog.V(2).InfoS("The object from the member cluster is a data object, consider it to be immediately available", + "gvr", *gvr, "inMemberClusterObj", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeAvailable, nil + } + klog.V(2).InfoS("Cannot determine the availability of the object from the member cluster; untrack its availability", + "gvr", *gvr, "resource", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeNotTrackable, nil + } +} + +// trackDeploymentAvailability tracks the availability of a deployment in the member cluster. +func trackDeploymentAvailability(inMemberClusterObj *unstructured.Unstructured) (ManifestProcessingAvailabilityResultType, error) { + var deploy appv1.Deployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(inMemberClusterObj.Object, &deploy); err != nil { + // Normally this branch should never run. + return ManifestProcessingAvailabilityResultTypeFailed, controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to convert the unstructured object to a deployment: %w", err)) + } + + // Check if the deployment is available. + requiredReplicas := int32(1) + if deploy.Spec.Replicas != nil { + requiredReplicas = *deploy.Spec.Replicas + } + if deploy.Status.ObservedGeneration == deploy.Generation && + requiredReplicas == deploy.Status.AvailableReplicas && + requiredReplicas == deploy.Status.UpdatedReplicas { + klog.V(2).InfoS("Deployment is available", "deployment", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeAvailable, nil + } + klog.V(2).InfoS("Deployment is not ready yet, will check later to see if it becomes available", "deployment", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeNotYetAvailable, nil +} + +// trackStatefulSetAvailability tracks the availability of a stateful set in the member cluster. +func trackStatefulSetAvailability(inMemberClusterObj *unstructured.Unstructured) (ManifestProcessingAvailabilityResultType, error) { + var statefulSet appv1.StatefulSet + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(inMemberClusterObj.Object, &statefulSet); err != nil { + // Normally this branch should never run. + return ManifestProcessingAvailabilityResultTypeFailed, controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to convert the unstructured object to a stateful set: %w", err)) + } + + // Check if the stateful set is available. + // + // A statefulSet is available if all if its replicas are available and the current replica count + // is equal to the updated replica count, which implies that all replicas are up to date. + requiredReplicas := int32(1) + if statefulSet.Spec.Replicas != nil { + requiredReplicas = *statefulSet.Spec.Replicas + } + if statefulSet.Status.ObservedGeneration == statefulSet.Generation && + statefulSet.Status.AvailableReplicas == requiredReplicas && + statefulSet.Status.CurrentReplicas == statefulSet.Status.UpdatedReplicas && + statefulSet.Status.CurrentRevision == statefulSet.Status.UpdateRevision { + klog.V(2).InfoS("StatefulSet is available", "statefulSet", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeAvailable, nil + } + klog.V(2).InfoS("Stateful set is not ready yet, will check later to see if it becomes available", "statefulSet", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeNotYetAvailable, nil +} + +// trackDaemonSetAvailability tracks the availability of a daemon set in the member cluster. +func trackDaemonSetAvailability(inMemberClusterObj *unstructured.Unstructured) (ManifestProcessingAvailabilityResultType, error) { + var daemonSet appv1.DaemonSet + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(inMemberClusterObj.Object, &daemonSet); err != nil { + // Normally this branch should never run. + return ManifestProcessingAvailabilityResultTypeFailed, controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to convert the unstructured object to a daemon set: %w", err)) + } + + // Check if the daemonSet is available. + // + // A daemonSet is available if all if its desired replicas (the count of which is equal to + // the number of applicable nodes in the cluster) are available and the current replica count + // is equal to the updated replica count, which implies that all replicas are up to date. + if daemonSet.Status.ObservedGeneration == daemonSet.Generation && + daemonSet.Status.NumberAvailable == daemonSet.Status.DesiredNumberScheduled && + daemonSet.Status.CurrentNumberScheduled == daemonSet.Status.UpdatedNumberScheduled { + klog.V(2).InfoS("DaemonSet is available", "daemonSet", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeAvailable, nil + } + klog.V(2).InfoS("Daemon set is not ready yet, will check later to see if it becomes available", "daemonSet", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeNotYetAvailable, nil +} + +// trackServiceAvailability tracks the availability of a service in the member cluster. +func trackServiceAvailability(inMemberClusterObj *unstructured.Unstructured) (ManifestProcessingAvailabilityResultType, error) { + var svc corev1.Service + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(inMemberClusterObj.Object, &svc); err != nil { + return ManifestProcessingAvailabilityResultTypeFailed, controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to convert the unstructured object to a service: %w", err)) + } + switch svc.Spec.Type { + case "": + fallthrough // The default service type is ClusterIP. + case corev1.ServiceTypeClusterIP: + fallthrough + case corev1.ServiceTypeNodePort: + // Fleet considers a ClusterIP or NodePort service to be available if it has at least one + // IP assigned. + if len(svc.Spec.ClusterIPs) > 0 && len(svc.Spec.ClusterIPs[0]) > 0 { + klog.V(2).InfoS("Service is available", "service", klog.KObj(inMemberClusterObj), "serviceType", svc.Spec.Type) + return ManifestProcessingAvailabilityResultTypeAvailable, nil + } + klog.V(2).InfoS("Service is not ready yet, will check later to see if it becomes available", "service", klog.KObj(inMemberClusterObj), "serviceType", svc.Spec.Type) + return ManifestProcessingAvailabilityResultTypeNotYetAvailable, nil + case corev1.ServiceTypeLoadBalancer: + // Fleet considers a loadBalancer service to be available if it has at least one load + // balancer IP or hostname assigned. + if len(svc.Status.LoadBalancer.Ingress) > 0 && + (len(svc.Status.LoadBalancer.Ingress[0].IP) > 0 || len(svc.Status.LoadBalancer.Ingress[0].Hostname) > 0) { + klog.V(2).InfoS("Service is available", "service", klog.KObj(inMemberClusterObj), "serviceType", svc.Spec.Type) + return ManifestProcessingAvailabilityResultTypeAvailable, nil + } + klog.V(2).InfoS("Service is not ready yet, will check later to see if it becomes available", "service", klog.KObj(inMemberClusterObj), "serviceType", svc.Spec.Type) + return ManifestProcessingAvailabilityResultTypeNotYetAvailable, nil + } + + // we don't know how to track the availability of when the service type is externalName + klog.V(2).InfoS("Cannot determine the availability of external name services; untrack its availability", "service", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeNotTrackable, nil +} + +// trackCRDAvailability tracks the availability of a custom resource definition in the member cluster. +func trackCRDAvailability(inMemberClusterObj *unstructured.Unstructured) (ManifestProcessingAvailabilityResultType, error) { + var crd apiextensionsv1.CustomResourceDefinition + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(inMemberClusterObj.Object, &crd); err != nil { + return ManifestProcessingAvailabilityResultTypeFailed, controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to convert the unstructured object to a custom resource definition: %w", err)) + } + + // If both conditions are True, the CRD has become available. + if apiextensionshelpers.IsCRDConditionTrue(&crd, apiextensionsv1.Established) && apiextensionshelpers.IsCRDConditionTrue(&crd, apiextensionsv1.NamesAccepted) { + klog.V(2).InfoS("CustomResourceDefinition is available", "customResourceDefinition", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeAvailable, nil + } + + klog.V(2).InfoS("Custom resource definition is not ready yet, will check later to see if it becomes available", klog.KObj(inMemberClusterObj)) + return ManifestProcessingAvailabilityResultTypeNotYetAvailable, nil +} + +// isDataResource checks if the resource is a data resource; such resources are +// available immediately after creation. +func isDataResource(gvr schema.GroupVersionResource) bool { + switch gvr { + case utils.NamespaceGVR: + return true + case utils.SecretGVR: + return true + case utils.ConfigMapGVR: + return true + case utils.RoleGVR: + return true + case utils.ClusterRoleGVR: + return true + case utils.RoleBindingGVR: + return true + case utils.ClusterRoleBindingGVR: + return true + } + return false +} diff --git a/pkg/controllers/workapplier/availability_tracker_test.go b/pkg/controllers/workapplier/availability_tracker_test.go new file mode 100644 index 000000000..3c3511932 --- /dev/null +++ b/pkg/controllers/workapplier/availability_tracker_test.go @@ -0,0 +1,943 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils" + "go.goms.io/fleet/pkg/utils/parallelizer" +) + +var ( + statefulSetTemplate = &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: nsName, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + } + + daemonSetTemplate = &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: nsName, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:latest", + }, + }, + }, + }, + }, + } + + crdTemplate = &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "foos.example.com", + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "example.com", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Kind: "Foo", + ListKind: "FooList", + Plural: "foos", + Singular: "foo", + }, + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": { + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "field": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +) + +// TestTrackDeploymentAvailability tests the trackDeploymentAvailability function. +func TestTrackDeploymentAvailability(t *testing.T) { + availableDeployWithFixedReplicaCount := deploy.DeepCopy() + availableDeployWithFixedReplicaCount.Status = appsv1.DeploymentStatus{ + Replicas: 1, + AvailableReplicas: 1, + UpdatedReplicas: 1, + } + + availableDeployWithDefaultReplicaCount := deploy.DeepCopy() + availableDeployWithDefaultReplicaCount.Spec.Replicas = nil + availableDeployWithDefaultReplicaCount.Status = appsv1.DeploymentStatus{ + Replicas: 1, + AvailableReplicas: 1, + UpdatedReplicas: 1, + } + + unavailableDeployWithStaleStatus := deploy.DeepCopy() + unavailableDeployWithStaleStatus.Generation = 2 + unavailableDeployWithStaleStatus.Status = appsv1.DeploymentStatus{ + ObservedGeneration: 1, + Replicas: 1, + AvailableReplicas: 1, + UpdatedReplicas: 1, + } + + unavailableDeployWithNotEnoughAvailableReplicas := deploy.DeepCopy() + unavailableDeployWithNotEnoughAvailableReplicas.Spec.Replicas = ptr.To(int32(5)) + unavailableDeployWithNotEnoughAvailableReplicas.Status = appsv1.DeploymentStatus{ + Replicas: 5, + AvailableReplicas: 2, + UpdatedReplicas: 5, + } + + unavailableDeployWithNotEnoughUpdatedReplicas := deploy.DeepCopy() + unavailableDeployWithNotEnoughUpdatedReplicas.Spec.Replicas = ptr.To(int32(5)) + unavailableDeployWithNotEnoughUpdatedReplicas.Status = appsv1.DeploymentStatus{ + Replicas: 5, + AvailableReplicas: 5, + UpdatedReplicas: 2, + } + + testCases := []struct { + name string + deploy *appsv1.Deployment + wantManifestProcessingAvailabilityResultType ManifestProcessingAvailabilityResultType + }{ + { + name: "available deployment (w/ fixed replica count)", + deploy: availableDeployWithFixedReplicaCount, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available deployment (w/ default replica count)", + deploy: availableDeployWithDefaultReplicaCount, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "unavailable deployment with stale status", + deploy: unavailableDeployWithStaleStatus, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable deployment with not enough available replicas", + deploy: unavailableDeployWithNotEnoughAvailableReplicas, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable deployment with not enough updated replicas", + deploy: unavailableDeployWithNotEnoughUpdatedReplicas, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotResTyp, err := trackDeploymentAvailability(toUnstructured(t, tc.deploy)) + if err != nil { + t.Fatalf("trackDeploymentAvailability() = %v, want no error", err) + } + if gotResTyp != tc.wantManifestProcessingAvailabilityResultType { + t.Errorf("manifestProcessingAvailabilityResultType = %v, want %v", gotResTyp, tc.wantManifestProcessingAvailabilityResultType) + } + }) + } +} + +// TestTrackStatefulSetAvailability tests the trackStatefulSetAvailability function. +func TestTrackStatefulSetAvailability(t *testing.T) { + availableStatefulSetWithFixedReplicaCount := statefulSetTemplate.DeepCopy() + availableStatefulSetWithFixedReplicaCount.Status = appsv1.StatefulSetStatus{ + Replicas: 1, + AvailableReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + CurrentRevision: "1", + UpdateRevision: "1", + } + + availableStatefulSetWithDefaultReplicaCount := statefulSetTemplate.DeepCopy() + availableStatefulSetWithDefaultReplicaCount.Spec.Replicas = nil + availableStatefulSetWithDefaultReplicaCount.Status = appsv1.StatefulSetStatus{ + Replicas: 1, + AvailableReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + CurrentRevision: "1", + UpdateRevision: "1", + } + + unavailableStatefulSetWithStaleStatus := statefulSetTemplate.DeepCopy() + unavailableStatefulSetWithStaleStatus.Generation = 2 + unavailableStatefulSetWithStaleStatus.Status = appsv1.StatefulSetStatus{ + ObservedGeneration: 1, + Replicas: 1, + AvailableReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + CurrentRevision: "1", + UpdateRevision: "1", + } + + unavailableStatefulSetWithNotEnoughAvailableReplicas := statefulSetTemplate.DeepCopy() + unavailableStatefulSetWithNotEnoughAvailableReplicas.Spec.Replicas = ptr.To(int32(5)) + unavailableStatefulSetWithNotEnoughAvailableReplicas.Status = appsv1.StatefulSetStatus{ + Replicas: 5, + AvailableReplicas: 2, + CurrentReplicas: 5, + UpdatedReplicas: 5, + CurrentRevision: "1", + UpdateRevision: "1", + } + + unavailableStatefulSetWithNotEnoughCurrentReplicas := statefulSetTemplate.DeepCopy() + unavailableStatefulSetWithNotEnoughCurrentReplicas.Spec.Replicas = ptr.To(int32(5)) + unavailableStatefulSetWithNotEnoughCurrentReplicas.Status = appsv1.StatefulSetStatus{ + Replicas: 5, + AvailableReplicas: 5, + CurrentReplicas: 2, + UpdatedReplicas: 5, + CurrentRevision: "1", + UpdateRevision: "1", + } + + unavailableStatefulSetWithNotLatestRevision := statefulSetTemplate.DeepCopy() + unavailableStatefulSetWithNotLatestRevision.Status = appsv1.StatefulSetStatus{ + Replicas: 1, + AvailableReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + CurrentRevision: "1", + UpdateRevision: "2", + } + + testCases := []struct { + name string + statefulSet *appsv1.StatefulSet + wantManifestProcessingAvailabilityResultType ManifestProcessingAvailabilityResultType + }{ + { + name: "available stateful set (w/ fixed replica count)", + statefulSet: availableStatefulSetWithFixedReplicaCount, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available stateful set (w/ default replica count)", + statefulSet: availableStatefulSetWithDefaultReplicaCount, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "unavailable stateful set with stale status", + statefulSet: unavailableStatefulSetWithStaleStatus, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable stateful set with not enough available replicas", + statefulSet: unavailableStatefulSetWithNotEnoughAvailableReplicas, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable stateful set with not enough current replicas", + statefulSet: unavailableStatefulSetWithNotEnoughCurrentReplicas, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable stateful set with not latest revision", + statefulSet: unavailableStatefulSetWithNotLatestRevision, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotResTyp, err := trackStatefulSetAvailability(toUnstructured(t, tc.statefulSet)) + if err != nil { + t.Fatalf("trackStatefulSetAvailability() = %v, want no error", err) + } + if gotResTyp != tc.wantManifestProcessingAvailabilityResultType { + t.Errorf("manifestProcessingAvailabilityResultType = %v, want %v", gotResTyp, tc.wantManifestProcessingAvailabilityResultType) + } + }) + } +} + +// TestTrackDaemonSetAvailability tests the trackDaemonSetAvailability function. +func TestTrackDaemonSetAvailability(t *testing.T) { + availableDaemonSet := daemonSetTemplate.DeepCopy() + availableDaemonSet.Status = appsv1.DaemonSetStatus{ + NumberAvailable: 1, + DesiredNumberScheduled: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 1, + } + + unavailableDaemonSetWithStaleStatus := daemonSetTemplate.DeepCopy() + unavailableDaemonSetWithStaleStatus.Generation = 2 + unavailableDaemonSetWithStaleStatus.Status = appsv1.DaemonSetStatus{ + ObservedGeneration: 1, + NumberAvailable: 1, + DesiredNumberScheduled: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 1, + } + + unavailableDaemonSetWithNotEnoughAvailablePods := daemonSetTemplate.DeepCopy() + unavailableDaemonSetWithNotEnoughAvailablePods.Status = appsv1.DaemonSetStatus{ + NumberAvailable: 2, + DesiredNumberScheduled: 5, + CurrentNumberScheduled: 5, + UpdatedNumberScheduled: 5, + } + + unavailableDaemonSetWithNotEnoughUpdatedPods := daemonSetTemplate.DeepCopy() + unavailableDaemonSetWithNotEnoughUpdatedPods.Status = appsv1.DaemonSetStatus{ + NumberAvailable: 5, + DesiredNumberScheduled: 5, + CurrentNumberScheduled: 5, + UpdatedNumberScheduled: 6, + } + + testCases := []struct { + name string + daemonSet *appsv1.DaemonSet + wantManifestProcessingAvailabilityResultType ManifestProcessingAvailabilityResultType + }{ + { + name: "available daemon set", + daemonSet: availableDaemonSet, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "unavailable daemon set with stale status", + daemonSet: unavailableDaemonSetWithStaleStatus, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable daemon set with not enough available pods", + daemonSet: unavailableDaemonSetWithNotEnoughAvailablePods, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable daemon set with not enough updated pods", + daemonSet: unavailableDaemonSetWithNotEnoughUpdatedPods, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotResTyp, err := trackDaemonSetAvailability(toUnstructured(t, tc.daemonSet)) + if err != nil { + t.Fatalf("trackDaemonSetAvailability() = %v, want no error", err) + } + if gotResTyp != tc.wantManifestProcessingAvailabilityResultType { + t.Errorf("manifestProcessingAvailabilityResultType = %v, want %v", gotResTyp, tc.wantManifestProcessingAvailabilityResultType) + } + }) + } +} + +// TestTrackServiceAvailability tests the trackServiceAvailability function. +func TestTrackServiceAvailability(t *testing.T) { + testCases := []struct { + name string + service *corev1.Service + wantManifestProcessingAvailabilityResultType ManifestProcessingAvailabilityResultType + }{ + { + name: "untrackable service (external name type)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ClusterIPs: []string{"192.168.1.1"}, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotTrackable, + }, + { + name: "available default typed service (IP assigned)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: "", + ClusterIPs: []string{"192.168.1.1"}, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available ClusterIP service (IP assigned)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available headless service", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available node port service (IP assigned)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + ClusterIP: "13.6.2.2", + ClusterIPs: []string{"192.168.1.1"}, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "unavailable ClusterIP service (no IP assigned)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "13.6.2.2", + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "available LoadBalancer service (IP assigned)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + { + IP: "10.1.2.4", + }, + }, + }, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available LoadBalancer service (hostname assigned)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + { + Hostname: "one.microsoft.com", + }, + }, + }, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "unavailable LoadBalancer service (ingress not ready)", + service: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{}, + }, + }, + }, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotResTyp, err := trackServiceAvailability(toUnstructured(t, tc.service)) + if err != nil { + t.Errorf("trackServiceAvailability() = %v, want no error", err) + } + if gotResTyp != tc.wantManifestProcessingAvailabilityResultType { + t.Errorf("manifestProcessingAvailabilityResultType = %v, want %v", gotResTyp, tc.wantManifestProcessingAvailabilityResultType) + } + }) + } +} + +// TestTrackCRDAvailability tests the trackCRDAvailability function. +func TestTrackCRDAvailability(t *testing.T) { + availableCRD := crdTemplate.DeepCopy() + availableCRD.Status = apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ + { + Type: apiextensionsv1.Established, + Status: apiextensionsv1.ConditionTrue, + }, + { + Type: apiextensionsv1.NamesAccepted, + Status: apiextensionsv1.ConditionTrue, + }, + }, + } + + unavailableCRDNotEstablished := crdTemplate.DeepCopy() + unavailableCRDNotEstablished.Status = apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ + { + Type: apiextensionsv1.Established, + Status: apiextensionsv1.ConditionFalse, + }, + { + Type: apiextensionsv1.NamesAccepted, + Status: apiextensionsv1.ConditionTrue, + }, + }, + } + + unavailableCRDNameNotAccepted := crdTemplate.DeepCopy() + unavailableCRDNameNotAccepted.Status = apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ + { + Type: apiextensionsv1.Established, + Status: apiextensionsv1.ConditionTrue, + }, + { + Type: apiextensionsv1.NamesAccepted, + Status: apiextensionsv1.ConditionFalse, + }, + }, + } + + testCases := []struct { + name string + crd *apiextensionsv1.CustomResourceDefinition + wantManifestProcessingAvailabilityResultType ManifestProcessingAvailabilityResultType + }{ + { + name: "available CRD", + crd: availableCRD, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "unavailable CRD (not established)", + crd: unavailableCRDNotEstablished, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + name: "unavailable CRD (name not accepted)", + crd: unavailableCRDNameNotAccepted, + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotResTyp, err := trackCRDAvailability(toUnstructured(t, tc.crd)) + if err != nil { + t.Fatalf("trackCRDAvailability() = %v, want no error", err) + } + if gotResTyp != tc.wantManifestProcessingAvailabilityResultType { + t.Errorf("manifestProcessingAvailabilityResultType = %v, want %v", gotResTyp, tc.wantManifestProcessingAvailabilityResultType) + } + }) + } +} + +// TestTrackInMemberClusterObjAvailabilityByGVR tests the trackInMemberClusterObjAvailabilityByGVR function. +func TestTrackInMemberClusterObjAvailabilityByGVR(t *testing.T) { + availableDeploy := deploy.DeepCopy() + availableDeploy.Status = appsv1.DeploymentStatus{ + Replicas: 1, + AvailableReplicas: 1, + UpdatedReplicas: 1, + } + + availableStatefulSet := statefulSetTemplate.DeepCopy() + availableStatefulSet.Status = appsv1.StatefulSetStatus{ + Replicas: 1, + AvailableReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + CurrentRevision: "1", + UpdateRevision: "1", + } + + availableSvc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + Spec: corev1.ServiceSpec{ + Type: "", + ClusterIPs: []string{"192.168.1.1"}, + }, + } + + availableDaemonSet := daemonSetTemplate.DeepCopy() + availableDaemonSet.Status = appsv1.DaemonSetStatus{ + NumberAvailable: 1, + DesiredNumberScheduled: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 1, + } + + availableCRD := crdTemplate.DeepCopy() + availableCRD.Status = apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ + { + Type: apiextensionsv1.Established, + Status: apiextensionsv1.ConditionTrue, + }, + { + Type: apiextensionsv1.NamesAccepted, + Status: apiextensionsv1.ConditionTrue, + }, + }, + } + + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: nsName, + }, + Data: map[string]string{ + "key": "value", + }, + } + + untrackableJob := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1", + Kind: "Job", + }, + } + + testCases := []struct { + name string + gvr schema.GroupVersionResource + inMemberClusterObj *unstructured.Unstructured + wantManifestProcessingAvailabilityResultType ManifestProcessingAvailabilityResultType + }{ + { + name: "available deployment", + gvr: utils.DeploymentGVR, + inMemberClusterObj: toUnstructured(t, availableDeploy), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available stateful set", + gvr: utils.StatefulSetGVR, + inMemberClusterObj: toUnstructured(t, availableStatefulSet), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available service", + gvr: utils.ServiceGVR, + inMemberClusterObj: toUnstructured(t, availableSvc), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available daemon set", + gvr: utils.DaemonSetGVR, + inMemberClusterObj: toUnstructured(t, availableDaemonSet), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "available custom resource definition", + gvr: utils.CustomResourceDefinitionGVR, + inMemberClusterObj: toUnstructured(t, availableCRD), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "data object (namespace)", + gvr: utils.NamespaceGVR, + inMemberClusterObj: toUnstructured(t, ns.DeepCopy()), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "data object (config map)", + gvr: utils.ConfigMapGVR, + inMemberClusterObj: toUnstructured(t, cm), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + name: "untrackable object (job)", + gvr: utils.JobGVR, + inMemberClusterObj: toUnstructured(t, untrackableJob), + wantManifestProcessingAvailabilityResultType: ManifestProcessingAvailabilityResultTypeNotTrackable, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotResTyp, err := trackInMemberClusterObjAvailabilityByGVR(&tc.gvr, tc.inMemberClusterObj) + if err != nil { + t.Fatalf("trackInMemberClusterObjAvailabilityByGVR() = %v, want no error", err) + } + if gotResTyp != tc.wantManifestProcessingAvailabilityResultType { + t.Errorf("manifestProcessingAvailabilityResultType = %v, want %v", gotResTyp, tc.wantManifestProcessingAvailabilityResultType) + } + }) + } +} + +// TestTrackInMemberClusterObjAvailability tests the trackInMemberClusterObjAvailability method. +func TestTrackInMemberClusterObjAvailability(t *testing.T) { + ctx := context.Background() + workRef := klog.KRef(memberReservedNSName, workName) + + availableDeploy := deploy.DeepCopy() + availableDeploy.Status = appsv1.DeploymentStatus{ + Replicas: 1, + AvailableReplicas: 1, + UpdatedReplicas: 1, + } + + unavailableDaemonSet := daemonSetTemplate.DeepCopy() + unavailableDaemonSet.Status = appsv1.DaemonSetStatus{ + NumberAvailable: 1, + DesiredNumberScheduled: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 2, + } + + untrackableJob := &batchv1.Job{} + + testCases := []struct { + name string + bundles []*manifestProcessingBundle + wantBundles []*manifestProcessingBundle + }{ + { + name: "mixed", + bundles: []*manifestProcessingBundle{ + // The IDs are set purely for the purpose of sorting the results. + + // An available deployment. + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + }, + gvr: &utils.DeploymentGVR, + inMemberClusterObj: toUnstructured(t, availableDeploy), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + }, + // A failed to get applied service. + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + }, + gvr: &utils.ServiceGVR, + inMemberClusterObj: nil, + applyResTyp: ManifestProcessingApplyResultTypeFailedToApply, + }, + // An unavailable daemon set. + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + }, + gvr: &utils.DaemonSetGVR, + inMemberClusterObj: toUnstructured(t, unavailableDaemonSet), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + }, + // An untrackable job. + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 3, + }, + gvr: &utils.JobGVR, + inMemberClusterObj: toUnstructured(t, untrackableJob), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + }, + }, + wantBundles: []*manifestProcessingBundle{ + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + }, + gvr: &utils.DeploymentGVR, + inMemberClusterObj: toUnstructured(t, availableDeploy), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeAvailable, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + }, + gvr: &utils.ServiceGVR, + inMemberClusterObj: nil, + applyResTyp: ManifestProcessingApplyResultTypeFailedToApply, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeSkipped, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + }, + gvr: &utils.DaemonSetGVR, + inMemberClusterObj: toUnstructured(t, unavailableDaemonSet), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 3, + }, + gvr: &utils.JobGVR, + inMemberClusterObj: toUnstructured(t, untrackableJob), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeNotTrackable, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + r := &Reconciler{ + parallelizer: parallelizer.NewParallelizer(2), + } + + r.trackInMemberClusterObjAvailability(ctx, tc.bundles, workRef) + + // A special less func to sort the bundles by their ordinal. + lessFuncManifestProcessingBundle := func(i, j *manifestProcessingBundle) bool { + return i.id.Ordinal < j.id.Ordinal + } + if diff := cmp.Diff( + tc.bundles, tc.wantBundles, + cmp.AllowUnexported(manifestProcessingBundle{}), + cmpopts.SortSlices(lessFuncManifestProcessingBundle), + ); diff != "" { + t.Errorf("bundles mismatches (-got, +want):\n%s", diff) + } + }) + } +} diff --git a/pkg/controllers/workapplier/controller.go b/pkg/controllers/workapplier/controller.go new file mode 100644 index 000000000..c831cee35 --- /dev/null +++ b/pkg/controllers/workapplier/controller.go @@ -0,0 +1,453 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "time" + + "go.uber.org/atomic" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrloption "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/controller" + "go.goms.io/fleet/pkg/utils/defaulter" + "go.goms.io/fleet/pkg/utils/parallelizer" +) + +const ( + patchDetailPerObjLimit = 100 + + minRequestAfterDuration = time.Second * 5 +) + +const ( + workFieldManagerName = "work-api-agent" +) + +// Reconciler reconciles a Work object. +type Reconciler struct { + hubClient client.Client + workNameSpace string + spokeDynamicClient dynamic.Interface + spokeClient client.Client + restMapper meta.RESTMapper + recorder record.EventRecorder + concurrentReconciles int + joined *atomic.Bool + parallelizer *parallelizer.Parallerlizer + + availabilityCheckRequeueAfter time.Duration + driftCheckRequeueAfter time.Duration +} + +func NewReconciler( + hubClient client.Client, workNameSpace string, + spokeDynamicClient dynamic.Interface, spokeClient client.Client, restMapper meta.RESTMapper, + recorder record.EventRecorder, + concurrentReconciles int, + workerCount int, + availabilityCheckRequestAfter time.Duration, + driftCheckRequestAfter time.Duration, +) *Reconciler { + acRequestAfter := availabilityCheckRequestAfter + if acRequestAfter < minRequestAfterDuration { + klog.V(2).InfoS("Availability check requeue after duration is too short; set to the longer default", "availabilityCheckRequestAfter", acRequestAfter) + acRequestAfter = minRequestAfterDuration + } + + dcRequestAfter := driftCheckRequestAfter + if dcRequestAfter < minRequestAfterDuration { + klog.V(2).InfoS("Drift check requeue after duration is too short; set to the longer default", "driftCheckRequestAfter", dcRequestAfter) + dcRequestAfter = minRequestAfterDuration + } + + return &Reconciler{ + hubClient: hubClient, + spokeDynamicClient: spokeDynamicClient, + spokeClient: spokeClient, + restMapper: restMapper, + recorder: recorder, + concurrentReconciles: concurrentReconciles, + parallelizer: parallelizer.NewParallelizer(workerCount), + workNameSpace: workNameSpace, + joined: atomic.NewBool(false), + availabilityCheckRequeueAfter: acRequestAfter, + driftCheckRequeueAfter: dcRequestAfter, + } +} + +const ( + allManifestsAppliedMessage = "All the specified manifests have been applied" + allAppliedObjectAvailableMessage = "All of the applied manifests are available" + someAppliedObjectUntrackableMessage = "Some of the applied manifests cannot be tracked for availability" + + notAllManifestsAppliedReason = "FailedToApplyAllManifests" + notAllManifestsAppliedMessage = "Failed to apply all the specified manifests (%d of %d manifests are applied)" + notAllAppliedObjectsAvailableReason = "NotAllAppliedObjectAreAvailable" + notAllAppliedObjectsAvailableMessage = "Not all of the applied manifests are available (%d of %d manifests are available)" + + someObjectsHaveDiffs = "Found configurations diffs on some objects (%d of %d objects have diffs)" +) + +var ( + // Some exported reasons. Currently only the untrackable reason is being actively used. + WorkNotTrackableReason = string(ManifestProcessingAvailabilityResultTypeNotTrackable) +) + +type manifestProcessingAppliedResultType string + +const ( + // The result types and descriptions for processing failures. + ManifestProcessingApplyResultTypeDecodingErred manifestProcessingAppliedResultType = "DecodingErred" + ManifestProcessingApplyResultTypeFoundGenerateNames manifestProcessingAppliedResultType = "FoundGenerateNames" + ManifestProcessingApplyResultTypeDuplicated manifestProcessingAppliedResultType = "Duplicated" + ManifestProcessingApplyResultTypeFailedToFindObjInMemberCluster manifestProcessingAppliedResultType = "FailedToFindObjInMemberCluster" + ManifestProcessingApplyResultTypeFailedToTakeOver manifestProcessingAppliedResultType = "FailedToTakeOver" + ManifestProcessingApplyResultTypeNotTakenOver manifestProcessingAppliedResultType = "NotTakenOver" + ManifestProcessingApplyResultTypeFailedToRunDriftDetection manifestProcessingAppliedResultType = "FailedToRunDriftDetection" + ManifestProcessingApplyResultTypeFoundDrifts manifestProcessingAppliedResultType = "FoundDrifts" + ManifestProcessingApplyResultTypeFailedToApply manifestProcessingAppliedResultType = "FailedToApply" + + // The result type and description for partially successfully processing attempts. + ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection manifestProcessingAppliedResultType = "AppliedWithFailedDriftDetection" + + ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetectionDescription = "Manifest has been applied successfully, but drift detection has failed" + + // The result type and description for successful processing attempts. + ManifestProcessingApplyResultTypeApplied manifestProcessingAppliedResultType = "Applied" + + ManifestProcessingApplyResultTypeAppliedDescription = "Manifest has been applied successfully" + + // A special result type for the case where no apply is performed (i.e., the ReportDiff mode). + ManifestProcessingApplyResultTypeNoApplyPerformed manifestProcessingAppliedResultType = "Skipped" +) + +type ManifestProcessingAvailabilityResultType string + +const ( + // The result type for availability check being skipped. + ManifestProcessingAvailabilityResultTypeSkipped ManifestProcessingAvailabilityResultType = "Skipped" + + // The result type for availability check failures. + ManifestProcessingAvailabilityResultTypeFailed ManifestProcessingAvailabilityResultType = "Failed" + + ManifestProcessingAvailabilityResultTypeFailedDescription = "Failed to track the availability of the applied manifest (error = %s)" + + // The result types for completed availability checks. + ManifestProcessingAvailabilityResultTypeAvailable ManifestProcessingAvailabilityResultType = "Available" + ManifestProcessingAvailabilityResultTypeNotYetAvailable ManifestProcessingAvailabilityResultType = "NotYetAvailable" + ManifestProcessingAvailabilityResultTypeNotTrackable ManifestProcessingAvailabilityResultType = "NotTrackable" + + ManifestProcessingAvailabilityResultTypeAvailableDescription = "Manifest is available" + ManifestProcessingAvailabilityResultTypeNotYetAvailableDescription = "Manifest is not yet available; Fleet will check again later" + ManifestProcessingAvailabilityResultTypeNotTrackableDescription = "Manifest's availability is not trackable; Fleet assumes that the applied manifest is available" +) + +type ManifestProcessingReportDiffResultType string + +const ( + // The result type for the cases where ReportDiff mode is not enabled. + ManifestProcessingReportDiffResultTypeNotEnabled ManifestProcessingReportDiffResultType = "NotEnabled" + + // The result type for diff reporting failures. + ManifestProcessingReportDiffResultTypeFailed ManifestProcessingReportDiffResultType = "Failed" + + ManifestProcessingReportDiffResultTypeFailedDescription = "Failed to report the diff between the hub cluster and the member cluster (error = %s)" + + // The result type for completed diff reportings. + ManifestProcessingReportDiffResultTypeFoundDiff ManifestProcessingReportDiffResultType = "FoundDiff" + ManifestProcessingReportDiffResultTypeNoDiffFound ManifestProcessingReportDiffResultType = "NoDiffFound" + + ManifestProcessingReportDiffResultTypeNoDiffFoundDescription = "No diff has been found between the hub cluster and the member cluster" + ManifestProcessingReportDiffResultTypeFoundDiffDescription = "Diff has been found between the hub cluster and the member cluster" +) + +type manifestProcessingBundle struct { + manifest *fleetv1beta1.Manifest + id *fleetv1beta1.WorkResourceIdentifier + manifestObj *unstructured.Unstructured + inMemberClusterObj *unstructured.Unstructured + gvr *schema.GroupVersionResource + applyResTyp manifestProcessingAppliedResultType + availabilityResTyp ManifestProcessingAvailabilityResultType + reportDiffResTyp ManifestProcessingReportDiffResultType + applyErr error + availabilityErr error + reportDiffErr error + drifts []fleetv1beta1.PatchDetail + diffs []fleetv1beta1.PatchDetail +} + +// Reconcile implement the control loop logic for Work object. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if !r.joined.Load() { + klog.V(2).InfoS("Work applier has not started yet", "work", req.NamespacedName) + return ctrl.Result{RequeueAfter: time.Second * 5}, nil + } + startTime := time.Now() + klog.V(2).InfoS("Work applier reconciliation starts", "work", req.NamespacedName) + defer func() { + latency := time.Since(startTime).Milliseconds() + klog.V(2).InfoS("Work applier reconciliation ends", "work", req.NamespacedName, "latency", latency) + }() + + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + err := r.hubClient.Get(ctx, req.NamespacedName, work) + switch { + case apierrors.IsNotFound(err): + klog.V(2).InfoS("Work object has been deleted", "work", req.NamespacedName) + return ctrl.Result{}, nil + case err != nil: + klog.ErrorS(err, "Failed to retrieve the work", "work", req.NamespacedName) + return ctrl.Result{}, controller.NewAPIServerError(true, err) + } + + workRef := klog.KObj(work) + + // Garbage collect the AppliedWork object if the Work object has been deleted. + if !work.DeletionTimestamp.IsZero() { + klog.V(2).InfoS("Work object has been marked for deletion; start garbage collection", work.Kind, workRef) + return r.garbageCollectAppliedWork(ctx, work) + } + + // ensure that the appliedWork and the finalizer exist + appliedWork, err := r.ensureAppliedWork(ctx, work) + if err != nil { + return ctrl.Result{}, err + } + expectedAppliedWorkOwnerRef := &metav1.OwnerReference{ + APIVersion: fleetv1beta1.GroupVersion.String(), + Kind: fleetv1beta1.AppliedWorkKind, + Name: appliedWork.GetName(), + UID: appliedWork.GetUID(), + BlockOwnerDeletion: ptr.To(false), + } + + // Set the default values for the Work object to avoid additional validation logic in the + // later steps. + defaulter.SetDefaultsWork(work) + + // Note (chenyu1): In the current version, for simplicity reasons, Fleet has dropped support + // for objects with generate names; any attempt to place such objects will yield an apply error. + // Originally this was supported, but a bug has stopped Fleet from handling such objects correctly. + // The code has been updated to automatically ignore identifiers with empty names so that + // reconciliation can resume in a previously erred setup. + // + // TO-DO (chenyu1): evaluate if it is necessary to add support for objects with generate + // names. + + // Prepare the bundles. + bundles := prepareManifestProcessingBundles(work) + + // Pre-process the manifests to apply. + // + // In this step, Fleet will: + // a) decode the manifests; and + // b) write ahead the manifest processing attempts; and + // c) remove any applied manifests left over from previous runs. + if err := r.preProcessManifests(ctx, bundles, work, expectedAppliedWorkOwnerRef); err != nil { + klog.ErrorS(err, "Failed to pre-process the manifests", "work", workRef) + return ctrl.Result{}, err + } + + // Process the manifests. + // + // In this step, Fleet will: + // a) find if there has been a corresponding object in the member cluster for each manifest; + // b) take over the object if applicable; + // c) report configuration differences if applicable; + // d) check for configuration drifts if applicable; + // e) apply each manifest. + r.processManifests(ctx, bundles, work, expectedAppliedWorkOwnerRef) + + // Track the availability information. + r.trackInMemberClusterObjAvailability(ctx, bundles, workRef) + + trackWorkApplyLatencyMetric(work) + + // Refresh the status of the Work object. + if err := r.refreshWorkStatus(ctx, work, bundles); err != nil { + return ctrl.Result{}, err + } + + // Refresh the status of the AppliedWork object. + if err := r.refreshAppliedWorkStatus(ctx, appliedWork, bundles); err != nil { + return ctrl.Result{}, err + } + + // If the Work object is not yet available, reconcile again. + if !isWorkObjectAvailable(work) { + klog.V(2).InfoS("Work object is not yet in an available state; requeue to monitor its availability", "work", workRef) + return ctrl.Result{RequeueAfter: r.availabilityCheckRequeueAfter}, nil + } + // Otherwise, reconcile again for drift detection purposes. + klog.V(2).InfoS("Work object is available; requeue to check for drifts", "work", workRef) + return ctrl.Result{RequeueAfter: r.driftCheckRequeueAfter}, nil +} + +// garbageCollectAppliedWork deletes the appliedWork and all the manifests associated with it from the cluster. +func (r *Reconciler) garbageCollectAppliedWork(ctx context.Context, work *fleetv1beta1.Work) (ctrl.Result, error) { + deletePolicy := metav1.DeletePropagationBackground + if !controllerutil.ContainsFinalizer(work, fleetv1beta1.WorkFinalizer) { + return ctrl.Result{}, nil + } + // delete the appliedWork which will remove all the manifests associated with it + // TODO: allow orphaned manifest + appliedWork := fleetv1beta1.AppliedWork{ + ObjectMeta: metav1.ObjectMeta{Name: work.Name}, + } + err := r.spokeClient.Delete(ctx, &appliedWork, &client.DeleteOptions{PropagationPolicy: &deletePolicy}) + switch { + case apierrors.IsNotFound(err): + klog.V(2).InfoS("The appliedWork is already deleted", "appliedWork", work.Name) + case err != nil: + klog.ErrorS(err, "Failed to delete the appliedWork", "appliedWork", work.Name) + return ctrl.Result{}, err + default: + klog.InfoS("Successfully deleted the appliedWork", "appliedWork", work.Name) + } + controllerutil.RemoveFinalizer(work, fleetv1beta1.WorkFinalizer) + return ctrl.Result{}, r.hubClient.Update(ctx, work, &client.UpdateOptions{}) +} + +// ensureAppliedWork makes sure that an associated appliedWork and a finalizer on the work resource exists on the cluster. +func (r *Reconciler) ensureAppliedWork(ctx context.Context, work *fleetv1beta1.Work) (*fleetv1beta1.AppliedWork, error) { + workRef := klog.KObj(work) + appliedWork := &fleetv1beta1.AppliedWork{} + hasFinalizer := false + if controllerutil.ContainsFinalizer(work, fleetv1beta1.WorkFinalizer) { + hasFinalizer = true + err := r.spokeClient.Get(ctx, types.NamespacedName{Name: work.Name}, appliedWork) + switch { + case apierrors.IsNotFound(err): + klog.ErrorS(err, "AppliedWork finalizer resource does not exist even with the finalizer, it will be recreated", "appliedWork", workRef.Name) + case err != nil: + klog.ErrorS(err, "Failed to retrieve the appliedWork ", "appliedWork", workRef.Name) + return nil, controller.NewAPIServerError(true, err) + default: + return appliedWork, nil + } + } + + // we create the appliedWork before setting the finalizer, so it should always exist unless it's deleted behind our back + appliedWork = &fleetv1beta1.AppliedWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: work.Name, + }, + Spec: fleetv1beta1.AppliedWorkSpec{ + WorkName: work.Name, + WorkNamespace: work.Namespace, + }, + } + if err := r.spokeClient.Create(ctx, appliedWork); err != nil && !apierrors.IsAlreadyExists(err) { + klog.ErrorS(err, "AppliedWork create failed", "appliedWork", workRef.Name) + return nil, err + } + if !hasFinalizer { + klog.InfoS("Add the finalizer to the work", "work", workRef) + work.Finalizers = append(work.Finalizers, fleetv1beta1.WorkFinalizer) + return appliedWork, r.hubClient.Update(ctx, work, &client.UpdateOptions{}) + } + klog.InfoS("Recreated the appliedWork resource", "appliedWork", workRef.Name) + return appliedWork, nil +} + +// prepareManifestProcessingBundles prepares the manifest processing bundles. +func prepareManifestProcessingBundles(work *fleetv1beta1.Work) []*manifestProcessingBundle { + // Pre-allocate the bundles. + bundles := make([]*manifestProcessingBundle, 0, len(work.Spec.Workload.Manifests)) + for idx := range work.Spec.Workload.Manifests { + manifest := work.Spec.Workload.Manifests[idx] + bundles = append(bundles, &manifestProcessingBundle{ + manifest: &manifest, + }) + } + return bundles +} + +// Join starts to reconcile +func (r *Reconciler) Join(_ context.Context) error { + if !r.joined.Load() { + klog.InfoS("Mark the apply work reconciler joined") + } + r.joined.Store(true) + return nil +} + +// Leave start +func (r *Reconciler) Leave(ctx context.Context) error { + var works fleetv1beta1.WorkList + if r.joined.Load() { + klog.InfoS("Mark the apply work reconciler left") + } + r.joined.Store(false) + // list all the work object we created in the member cluster namespace + listOpts := []client.ListOption{ + client.InNamespace(r.workNameSpace), + } + if err := r.hubClient.List(ctx, &works, listOpts...); err != nil { + klog.ErrorS(err, "Failed to list all the work object", "clusterNS", r.workNameSpace) + return client.IgnoreNotFound(err) + } + // we leave the resources on the member cluster for now + for _, work := range works.Items { + staleWork := work.DeepCopy() + if controllerutil.ContainsFinalizer(staleWork, fleetv1beta1.WorkFinalizer) { + controllerutil.RemoveFinalizer(staleWork, fleetv1beta1.WorkFinalizer) + if updateErr := r.hubClient.Update(ctx, staleWork, &client.UpdateOptions{}); updateErr != nil { + klog.ErrorS(updateErr, "Failed to remove the work finalizer from the work", + "clusterNS", r.workNameSpace, "work", klog.KObj(staleWork)) + return updateErr + } + } + } + klog.V(2).InfoS("Successfully removed all the work finalizers in the cluster namespace", + "clusterNS", r.workNameSpace, "number of work", len(works.Items)) + return nil +} + +// SetupWithManager wires up the controller. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + WithOptions(ctrloption.Options{ + MaxConcurrentReconciles: r.concurrentReconciles, + }). + For(&fleetv1beta1.Work{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} diff --git a/pkg/controllers/workapplier/controller_integration_migrated_1_test.go b/pkg/controllers/workapplier/controller_integration_migrated_1_test.go new file mode 100644 index 000000000..a744c97d2 --- /dev/null +++ b/pkg/controllers/workapplier/controller_integration_migrated_1_test.go @@ -0,0 +1,612 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + testv1alpha1 "go.goms.io/fleet/test/apis/v1alpha1" + "go.goms.io/fleet/test/utils/controller" +) + +const timeout = time.Second * 10 +const interval = time.Millisecond * 250 + +var _ = Describe("Work Controller", func() { + var cm *corev1.ConfigMap + var work *fleetv1beta1.Work + const defaultNS = "default" + + Context("Test single work propagation", func() { + It("Should have a configmap deployed correctly", func() { + cmName := "testcm" + cmNamespace := defaultNS + cm = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: cmNamespace, + }, + Data: map[string]string{ + "test": "test", + }, + } + + By("create the work") + work = createWorkWithManifest(testWorkNamespace, cm) + err := k8sClient.Create(context.Background(), work) + Expect(err).ToNot(HaveOccurred()) + + resultWork := waitForWorkToBeAvailable(work.GetName(), work.GetNamespace()) + Expect(len(resultWork.Status.ManifestConditions)).Should(Equal(1)) + expectedResourceID := fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "ConfigMap", + Resource: "configmaps", + Namespace: cmNamespace, + Name: cm.Name, + } + Expect(cmp.Diff(resultWork.Status.ManifestConditions[0].Identifier, expectedResourceID)).Should(BeEmpty()) + expected := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + Expect(controller.CompareConditions(expected, resultWork.Status.ManifestConditions[0].Conditions)).Should(BeEmpty()) + expected = []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + Expect(controller.CompareConditions(expected, resultWork.Status.Conditions)).Should(BeEmpty()) + + By("Check applied config map") + var configMap corev1.ConfigMap + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: cmName, Namespace: cmNamespace}, &configMap)).Should(Succeed()) + Expect(cmp.Diff(configMap.Labels, cm.Labels)).Should(BeEmpty()) + Expect(cmp.Diff(configMap.Data, cm.Data)).Should(BeEmpty()) + + Expect(k8sClient.Delete(ctx, work)).Should(Succeed(), "Failed to deleted the work") + }) + + It("Should pick up the built-in manifest change correctly", func() { + cmName := "testconfig" + cmNamespace := defaultNS + cm = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: cmNamespace, + Labels: map[string]string{ + "labelKey1": "value1", + "labelKey2": "value2", + }, + Annotations: map[string]string{ + "annotationKey1": "annotation1", + "annotationKey2": "annotation2", + }, + }, + Data: map[string]string{ + "data1": "test1", + }, + } + + By("create the work") + work = createWorkWithManifest(testWorkNamespace, cm) + Expect(k8sClient.Create(context.Background(), work)).ToNot(HaveOccurred()) + + By("wait for the work to be available") + waitForWorkToBeAvailable(work.GetName(), work.GetNamespace()) + + By("Check applied config map") + verifyAppliedConfigMap(cm) + + By("Modify the configMap manifest") + // add new data + cm.Data["data2"] = "test2" + // modify one data + cm.Data["data1"] = "newValue" + // modify label key1 + cm.Labels["labelKey1"] = "newValue" + // remove label key2 + delete(cm.Labels, "labelKey2") + // add annotations key3 + cm.Annotations["annotationKey3"] = "annotation3" + // remove annotations key1 + delete(cm.Annotations, "annotationKey1") + + By("update the work") + resultWork := waitForWorkToApply(work.GetName(), work.GetNamespace()) + rawCM, err := json.Marshal(cm) + Expect(err).Should(Succeed()) + resultWork.Spec.Workload.Manifests[0].Raw = rawCM + Expect(k8sClient.Update(ctx, resultWork)).Should(Succeed()) + + By("wait for the change of the work to be applied") + waitForWorkToApply(work.GetName(), work.GetNamespace()) + + By("verify that applied configMap took all the changes") + verifyAppliedConfigMap(cm) + + Expect(k8sClient.Delete(ctx, work)).Should(Succeed(), "Failed to deleted the work") + }) + + It("Should merge the third party change correctly", func() { + cmName := "test-merge" + cmNamespace := defaultNS + cm = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: cmNamespace, + Labels: map[string]string{ + "labelKey1": "value1", + "labelKey2": "value2", + "labelKey3": "value3", + }, + }, + Data: map[string]string{ + "data1": "test1", + }, + } + + By("create the work") + work = createWorkWithManifest(testWorkNamespace, cm) + err := k8sClient.Create(context.Background(), work) + Expect(err).ToNot(HaveOccurred()) + + By("wait for the work to be applied") + waitForWorkToApply(work.GetName(), work.GetNamespace()) + + By("Check applied configMap") + appliedCM := verifyAppliedConfigMap(cm) + + By("Modify and update the applied configMap") + // add a new data + appliedCM.Data["data2"] = "another value" + // add a new data + appliedCM.Data["data3"] = "added data by third party" + // modify label key1 + appliedCM.Labels["labelKey1"] = "third-party-label" + // remove label key2 and key3 + delete(cm.Labels, "labelKey2") + delete(cm.Labels, "labelKey3") + Expect(k8sClient.Update(context.Background(), appliedCM)).Should(Succeed()) + + By("Get the last applied config map and verify it's updated") + var modifiedCM corev1.ConfigMap + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &modifiedCM)).Should(Succeed()) + Expect(cmp.Diff(appliedCM.Labels, modifiedCM.Labels)).Should(BeEmpty()) + Expect(cmp.Diff(appliedCM.Data, modifiedCM.Data)).Should(BeEmpty()) + + By("Modify the manifest") + // modify one data + cm.Data["data1"] = "modifiedValue" + // add a conflict data + cm.Data["data2"] = "added by manifest" + // change label key3 with a new value + cm.Labels["labelKey3"] = "added-back-by-manifest" + + By("update the work") + resultWork := waitForWorkToApply(work.GetName(), work.GetNamespace()) + rawCM, err := json.Marshal(cm) + Expect(err).Should(Succeed()) + resultWork.Spec.Workload.Manifests[0].Raw = rawCM + Expect(k8sClient.Update(context.Background(), resultWork)).Should(Succeed()) + + By("wait for the change of the work to be applied") + waitForWorkToApply(work.GetName(), work.GetNamespace()) + + By("Get the last applied config map") + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: cmName, Namespace: cmNamespace}, appliedCM)).Should(Succeed()) + + By("Check the config map data") + // data1's value picks up our change + // data2 is value is overridden by our change + // data3 is added by the third party + expectedData := map[string]string{ + "data1": "modifiedValue", + "data2": "added by manifest", + "data3": "added data by third party", + } + Expect(cmp.Diff(appliedCM.Data, expectedData)).Should(BeEmpty()) + + By("Check the config map label") + // key1's value is override back even if we didn't change it + // key2 is deleted by third party since we didn't change it + // key3's value added back after we change the value + expectedLabel := map[string]string{ + "labelKey1": "value1", + "labelKey3": "added-back-by-manifest", + } + Expect(cmp.Diff(appliedCM.Labels, expectedLabel)).Should(BeEmpty()) + + Expect(k8sClient.Delete(ctx, work)).Should(Succeed(), "Failed to deleted the work") + }) + + It("Should pick up the crd change correctly", func() { + testResourceName := "test-resource-name" + testResourceNamespace := defaultNS + testResource := &testv1alpha1.TestResource{ + TypeMeta: metav1.TypeMeta{ + APIVersion: testv1alpha1.GroupVersion.String(), + Kind: "TestResource", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: testResourceName, + Namespace: testResourceNamespace, + }, + Spec: testv1alpha1.TestResourceSpec{ + Foo: "foo", + Bar: "bar", + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "region", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"us", "eu"}, + }, + { + Key: "prod", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + }, + } + + By("create the work") + work = createWorkWithManifest(testWorkNamespace, testResource) + err := k8sClient.Create(context.Background(), work) + Expect(err).ToNot(HaveOccurred()) + + By("wait for the work to be applied") + waitForWorkToBeAvailable(work.GetName(), work.GetNamespace()) + + By("Check applied TestResource") + var appliedTestResource testv1alpha1.TestResource + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &appliedTestResource)).Should(Succeed()) + + By("verify the TestResource spec") + Expect(cmp.Diff(appliedTestResource.Spec, testResource.Spec)).Should(BeEmpty()) + + By("Modify and update the applied TestResource") + // add/modify/remove a match + appliedTestResource.Spec.LabelSelector.MatchExpressions = []metav1.LabelSelectorRequirement{ + { + Key: "region", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"asia"}, + }, + { + Key: "extra", + Operator: metav1.LabelSelectorOpExists, + }, + } + appliedTestResource.Spec.Items = []string{"a", "b"} + appliedTestResource.Spec.Foo = "foo1" + appliedTestResource.Spec.Bar = "bar1" + Expect(k8sClient.Update(context.Background(), &appliedTestResource)).Should(Succeed()) + + By("Verify applied TestResource modified") + var modifiedTestResource testv1alpha1.TestResource + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &modifiedTestResource)).Should(Succeed()) + Expect(cmp.Diff(appliedTestResource.Spec, modifiedTestResource.Spec)).Should(BeEmpty()) + + By("Modify the TestResource") + testResource.Spec.LabelSelector.MatchExpressions = []metav1.LabelSelectorRequirement{ + { + Key: "region", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"us", "asia", "eu"}, + }, + } + testResource.Spec.Foo = "foo2" + testResource.Spec.Bar = "bar2" + By("update the work") + resultWork := waitForWorkToApply(work.GetName(), work.GetNamespace()) + rawTR, err := json.Marshal(testResource) + Expect(err).Should(Succeed()) + resultWork.Spec.Workload.Manifests[0].Raw = rawTR + Expect(k8sClient.Update(context.Background(), resultWork)).Should(Succeed()) + waitForWorkToApply(work.GetName(), work.GetNamespace()) + + By("Get the last applied TestResource") + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &appliedTestResource)).Should(Succeed()) + + By("Check the TestResource spec, its an override for arrays") + expectedItems := []string{"a", "b"} + Expect(cmp.Diff(appliedTestResource.Spec.Items, expectedItems)).Should(BeEmpty()) + Expect(cmp.Diff(appliedTestResource.Spec.LabelSelector, testResource.Spec.LabelSelector)).Should(BeEmpty()) + Expect(cmp.Diff(appliedTestResource.Spec.Foo, "foo2")).Should(BeEmpty()) + Expect(cmp.Diff(appliedTestResource.Spec.Bar, "bar2")).Should(BeEmpty()) + + Expect(k8sClient.Delete(ctx, work)).Should(Succeed(), "Failed to deleted the work") + }) + + It("Check that the apply still works if the last applied annotation does not exist", func() { + ctx = context.Background() + cmName := "test-merge-without-lastapply" + cmNamespace := defaultNS + cm = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: cmNamespace, + Labels: map[string]string{ + "labelKey1": "value1", + "labelKey2": "value2", + "labelKey3": "value3", + }, + }, + Data: map[string]string{ + "data1": "test1", + }, + } + + By("create the work") + work = createWorkWithManifest(testWorkNamespace, cm) + err := k8sClient.Create(ctx, work) + Expect(err).Should(Succeed()) + + By("wait for the work to be applied") + waitForWorkToApply(work.GetName(), work.GetNamespace()) + + By("Check applied configMap") + appliedCM := verifyAppliedConfigMap(cm) + + By("Delete the last applied annotation from the current resource") + delete(appliedCM.Annotations, fleetv1beta1.LastAppliedConfigAnnotation) + Expect(k8sClient.Update(ctx, appliedCM)).Should(Succeed()) + + By("Get the last applied config map and verify it does not have the last applied annotation") + var modifiedCM corev1.ConfigMap + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &modifiedCM)).Should(Succeed()) + Expect(modifiedCM.Annotations[fleetv1beta1.LastAppliedConfigAnnotation]).Should(BeEmpty()) + + By("Modify the manifest") + // modify one data + cm.Data["data1"] = "modifiedValue" + // add a conflict data + cm.Data["data2"] = "added by manifest" + // change label key3 with a new value + cm.Labels["labelKey3"] = "added-back-by-manifest" + + By("update the work") + resultWork := waitForWorkToApply(work.GetName(), work.GetNamespace()) + rawCM, err := json.Marshal(cm) + Expect(err).Should(Succeed()) + resultWork.Spec.Workload.Manifests[0].Raw = rawCM + Expect(k8sClient.Update(ctx, resultWork)).Should(Succeed()) + + By("wait for the change of the work to be applied") + waitForWorkToApply(work.GetName(), work.GetNamespace()) + + By("Check applied configMap is modified even without the last applied annotation") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cmNamespace}, appliedCM)).Should(Succeed()) + verifyAppliedConfigMap(cm) + + Expect(k8sClient.Delete(ctx, work)).Should(Succeed(), "Failed to deleted the work") + }) + + It("Check that failed to apply manifest has the proper identification", func() { + testResourceName := "test-resource-name-failed" + // to ensure apply fails. + namespace := "random-test-namespace" + testResource := &testv1alpha1.TestResource{ + TypeMeta: metav1.TypeMeta{ + APIVersion: testv1alpha1.GroupVersion.String(), + Kind: "TestResource", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: testResourceName, + Namespace: namespace, + }, + Spec: testv1alpha1.TestResourceSpec{ + Foo: "foo", + }, + } + work = createWorkWithManifest(testWorkNamespace, testResource) + err := k8sClient.Create(context.Background(), work) + Expect(err).ToNot(HaveOccurred()) + + By("wait for the work to be applied, apply condition set to failed") + var resultWork fleetv1beta1.Work + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: work.Name, Namespace: work.GetNamespace()}, &resultWork) + if err != nil { + return false + } + applyCond := meta.FindStatusCondition(resultWork.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied) + if applyCond == nil || applyCond.Status != metav1.ConditionFalse || applyCond.ObservedGeneration != resultWork.Generation { + return false + } + if !meta.IsStatusConditionFalse(resultWork.Status.ManifestConditions[0].Conditions, fleetv1beta1.WorkConditionTypeApplied) { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + expectedResourceID := fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: testv1alpha1.GroupVersion.Group, + Version: testv1alpha1.GroupVersion.Version, + Resource: "testresources", + Kind: testResource.Kind, + Namespace: testResource.GetNamespace(), + Name: testResource.GetName(), + } + Expect(cmp.Diff(resultWork.Status.ManifestConditions[0].Identifier, expectedResourceID)).Should(BeEmpty()) + }) + }) + + // This test will set the work controller to leave and then join again. + // It cannot run parallel with other tests. + Context("Test multiple work propagation", Serial, func() { + var works []*fleetv1beta1.Work + + AfterEach(func() { + for _, staleWork := range works { + err := k8sClient.Delete(context.Background(), staleWork) + Expect(err).ToNot(HaveOccurred()) + } + }) + + It("Test join and leave work correctly", func() { + By("create the works") + var configMap corev1.ConfigMap + cmNamespace := defaultNS + var cmNames []string + numWork := 10 + data := map[string]string{ + "test-key-1": "test-value-1", + "test-key-2": "test-value-2", + "test-key-3": "test-value-3", + } + + for i := 0; i < numWork; i++ { + cmName := "testcm-" + utilrand.String(10) + cmNames = append(cmNames, cmName) + cm = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: cmNamespace, + }, + Data: data, + } + // make sure we can call join as many as possible + Expect(workController.Join(ctx)).Should(Succeed()) + work = createWorkWithManifest(testWorkNamespace, cm) + err := k8sClient.Create(ctx, work) + Expect(err).ToNot(HaveOccurred()) + By(fmt.Sprintf("created the work = %s", work.GetName())) + works = append(works, work) + } + + By("make sure the works are handled") + for i := 0; i < numWork; i++ { + waitForWorkToBeHandled(works[i].GetName(), works[i].GetNamespace()) + } + + By("mark the work controller as leave") + Eventually(func() error { + return workController.Leave(ctx) + }, timeout, interval).Should(Succeed()) + + By("make sure the manifests have no finalizer and its status match the member cluster") + newData := map[string]string{ + "test-key-1": "test-value-1", + "test-key-2": "test-value-2", + "test-key-3": "test-value-3", + "new-test-key-1": "test-value-4", + "new-test-key-2": "test-value-5", + } + for i := 0; i < numWork; i++ { + var resultWork fleetv1beta1.Work + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: works[i].GetName(), Namespace: testWorkNamespace}, &resultWork)).Should(Succeed()) + Expect(controllerutil.ContainsFinalizer(&resultWork, fleetv1beta1.WorkFinalizer)).Should(BeFalse()) + // make sure that leave can be called as many times as possible + // The work may be updated and may hit 409 error. + Eventually(func() error { + return workController.Leave(ctx) + }, timeout, interval).Should(Succeed(), "Failed to set the work controller to leave") + By(fmt.Sprintf("change the work = %s", work.GetName())) + cm = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmNames[i], + Namespace: cmNamespace, + }, + Data: newData, + } + rawCM, err := json.Marshal(cm) + Expect(err).Should(Succeed()) + resultWork.Spec.Workload.Manifests[0].Raw = rawCM + Expect(k8sClient.Update(ctx, &resultWork)).Should(Succeed()) + } + + By("make sure the update in the work is not picked up") + Consistently(func() bool { + for i := 0; i < numWork; i++ { + By(fmt.Sprintf("updated the work = %s", works[i].GetName())) + var resultWork fleetv1beta1.Work + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: works[i].GetName(), Namespace: testWorkNamespace}, &resultWork) + Expect(err).Should(Succeed()) + Expect(controllerutil.ContainsFinalizer(&resultWork, fleetv1beta1.WorkFinalizer)).Should(BeFalse()) + applyCond := meta.FindStatusCondition(resultWork.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied) + if applyCond != nil && applyCond.Status == metav1.ConditionTrue && applyCond.ObservedGeneration == resultWork.Generation { + return false + } + By("check if the config map is not changed") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cmNames[i], Namespace: cmNamespace}, &configMap)).Should(Succeed()) + Expect(cmp.Diff(configMap.Data, data)).Should(BeEmpty()) + } + return true + }, timeout, interval).Should(BeTrue()) + + By("enable the work controller again") + Expect(workController.Join(ctx)).Should(Succeed()) + + By("make sure the work change get picked up") + for i := 0; i < numWork; i++ { + resultWork := waitForWorkToApply(works[i].GetName(), works[i].GetNamespace()) + Expect(len(resultWork.Status.ManifestConditions)).Should(Equal(1)) + Expect(meta.IsStatusConditionTrue(resultWork.Status.ManifestConditions[0].Conditions, fleetv1beta1.WorkConditionTypeApplied)).Should(BeTrue()) + By("the work is applied, check if the applied config map is updated") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cmNames[i], Namespace: cmNamespace}, &configMap)).Should(Succeed()) + Expect(cmp.Diff(configMap.Data, newData)).Should(BeEmpty()) + } + }) + }) +}) diff --git a/pkg/controllers/workapplier/controller_integration_migrated_2_test.go b/pkg/controllers/workapplier/controller_integration_migrated_2_test.go new file mode 100644 index 000000000..a5567cdc5 --- /dev/null +++ b/pkg/controllers/workapplier/controller_integration_migrated_2_test.go @@ -0,0 +1,221 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilrand "k8s.io/apimachinery/pkg/util/rand" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + testv1alpha1 "go.goms.io/fleet/test/apis/v1alpha1" +) + +var _ = Describe("Work Status Reconciler", func() { + var resourceNamespace string + var work *fleetv1beta1.Work + var cm, cm2 *corev1.ConfigMap + var rns corev1.Namespace + + BeforeEach(func() { + resourceNamespace = utilrand.String(5) + rns = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceNamespace, + }, + } + Expect(k8sClient.Create(context.Background(), &rns)).Should(Succeed(), "Failed to create the resource namespace") + + // Create the Work object with some type of Manifest resource. + cm = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap-" + utilrand.String(5), + Namespace: resourceNamespace, + }, + Data: map[string]string{ + "test": "test", + }, + } + cm2 = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap2-" + utilrand.String(5), + Namespace: resourceNamespace, + }, + Data: map[string]string{ + "test": "test", + }, + } + + By("Create work that contains two configMaps") + work = &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: "work-" + utilrand.String(5), + Namespace: testWorkNamespace, + }, + Spec: fleetv1beta1.WorkSpec{ + Workload: fleetv1beta1.WorkloadTemplate{ + Manifests: []fleetv1beta1.Manifest{ + { + RawExtension: runtime.RawExtension{Object: cm}, + }, + { + RawExtension: runtime.RawExtension{Object: cm2}, + }, + }, + }, + }, + } + }) + + AfterEach(func() { + // TODO: Ensure that all resources are being deleted. + Expect(k8sClient.Delete(context.Background(), work)).Should(Succeed()) + Expect(k8sClient.Delete(context.Background(), &rns)).Should(Succeed()) + }) + + It("Should delete the manifest from the member cluster after it is removed from work", func() { + By("Apply the work") + Expect(k8sClient.Create(context.Background(), work)).ToNot(HaveOccurred()) + + By("Make sure that the work is applied") + currentWork := waitForWorkToApply(work.Name, testWorkNamespace) + var appliedWork fleetv1beta1.AppliedWork + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(len(appliedWork.Status.AppliedResources)).Should(Equal(2)) + + By("Remove configMap 2 from the work") + currentWork.Spec.Workload.Manifests = []fleetv1beta1.Manifest{ + { + RawExtension: runtime.RawExtension{Object: cm}, + }, + } + Expect(k8sClient.Update(context.Background(), currentWork)).Should(Succeed()) + + By("Verify that the resource is removed from the cluster") + Eventually(func() bool { + var configMap corev1.ConfigMap + return apierrors.IsNotFound(k8sClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap)) + }, timeout, interval).Should(BeTrue()) + + By("Verify that the appliedWork status is correct") + Eventually(func() bool { + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + return len(appliedWork.Status.AppliedResources) == 1 + }, timeout, interval).Should(BeTrue()) + Expect(appliedWork.Status.AppliedResources[0].Name).Should(Equal(cm.GetName())) + Expect(appliedWork.Status.AppliedResources[0].Namespace).Should(Equal(cm.GetNamespace())) + Expect(appliedWork.Status.AppliedResources[0].Version).Should(Equal(cm.GetObjectKind().GroupVersionKind().Version)) + Expect(appliedWork.Status.AppliedResources[0].Group).Should(Equal(cm.GetObjectKind().GroupVersionKind().Group)) + Expect(appliedWork.Status.AppliedResources[0].Kind).Should(Equal(cm.GetObjectKind().GroupVersionKind().Kind)) + }) + + It("Should delete the manifest from the member cluster even if there is apply failure", func() { + By("Apply the work") + Expect(k8sClient.Create(context.Background(), work)).ToNot(HaveOccurred()) + + By("Make sure that the work is applied") + currentWork := waitForWorkToApply(work.Name, testWorkNamespace) + var appliedWork fleetv1beta1.AppliedWork + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(len(appliedWork.Status.AppliedResources)).Should(Equal(2)) + + By("replace configMap with a bad object from the work") + testResource := &testv1alpha1.TestResource{ + TypeMeta: metav1.TypeMeta{ + APIVersion: testv1alpha1.GroupVersion.String(), + Kind: "TestResource", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testresource-" + utilrand.String(5), + // to ensure the resource is not applied. + Namespace: "random-test-namespace", + }, + } + currentWork.Spec.Workload.Manifests = []fleetv1beta1.Manifest{ + { + RawExtension: runtime.RawExtension{Object: testResource}, + }, + } + Expect(k8sClient.Update(context.Background(), currentWork)).Should(Succeed()) + + By("Verify that the configMaps are removed from the cluster even if the new resource didn't apply") + Eventually(func() bool { + var configMap corev1.ConfigMap + return apierrors.IsNotFound(k8sClient.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap)) + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + var configMap corev1.ConfigMap + return apierrors.IsNotFound(k8sClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap)) + }, timeout, interval).Should(BeTrue()) + + By("Verify that the appliedWork status is correct") + Eventually(func() bool { + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + return len(appliedWork.Status.AppliedResources) == 0 + }, timeout, interval).Should(BeTrue()) + }) + + It("Test the order of the manifest in the work alone does not trigger any operation in the member cluster", func() { + By("Apply the work") + Expect(k8sClient.Create(context.Background(), work)).ToNot(HaveOccurred()) + + By("Make sure that the work is applied") + currentWork := waitForWorkToApply(work.Name, testWorkNamespace) + var appliedWork fleetv1beta1.AppliedWork + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(len(appliedWork.Status.AppliedResources)).Should(Equal(2)) + + By("Make sure that the manifests exist on the member cluster") + Eventually(func() bool { + var configMap corev1.ConfigMap + return k8sClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap) == nil && + k8sClient.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap) == nil + }, timeout, interval).Should(BeTrue()) + + By("Change the order of the two configs in the work") + currentWork.Spec.Workload.Manifests = []fleetv1beta1.Manifest{ + { + RawExtension: runtime.RawExtension{Object: cm2}, + }, + { + RawExtension: runtime.RawExtension{Object: cm}, + }, + } + Expect(k8sClient.Update(context.Background(), currentWork)).Should(Succeed()) + + By("Verify that nothing is removed from the cluster") + Consistently(func() bool { + var configMap corev1.ConfigMap + return k8sClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap) == nil && + k8sClient.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap) == nil + }, timeout, time.Millisecond*25).Should(BeTrue()) + + By("Verify that the appliedWork status is correct") + Eventually(func() bool { + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + return len(appliedWork.Status.AppliedResources) == 2 + }, timeout, interval).Should(BeTrue()) + Expect(appliedWork.Status.AppliedResources[0].Name).Should(Equal(cm2.GetName())) + Expect(appliedWork.Status.AppliedResources[1].Name).Should(Equal(cm.GetName())) + }) +}) diff --git a/pkg/controllers/workapplier/controller_integration_migrated_helper_test.go b/pkg/controllers/workapplier/controller_integration_migrated_helper_test.go new file mode 100644 index 000000000..45fa25802 --- /dev/null +++ b/pkg/controllers/workapplier/controller_integration_migrated_helper_test.go @@ -0,0 +1,128 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/condition" +) + +// createWorkWithManifest creates a work given a manifest +func createWorkWithManifest(workNamespace string, manifest runtime.Object) *fleetv1beta1.Work { + manifestCopy := manifest.DeepCopyObject() + newWork := fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: "work-" + utilrand.String(5), + Namespace: workNamespace, + }, + Spec: fleetv1beta1.WorkSpec{ + Workload: fleetv1beta1.WorkloadTemplate{ + Manifests: []fleetv1beta1.Manifest{ + { + RawExtension: runtime.RawExtension{Object: manifestCopy}, + }, + }, + }, + }, + } + return &newWork +} + +// verifyAppliedConfigMap verifies that the applied CM is the same as the CM we want to apply +func verifyAppliedConfigMap(cm *corev1.ConfigMap) *corev1.ConfigMap { + var appliedCM corev1.ConfigMap + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &appliedCM)).Should(Succeed()) + + By("Check the config map label") + Expect(cmp.Diff(appliedCM.Labels, cm.Labels)).Should(BeEmpty()) + + By("Check the config map annotation value") + Expect(len(appliedCM.Annotations)).Should(Equal(len(cm.Annotations) + 2)) // we added 2 more annotations + for key := range cm.Annotations { + Expect(appliedCM.Annotations[key]).Should(Equal(cm.Annotations[key])) + } + Expect(appliedCM.Annotations[fleetv1beta1.ManifestHashAnnotation]).ShouldNot(BeEmpty()) + Expect(appliedCM.Annotations[fleetv1beta1.LastAppliedConfigAnnotation]).ShouldNot(BeEmpty()) + + By("Check the config map data") + Expect(cmp.Diff(appliedCM.Data, cm.Data)).Should(BeEmpty()) + return &appliedCM +} + +// waitForWorkToApply waits for a work to be applied +func waitForWorkToApply(workName, workNS string) *fleetv1beta1.Work { + var resultWork fleetv1beta1.Work + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: workName, Namespace: workNS}, &resultWork) + if err != nil { + return false + } + applyCond := meta.FindStatusCondition(resultWork.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied) + if applyCond == nil || applyCond.Status != metav1.ConditionTrue || applyCond.ObservedGeneration != resultWork.Generation { + By(fmt.Sprintf("applyCond not true: %v", applyCond)) + return false + } + for _, manifestCondition := range resultWork.Status.ManifestConditions { + if !meta.IsStatusConditionTrue(manifestCondition.Conditions, fleetv1beta1.WorkConditionTypeApplied) { + By(fmt.Sprintf("manifest applyCond not true %v : %v", manifestCondition.Identifier, manifestCondition.Conditions)) + return false + } + } + return true + }, timeout, interval).Should(BeTrue()) + return &resultWork +} + +// waitForWorkToAvailable waits for a work to have an available condition to be true +func waitForWorkToBeAvailable(workName, workNS string) *fleetv1beta1.Work { + var resultWork fleetv1beta1.Work + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: workName, Namespace: workNS}, &resultWork) + if err != nil { + return false + } + availCond := meta.FindStatusCondition(resultWork.Status.Conditions, fleetv1beta1.WorkConditionTypeAvailable) + if !condition.IsConditionStatusTrue(availCond, resultWork.Generation) { + By(fmt.Sprintf("availCond not true: %v", availCond)) + return false + } + for _, manifestCondition := range resultWork.Status.ManifestConditions { + if !meta.IsStatusConditionTrue(manifestCondition.Conditions, fleetv1beta1.WorkConditionTypeAvailable) { + By(fmt.Sprintf("manifest availCond not true %v : %v", manifestCondition.Identifier, manifestCondition.Conditions)) + return false + } + } + return true + }, timeout, interval).Should(BeTrue()) + return &resultWork +} + +// waitForWorkToBeHandled waits for a work to have a finalizer +func waitForWorkToBeHandled(workName, workNS string) *fleetv1beta1.Work { + var resultWork fleetv1beta1.Work + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: workName, Namespace: workNS}, &resultWork) + if err != nil { + return false + } + return controllerutil.ContainsFinalizer(&resultWork, fleetv1beta1.WorkFinalizer) + }, timeout, interval).Should(BeTrue()) + return &resultWork +} diff --git a/pkg/controllers/workapplier/controller_integration_test.go b/pkg/controllers/workapplier/controller_integration_test.go new file mode 100644 index 000000000..83454486e --- /dev/null +++ b/pkg/controllers/workapplier/controller_integration_test.go @@ -0,0 +1,3994 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +const ( + workNameTemplate = "work-%s" + nsNameTemplate = "ns-%s" + deployNameTemplate = "deploy-%s" +) + +const ( + eventuallyDuration = time.Second * 30 + eventuallyInternal = time.Second * 1 + consistentlyDuration = time.Second * 5 + consistentlyInternal = time.Millisecond * 500 +) + +var ( + ignoreFieldObjectMetaAutoGenFields = cmpopts.IgnoreFields(metav1.ObjectMeta{}, "CreationTimestamp", "Generation", "ResourceVersion", "SelfLink", "UID", "ManagedFields") + ignoreFieldAppliedWorkStatus = cmpopts.IgnoreFields(fleetv1beta1.AppliedWork{}, "Status") + ignoreFieldConditionLTTMsg = cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "Message") + ignoreDriftDetailsObsTime = cmpopts.IgnoreFields(fleetv1beta1.DriftDetails{}, "ObservationTime", "FirstDriftedObservedTime") + ignoreDiffDetailsObsTime = cmpopts.IgnoreFields(fleetv1beta1.DiffDetails{}, "ObservationTime", "FirstDiffedObservedTime") + + lessFuncPatchDetail = func(a, b fleetv1beta1.PatchDetail) bool { + return a.Path < b.Path + } +) + +var ( + dummyLabelKey = "foo" + dummyLabelValue1 = "bar" + dummyLabelValue2 = "baz" + dummyLabelValue3 = "quz" + dummyLabelValue4 = "qux" +) + +// createWorkObject creates a new Work object with the given name, manifests, and apply strategy. +func createWorkObject(workName string, applyStrategy *fleetv1beta1.ApplyStrategy, rawManifestJSON ...[]byte) { + manifests := make([]fleetv1beta1.Manifest, len(rawManifestJSON)) + for idx := range rawManifestJSON { + manifests[idx] = fleetv1beta1.Manifest{ + RawExtension: runtime.RawExtension{ + Raw: rawManifestJSON[idx], + }, + } + } + + work := &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + }, + Spec: fleetv1beta1.WorkSpec{ + Workload: fleetv1beta1.WorkloadTemplate{ + Manifests: manifests, + }, + ApplyStrategy: applyStrategy, + }, + } + Expect(hubClient.Create(ctx, work)).To(Succeed()) +} + +func updateWorkObject(workName string, applyStrategy *fleetv1beta1.ApplyStrategy, rawManifestJSON ...[]byte) { + manifests := make([]fleetv1beta1.Manifest, len(rawManifestJSON)) + for idx := range rawManifestJSON { + manifests[idx] = fleetv1beta1.Manifest{ + RawExtension: runtime.RawExtension{ + Raw: rawManifestJSON[idx], + }, + } + } + + work := &fleetv1beta1.Work{} + Expect(hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work)).To(Succeed()) + + work.Spec.Workload.Manifests = manifests + work.Spec.ApplyStrategy = applyStrategy + Expect(hubClient.Update(ctx, work)).To(Succeed()) +} + +func marshalK8sObjJSON(obj runtime.Object) []byte { + unstructuredObjMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + Expect(err).To(BeNil(), "Failed to convert the object to an unstructured object") + unstructuredObj := &unstructured.Unstructured{Object: unstructuredObjMap} + json, err := unstructuredObj.MarshalJSON() + Expect(err).To(BeNil(), "Failed to marshal the unstructured object to JSON") + return json +} + +func workFinalizerAddedActual(workName string) func() error { + return func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + // Check that the cleanup finalizer has been added. + if !controllerutil.ContainsFinalizer(work, fleetv1beta1.WorkFinalizer) { + return fmt.Errorf("cleanup finalizer has not been added") + } + return nil + } +} + +func appliedWorkCreatedActual(workName string) func() error { + return func() error { + // Retrieve the AppliedWork object. + appliedWork := &fleetv1beta1.AppliedWork{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, appliedWork); err != nil { + return fmt.Errorf("failed to retrieve the AppliedWork object: %w", err) + } + + wantAppliedWork := &fleetv1beta1.AppliedWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + }, + Spec: fleetv1beta1.AppliedWorkSpec{ + WorkName: workName, + WorkNamespace: memberReservedNSName, + }, + } + if diff := cmp.Diff( + appliedWork, wantAppliedWork, + ignoreFieldObjectMetaAutoGenFields, + ignoreFieldAppliedWorkStatus, + ); diff != "" { + return fmt.Errorf("appliedWork diff (-got +want):\n%s", diff) + } + return nil + } +} + +func prepareAppliedWorkOwnerRef(workName string) *metav1.OwnerReference { + // Retrieve the AppliedWork object. + appliedWork := &fleetv1beta1.AppliedWork{} + Expect(memberClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, appliedWork)).To(Succeed(), "Failed to retrieve the AppliedWork object") + + // Prepare the expected OwnerReference. + return &metav1.OwnerReference{ + APIVersion: fleetv1beta1.GroupVersion.String(), + Kind: "AppliedWork", + Name: appliedWork.Name, + UID: appliedWork.GetUID(), + BlockOwnerDeletion: ptr.To(false), + } +} + +func regularNSObjectAppliedActual(nsName string, appliedWorkOwnerRef *metav1.OwnerReference) func() error { + return func() error { + // Retrieve the NS object. + gotNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, gotNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Check that the NS object has been created as expected. + + // To ignore default values automatically, here the test suite rebuilds the objects. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: gotNS.Name, + OwnerReferences: gotNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + } +} + +func regularDeploymentObjectAppliedActual(nsName, deployName string, appliedWorkOwnerRef *metav1.OwnerReference) func() error { + return func() error { + // Retrieve the Deployment object. + gotDeploy := &appsv1.Deployment{} + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + // Check that the Deployment object has been created as expected. + + // To ignore default values automatically, here the test suite rebuilds the objects. + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + if len(gotDeploy.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("number of containers in the Deployment object, got %d, want %d", len(gotDeploy.Spec.Template.Spec.Containers), 1) + } + if len(gotDeploy.Spec.Template.Spec.Containers[0].Ports) != 1 { + return fmt.Errorf("number of ports in the first container, got %d, want %d", len(gotDeploy.Spec.Template.Spec.Containers[0].Ports), 1) + } + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: gotDeploy.Namespace, + Name: gotDeploy.Name, + OwnerReferences: gotDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: gotDeploy.Spec.Replicas, + Selector: gotDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": gotDeploy.Spec.Template.ObjectMeta.Labels["app"], + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gotDeploy.Spec.Template.Spec.Containers[0].Name, + Image: gotDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: gotDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + } +} + +func markDeploymentAsAvailable(nsName, deployName string) { + // Retrieve the Deployment object. + gotDeploy := &appsv1.Deployment{} + Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + + // Mark the Deployment object as available. + now := metav1.Now() + requiredReplicas := int32(1) + if gotDeploy.Spec.Replicas != nil { + requiredReplicas = *gotDeploy.Spec.Replicas + } + gotDeploy.Status = appsv1.DeploymentStatus{ + ObservedGeneration: gotDeploy.Generation, + Replicas: requiredReplicas, + UpdatedReplicas: requiredReplicas, + ReadyReplicas: requiredReplicas, + AvailableReplicas: requiredReplicas, + UnavailableReplicas: 0, + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentAvailable, + Status: corev1.ConditionTrue, + Reason: "MarkedAsAvailable", + Message: "Deployment has been marked as available", + LastUpdateTime: now, + LastTransitionTime: now, + }, + }, + } + Expect(memberClient.Status().Update(ctx, gotDeploy)).To(Succeed(), "Failed to mark the Deployment object as available") +} + +func workStatusUpdated( + workName string, + workConds []metav1.Condition, + manifestConds []fleetv1beta1.ManifestCondition, + noLaterThanObservationTime *metav1.Time, + noLaterThanFirstObservedTime *metav1.Time, +) func() error { + return func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + // Prepare the expected Work object status. + + // Update the conditions with the observed generation. + // + // Note that the observed generation of a manifest condition is that of an applied + // resource, not that of the Work object. + for idx := range workConds { + workConds[idx].ObservedGeneration = work.Generation + } + wantWorkStatus := fleetv1beta1.WorkStatus{ + Conditions: workConds, + ManifestConditions: manifestConds, + } + + // Check that the Work object status has been updated as expected. + if diff := cmp.Diff( + work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + ); diff != "" { + return fmt.Errorf("work status diff (-got, +want):\n%s", diff) + } + + // For each manifest condition, verify the timestamps. + for idx := range work.Status.ManifestConditions { + manifestCond := &work.Status.ManifestConditions[idx] + if manifestCond.DriftDetails != nil { + if noLaterThanObservationTime != nil && manifestCond.DriftDetails.ObservationTime.After(noLaterThanObservationTime.Time) { + return fmt.Errorf("drift observation time is later than expected (observed: %v, no later than: %v)", manifestCond.DriftDetails.ObservationTime, noLaterThanObservationTime) + } + + if noLaterThanFirstObservedTime != nil && manifestCond.DriftDetails.FirstDriftedObservedTime.After(noLaterThanFirstObservedTime.Time) { + return fmt.Errorf("first drifted observation time is later than expected (observed: %v, no later than: %v)", manifestCond.DriftDetails.FirstDriftedObservedTime, noLaterThanFirstObservedTime) + } + + if !manifestCond.DriftDetails.ObservationTime.After(manifestCond.DriftDetails.FirstDriftedObservedTime.Time) { + return fmt.Errorf("drift observation time is later than first drifted observation time (observed: %v, first observed: %v)", manifestCond.DriftDetails.ObservationTime, manifestCond.DriftDetails.FirstDriftedObservedTime) + } + } + + if manifestCond.DiffDetails != nil { + if noLaterThanObservationTime != nil && manifestCond.DiffDetails.ObservationTime.After(noLaterThanObservationTime.Time) { + return fmt.Errorf("diff observation time is later than expected (observed: %v, no later than: %v)", manifestCond.DiffDetails.ObservationTime, noLaterThanObservationTime) + } + + if noLaterThanFirstObservedTime != nil && manifestCond.DiffDetails.FirstDiffedObservedTime.After(noLaterThanFirstObservedTime.Time) { + return fmt.Errorf("first diffed observation time is later than expected (observed: %v, no later than: %v)", manifestCond.DiffDetails.FirstDiffedObservedTime, noLaterThanFirstObservedTime) + } + + if !manifestCond.DiffDetails.ObservationTime.After(manifestCond.DiffDetails.FirstDiffedObservedTime.Time) { + return fmt.Errorf("diff observation time is later than first diffed observation time (observed: %v, first observed: %v)", manifestCond.DiffDetails.ObservationTime, manifestCond.DiffDetails.FirstDiffedObservedTime) + } + } + } + return nil + } +} + +func appliedWorkStatusUpdated(workName string, appliedResourceMeta []fleetv1beta1.AppliedResourceMeta) func() error { + return func() error { + // Retrieve the AppliedWork object. + appliedWork := &fleetv1beta1.AppliedWork{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, appliedWork); err != nil { + return fmt.Errorf("failed to retrieve the AppliedWork object: %w", err) + } + + // Prepare the expected AppliedWork object status. + wantAppliedWorkStatus := fleetv1beta1.AppliedWorkStatus{ + AppliedResources: appliedResourceMeta, + } + if diff := cmp.Diff(appliedWork.Status, wantAppliedWorkStatus); diff != "" { + return fmt.Errorf("appliedWork status diff (-got, +want):\n%s", diff) + } + return nil + } +} + +func cleanupWorkObject(workName string) { + // Retrieve the Work object. + work := &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + }, + } + Expect(hubClient.Delete(ctx, work)).To(Succeed(), "Failed to delete the Work object") + + // Wait for the removal of the Work object. + workRemovedActual := func() error { + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); !errors.IsNotFound(err) { + return fmt.Errorf("work object still exists or an unexpected error occurred: %w", err) + } + return nil + } + Eventually(workRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the Work object") +} + +func appliedWorkRemovedActual(workName string) func() error { + return func() error { + // Retrieve the AppliedWork object. + appliedWork := &fleetv1beta1.AppliedWork{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); !errors.IsNotFound(err) { + return fmt.Errorf("appliedWork object still exists or an unexpected error occurred: %w", err) + } + return nil + } +} + +func regularDeployRemovedActual(nsName, deployName string) func() error { + return func() error { + // Retrieve the Deployment object. + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: deployName, + }, + } + if err := memberClient.Delete(ctx, deploy); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete the Deployment object: %w", err) + } + + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, deploy); !errors.IsNotFound(err) { + return fmt.Errorf("deployment object still exists or an unexpected error occurred: %w", err) + } + return nil + } +} + +func regularNSObjectNotAppliedActual(nsName string) func() error { + return func() error { + // Retrieve the NS object. + ns := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, ns); !errors.IsNotFound(err) { + return fmt.Errorf("namespace object exists or an unexpected error occurred: %w", err) + } + return nil + } +} + +func regularDeployNotRemovedActual(nsName, deployName string) func() error { + return func() error { + // Retrieve the Deployment object. + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: deployName, + }, + } + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, deploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + return nil + } +} + +var _ = Describe("applying manifests", func() { + Context("apply new manifests (regular)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "a1") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "a1") + deployName := fmt.Sprintf(deployNameTemplate, "a1") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Create a new Work object with all the manifest JSONs. + createWorkObject(workName, nil, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + + // Ensure that the Deployment object has been applied as expected. + regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) + Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the deployment object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + }) + + It("can mark the deployment as available", func() { + markDeploymentAsAvailable(nsName, deployName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 1, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + UID: regularDeploy.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that all applied manifests have been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) + Eventually(regularDeployRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("garbage collect removed manifests", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "a3") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "a3") + deployName := fmt.Sprintf(deployNameTemplate, "a3") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Create a new Work object with all the manifest JSONs. + createWorkObject(workName, nil, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + + // Ensure that the Deployment object has been applied as expected. + regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) + Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the deployment object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + }) + + It("can mark the deployment as available", func() { + markDeploymentAsAvailable(nsName, deployName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 1, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + UID: regularDeploy.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can delete some manifests", func() { + // Update the work object and remove the Deployment manifest. + + // Re-prepare the JSON to make sure that type meta info. is included correctly. + regularNS := ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + updateWorkObject(workName, nil, regularNSJSON) + }) + + It("should garbage collect removed manifests", func() { + deployRemovedActual := regularDeployRemovedActual(nsName, deployName) + Eventually(deployRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the deployment object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that all applied manifests have been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) +}) + +var _ = Describe("drift detection and takeover", func() { + Context("take over pre-existing resources (take over if no diff, no diff present)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b1") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b1") + deployName := fmt.Sprintf(deployNameTemplate, "b1") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + regularNS = ns.DeepCopy() + regularNS.Name = nsName + + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + + // Prepare the JSONs for the resources. + regularNSJSON := marshalK8sObjJSON(regularNS) + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Create the resources on the member cluster side. + Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + + markDeploymentAsAvailable(nsName, deployName) + + // Create the Work object. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, + } + createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + + // Ensure that the Deployment object has been applied as expected. + regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) + Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the deployment object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + }) + + It("can mark the deployment as available", func() { + markDeploymentAsAvailable(nsName, deployName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 2, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + UID: regularDeploy.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that all applied manifests have been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) + Eventually(regularDeployRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("take over pre-existing resources (take over if no diff, with diff present, partial comparison)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b2") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b2") + deployName := fmt.Sprintf(deployNameTemplate, "b2") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + regularNS = ns.DeepCopy() + regularNS.Name = nsName + + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + + // Prepare the JSONs for the resources. + regularNSJSON := marshalK8sObjJSON(regularNS) + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Make cluster specific changes. + + // Labels is not a managed field; with partial comparison this variance will be + // ignored. + regularNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + } + // Replicas is a managed field; with partial comparison this variance will be noted. + regularDeploy.Spec.Replicas = ptr.To(int32(2)) + + // Create the resources on the member cluster side. + Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + + markDeploymentAsAvailable(nsName, deployName) + + // Create the Work object. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, + } + createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply some manifests (while preserving diffs in unmanaged fields)", func() { + // Verify that the object has been taken over, but all the unmanaged fields are + // left alone. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + // The label below is added by K8s itself (system-managed well-known label). + "kubernetes.io/metadata.name": "ns-b2", + } + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + Eventually(func() error { + // Retrieve the NS object. + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + Labels: regularNS.Labels, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to take over the NS object") + }) + + It("should not take over some objects", func() { + // Verify that the object has not been taken over. + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.Spec.Replicas = ptr.To(int32(2)) + + Consistently(func() error { + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + if len(regularDeploy.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("number of containers in the Deployment object, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers), 1) + } + if len(regularDeploy.Spec.Template.Spec.Containers[0].Ports) != 1 { + return fmt.Errorf("number of ports in the first container, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers[0].Ports), 1) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: regularDeploy.Namespace, + Name: regularDeploy.Name, + OwnerReferences: regularDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: regularDeploy.Spec.Replicas, + Selector: regularDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: regularDeploy.Spec.Template.ObjectMeta.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: regularDeploy.Spec.Template.Spec.Containers[0].Name, + Image: regularDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: regularDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to leave the Deployment object alone") + }) + + It("should update the Work object status", func() { + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFailedToTakeOver), + ObservedGeneration: 1, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularDeploy.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInMember: "2", + ValueInHub: "1", + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // Ensure that the Deployment object has been left alone. + regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) + Consistently(regularDeployNotRemovedActual, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("take over pre-existing resources (take over if no diff, with diff, full comparison)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b3") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b3") + deployName := fmt.Sprintf(deployNameTemplate, "b3") + + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + regularNS = ns.DeepCopy() + regularNS.Name = nsName + + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + + // Prepare the JSONs for the resources. + regularNSJSON := marshalK8sObjJSON(regularNS) + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Make cluster specific changes. + + // Labels is not a managed field; with partial comparison this variance will be + // ignored. + regularNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + } + // Replicas is a managed field; with partial comparison this variance will be noted. + regularDeploy.Spec.Replicas = ptr.To(int32(2)) + + // Create the resources on the member cluster side. + Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + + markDeploymentAsAvailable(nsName, deployName) + + // Create the Work object. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypeFullComparison, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, + } + createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + }) + + It("should not take over any object", func() { + // Verify that the NS object has not been taken over. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + // The label below is added by K8s itself (system-managed well-known label). + "kubernetes.io/metadata.name": "ns-b3", + } + + Consistently(func() error { + // Retrieve the NS object. + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + Labels: regularNS.Labels, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to take over the NS object") + + // Verify that the Deployment object has not been taken over. + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.Spec.Replicas = ptr.To(int32(2)) + + Consistently(func() error { + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + if len(regularDeploy.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("number of containers in the Deployment object, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers), 1) + } + if len(regularDeploy.Spec.Template.Spec.Containers[0].Ports) != 1 { + return fmt.Errorf("number of ports in the first container, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers[0].Ports), 1) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: regularDeploy.Namespace, + Name: regularDeploy.Name, + OwnerReferences: regularDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: regularDeploy.Spec.Replicas, + Selector: regularDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: regularDeploy.Spec.Template.ObjectMeta.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: regularDeploy.Spec.Template.Spec.Containers[0].Name, + Image: regularDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: regularDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInternal, "Failed to leave the Deployment object alone") + }) + + It("should update the Work object status", func() { + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFailedToTakeOver), + ObservedGeneration: 0, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularNS.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/metadata/labels/foo", + ValueInMember: dummyLabelValue1, + }, + // TO-DO (chenyu1): This is a namespace specific field; consider + // if this should be added as an exception which allows ignoring + // this diff automatically. + { + Path: "/spec/finalizers", + ValueInMember: "[kubernetes]", + }, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFailedToTakeOver), + ObservedGeneration: 1, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularDeploy.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + {Path: "/spec/progressDeadlineSeconds", ValueInMember: "600"}, + { + Path: "/spec/replicas", + ValueInMember: "2", + ValueInHub: "1", + }, + {Path: "/spec/revisionHistoryLimit", ValueInMember: "10"}, + { + Path: "/spec/strategy/rollingUpdate", + ValueInMember: "map[maxSurge:25% maxUnavailable:25%]", + }, + {Path: "/spec/strategy/type", ValueInMember: "RollingUpdate"}, + { + Path: "/spec/template/spec/containers/0/imagePullPolicy", + ValueInMember: "Always", + }, + {Path: "/spec/template/spec/containers/0/ports/0/protocol", ValueInMember: "TCP"}, + { + Path: "/spec/template/spec/containers/0/terminationMessagePath", + ValueInMember: "/dev/termination-log", + }, + { + Path: "/spec/template/spec/containers/0/terminationMessagePolicy", + ValueInMember: "File", + }, + {Path: "/spec/template/spec/dnsPolicy", ValueInMember: "ClusterFirst"}, + {Path: "/spec/template/spec/restartPolicy", ValueInMember: "Always"}, + {Path: "/spec/template/spec/schedulerName", ValueInMember: "default-scheduler"}, + {Path: "/spec/template/spec/securityContext", ValueInMember: "map[]"}, + {Path: "/spec/template/spec/terminationGracePeriodSeconds", ValueInMember: "30"}, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // No object can be applied, hence no resource are bookkept in the AppliedWork object status. + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // Ensure that the Deployment object has been left alone. + regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) + Consistently(regularDeployNotRemovedActual, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("detect drifts (apply if no drift, drift occurred, partial comparison)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b6") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b6") + deployName := fmt.Sprintf(deployNameTemplate, "b6") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, + } + createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + + // Ensure that the Deployment object has been applied as expected. + regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) + Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the deployment object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + }) + + It("can mark the deployment as available", func() { + markDeploymentAsAvailable(nsName, deployName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 1, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + UID: regularDeploy.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + // Use Eventually blocks to avoid conflicts. + Eventually(func() error { + // Retrieve the Deployment object. + updatedDeploy := &appsv1.Deployment{} + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + // Make changes to the Deployment object. + updatedDeploy.Spec.Replicas = ptr.To(int32(2)) + + // Update the Deployment object. + if err := memberClient.Update(ctx, updatedDeploy); err != nil { + return fmt.Errorf("failed to update the Deployment object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the Deployment object") + + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Make changes to the NS object. + if updatedNS.Labels == nil { + updatedNS.Labels = map[string]string{} + } + updatedNS.Labels[dummyLabelKey] = dummyLabelValue1 + + // Update the NS object. + if err := memberClient.Update(ctx, updatedNS); err != nil { + return fmt.Errorf("failed to update the NS object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the NS object") + }) + + It("should continue to apply some manifest (while preserving drifts in unmanaged fields)", func() { + // Verify that the object are still being applied, with the drifts in unmanaged fields + // untouched. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + // The label below is added by K8s itself (system-managed well-known label). + "kubernetes.io/metadata.name": "ns-b6", + } + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + Consistently(func() error { + // Retrieve the NS object. + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + Labels: regularNS.Labels, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to take over the NS object") + }) + + It("should stop applying some objects", func() { + // Verify that the changes in managed fields are not overwritten. + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + wantDeploy.Spec.Replicas = ptr.To(int32(2)) + + Consistently(func() error { + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + if len(regularDeploy.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("number of containers in the Deployment object, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers), 1) + } + if len(regularDeploy.Spec.Template.Spec.Containers[0].Ports) != 1 { + return fmt.Errorf("number of ports in the first container, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers[0].Ports), 1) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: regularDeploy.Namespace, + Name: regularDeploy.Name, + OwnerReferences: regularDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: regularDeploy.Spec.Replicas, + Selector: regularDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: regularDeploy.Spec.Template.ObjectMeta.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: regularDeploy.Spec.Template.Spec.Containers[0].Name, + Image: regularDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: regularDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to leave the Deployment object alone") + }) + + It("should update the Work object status", func() { + // Shift the timestamp to account for drift detection delays. + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + ObservedGeneration: 2, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularDeploy.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInMember: "2", + ValueInHub: "1", + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // Ensure that the Deployment object has been left alone. + regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) + Consistently(regularDeployNotRemovedActual, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + // For simplicity reasons, this test case will only involve a NS object. + Context("detect drifts (apply if no drift, drift occurred, full comparison)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b7") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b7") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNS.Spec.Finalizers = []corev1.FinalizerName{"kubernetes"} + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypeFullComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, + } + createWorkObject(workName, applyStrategy, regularNSJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Make changes to the NS object. + if updatedNS.Labels == nil { + updatedNS.Labels = map[string]string{} + } + updatedNS.Labels[dummyLabelKey] = dummyLabelValue1 + + // Update the NS object. + if err := memberClient.Update(ctx, updatedNS); err != nil { + return fmt.Errorf("failed to update the NS object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the NS object") + }) + + It("should stop applying some objects", func() { + // Verify that the changes in unmanaged fields are not overwritten. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + wantNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + // The label below is added by K8s itself (system-managed well-known label). + "kubernetes.io/metadata.name": "ns-b7", + } + + Consistently(func() error { + // Retrieve the NS object. + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + Labels: regularNS.Labels, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to leave the NS object alone") + }) + + It("should update the Work object status", func() { + // Shift the timestamp to account for drift detection delays. + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularNS.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/metadata/labels/foo", + ValueInMember: dummyLabelValue1, + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // No object can be applied, hence no resource are bookkept in the AppliedWork object status. + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + // For simplicity reasons, this test case will only involve a NS object. + Context("overwrite drifts (always apply, partial comparison)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b8") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b8") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + } + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeAlways, + } + createWorkObject(workName, applyStrategy, regularNSJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Make changes to the NS object. + if updatedNS.Labels == nil { + updatedNS.Labels = map[string]string{} + } + updatedNS.Labels[dummyLabelKey] = dummyLabelValue2 + + // Update the NS object. + if err := memberClient.Update(ctx, updatedNS); err != nil { + return fmt.Errorf("failed to update the NS object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the NS object") + }) + + It("should continue to apply some manifest (while overwriting drifts in managed fields)", func() { + // Verify that the object are still being applied, with the drifts in managed fields + // overwritten. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + // The label below is added by K8s itself (system-managed well-known label). + "kubernetes.io/metadata.name": "ns-b8", + } + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + nsOverwrittenActual := func() error { + // Retrieve the NS object. + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + Labels: regularNS.Labels, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + } + Eventually(nsOverwrittenActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the NS object") + Consistently(nsOverwrittenActual, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to apply the NS object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + // For simplicity reasons, this test case will only involve a NS object. + Context("overwrite drifts (apply if no drift, drift occurred before manifest version bump, partial comparison)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b9") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b9") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + } + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, + } + createWorkObject(workName, applyStrategy, marshalK8sObjJSON(regularNS)) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Make changes to the NS object. + if updatedNS.Labels == nil { + updatedNS.Labels = map[string]string{} + } + updatedNS.Labels[dummyLabelKey] = dummyLabelValue2 + + // Update the NS object. + if err := memberClient.Update(ctx, updatedNS); err != nil { + return fmt.Errorf("failed to update the NS object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the NS object") + }) + + It("should stop applying some objects", func() { + // Verify that the changes in unmanaged fields are not overwritten. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + wantNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue2, + // The label below is added by K8s itself (system-managed well-known label). + "kubernetes.io/metadata.name": "ns-b9", + } + + Consistently(func() error { + // Retrieve the NS object. + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + Labels: regularNS.Labels, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to leave the NS object alone") + }) + + It("should update the Work object status", func() { + // Shift the timestamp to account for drift detection delays. + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularNS.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/metadata/labels/foo", + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // No object can be applied, hence no resource are bookkept in the AppliedWork object status. + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can update the Work object", func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue3, + } + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, + } + updateWorkObject(workName, applyStrategy, marshalK8sObjJSON(regularNS)) + }) + + It("should apply the new manifests and overwrite all drifts in managed fields", func() { + // Verify that the new manifests are applied. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue3, + // The label below is added by K8s itself (system-managed well-known label). + "kubernetes.io/metadata.name": "ns-b9", + } + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + Eventually(func() error { + // Retrieve the NS object. + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + Labels: regularNS.Labels, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply new manifests") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + // For simplicity reasons, this test case will only involve a NS object. + Context("first drifted time preservation", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b10") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b10") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + } + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, + } + createWorkObject(workName, applyStrategy, marshalK8sObjJSON(regularNS)) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Make changes to the NS object. + if updatedNS.Labels == nil { + updatedNS.Labels = map[string]string{} + } + updatedNS.Labels[dummyLabelKey] = dummyLabelValue2 + + // Update the NS object. + if err := memberClient.Update(ctx, updatedNS); err != nil { + return fmt.Errorf("failed to update the NS object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the NS object") + }) + + var firstDriftedMustBeforeTimestamp metav1.Time + + It("should update the Work object status", func() { + // Shift the timestamp to account for drift detection delays. + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularNS.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/metadata/labels/foo", + ValueInHub: dummyLabelValue1, + ValueInMember: dummyLabelValue2, + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + + // Track the timestamp that was just after the drift was first detected. + firstDriftedMustBeforeTimestamp = metav1.Now() + }) + + It("can make changes to the objects, again", func() { + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Make changes to the NS object. + if updatedNS.Labels == nil { + updatedNS.Labels = map[string]string{} + } + updatedNS.Labels[dummyLabelKey] = dummyLabelValue4 + + // Update the NS object. + if err := memberClient.Update(ctx, updatedNS); err != nil { + return fmt.Errorf("failed to update the NS object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the NS object") + }) + + It("should update the Work object status (must track timestamps correctly)", func() { + // Shift the timestamp to account for drift detection delays. + driftObservedMustBeforeTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularNS.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/metadata/labels/foo", + ValueInMember: dummyLabelValue4, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &driftObservedMustBeforeTimestamp, &firstDriftedMustBeforeTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + }) + + Context("never take over", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "b12") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "b12") + deployName := fmt.Sprintf(deployNameTemplate, "b12") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + + // Prepare the JSONs for the resources. + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Create the resources on the member cluster side. + Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + } + createWorkObject(workName, applyStrategy, regularNSJSON, marshalK8sObjJSON(regularDeploy)) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should apply the manifests that haven not been created yet", func() { + // Ensure that the Deployment object has been applied as expected. + regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) + Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to apply the deployment object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + }) + + It("should not apply the manifests that have corresponding resources", func() { + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Rebuild the NS object to ignore default values automatically. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: updatedNS.Name, + OwnerReferences: updatedNS.OwnerReferences, + }, + } + + wantNS := ns.DeepCopy() + wantNS.Name = nsName + if diff := cmp.Diff(rebuiltGotNS, wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to leave the NS object alone") + }) + + It("can mark the deployment as available", func() { + markDeploymentAsAvailable(nsName, deployName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeNotTakenOver), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 1, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + UID: regularDeploy.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that all applied manifests have been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) + Eventually(regularDeployRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) +}) + +var _ = Describe("report diff", func() { + // For simplicity reasons, this test case will only involve a NS object. + Context("report diff only (new object)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "c1") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "c1") + + var regularNS *corev1.Namespace + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + } + createWorkObject(workName, applyStrategy, regularNSJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should not apply the manifests", func() { + // Ensure that the NS object has not been applied. + regularNSObjectNotAppliedActual := regularNSObjectNotAppliedActual(nsName) + Eventually(regularNSObjectNotAppliedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to avoid applying the namespace object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + ObservedGeneration: 0, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/", + ValueInHub: "(the whole object)", + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("report diff only (with diff present, diff disappears later, partial comparison)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "c2") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "c2") + deployName := fmt.Sprintf(deployNameTemplate, "c2") + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Create the objects first in the member cluster. + Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + + regularDeploy.Spec.Replicas = ptr.To(int32(2)) + Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + } + createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should own the objects, but not apply any manifests", func() { + // Verify that the Deployment manifest has not been applied, yet Fleet has assumed + // its ownership. + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + wantDeploy.Spec.Replicas = ptr.To(int32(2)) + + deployOwnedButNotApplied := func() error { + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + if len(regularDeploy.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("number of containers in the Deployment object, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers), 1) + } + if len(regularDeploy.Spec.Template.Spec.Containers[0].Ports) != 1 { + return fmt.Errorf("number of ports in the first container, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers[0].Ports), 1) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: regularDeploy.Namespace, + Name: regularDeploy.Name, + OwnerReferences: regularDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: regularDeploy.Spec.Replicas, + Selector: regularDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: regularDeploy.Spec.Template.ObjectMeta.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: regularDeploy.Spec.Template.Spec.Containers[0].Name, + Image: regularDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: regularDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + } + + Eventually(deployOwnedButNotApplied, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to own the Deployment object without applying the manifest") + Consistently(deployOwnedButNotApplied, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to own the Deployment object without applying the manifest") + + // Verify that Fleet has assumed ownership of the NS object. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + nsOwnedButNotApplied := func() error { + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + } + Eventually(nsOwnedButNotApplied, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to own the NS object without applying the manifest") + Consistently(nsOwnedButNotApplied, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to own the NS object without applying the manifest") + }) + + It("should update the Work object status", func() { + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + ObservedGeneration: 1, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularDeploy.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInMember: "2", + ValueInHub: "1", + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + // Use Eventually blocks to avoid conflicts. + Eventually(func() error { + // Retrieve the Deployment object. + updatedDeploy := &appsv1.Deployment{} + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + // Make changes to the Deployment object. + updatedDeploy.Spec.Replicas = ptr.To(int32(1)) + + // Update the Deployment object. + if err := memberClient.Update(ctx, updatedDeploy); err != nil { + return fmt.Errorf("failed to update the Deployment object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update the Deployment object") + }) + + It("can mark the deployment as available", func() { + markDeploymentAsAvailable(nsName, deployName) + }) + + It("should update the Work object status", func() { + // Shift the timestamp to account for drift/diff detection delays. + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound), + ObservedGeneration: 2, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + // Ensure that the Deployment object has been left alone. + regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) + Consistently(regularDeployNotRemovedActual, consistentlyDuration, consistentlyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("report diff only (w/ not taken over resources, partial comparison, a.k.a. do not touch anything and just report diff)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, "c6") + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, "c6") + deployName := fmt.Sprintf(deployNameTemplate, "c6") + + var regularNS *corev1.Namespace + var regularDeploy *appsv1.Deployment + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Create the objects first in the member cluster. + Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + + regularDeploy.Spec.Replicas = ptr.To(int32(2)) + Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + } + createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to prepare an AppliedWork object") + }) + + It("should not apply any manifest", func() { + // Verify that the NS manifest has not been applied. + Eventually(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Rebuild the NS object to ignore default values automatically. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: updatedNS.Name, + OwnerReferences: updatedNS.OwnerReferences, + }, + } + wantNS := ns.DeepCopy() + wantNS.Name = nsName + if diff := cmp.Diff(rebuiltGotNS, wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to leave the NS object alone") + + // Verify that the Deployment manifest has not been applied. + Eventually(func() error { + // Retrieve the Deployment object. + updatedDeploy := &appsv1.Deployment{} + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + // Rebuild the Deployment object to ignore default values automatically. + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: updatedDeploy.Namespace, + Name: updatedDeploy.Name, + OwnerReferences: updatedDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: updatedDeploy.Spec.Replicas, + Selector: updatedDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: updatedDeploy.Spec.Template.ObjectMeta.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: updatedDeploy.Spec.Template.Spec.Containers[0].Name, + Image: updatedDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: updatedDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.Spec.Replicas = ptr.To(int32(2)) + + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + }, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to leave the Deployment object alone") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + ObservedGeneration: 1, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInHub: "1", + ValueInMember: "2", + }, + }, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + cleanupWorkObject(workName) + + // Ensure that all applied manifests have been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the AppliedWork object") + + regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) + Eventually(regularDeployRemovedActual, eventuallyDuration, eventuallyInternal).Should(Succeed(), "Failed to remove the deployment object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) +}) diff --git a/pkg/controllers/workapplier/controller_test.go b/pkg/controllers/workapplier/controller_test.go new file mode 100644 index 000000000..89df03b0d --- /dev/null +++ b/pkg/controllers/workapplier/controller_test.go @@ -0,0 +1,238 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "fmt" + "log" + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" +) + +const ( + workName = "work-1" + + deployName = "deploy-1" + configMapName = "configmap-1" + nsName = "ns-1" +) + +var ( + nsGVR = schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "namespaces", + } + + deploy = &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: deployName, + Namespace: nsName, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + } + deployUnstructured *unstructured.Unstructured + deployJSON []byte + + ns = &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + }, + } + nsUnstructured *unstructured.Unstructured + nsJSON []byte + + dummyOwnerRef = metav1.OwnerReference{ + APIVersion: "dummy.owner/v1", + Kind: "DummyOwner", + Name: "dummy-owner", + UID: "1234-5678-90", + } +) + +var ( + appliedWorkOwnerRef = &metav1.OwnerReference{ + APIVersion: "placement.kubernetes-fleet.io/v1beta1", + Kind: "AppliedWork", + Name: workName, + UID: "uid", + } +) + +var ( + ignoreFieldTypeMetaInNamespace = cmpopts.IgnoreFields(corev1.Namespace{}, "TypeMeta") + + lessFuncAppliedResourceMeta = func(i, j fleetv1beta1.AppliedResourceMeta) bool { + iStr := fmt.Sprintf("%s/%s/%s/%s/%s", i.Group, i.Version, i.Kind, i.Namespace, i.Name) + jStr := fmt.Sprintf("%s/%s/%s/%s/%s", j.Group, j.Version, j.Kind, j.Namespace, j.Name) + return iStr < jStr + } +) + +func nsWRI(ordinal int, nsName string) *fleetv1beta1.WorkResourceIdentifier { + return &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: ordinal, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + } +} + +func deployWRI(ordinal int, nsName, deployName string) *fleetv1beta1.WorkResourceIdentifier { + return &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: ordinal, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + } +} + +func manifestAppliedCond(workGeneration int64, status metav1.ConditionStatus, reason, message string) metav1.Condition { + return metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: status, + ObservedGeneration: workGeneration, + Reason: reason, + Message: message, + } +} + +func TestMain(m *testing.M) { + // Add custom APIs to the runtime scheme. + if err := fleetv1beta1.AddToScheme(scheme.Scheme); err != nil { + log.Fatalf("failed to add custom APIs (placement/v1beta1) to the runtime scheme: %v", err) + } + + // Initialize the variables. + initializeVariables() + + os.Exit(m.Run()) +} + +func initializeVariables() { + var err error + + // Regular objects. + // Deployment. + deployGenericMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(deploy) + if err != nil { + log.Fatalf("failed to convert deployment to unstructured: %v", err) + } + deployUnstructured = &unstructured.Unstructured{Object: deployGenericMap} + + deployJSON, err = deployUnstructured.MarshalJSON() + if err != nil { + log.Fatalf("failed to marshal deployment to JSON: %v", err) + } + + // Namespace. + nsGenericMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ns) + if err != nil { + log.Fatalf("failed to convert namespace to unstructured: %v", err) + } + nsUnstructured = &unstructured.Unstructured{Object: nsGenericMap} + nsJSON, err = nsUnstructured.MarshalJSON() + if err != nil { + log.Fatalf("failed to marshal namespace to JSON: %v", err) + } +} + +// TestPrepareManifestProcessingBundles tests the prepareManifestProcessingBundles function. +func TestPrepareManifestProcessingBundles(t *testing.T) { + deployJSON := deployJSON + nsJSON := nsJSON + memberReservedNSName := "fleet-member-experimental" + + work := &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + }, + Spec: fleetv1beta1.WorkSpec{ + Workload: fleetv1beta1.WorkloadTemplate{ + Manifests: []fleetv1beta1.Manifest{ + { + RawExtension: runtime.RawExtension{ + Raw: nsJSON, + }, + }, + { + RawExtension: runtime.RawExtension{ + Raw: deployJSON, + }, + }, + }, + }, + }, + } + + bundles := prepareManifestProcessingBundles(work) + wantBundles := []*manifestProcessingBundle{ + { + manifest: &work.Spec.Workload.Manifests[0], + }, + { + manifest: &work.Spec.Workload.Manifests[1], + }, + } + if diff := cmp.Diff(bundles, wantBundles, cmp.AllowUnexported(manifestProcessingBundle{})); diff != "" { + t.Errorf("prepareManifestProcessingBundles() mismatches (-got +want):\n%s", diff) + } +} diff --git a/pkg/controllers/workapplier/drift_detection_takeover.go b/pkg/controllers/workapplier/drift_detection_takeover.go new file mode 100644 index 000000000..cc541873f --- /dev/null +++ b/pkg/controllers/workapplier/drift_detection_takeover.go @@ -0,0 +1,408 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + "strings" + + "github.com/qri-io/jsonpointer" + "github.com/wI2L/jsondiff" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +const ( + k8sReservedLabelAnnotationFullDomain = "kubernetes.io/" + k8sReservedLabelAnnotationAbbrDomain = "k8s.io/" + fleetReservedLabelAnnotationDomain = "kubernetes-fleet.io/" +) + +// takeOverPreExistingObject takes over a pre-existing object in the member cluster. +func (r *Reconciler) takeOverPreExistingObject( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObj, inMemberClusterObj *unstructured.Unstructured, + applyStrategy *fleetv1beta1.ApplyStrategy, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) (*unstructured.Unstructured, []fleetv1beta1.PatchDetail, error) { + inMemberClusterObjCopy := inMemberClusterObj.DeepCopy() + existingOwnerRefs := inMemberClusterObjCopy.GetOwnerReferences() + + // At this moment Fleet is set to leave applied resources in the member cluster if the cluster + // decides to leave its fleet; when this happens, the resources will be owned by an AppliedWork + // object that might not have a corresponding Work object anymore as the cluster might have + // been de-selected. If the cluster decides to re-join the fleet, and the same set of manifests + // are being applied again, one may encounter unexpected errors due to the presence of the + // left behind AppliedWork object. To address this issue, Fleet here will perform a cleanup, + // removing any owner reference that points to an orphaned AppliedWork object. + existingOwnerRefs, err := r.removeLeftBehindAppliedWorkOwnerRefs(ctx, existingOwnerRefs) + if err != nil { + return nil, nil, fmt.Errorf("failed to remove left-behind AppliedWork owner references: %w", err) + } + + // Check this object is already owned by another object (or controller); if so, Fleet will only + // add itself as an additional owner if co-ownership is allowed. + if len(existingOwnerRefs) >= 1 && !applyStrategy.AllowCoOwnership { + // The object is already owned by another object, and co-ownership is forbidden. + // No takeover will be performed. + // + // Note that This will be registered as an (apply) error. + return nil, nil, fmt.Errorf("the object is already owned by some other sources(s) and co-ownership is disallowed") + } + + // Check if the object is already owned by Fleet, but the owner is a different AppliedWork + // object, i.e., the object has been placed in duplicate. + // + // Originally Fleet does allow placing the same object multiple times; each placement + // attempt might have its own object spec (envelopes might be used and in each envelope + // the object looks different). This would lead to the object being constantly overwritten, + // but no error would be raised on the user-end. With the drift detection feature, however, + // this scenario would lead to constant flipping of the drift reporting, which could lead to + // user confusion. To address this corner case, Fleet would now deny placing the same object + // twice. + if isPlacedByFleetInDuplicate(existingOwnerRefs, expectedAppliedWorkOwnerRef) { + return nil, nil, fmt.Errorf("the object is already owned by another Fleet AppliedWork object") + } + + // Check if the takeover action requires additional steps (configuration difference inspection). + // + // Note that the default takeover action is AlwaysApply. + if applyStrategy.WhenToTakeOver == fleetv1beta1.WhenToTakeOverTypeIfNoDiff { + configDiffs, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, gvr, manifestObj, inMemberClusterObjCopy, applyStrategy.ComparisonOption) + switch { + case err != nil: + return nil, nil, fmt.Errorf("failed to calculate configuration diffs between the manifest object and the object from the member cluster: %w", err) + case len(configDiffs) > 0: + return nil, configDiffs, nil + } + } + + // Take over the object. + updatedOwnerRefs := append(existingOwnerRefs, *expectedAppliedWorkOwnerRef) + inMemberClusterObjCopy.SetOwnerReferences(updatedOwnerRefs) + takenOverInMemberClusterObj, err := r.spokeDynamicClient. + Resource(*gvr).Namespace(inMemberClusterObjCopy.GetNamespace()). + Update(ctx, inMemberClusterObjCopy, metav1.UpdateOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to take over the object: %w", err) + } + + return takenOverInMemberClusterObj, nil, nil +} + +// diffBetweenManifestAndInMemberClusterObjects calculates the differences between the manifest object +// and its corresponding object in the member cluster. +func (r *Reconciler) diffBetweenManifestAndInMemberClusterObjects( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObj, inMemberClusterObj *unstructured.Unstructured, + cmpOption fleetv1beta1.ComparisonOptionType, +) ([]fleetv1beta1.PatchDetail, error) { + switch cmpOption { + case fleetv1beta1.ComparisonOptionTypePartialComparison: + return r.partialDiffBetweenManifestAndInMemberClusterObjects(ctx, gvr, manifestObj, inMemberClusterObj) + case fleetv1beta1.ComparisonOptionTypeFullComparison: + // For the full comparison, Fleet compares directly the JSON representations of the + // manifest object and the object in the member cluster. + return preparePatchDetails(manifestObj, inMemberClusterObj) + default: + return nil, fmt.Errorf("an invalid comparison option is specified") + } +} + +// partialDiffBetweenManifestAndInMemberClusterObjects calculates the differences between the +// manifest object and its corresponding object in the member cluster by performing a dry-run +// apply op; this would ignore differences in the unmanaged fields and report only those +// in the managed fields. +func (r *Reconciler) partialDiffBetweenManifestAndInMemberClusterObjects( + ctx context.Context, + gvr *schema.GroupVersionResource, + manifestObj, inMemberClusterObj *unstructured.Unstructured, +) ([]fleetv1beta1.PatchDetail, error) { + // Fleet calculates the partial diff between two objects by running apply ops in the dry-run + // mode. + appliedObj, err := r.applyInDryRunMode(ctx, gvr, manifestObj, inMemberClusterObj) + if err != nil { + return nil, fmt.Errorf("failed to apply the manifest in dry-run mode: %w", err) + } + + // After the dry-run apply op, all the managed fields should have been overwritten using the + // values from the manifest object, while leaving all the unmanaged fields untouched. This + // would allow Fleet to compare the object returned by the dry-run apply op with the object + // that is currently in the member cluster; if all the fields are consistent, it is safe + // for us to assume that there are no drifts, otherwise, any fields that are different + // imply that running an actual apply op would lead to unexpected changes, which signifies + // the presence of partial drifts (drifts in managed fields). + + // Prepare the patch details. + return preparePatchDetails(appliedObj, inMemberClusterObj) +} + +// organizeJSONPatchIntoFleetPatchDetails organizes the JSON patch operations into Fleet patch details. +func organizeJSONPatchIntoFleetPatchDetails(patch jsondiff.Patch, manifestObjMap map[string]interface{}) ([]fleetv1beta1.PatchDetail, error) { + // Pre-allocate the slice for the patch details. The organization procedure typically will yield + // the same number of PatchDetail items as the JSON patch operations. + details := make([]fleetv1beta1.PatchDetail, 0, len(patch)) + + // A side note: here Fleet takes an expedient approach processing null JSON paths, by treating + // null paths as empty strings. + + // Process only the first 100 ops. + // TO-DO (chenyu1): Impose additional size limits. + for idx := 0; idx < len(patch) && idx < patchDetailPerObjLimit; idx++ { + op := patch[idx] + pathPtr, err := jsonpointer.Parse(op.Path) + if err != nil { + // An invalid path is found; normally this should not happen. + return nil, fmt.Errorf("failed to parse the JSON path: %w", err) + } + fromPtr, err := jsonpointer.Parse(op.From) + if err != nil { + // An invalid path is found; normally this should not happen. + return nil, fmt.Errorf("failed to parse the JSON path: %w", err) + } + + switch op.Type { + case jsondiff.OperationAdd: + details = append(details, fleetv1beta1.PatchDetail{ + Path: op.Path, + ValueInMember: fmt.Sprint(op.Value), + }) + case jsondiff.OperationRemove: + // Fleet here skips validation as the JSON data is just marshalled. + hubValue, err := pathPtr.Eval(manifestObjMap) + if err != nil { + return nil, fmt.Errorf("failed to evaluate the JSON path %s in the manifest object: %w", op.Path, err) + } + details = append(details, fleetv1beta1.PatchDetail{ + Path: op.Path, + ValueInHub: fmt.Sprintf("%v", hubValue), + }) + case jsondiff.OperationReplace: + // Fleet here skips validation as the JSON data is just marshalled. + hubValue, err := pathPtr.Eval(manifestObjMap) + if err != nil { + return nil, fmt.Errorf("failed to evaluate the JSON path %s in the manifest object: %w", op.Path, err) + } + details = append(details, fleetv1beta1.PatchDetail{ + Path: op.Path, + ValueInMember: fmt.Sprint(op.Value), + ValueInHub: fmt.Sprintf("%v", hubValue), + }) + case jsondiff.OperationMove: + // Normally the Move operation will not be returned as factorization is disabled + // for the JSON patch calculation process; however, Fleet here still processes them + // just in case. + // + // Each Move operation will be parsed into two separate operations. + hubValue, err := fromPtr.Eval(manifestObjMap) + if err != nil { + return nil, fmt.Errorf("failed to evaluate the JSON path %s in the manifest object: %w", op.Path, err) + } + details = append(details, fleetv1beta1.PatchDetail{ + Path: op.From, + ValueInHub: fmt.Sprintf("%v", hubValue), + }) + details = append(details, fleetv1beta1.PatchDetail{ + Path: op.Path, + ValueInMember: fmt.Sprintf("%v", hubValue), + }) + case jsondiff.OperationCopy: + // Normally the Copy operation will not be returned as factorization is disabled + // for the JSON patch calculation process; however, Fleet here still processes them + // just in case. + // + // Each Copy operation will be parsed into an Add operation. + hubValue, err := fromPtr.Eval(manifestObjMap) + if err != nil { + return nil, fmt.Errorf("failed to evaluate the JSON path %s in the applied object: %w", op.Path, err) + } + details = append(details, fleetv1beta1.PatchDetail{ + Path: op.Path, + ValueInMember: fmt.Sprintf("%v", hubValue), + }) + case jsondiff.OperationTest: + // The Test op is a no-op in Fleet's use case. Normally it will not be returned, either. + default: + // An unexpected op is returned. + return nil, fmt.Errorf("an unexpected JSON patch operation is returned (%s)", op) + } + } + + return details, nil +} + +// preparePatchDetails calculates the differences between two objects in the form +// of Fleet patch details. +func preparePatchDetails(srcObj, destObj *unstructured.Unstructured) ([]fleetv1beta1.PatchDetail, error) { + // Discard certain fields from both objects before comparison. + srcObjCopy := discardFieldsIrrelevantInComparisonFrom(srcObj) + destObjCopy := discardFieldsIrrelevantInComparisonFrom(destObj) + + // Marshal the objects into JSON. + srcObjJSONBytes, err := srcObjCopy.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal the source object into JSON: %w", err) + } + + destObjJSONBytes, err := destObjCopy.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal the destination object into JSON: %w", err) + } + + // Compare the JSON representations. + patch, err := jsondiff.CompareJSON(srcObjJSONBytes, destObjJSONBytes) + if err != nil { + return nil, fmt.Errorf("failed to compare the JSON representations of the source and destination objects: %w", err) + } + + details, err := organizeJSONPatchIntoFleetPatchDetails(patch, srcObjCopy.Object) + if err != nil { + return nil, fmt.Errorf("failed to organize JSON patch operations into Fleet patch details: %w", err) + } + return details, nil +} + +// removeLeftBehindAppliedWorkOwnerRefs removes owner references that point to orphaned AppliedWork objects. +func (r *Reconciler) removeLeftBehindAppliedWorkOwnerRefs(ctx context.Context, ownerRefs []metav1.OwnerReference) ([]metav1.OwnerReference, error) { + updatedOwnerRefs := make([]metav1.OwnerReference, 0, len(ownerRefs)) + for idx := range ownerRefs { + ownerRef := ownerRefs[idx] + if ownerRef.APIVersion != fleetv1beta1.GroupVersion.String() && ownerRef.Kind != fleetv1beta1.AppliedWorkKind { + // Skip non AppliedWork owner references. + updatedOwnerRefs = append(updatedOwnerRefs, ownerRef) + continue + } + + // Check if the AppliedWork object has a corresponding Work object. + workObj := &fleetv1beta1.Work{} + err := r.hubClient.Get(ctx, types.NamespacedName{Namespace: r.workNameSpace, Name: ownerRef.Name}, workObj) + switch { + case err != nil && !errors.IsNotFound(err): + // An unexpected error occurred. + return nil, fmt.Errorf("failed to get the Work object: %w", err) + case err == nil: + // The AppliedWork owner reference is valid; no need for removal. + // + // Note that no UID check is performed here; Fleet can (and will) reuse the same AppliedWork + // as long as it has the same name as a Work object, even if the AppliedWork object is not + // originally derived from it. This is safe as the AppliedWork object is in essence a delegate + // and does not keep any additional information. + updatedOwnerRefs = append(updatedOwnerRefs, ownerRef) + continue + default: + // The AppliedWork owner reference is invalid; the Work object does not exist. + // Remove the owner reference. + klog.V(2).InfoS("Found an owner reference that points to an orphaned AppliedWork object", "ownerRef", ownerRef) + continue + } + } + + return updatedOwnerRefs, nil +} + +// discardFieldsIrrelevantInComparisonFrom discards fields that are irrelevant when comparing +// the manifest and live objects (or two manifest objects). +// +// Note that this method will return an object copy; the original object will be left untouched. +func discardFieldsIrrelevantInComparisonFrom(obj *unstructured.Unstructured) *unstructured.Unstructured { + // Create a deep copy of the object. + objCopy := obj.DeepCopy() + + // Remove object meta fields that are irrelevant in comparison. + + // Clear out the object's name/namespace. For regular objects, the names/namespaces will + // always be same between the manifest and the live objects, as guaranteed by the object + // retrieval step earlier, so a comparison on these fields is unnecessary anyway. + objCopy.SetName("") + objCopy.SetNamespace("") + + // Clear out the object's generate name. This is a field that is irrelevant in comparison. + objCopy.SetGenerateName("") + + // Remove certain labels and annotations. + // + // Fleet will remove labels/annotations that are reserved for Fleet own use cases, plus + // well-known Kubernetes labels and annotations, as these cannot (should not) be set by users + // directly. + annotations := objCopy.GetAnnotations() + cleanedAnnotations := map[string]string{} + for k, v := range annotations { + if strings.Contains(k, k8sReservedLabelAnnotationFullDomain) { + // Skip Kubernetes reserved annotations. + continue + } + + if strings.Contains(k, k8sReservedLabelAnnotationAbbrDomain) { + // Skip Kubernetes reserved annotations. + continue + } + + if strings.Contains(k, fleetReservedLabelAnnotationDomain) { + // Skip Fleet reserved annotations. + continue + } + cleanedAnnotations[k] = v + } + objCopy.SetAnnotations(cleanedAnnotations) + + labels := objCopy.GetLabels() + cleanedLabels := map[string]string{} + for k, v := range labels { + if strings.Contains(k, k8sReservedLabelAnnotationFullDomain) { + // Skip Kubernetes reserved labels. + continue + } + + if strings.Contains(k, k8sReservedLabelAnnotationAbbrDomain) { + // Skip Kubernetes reserved labels. + continue + } + + if strings.Contains(k, fleetReservedLabelAnnotationDomain) { + // Skip Fleet reserved labels. + continue + } + cleanedLabels[k] = v + } + objCopy.SetLabels(cleanedLabels) + + // Fields below are system-reserved fields in object meta. Technically speaking they can be + // set in the manifests, but this is a very uncommon practice, and currently Fleet will clear + // these fields (except for the finalizers) before applying the manifests]. + // As a result, for now Fleet will ignore them in the comparison process as well. + // + // TO-DO (chenyu1): evaluate if this is a correct assumption for most (if not all) Fleet + // users. + objCopy.SetFinalizers([]string{}) + objCopy.SetManagedFields([]metav1.ManagedFieldsEntry{}) + objCopy.SetOwnerReferences([]metav1.OwnerReference{}) + + // Fields below are read-only fields in object meta. Fleet will ignore them in the comparison + // process. + objCopy.SetCreationTimestamp(metav1.Time{}) + // Deleted objects are handled separately in the apply process; for comparison purposes, + // Fleet will ignore the deletion timestamp and grace period seconds. + objCopy.SetDeletionTimestamp(nil) + objCopy.SetDeletionGracePeriodSeconds(nil) + objCopy.SetGeneration(0) + objCopy.SetResourceVersion("") + objCopy.SetSelfLink("") + objCopy.SetUID("") + + // Remove the status field. + unstructured.RemoveNestedField(objCopy.Object, "status") + + return objCopy +} diff --git a/pkg/controllers/workapplier/drift_detection_takeover_test.go b/pkg/controllers/workapplier/drift_detection_takeover_test.go new file mode 100644 index 000000000..6dd5f1980 --- /dev/null +++ b/pkg/controllers/workapplier/drift_detection_takeover_test.go @@ -0,0 +1,831 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/wI2L/jsondiff" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +var ( + svcName = "web" + + svc = corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: svcName, + }, + Spec: corev1.ServiceSpec{}, + } +) + +func toUnstructured(t *testing.T, obj runtime.Object) *unstructured.Unstructured { + unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + t.Fatalf("failed to convert obj into a map: %v", err) + } + return &unstructured.Unstructured{Object: unstructuredObj} +} + +// Note (chenyu1): The fake client Fleet uses for unit tests has trouble processing certain +// requests at the moment; affected test cases will be covered in +// integration tests (w/ real clients) instead. + +// TestTakeOverPreExistingObject tests the takeOverPreExistingObject method. +func TestTakeOverPreExistingObject(t *testing.T) { + ctx := context.Background() + + nsWithNonFleetOwnerUnstructured := nsUnstructured.DeepCopy() + nsWithNonFleetOwnerUnstructured.SetOwnerReferences([]metav1.OwnerReference{ + dummyOwnerRef, + }) + + nsWithFleetOwnerUnstructured := nsUnstructured.DeepCopy() + nsWithFleetOwnerUnstructured.SetOwnerReferences([]metav1.OwnerReference{ + { + APIVersion: "placement.kubernetes-fleet.io/v1beta1", + Kind: "AppliedWork", + Name: "dummy-work", + UID: "0987-6543-21", + }, + }) + + wantTakenOverObj := nsUnstructured.DeepCopy() + wantTakenOverObj.SetOwnerReferences([]metav1.OwnerReference{ + *appliedWorkOwnerRef, + }) + + wantTakenOverObjWithAdditionalNonFleetOwner := nsUnstructured.DeepCopy() + wantTakenOverObjWithAdditionalNonFleetOwner.SetOwnerReferences([]metav1.OwnerReference{ + dummyOwnerRef, + *appliedWorkOwnerRef, + }) + + nsWithLabelsUnstructured := nsUnstructured.DeepCopy() + nsWithLabelsUnstructured.SetLabels(map[string]string{ + dummyLabelKey: dummyLabelValue1, + }) + + testCases := []struct { + name string + gvr *schema.GroupVersionResource + manifestObj *unstructured.Unstructured + inMemberClusterObj *unstructured.Unstructured + workObj *fleetv1beta1.Work + applyStrategy *fleetv1beta1.ApplyStrategy + expectedAppliedWorkOwnerRef *metav1.OwnerReference + wantErred bool + wantTakeOverObj *unstructured.Unstructured + wantPatchDetails []fleetv1beta1.PatchDetail + }{ + { + name: "existing non-Fleet owner, co-ownership not allowed", + gvr: &nsGVR, + manifestObj: nsUnstructured, + inMemberClusterObj: nsWithNonFleetOwnerUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + AllowCoOwnership: false, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantErred: true, + }, + { + name: "existing Fleet owner, co-ownership allowed", + gvr: &nsGVR, + manifestObj: nsUnstructured, + inMemberClusterObj: nsWithFleetOwnerUnstructured, + workObj: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-work", + Namespace: memberReservedNSName, + }, + }, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + AllowCoOwnership: true, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantErred: true, + }, + { + name: "no owner, always take over", + gvr: &nsGVR, + manifestObj: nsUnstructured, + inMemberClusterObj: nsUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantTakeOverObj: wantTakenOverObj, + }, + { + name: "existing non-Fleet owner,co-ownership allowed, always take over", + gvr: &nsGVR, + manifestObj: nsUnstructured, + inMemberClusterObj: nsWithNonFleetOwnerUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + AllowCoOwnership: true, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantTakeOverObj: wantTakenOverObjWithAdditionalNonFleetOwner, + }, + // The fake client Fleet uses for unit tests has trouble processing dry-run requests (server + // side apply); such test cases will be handled in integration tests instead. + { + name: "no owner, take over if no diff, diff found, full comparison", + gvr: &nsGVR, + manifestObj: nsUnstructured, + inMemberClusterObj: nsWithLabelsUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, + ComparisonOption: fleetv1beta1.ComparisonOptionTypeFullComparison, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/metadata/labels/foo", + ValueInMember: "bar", + }, + }, + }, + { + name: "no owner, take over if no diff, no diff", + gvr: &nsGVR, + manifestObj: nsUnstructured, + inMemberClusterObj: nsUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, + ComparisonOption: fleetv1beta1.ComparisonOptionTypeFullComparison, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantTakeOverObj: wantTakenOverObj, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeHubClientBuilder := ctrlfake.NewClientBuilder().WithScheme(scheme.Scheme) + if tc.workObj != nil { + fakeHubClientBuilder = fakeHubClientBuilder.WithObjects(tc.workObj) + } + fakeHubClient := fakeHubClientBuilder.Build() + fakeMemberClient := fake.NewSimpleDynamicClient(scheme.Scheme, tc.inMemberClusterObj) + r := &Reconciler{ + hubClient: fakeHubClient, + spokeDynamicClient: fakeMemberClient, + workNameSpace: memberReservedNSName, + } + + takenOverObj, patchDetails, err := r.takeOverPreExistingObject( + ctx, + tc.gvr, + tc.manifestObj, tc.inMemberClusterObj, + tc.applyStrategy, + tc.expectedAppliedWorkOwnerRef) + if tc.wantErred { + if err == nil { + t.Errorf("takeOverPreExistingObject() = nil, want erred") + } + return + } + + if err != nil { + t.Errorf("takeOverPreExistingObject() = %v, want no error", err) + } + if diff := cmp.Diff(takenOverObj, tc.wantTakeOverObj); diff != "" { + t.Errorf("takenOverObject mismatches (-got, +want):\n%s", diff) + } + if diff := cmp.Diff(patchDetails, tc.wantPatchDetails); diff != "" { + t.Errorf("patchDetails mismatches (-got, +want):\n%s", diff) + } + }) + } +} + +// TestPreparePatchDetails tests the preparePatchDetails function. +func TestPreparePatchDetails(t *testing.T) { + deploy1Manifest := deploy.DeepCopy() + deploy1Manifest.Spec.Strategy.Type = appsv1.RecreateDeploymentStrategyType + + deploy2Manifest := deploy.DeepCopy() + deploy2Manifest.Spec.RevisionHistoryLimit = ptr.To(int32(10)) + + deploy3Manifest := deploy.DeepCopy() + deploy3Manifest.Spec.Paused = true + + deploy4Manifest := deploy.DeepCopy() + deploy4Manifest.Spec.Selector.MatchLabels = map[string]string{ + "app": "envoy", + "team": "red", + } + + svc1Manifest := svc.DeepCopy() + svc1Manifest.Spec.Ports = []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + } + + svc2Manifest := svc.DeepCopy() + svc2Manifest.Spec.Ports = []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + { + Name: "custom", + Protocol: corev1.ProtocolUDP, + Port: 10000, + TargetPort: intstr.FromInt(10000), + }, + { + Name: "https", + Protocol: corev1.ProtocolTCP, + Port: 443, + TargetPort: intstr.FromString("https"), + }, + } + svc2InMember := svc.DeepCopy() + svc2InMember.Spec.Ports = []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + { + Name: "https", + Protocol: corev1.ProtocolTCP, + Port: 443, + TargetPort: intstr.FromString("https"), + }, + } + + svc3Manifest := svc.DeepCopy() + svc3Manifest.Spec.Selector = map[string]string{ + "app": "nginx", + } + + svc4Manifest := svc.DeepCopy() + svc4Manifest.Spec.Ports = []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + { + Name: "https", + Protocol: corev1.ProtocolTCP, + Port: 443, + TargetPort: intstr.FromString("https"), + }, + } + svc4InMember := svc4Manifest.DeepCopy() + svc4InMember.Spec.Ports[0].Port = 8080 + svc4InMember.Spec.Ports[1].TargetPort = intstr.FromInt(8443) + + testCases := []struct { + name string + manifestObj *unstructured.Unstructured + inMemberClusterObj *unstructured.Unstructured + wantPatchDetails []fleetv1beta1.PatchDetail + }{ + { + name: "field addition, object field (string)", + manifestObj: toUnstructured(t, deploy1Manifest), + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/strategy/type", + ValueInHub: "Recreate", + }, + }, + }, + { + name: "field addition, object field (numeral)", + manifestObj: toUnstructured(t, deploy2Manifest), + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/revisionHistoryLimit", + ValueInHub: "10", + }, + }, + }, + { + name: "field addition, object field (bool)", + manifestObj: toUnstructured(t, deploy3Manifest), + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/paused", + ValueInHub: "true", + }, + }, + }, + { + name: "field addition, array", + manifestObj: toUnstructured(t, svc1Manifest), + inMemberClusterObj: toUnstructured(t, svc.DeepCopy()), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/ports", + ValueInHub: "[map[name:http port:80 protocol:TCP targetPort:8080]]", + }, + }, + }, + { + name: "field addition, array item", + manifestObj: toUnstructured(t, svc2Manifest), + inMemberClusterObj: toUnstructured(t, svc2InMember), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/ports/2", + ValueInHub: "map[name:https port:443 protocol:TCP targetPort:https]", + }, + { + Path: "/spec/ports/1/name", + ValueInMember: "https", + ValueInHub: "custom", + }, + { + Path: "/spec/ports/1/port", + ValueInMember: "443", + ValueInHub: "10000", + }, + { + Path: "/spec/ports/1/protocol", + ValueInMember: "TCP", + ValueInHub: "UDP", + }, + { + Path: "/spec/ports/1/targetPort", + ValueInMember: "https", + ValueInHub: "10000", + }, + }, + }, + { + name: "field addition, object", + manifestObj: toUnstructured(t, svc3Manifest), + inMemberClusterObj: toUnstructured(t, svc.DeepCopy()), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/selector", + ValueInHub: "map[app:nginx]", + }, + }, + }, + { + name: "field addition, nested (array)", + manifestObj: toUnstructured(t, svc4Manifest), + inMemberClusterObj: toUnstructured(t, svc4InMember), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/ports/0/port", + ValueInMember: "8080", + ValueInHub: "80", + }, + { + Path: "/spec/ports/1/targetPort", + ValueInMember: "8443", + ValueInHub: "https", + }, + }, + }, + { + name: "field addition, nested (object)", + manifestObj: toUnstructured(t, deploy4Manifest), + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/selector/matchLabels/app", + ValueInMember: "nginx", + ValueInHub: "envoy", + }, + { + Path: "/spec/selector/matchLabels/team", + ValueInHub: "red", + }, + }, + }, + // TO-DO (chenyu1): add more test cases. + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + patchDetails, err := preparePatchDetails(tc.manifestObj, tc.inMemberClusterObj) + if err != nil { + t.Fatalf("preparePatchDetails() = %v, want no error", err) + } + + if diff := cmp.Diff(patchDetails, tc.wantPatchDetails, cmpopts.SortSlices(lessFuncPatchDetail)); diff != "" { + t.Fatalf("patchDetails mismatches (-got, +want):\n%s", diff) + } + }) + } +} + +// TestDiscardFieldsIrrelevantInComparisonFrom tests the discardFieldsIrrelevantInComparisonFrom function. +func TestDiscardFieldsIrrelevantInComparisonFrom(t *testing.T) { + // This test spec uses a Deployment object as the target. + generateName := "app-" + dummyManager := "dummy-manager" + now := metav1.Now() + dummyGeneration := int64(1) + dummyResourceVersion := "abc" + dummySelfLink := "self-link" + dummyUID := "123-xyz-abcd" + + deploy := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: deployName, + Namespace: nsName, + GenerateName: generateName, + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", k8sReservedLabelAnnotationFullDomain, dummyLabelKey): dummyLabelValue1, + fmt.Sprintf("%s/%s", k8sReservedLabelAnnotationAbbrDomain, dummyLabelKey): dummyLabelValue1, + fmt.Sprintf("%s/%s", fleetReservedLabelAnnotationDomain, dummyLabelKey): dummyLabelValue1, + dummyLabelKey: dummyLabelValue1, + }, + Labels: map[string]string{ + fmt.Sprintf("%s/%s", k8sReservedLabelAnnotationFullDomain, dummyLabelKey): dummyLabelValue1, + fmt.Sprintf("%s/%s", k8sReservedLabelAnnotationAbbrDomain, dummyLabelKey): dummyLabelValue1, + fmt.Sprintf("%s/%s", fleetReservedLabelAnnotationDomain, dummyLabelKey): dummyLabelValue1, + dummyLabelKey: dummyLabelValue2, + }, + Finalizers: []string{ + dummyLabelKey, + }, + ManagedFields: []metav1.ManagedFieldsEntry{ + { + Manager: dummyManager, + Operation: metav1.ManagedFieldsOperationUpdate, + APIVersion: "v1", + Time: &now, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{}, + }, + }, + OwnerReferences: []metav1.OwnerReference{ + dummyOwnerRef, + }, + CreationTimestamp: now, + DeletionTimestamp: &now, + DeletionGracePeriodSeconds: ptr.To(int64(30)), + Generation: dummyGeneration, + ResourceVersion: dummyResourceVersion, + SelfLink: dummySelfLink, + UID: types.UID(dummyUID), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + }, + Status: appsv1.DeploymentStatus{ + Replicas: 1, + }, + } + wantDeploy := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + dummyLabelKey: dummyLabelValue1, + }, + Labels: map[string]string{ + dummyLabelKey: dummyLabelValue2, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + }, + Status: appsv1.DeploymentStatus{}, + } + + testCases := []struct { + name string + unstructuredObj *unstructured.Unstructured + wantUnstructuredObj *unstructured.Unstructured + }{ + { + name: "deploy", + unstructuredObj: toUnstructured(t, deploy), + wantUnstructuredObj: toUnstructured(t, wantDeploy), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := discardFieldsIrrelevantInComparisonFrom(tc.unstructuredObj) + + // There are certain fields that need to be set manually as they might got omitted + // when being cast to an Unstructured object. + tc.wantUnstructuredObj.SetFinalizers([]string{}) + tc.wantUnstructuredObj.SetCreationTimestamp(metav1.Time{}) + tc.wantUnstructuredObj.SetManagedFields([]metav1.ManagedFieldsEntry{}) + tc.wantUnstructuredObj.SetOwnerReferences([]metav1.OwnerReference{}) + unstructured.RemoveNestedField(tc.wantUnstructuredObj.Object, "status") + + if diff := cmp.Diff(got, tc.wantUnstructuredObj, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("discardFieldsIrrelevantInComparisonFrom() mismatches (-got, +want):\n%s", diff) + } + }) + } +} + +// TestRemoveLeftBehindAppliedWorkOwnerRefs tests the removeLeftBehindAppliedWorkOwnerRefs method. +func TestRemoveLeftBehindAppliedWorkOwnerRefs(t *testing.T) { + ctx := context.Background() + + orphanedAppliedWorkOwnerRef := metav1.OwnerReference{ + APIVersion: fleetv1beta1.GroupVersion.String(), + Kind: fleetv1beta1.AppliedWorkKind, + Name: "orphaned-work", + UID: "123-xyz-abcd", + } + + testCases := []struct { + name string + ownerRefs []metav1.OwnerReference + wantOwnerRefs []metav1.OwnerReference + workObj *fleetv1beta1.Work + }{ + { + name: "mixed", + ownerRefs: []metav1.OwnerReference{ + dummyOwnerRef, + *appliedWorkOwnerRef, + orphanedAppliedWorkOwnerRef, + }, + wantOwnerRefs: []metav1.OwnerReference{ + dummyOwnerRef, + *appliedWorkOwnerRef, + }, + workObj: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeHubClientBuilder := ctrlfake.NewClientBuilder().WithScheme(scheme.Scheme) + if tc.workObj != nil { + fakeHubClientBuilder = fakeHubClientBuilder.WithObjects(tc.workObj) + } + fakeHubClient := fakeHubClientBuilder.Build() + + r := &Reconciler{ + hubClient: fakeHubClient, + workNameSpace: memberReservedNSName, + } + + gotOwnerRefs, err := r.removeLeftBehindAppliedWorkOwnerRefs(ctx, tc.ownerRefs) + if err != nil { + t.Fatalf("removeLeftBehindAppliedWorkOwnerRefs() = %v, want no error", err) + } + + if diff := cmp.Diff(gotOwnerRefs, tc.wantOwnerRefs); diff != "" { + t.Errorf("owner refs mismatches (-got, +want):\n%s", diff) + } + }) + } +} + +// TestOrganizeJSONPatchIntoFleetPatchDetails tests the organizeJSONPatchIntoFleetPatchDetails function. +func TestOrganizeJSONPatchIntoFleetPatchDetails(t *testing.T) { + testCases := []struct { + name string + patch jsondiff.Patch + manifestObjMap map[string]interface{} + wantPatchDetails []fleetv1beta1.PatchDetail + }{ + { + name: "add", + patch: jsondiff.Patch{ + { + Type: jsondiff.OperationAdd, + Path: "/a", + Value: "1", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/a", + ValueInMember: "1", + }, + }, + }, + { + name: "remove", + patch: jsondiff.Patch{ + { + Type: jsondiff.OperationRemove, + Path: "/b", + }, + }, + manifestObjMap: map[string]interface{}{ + "b": "2", + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/b", + ValueInHub: "2", + }, + }, + }, + { + name: "Replace", + patch: jsondiff.Patch{ + { + Type: jsondiff.OperationReplace, + Path: "/c", + Value: "3", + }, + }, + manifestObjMap: map[string]interface{}{ + "c": "4", + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/c", + ValueInMember: "3", + ValueInHub: "4", + }, + }, + }, + { + name: "Move", + patch: jsondiff.Patch{ + { + Type: jsondiff.OperationMove, + From: "/d", + Path: "/e", + }, + }, + manifestObjMap: map[string]interface{}{ + "d": "6", + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/d", + ValueInHub: "6", + }, + { + Path: "/e", + ValueInMember: "6", + }, + }, + }, + { + name: "Copy", + patch: jsondiff.Patch{ + { + Type: jsondiff.OperationCopy, + From: "/f", + Path: "/g", + }, + }, + manifestObjMap: map[string]interface{}{ + "f": "7", + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/g", + ValueInMember: "7", + }, + }, + }, + { + name: "Test", + patch: jsondiff.Patch{ + { + Type: jsondiff.OperationTest, + Path: "/h", + Value: "8", + }, + }, + manifestObjMap: map[string]interface{}{}, + wantPatchDetails: []fleetv1beta1.PatchDetail{}, + }, + { + name: "Mixed", + patch: jsondiff.Patch{ + { + Type: jsondiff.OperationAdd, + Path: "/a", + Value: "1", + }, + { + Type: jsondiff.OperationRemove, + Path: "/b", + }, + { + Type: jsondiff.OperationReplace, + Path: "/c", + Value: "3", + }, + { + Type: jsondiff.OperationMove, + From: "/d", + Path: "/e", + }, + { + Type: jsondiff.OperationCopy, + From: "/f", + Path: "/g", + }, + { + Type: jsondiff.OperationTest, + Path: "/h", + Value: "8", + }, + }, + manifestObjMap: map[string]interface{}{ + "b": "2", + "c": "4", + "d": "6", + "f": "7", + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/a", + ValueInMember: "1", + }, + { + Path: "/b", + ValueInHub: "2", + }, + { + Path: "/c", + ValueInMember: "3", + ValueInHub: "4", + }, + { + Path: "/d", + ValueInHub: "6", + }, + { + Path: "/e", + ValueInMember: "6", + }, + { + Path: "/g", + ValueInMember: "7", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotPatchDetails, err := organizeJSONPatchIntoFleetPatchDetails(tc.patch, tc.manifestObjMap) + if err != nil { + t.Fatalf("organizeJSONPatchIntoFleetPatchDetails() = %v, want no error", err) + } + + if diff := cmp.Diff(gotPatchDetails, tc.wantPatchDetails); diff != "" { + t.Errorf("patchDetails mismatches (-got, +want):\n%s", diff) + } + }) + } +} diff --git a/pkg/controllers/workapplier/metrics.go b/pkg/controllers/workapplier/metrics.go new file mode 100644 index 000000000..92abed61b --- /dev/null +++ b/pkg/controllers/workapplier/metrics.go @@ -0,0 +1,34 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "time" + + "k8s.io/klog/v2" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/metrics" + "go.goms.io/fleet/pkg/utils" +) + +func trackWorkApplyLatencyMetric(work *fleetv1beta1.Work) { + // Calculate the work apply latency. + lastUpdateTime, ok := work.GetAnnotations()[utils.LastWorkUpdateTimeAnnotationKey] + if ok { + workUpdateTime, parseErr := time.Parse(time.RFC3339, lastUpdateTime) + if parseErr != nil { + klog.ErrorS(parseErr, "Failed to parse the last update timestamp on the work", "work", klog.KObj(work)) + return + } + + latency := time.Since(workUpdateTime) + metrics.WorkApplyTime.WithLabelValues(work.GetName()).Observe(latency.Seconds()) + klog.V(2).InfoS("Work has been applied", "work", klog.KObj(work), "latency", latency.Milliseconds()) + } + + klog.V(2).InfoS("No last update timestamp found on the Work object", "work", klog.KObj(work)) +} diff --git a/pkg/controllers/workapplier/preprocess.go b/pkg/controllers/workapplier/preprocess.go new file mode 100644 index 000000000..a62bfdf78 --- /dev/null +++ b/pkg/controllers/workapplier/preprocess.go @@ -0,0 +1,547 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + "reflect" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/controller" + "go.goms.io/fleet/pkg/utils/defaulter" +) + +const ( + // A list of condition related values. + ManifestAppliedCondPreparingToProcessReason = "PreparingToProcess" + ManifestAppliedCondPreparingToProcessMessage = "The manifest is being prepared for processing." +) + +// preProcessManifests pre-processes manifests for the later ops. +func (r *Reconciler) preProcessManifests( + ctx context.Context, + bundles []*manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) error { + // Decode the manifests. + // Run the decoding in parallel to boost performance. + // + // This is concurrency safe as the bundles slice has been pre-allocated. + + // Prepare a child context. + // Cancel the child context anyway to avoid leaks. + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + + doWork := func(pieces int) { + // At this moment the bundles are just created. + bundle := bundles[pieces] + + gvr, manifestObj, err := r.decodeManifest(bundle.manifest) + // Build the identifier. Note that this would return an identifier even if the decoding + // fails. + bundle.id = buildWorkResourceIdentifier(pieces, gvr, manifestObj) + if err != nil { + klog.ErrorS(err, "Failed to decode the manifest", "ordinal", pieces, "work", klog.KObj(work)) + bundle.applyErr = fmt.Errorf("failed to decode manifest: %w", err) + bundle.applyResTyp = ManifestProcessingApplyResultTypeDecodingErred + return + } + + // Reject objects with generate names. + if len(manifestObj.GetGenerateName()) > 0 { + klog.V(2).InfoS("Reject objects with generate names", "manifestObj", klog.KObj(manifestObj), "work", klog.KObj(work)) + bundle.applyErr = fmt.Errorf("objects with generate names are not supported") + bundle.applyResTyp = ManifestProcessingApplyResultTypeFoundGenerateNames + return + } + + bundle.manifestObj = manifestObj + bundle.gvr = gvr + klog.V(2).InfoS("Decoded a manifest", + "manifestObj", klog.KObj(manifestObj), + "GVR", *gvr, + "work", klog.KObj(work)) + } + r.parallelizer.ParallelizeUntil(childCtx, len(bundles), doWork, "decodingManifests") + + // Write ahead the manifest processing attempts in the Work object status. In the process + // Fleet will also perform a cleanup to remove any left-over manifests that are applied + // from previous runs. + // + // This is set up to address a corner case where the agent could crash right after manifests + // are applied but before the status is properly updated, and upon the agent's restart, the + // list of manifests has changed (some manifests have been removed). This would lead to a + // situation where Fleet would lose track of the removed manifests. + // + // To avoid conflicts (or the hassle of preparing individual patches), the status update is + // done in batch. + return r.writeAheadManifestProcessingAttempts(ctx, bundles, work, expectedAppliedWorkOwnerRef) +} + +// writeAheadManifestProcessingAttempts helps write ahead manifest processing attempts so that +// Fleet can always track applied manifests, even upon untimely crashes. This method will +// also check for any leftover apply attempts from previous runs and clean them up (if the +// correspond object has been applied). +func (r *Reconciler) writeAheadManifestProcessingAttempts( + ctx context.Context, + bundles []*manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) error { + workRef := klog.KObj(work) + + // As a shortcut, if there's no spec change in the Work object and the status indicates that + // a previous apply attempt has been recorded (**successful or not**), Fleet will skip the write-ahead + // op. + workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied) + if workAppliedCond != nil && workAppliedCond.ObservedGeneration == work.Generation { + klog.V(2).InfoS("Attempt to apply the current set of manifests has been made before and the results have been recorded; will skip the write-ahead process", "work", workRef) + return nil + } + + // As another shortcut, if the Work object has an apply strategy that has the ReportDiff + // mode on, Fleet will skip the write-ahead op. + // + // Note that in this scenario Fleet will not attempt to remove any left over manifests; + // such manifests will only get cleaned up when the ReportDiff mode is turned off, or the + // CRP itself is deleted. + if work.Spec.ApplyStrategy != nil && work.Spec.ApplyStrategy.Type == fleetv1beta1.ApplyStrategyTypeReportDiff { + klog.V(2).InfoS("The apply strategy is set to report diff; will skip the write-ahead process", "work", workRef) + return nil + } + + // Prepare the status update (the new manifest conditions) for the write-ahead process. + // + // Note that even though we pre-allocate the slice, the length is set to 0. This is to + // accommodate the case where there might manifests that have failed pre-processing; + // such manifests will not be included in this round's status update. + manifestCondsForWA := make([]fleetv1beta1.ManifestCondition, 0, len(bundles)) + + // Prepare an query index of existing manifest conditions on the Work object for quicker + // lookups. + existingManifestCondQIdx := prepareExistingManifestCondQIdx(work.Status.ManifestConditions) + + // For each manifest, verify if it has been tracked in the newly prepared manifest conditions. + // This helps signal duplicated resources in the Work object. + checked := make(map[string]bool, len(bundles)) + for idx := range bundles { + bundle := bundles[idx] + if bundle.applyErr != nil { + // Skip a manifest if it cannot be pre-processed, i.e., it can only be identified by + // its ordinal. + // + // Such manifests would still be reported in the status (see the later parts of the + // reconciliation loop), it is just that they are not relevant in the write-ahead + // process. + continue + } + + // Register the manifest in the checked map; if another manifest with the same identifier + // has been checked before, Fleet would mark the current manifest as a duplicate and skip + // it. This is to address a corner case where users might have specified the same manifest + // twice in resource envelopes; duplication will not occur if the manifests are directly + // created in the hub cluster. + // + // A side note: Golang does support using structs as map keys; preparing the string + // representations of structs as keys can help performance, though not by much. The reason + // why string representations are used here is not for performance, though; instead, it + // is to address the issue that for this comparison, ordinals should be ignored. + wriStr, err := formatWRIString(bundle.id) + if err != nil { + // Normally this branch will never run as all manifests that cannot be decoded has been + // skipped in the check above. Here Fleet simply skips the manifest. + klog.ErrorS(err, "Failed to format the work resource identifier string", + "ordinal", idx, "work", workRef) + continue + } + if _, found := checked[wriStr]; found { + klog.V(2).InfoS("A duplicate manifest has been found", + "ordinal", idx, "work", workRef, "WRI", wriStr) + bundle.applyErr = fmt.Errorf("a duplicate manifest has been found") + bundle.applyResTyp = ManifestProcessingApplyResultTypeDuplicated + continue + } + checked[wriStr] = true + + // Prepare the manifest conditions for the write-ahead process. + manifestCondForWA := prepareManifestCondForWA(wriStr, bundle.id, work.Generation, existingManifestCondQIdx, work.Status.ManifestConditions) + manifestCondsForWA = append(manifestCondsForWA, manifestCondForWA) + + klog.V(2).InfoS("Prepared write-ahead information for a manifest", + "manifestObj", klog.KObj(bundle.manifestObj), "WRI", wriStr, "work", workRef) + } + + // Identify any manifests from previous runs that might have been applied and are now left + // over in the member cluster. + leftOverManifests := findLeftOverManifests(manifestCondsForWA, existingManifestCondQIdx, work.Status.ManifestConditions) + if err := r.removeLeftOverManifests(ctx, leftOverManifests, expectedAppliedWorkOwnerRef); err != nil { + klog.Errorf("Failed to remove left-over manifests (work=%+v, leftOverManifestCount=%d, removalFailureCount=%d)", + workRef, len(leftOverManifests), len(err.Errors())) + return fmt.Errorf("failed to remove left-over manifests: %w", err) + } + klog.V(2).InfoS("Left-over manifests are found and removed", + "leftOverManifestCount", len(leftOverManifests), "work", workRef) + + // Update the status. + // + // Note that the Work object might have been refreshed by controllers on the hub cluster + // before this step runs; in this case the current reconciliation loop must be abandoned. + if work.Status.Conditions == nil { + // As a sanity check, set an empty set of conditions. Currently the API definition does + // not allow nil conditions. + work.Status.Conditions = []metav1.Condition{} + } + work.Status.ManifestConditions = manifestCondsForWA + if err := r.hubClient.Status().Update(ctx, work); err != nil { + return controller.NewAPIServerError(false, fmt.Errorf("failed to write ahead manifest processing attempts: %w", err)) + } + klog.V(2).InfoS("Write-ahead process completed", "work", workRef) + + // Set the defaults again as the result yielded by the status update might have changed the object. + defaulter.SetDefaultsWork(work) + return nil +} + +// Decodes the manifest JSON into a Kubernetes unstructured object. +func (r *Reconciler) decodeManifest(manifest *fleetv1beta1.Manifest) (*schema.GroupVersionResource, *unstructured.Unstructured, error) { + unstructuredObj := &unstructured.Unstructured{} + if err := unstructuredObj.UnmarshalJSON(manifest.Raw); err != nil { + return &schema.GroupVersionResource{}, nil, fmt.Errorf("failed to unmarshal JSON: %w", err) + } + + mapping, err := r.restMapper.RESTMapping(unstructuredObj.GroupVersionKind().GroupKind(), unstructuredObj.GroupVersionKind().Version) + if err != nil { + return &schema.GroupVersionResource{}, unstructuredObj, fmt.Errorf("failed to find GVR from member cluster client REST mapping: %w", err) + } + + return &mapping.Resource, unstructuredObj, nil +} + +// buildWorkResourceIdentifier builds a work resource identifier for a manifest. +// +// Note that if the manifest cannot be decoded/applied, this function will return an identifier with +// the available information on hand. +func buildWorkResourceIdentifier( + manifestIdx int, + gvr *schema.GroupVersionResource, + manifestObj *unstructured.Unstructured, +) *fleetv1beta1.WorkResourceIdentifier { + // The ordinal field is always set. + identifier := &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: manifestIdx, + } + + // Set the GVK, name, namespace, and generate name information if the manifest can be decoded + // as a Kubernetes unstructured object. + // + // Note that: + // * For cluster-scoped objects, the namespace field will be empty. + // * For objects with generated names, the name field will be empty. + // * For regular objects (i.e., objects with a pre-defined name), the generate name field will be empty. + if manifestObj != nil { + identifier.Group = manifestObj.GroupVersionKind().Group + identifier.Version = manifestObj.GroupVersionKind().Version + identifier.Kind = manifestObj.GetKind() + identifier.Name = manifestObj.GetName() + identifier.Namespace = manifestObj.GetNamespace() + } + + // Set the GVR information if the manifest object can be REST mapped. + if gvr != nil { + identifier.Resource = gvr.Resource + } + + return identifier +} + +// prepareExistingManifestCondQIdx returns a map that allows quicker look up of a manifest +// condition given a work resource identifier. +func prepareExistingManifestCondQIdx(existingManifestConditions []fleetv1beta1.ManifestCondition) map[string]int { + existingManifestConditionQIdx := make(map[string]int) + for idx := range existingManifestConditions { + manifestCond := existingManifestConditions[idx] + + wriStr, err := formatWRIString(&manifestCond.Identifier) + if err != nil { + // There might be manifest conditions without a valid identifier in the existing set of + // manifest conditions (e.g., decoding error has occurred in the previous run). + // Fleet will skip these manifest conditions. This is not considered as an error. + continue + } + + existingManifestConditionQIdx[wriStr] = idx + } + return existingManifestConditionQIdx +} + +// prepareManifestCondForWA prepares a manifest condition for the write-ahead process. +func prepareManifestCondForWA( + wriStr string, wri *fleetv1beta1.WorkResourceIdentifier, + workGeneration int64, + existingManifestCondQIdx map[string]int, + existingManifestConds []fleetv1beta1.ManifestCondition, +) fleetv1beta1.ManifestCondition { + // For each manifest to process, check if there is a corresponding entry in the existing set + // of manifest conditions. If so, Fleet will port information back to keep track of the + // previous processing results; otherwise, Fleet will report that it is preparing to process + // the manifest. + existingManifestConditionIdx, found := existingManifestCondQIdx[wriStr] + if found { + // The current manifest condition has a corresponding entry in the existing set of manifest + // conditions. + // + // Fleet simply ports the information back. + return existingManifestConds[existingManifestConditionIdx] + } + + // No corresponding entry is found in the existing set of manifest conditions. + // + // Prepare a manifest condition that indicates that Fleet is preparing to be process the manifest. + return fleetv1beta1.ManifestCondition{ + Identifier: *wri, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + ObservedGeneration: workGeneration, + Reason: ManifestAppliedCondPreparingToProcessReason, + Message: ManifestAppliedCondPreparingToProcessMessage, + LastTransitionTime: metav1.Now(), + }, + }, + } +} + +// findLeftOverManifests returns the manifests that have been left over on the member cluster side. +func findLeftOverManifests( + manifestCondsForWA []fleetv1beta1.ManifestCondition, + existingManifestCondQIdx map[string]int, + existingManifestConditions []fleetv1beta1.ManifestCondition, +) []fleetv1beta1.AppliedResourceMeta { + // Build an index for quicker lookup in the newly prepared write-ahead manifest conditions. + // Here Fleet uses the string representations as map keys to omit ordinals from any lookup. + // + // Note that before this step, Fleet has already filtered out duplicate manifests. + manifestCondsForWAQIdx := make(map[string]int) + for idx := range manifestCondsForWA { + manifestCond := manifestCondsForWA[idx] + + wriStr, err := formatWRIString(&manifestCond.Identifier) + if err != nil { + // Normally this branch will never run as all manifests that cannot be decoded has been + // skipped before this function is called. Here Fleet simply skips the manifest as it + // has no effect on the process. + klog.ErrorS(err, "failed to format the work resource identifier string", "manifest", manifestCond.Identifier) + continue + } + + manifestCondsForWAQIdx[wriStr] = idx + } + + // For each manifest condition in the existing set of manifest conditions, check if + // there is a corresponding entry in the set of manifest conditions prepared for the write-ahead + // process. If not, Fleet will consider that the manifest has been left over on the member + // cluster side and should be removed. + + // Use an AppliedResourceMeta slice to allow code sharing. + leftOverManifests := []fleetv1beta1.AppliedResourceMeta{} + for existingManifestWRIStr, existingManifestCondIdx := range existingManifestCondQIdx { + _, found := manifestCondsForWAQIdx[existingManifestWRIStr] + if !found { + existingManifestCond := existingManifestConditions[existingManifestCondIdx] + // The current manifest condition does not have a corresponding entry in the set of manifest + // conditions prepared for the write-ahead process. + + // Verify if the manifest condition indicates that the manifest could have been + // applied. + applied := meta.FindStatusCondition(existingManifestCond.Conditions, fleetv1beta1.WorkConditionTypeApplied) + if applied.Status == metav1.ConditionTrue || applied.Reason == ManifestAppliedCondPreparingToProcessReason { + // Fleet assumes that the manifest has been applied if: + // a) it has an applied condition set to the True status; or + // b) it has an applied condition which signals that the object is preparing to be processed. + // + // Note that the manifest condition might not be up-to-date, so Fleet will not + // check on the generation information. + leftOverManifests = append(leftOverManifests, fleetv1beta1.AppliedResourceMeta{ + WorkResourceIdentifier: existingManifestCond.Identifier, + // UID information might not be available at this moment; the cleanup process + // will perform additional checks anyway to guard against the + // create-delete-recreate cases and/or same name but different setup cases. + // + // As a side note, it is true that the AppliedWork object status might have + // the UID information; Fleet cannot rely on that though, as the AppliedWork + // status is not guaranteed to be tracking the result of the last apply op. + // Should the Fleet agent restarts multiple times before it gets a chance to + // write the AppliedWork object statys, the UID information in the status + // might be several generations behind. + }) + } + } + } + return leftOverManifests +} + +// removeLeftOverManifests removes applied left-over manifests from the member cluster. +func (r *Reconciler) removeLeftOverManifests( + ctx context.Context, + leftOverManifests []fleetv1beta1.AppliedResourceMeta, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) utilerrors.Aggregate { + // Remove all the manifests in parallel. + // + // This is concurrency safe as each worker processes its own applied manifest and writes + // to its own error slot. + + // Prepare a child context. + // Cancel the child context anyway to avoid leaks. + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Pre-allocate the slice. + errs := make([]error, len(leftOverManifests)) + doWork := func(pieces int) { + appliedManifestMeta := leftOverManifests[pieces] + + // Remove the left-over manifest. + err := r.removeOneLeftOverManifest(ctx, appliedManifestMeta, expectedAppliedWorkOwnerRef) + if err != nil { + errs[pieces] = fmt.Errorf("failed to remove the left-over manifest (regular object): %w", err) + } + } + r.parallelizer.ParallelizeUntil(childCtx, len(leftOverManifests), doWork, "removeLeftOverManifests") + + return utilerrors.NewAggregate(errs) +} + +// removeOneLeftOverManifestWithGenerateName removes an applied manifest object that is left over +// in the member cluster. +func (r *Reconciler) removeOneLeftOverManifest( + ctx context.Context, + leftOverManifest fleetv1beta1.AppliedResourceMeta, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) error { + // Build the GVR. + gvr := schema.GroupVersionResource{ + Group: leftOverManifest.Group, + Version: leftOverManifest.Version, + Resource: leftOverManifest.Resource, + } + manifestNamespace := leftOverManifest.Namespace + manifestName := leftOverManifest.Name + + inMemberClusterObj, err := r.spokeDynamicClient. + Resource(gvr). + Namespace(manifestNamespace). + Get(ctx, manifestName, metav1.GetOptions{}) + switch { + case err != nil && apierrors.IsNotFound(err): + // The object has been deleted from the member cluster; no further action is needed. + return nil + case err != nil: + // Failed to retrieve the object from the member cluster. + return fmt.Errorf("failed to retrieve the object from the member cluster (gvr=%+v, manifestObj=%+v): %w", gvr, klog.KRef(manifestNamespace, manifestName), err) + case inMemberClusterObj.GetDeletionTimestamp() != nil: + // The object has been marked for deletion; no further action is needed. + return nil + } + + // There are occasions, though rare, where the object has the same GVR + namespace + name + // combo but is not the applied object Fleet tries to find. This could happen if the object + // has been deleted and then re-created manually without Fleet's acknowledgement. In such cases + // Fleet would ignore the object, and this is not registered as an error. + if !isInMemberClusterObjectDerivedFromManifestObj(inMemberClusterObj, expectedAppliedWorkOwnerRef) { + // The object is not derived from the manifest object. + klog.V(2).InfoS("The object to remove is not derived from the manifest object; will not proceed with the removal", + "gvr", gvr, "manifestObj", + klog.KRef(manifestNamespace, manifestName), "inMemberClusterObj", klog.KObj(inMemberClusterObj), + "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return nil + } + + switch { + case len(inMemberClusterObj.GetOwnerReferences()) > 1: + // Fleet is not the sole owner of the object; in this case, Fleet will only drop the + // ownership. + klog.V(2).InfoS("The object to remove is co-owned by other sources; Fleet will drop the ownership", + "gvr", gvr, "manifestObj", + klog.KRef(manifestNamespace, manifestName), "inMemberClusterObj", klog.KObj(inMemberClusterObj), + "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + removeOwnerRef(inMemberClusterObj, expectedAppliedWorkOwnerRef) + if _, err := r.spokeDynamicClient.Resource(gvr).Namespace(manifestNamespace).Update(ctx, inMemberClusterObj, metav1.UpdateOptions{}); err != nil && !apierrors.IsNotFound(err) { + // Failed to drop the ownership. + return fmt.Errorf("failed to drop the ownership of the object (gvr=%+v, manifestObj=%+v, inMemberClusterObj=%+v, expectedAppliedWorkOwnerRef=%+v): %w", + gvr, klog.KRef(manifestNamespace, manifestName), klog.KObj(inMemberClusterObj), *expectedAppliedWorkOwnerRef, err) + } + default: + // Fleet is the sole owner of the object; in this case, Fleet will delete the object. + klog.V(2).InfoS("The object to remove is solely owned by Fleet; Fleet will delete the object", + "gvr", gvr, "manifestObj", + klog.KRef(manifestNamespace, manifestName), "inMemberClusterObj", klog.KObj(inMemberClusterObj), + "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + inMemberClusterObjUID := inMemberClusterObj.GetUID() + deleteOpts := metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{ + // Add a UID pre-condition to guard against the case where the object has changed + // right before the deletion request is sent. + // + // Technically speaking resource version based concurrency control should also be + // enabled here; Fleet drops the check to avoid conflicts; this is safe as the Fleet + // ownership is considered to be a reserved field and other changes on the object are + // irrelevant to this step. + UID: &inMemberClusterObjUID, + }, + } + if err := r.spokeDynamicClient.Resource(gvr).Namespace(manifestNamespace).Delete(ctx, manifestName, deleteOpts); err != nil && !apierrors.IsNotFound(err) { + // Failed to delete the object from the member cluster. + return fmt.Errorf("failed to delete the object (gvr=%+v, manifestObj=%+v, inMemberClusterObj=%+v, expectedAppliedWorkOwnerRef=%+v): %w", + gvr, klog.KRef(manifestNamespace, manifestName), klog.KObj(inMemberClusterObj), *expectedAppliedWorkOwnerRef, err) + } + } + return nil +} + +// isInMemberClusterObjectDerivedFromManifestObj checks if an object in the member cluster is derived +// from a specific manifest object. +func isInMemberClusterObjectDerivedFromManifestObj(inMemberClusterObj *unstructured.Unstructured, expectedAppliedWorkOwnerRef *metav1.OwnerReference) bool { + // Do a sanity check. + if inMemberClusterObj == nil { + return false + } + + // Verify if the owner reference still stands. + curOwners := inMemberClusterObj.GetOwnerReferences() + for idx := range curOwners { + if reflect.DeepEqual(curOwners[idx], *expectedAppliedWorkOwnerRef) { + return true + } + } + return false +} + +// removeOwnerRef removes the given owner reference from the object. +func removeOwnerRef(obj *unstructured.Unstructured, expectedAppliedWorkOwnerRef *metav1.OwnerReference) { + ownerRefs := obj.GetOwnerReferences() + updatedOwnerRefs := make([]metav1.OwnerReference, 0, len(ownerRefs)) + + // Re-build the owner references; remove the given one from the list. + for idx := range ownerRefs { + if !reflect.DeepEqual(ownerRefs[idx], *expectedAppliedWorkOwnerRef) { + updatedOwnerRefs = append(updatedOwnerRefs, ownerRefs[idx]) + } + } + obj.SetOwnerReferences(updatedOwnerRefs) +} diff --git a/pkg/controllers/workapplier/preprocess_test.go b/pkg/controllers/workapplier/preprocess_test.go new file mode 100644 index 000000000..1728d0564 --- /dev/null +++ b/pkg/controllers/workapplier/preprocess_test.go @@ -0,0 +1,571 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/parallelizer" +) + +// TestBuildWorkResourceIdentifier tests the buildWorkResourceIdentifier function. +func TestBuildWorkResourceIdentifier(t *testing.T) { + testCases := []struct { + name string + manifestIdx int + gvr *schema.GroupVersionResource + manifestObj *unstructured.Unstructured + wantWRI *fleetv1beta1.WorkResourceIdentifier + }{ + { + name: "ordinal only", + manifestIdx: 0, + wantWRI: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + }, + }, + { + name: "ordinal and manifest object", + manifestIdx: 1, + manifestObj: nsUnstructured, + wantWRI: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "", + Version: "v1", + Kind: "Namespace", + Name: nsName, + }, + }, + { + name: "ordinal, manifest object, and GVR", + manifestIdx: 2, + gvr: &schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + manifestObj: deployUnstructured, + wantWRI: deployWRI(2, nsName, deployName), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + wri := buildWorkResourceIdentifier(tc.manifestIdx, tc.gvr, tc.manifestObj) + if diff := cmp.Diff(wri, tc.wantWRI); diff != "" { + t.Errorf("buildWorkResourceIdentifier() mismatches (-got +want):\n%s", diff) + } + }) + } +} + +// TestRemoveLeftOverManifests tests the removeLeftOverManifests method. +func TestRemoveLeftOverManifests(t *testing.T) { + ctx := context.Background() + + additionalOwnerRef := &metav1.OwnerReference{ + APIVersion: "v1", + Kind: "SuperNamespace", + Name: "super-ns", + UID: "super-ns-uid", + } + + nsName0 := fmt.Sprintf(nsNameTemplate, "0") + nsName1 := fmt.Sprintf(nsNameTemplate, "1") + nsName2 := fmt.Sprintf(nsNameTemplate, "2") + nsName3 := fmt.Sprintf(nsNameTemplate, "3") + + testCases := []struct { + name string + leftOverManifests []fleetv1beta1.AppliedResourceMeta + inMemberClusterObjs []runtime.Object + wantInMemberClusterObjs []corev1.Namespace + wantRemovedInMemberClusterObjs []corev1.Namespace + }{ + { + name: "mixed", + leftOverManifests: []fleetv1beta1.AppliedResourceMeta{ + // The object is present. + { + WorkResourceIdentifier: *nsWRI(0, nsName0), + }, + // The object cannot be found. + { + WorkResourceIdentifier: *nsWRI(1, nsName1), + }, + // The object is not owned by Fleet. + { + WorkResourceIdentifier: *nsWRI(2, nsName2), + }, + // The object has multiple owners. + { + WorkResourceIdentifier: *nsWRI(3, nsName3), + }, + }, + inMemberClusterObjs: []runtime.Object{ + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName0, + }, + }, + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName2, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + }, + }, + }, + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName3, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + *appliedWorkOwnerRef, + }, + }, + }, + }, + wantInMemberClusterObjs: []corev1.Namespace{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: nsName2, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: nsName3, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + }, + }, + }, + }, + wantRemovedInMemberClusterObjs: []corev1.Namespace{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: nsName0, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewSimpleDynamicClient(scheme.Scheme, tc.inMemberClusterObjs...) + r := &Reconciler{ + spokeDynamicClient: fakeClient, + parallelizer: parallelizer.NewParallelizer(2), + } + if err := r.removeLeftOverManifests(ctx, tc.leftOverManifests, appliedWorkOwnerRef); err != nil { + t.Errorf("removeLeftOverManifests() = %v, want no error", err) + } + + for idx := range tc.wantInMemberClusterObjs { + wantNS := tc.wantInMemberClusterObjs[idx] + + gotUnstructured, err := fakeClient. + Resource(nsGVR). + Namespace(wantNS.GetNamespace()). + Get(ctx, wantNS.GetName(), metav1.GetOptions{}) + if err != nil { + t.Errorf("Get Namespace(%v) = %v, want no error", klog.KObj(&wantNS), err) + continue + } + + gotNS := wantNS.DeepCopy() + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(gotUnstructured.Object, &gotNS); err != nil { + t.Errorf("FromUnstructured() = %v, want no error", err) + } + + if diff := cmp.Diff(gotNS, &wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { + t.Errorf("NS(%v) mismatches (-got +want):\n%s", klog.KObj(&wantNS), diff) + } + } + + for idx := range tc.wantRemovedInMemberClusterObjs { + wantRemovedNS := tc.wantRemovedInMemberClusterObjs[idx] + + gotUnstructured, err := fakeClient. + Resource(nsGVR). + Namespace(wantRemovedNS.GetNamespace()). + Get(ctx, wantRemovedNS.GetName(), metav1.GetOptions{}) + if err != nil { + t.Errorf("Get Namespace(%v) = %v, want no error", klog.KObj(&wantRemovedNS), err) + } + + gotRemovedNS := wantRemovedNS.DeepCopy() + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(gotUnstructured.Object, &gotRemovedNS); err != nil { + t.Errorf("FromUnstructured() = %v, want no error", err) + } + + if !gotRemovedNS.DeletionTimestamp.IsZero() { + t.Errorf("Namespace(%v) has not been deleted", klog.KObj(&wantRemovedNS)) + } + } + }) + } +} + +// TestRemoveOneLeftOverManifest tests the removeOneLeftOverManifest method. +func TestRemoveOneLeftOverManifest(t *testing.T) { + ctx := context.Background() + now := metav1.Now().Rfc3339Copy() + leftOverManifest := fleetv1beta1.AppliedResourceMeta{ + WorkResourceIdentifier: *nsWRI(0, nsName), + } + additionalOwnerRef := &metav1.OwnerReference{ + APIVersion: "v1", + Kind: "SuperNamespace", + Name: "super-ns", + UID: "super-ns-uid", + } + + testCases := []struct { + name string + // To simplify things, for this test Fleet uses a fixed concrete type. + inMemberClusterObj *corev1.Namespace + wantInMemberClusterObj *corev1.Namespace + }{ + { + name: "not found", + }, + { + name: "already deleted", + inMemberClusterObj: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + DeletionTimestamp: &now, + }, + }, + wantInMemberClusterObj: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + DeletionTimestamp: &now, + }, + }, + }, + { + name: "not derived from manifest object", + inMemberClusterObj: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + }, + }, + }, + wantInMemberClusterObj: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + }, + }, + }, + }, + { + name: "multiple owners", + inMemberClusterObj: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + *appliedWorkOwnerRef, + }, + }, + }, + wantInMemberClusterObj: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + OwnerReferences: []metav1.OwnerReference{ + *additionalOwnerRef, + }, + }, + }, + }, + { + name: "deletion", + inMemberClusterObj: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + OwnerReferences: []metav1.OwnerReference{ + *appliedWorkOwnerRef, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var fakeClient *fake.FakeDynamicClient + if tc.inMemberClusterObj != nil { + fakeClient = fake.NewSimpleDynamicClient(scheme.Scheme, tc.inMemberClusterObj) + } else { + fakeClient = fake.NewSimpleDynamicClient(scheme.Scheme) + } + + r := &Reconciler{ + spokeDynamicClient: fakeClient, + } + if err := r.removeOneLeftOverManifest(ctx, leftOverManifest, appliedWorkOwnerRef); err != nil { + t.Errorf("removeOneLeftOverManifest() = %v, want no error", err) + } + + if tc.inMemberClusterObj != nil { + var gotUnstructured *unstructured.Unstructured + var err error + // The method is expected to modify the object. + gotUnstructured, err = fakeClient. + Resource(nsGVR). + Namespace(tc.inMemberClusterObj.GetNamespace()). + Get(ctx, tc.inMemberClusterObj.GetName(), metav1.GetOptions{}) + switch { + case errors.IsNotFound(err) && tc.wantInMemberClusterObj == nil: + // The object is expected to be deleted. + return + case errors.IsNotFound(err): + // An object is expected to be found. + t.Errorf("Get(%v) = %v, want no error", klog.KObj(tc.inMemberClusterObj), err) + return + case err != nil: + // An unexpected error occurred. + t.Errorf("Get(%v) = %v, want no error", klog.KObj(tc.inMemberClusterObj), err) + return + } + + got := tc.wantInMemberClusterObj.DeepCopy() + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(gotUnstructured.Object, &got); err != nil { + t.Errorf("FromUnstructured() = %v, want no error", err) + return + } + + if diff := cmp.Diff(got, tc.wantInMemberClusterObj, ignoreFieldTypeMetaInNamespace); diff != "" { + t.Errorf("NS(%v) mismatches (-got +want):\n%s", klog.KObj(tc.inMemberClusterObj), diff) + } + return + } + }) + } +} + +// TestPrepareExistingManifestCondQIdx tests the prepareExistingManifestCondQIdx function. +func TestPrepareExistingManifestCondQIdx(t *testing.T) { + testCases := []struct { + name string + existingManifestConds []fleetv1beta1.ManifestCondition + wantQIdx map[string]int + }{ + { + name: "mixed", + existingManifestConds: []fleetv1beta1.ManifestCondition{ + { + Identifier: *nsWRI(0, nsName), + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + }, + }, + { + Identifier: *deployWRI(2, nsName, deployName), + }, + }, + wantQIdx: map[string]int{ + fmt.Sprintf("GV=/v1, Kind=Namespace, Namespace=, Name=%s", nsName): 0, + fmt.Sprintf("GV=apps/v1, Kind=Deployment, Namespace=%s, Name=%s", nsName, deployName): 2, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + qIdx := prepareExistingManifestCondQIdx(tc.existingManifestConds) + if diff := cmp.Diff(qIdx, tc.wantQIdx); diff != "" { + t.Errorf("prepareExistingManifestCondQIdx() mismatches (-got +want):\n%s", diff) + } + }) + } +} + +// TestPrepareManifestCondForWA tests the prepareManifestCondForWA function. +func TestPrepareManifestCondForWA(t *testing.T) { + workGeneration := int64(0) + + testCases := []struct { + name string + wriStr string + wri *fleetv1beta1.WorkResourceIdentifier + workGeneration int64 + existingManifestCondQIdx map[string]int + existingManifestConds []fleetv1beta1.ManifestCondition + wantManifestCondForWA *fleetv1beta1.ManifestCondition + }{ + { + name: "match found", + wriStr: fmt.Sprintf("GV=/v1, Kind=Namespace, Namespace=, Name=%s", nsName), + wri: nsWRI(0, nsName), + workGeneration: workGeneration, + existingManifestCondQIdx: map[string]int{ + fmt.Sprintf("GV=/v1, Kind=Namespace, Namespace=, Name=%s", nsName): 0, + }, + existingManifestConds: []fleetv1beta1.ManifestCondition{ + { + Identifier: *nsWRI(0, nsName), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration, metav1.ConditionTrue, string(ManifestProcessingApplyResultTypeApplied), ManifestProcessingApplyResultTypeAppliedDescription), + }, + }, + }, + wantManifestCondForWA: &fleetv1beta1.ManifestCondition{ + Identifier: *nsWRI(0, nsName), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration, metav1.ConditionTrue, string(ManifestProcessingApplyResultTypeApplied), ManifestProcessingApplyResultTypeAppliedDescription), + }, + }, + }, + { + name: "match not found", + wriStr: fmt.Sprintf("GV=apps/v1, Kind=Deployment, Namespace=%s, Name=%s", nsName, deployName), + wri: deployWRI(1, nsName, deployName), + workGeneration: workGeneration, + wantManifestCondForWA: &fleetv1beta1.ManifestCondition{ + Identifier: *deployWRI(1, nsName, deployName), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration, metav1.ConditionFalse, ManifestAppliedCondPreparingToProcessReason, ManifestAppliedCondPreparingToProcessMessage), + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + manifestCondForWA := prepareManifestCondForWA(tc.wriStr, tc.wri, tc.workGeneration, tc.existingManifestCondQIdx, tc.existingManifestConds) + if diff := cmp.Diff(&manifestCondForWA, tc.wantManifestCondForWA, ignoreFieldConditionLTTMsg); diff != "" { + t.Errorf("prepareManifestCondForWA() mismatches (-got +want):\n%s", diff) + } + }) + } +} + +// TestFindLeftOverManifests tests the findLeftOverManifests function. +func TestFindLeftOverManifests(t *testing.T) { + workGeneration0 := int64(0) + workGeneration1 := int64(1) + + nsName0 := fmt.Sprintf(nsNameTemplate, "0") + nsName1 := fmt.Sprintf(nsNameTemplate, "1") + nsName2 := fmt.Sprintf(nsNameTemplate, "2") + nsName3 := fmt.Sprintf(nsNameTemplate, "3") + nsName4 := fmt.Sprintf(nsNameTemplate, "4") + + testCases := []struct { + name string + manifestCondsForWA []fleetv1beta1.ManifestCondition + existingManifestCondQIdx map[string]int + existingManifestConditions []fleetv1beta1.ManifestCondition + wantLeftOverManifests []fleetv1beta1.AppliedResourceMeta + }{ + { + name: "mixed", + manifestCondsForWA: []fleetv1beta1.ManifestCondition{ + // New manifest. + { + Identifier: *nsWRI(0, nsName0), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration1, metav1.ConditionFalse, ManifestAppliedCondPreparingToProcessReason, ManifestAppliedCondPreparingToProcessMessage), + }, + }, + // Existing manifest. + { + Identifier: *nsWRI(1, nsName1), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration0, metav1.ConditionTrue, string(ManifestProcessingApplyResultTypeApplied), ManifestProcessingApplyResultTypeAppliedDescription), + }, + }, + }, + existingManifestConditions: []fleetv1beta1.ManifestCondition{ + // Manifest condition that signals a decoding error. + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + }, + }, + // Manifest condition that corresponds to the existing manifest. + { + Identifier: *nsWRI(1, nsName1), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration0, metav1.ConditionTrue, string(ManifestProcessingApplyResultTypeApplied), ManifestProcessingApplyResultTypeAppliedDescription), + }, + }, + // Manifest condition that corresponds to a previously applied and now gone manifest. + { + Identifier: *nsWRI(2, nsName2), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration0, metav1.ConditionTrue, string(ManifestProcessingApplyResultTypeApplied), ManifestProcessingApplyResultTypeAppliedDescription), + }, + }, + // Manifest condition that corresponds to a gone manifest that failed to be applied. + { + Identifier: *nsWRI(3, nsName3), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration0, metav1.ConditionFalse, string(ManifestProcessingApplyResultTypeFailedToApply), ""), + }, + }, + // Manifest condition that corresponds to a gone manifest that has been marked as to be applied (preparing to be processed). + { + Identifier: *nsWRI(4, nsName4), + Conditions: []metav1.Condition{ + manifestAppliedCond(workGeneration0, metav1.ConditionFalse, ManifestAppliedCondPreparingToProcessReason, ManifestAppliedCondPreparingToProcessMessage), + }, + }, + }, + existingManifestCondQIdx: map[string]int{ + fmt.Sprintf("GV=/v1, Kind=Namespace, Namespace=, Name=%s", nsName1): 1, + fmt.Sprintf("GV=/v1, Kind=Namespace, Namespace=, Name=%s", nsName2): 2, + fmt.Sprintf("GV=/v1, Kind=Namespace, Namespace=, Name=%s", nsName3): 3, + fmt.Sprintf("GV=/v1, Kind=Namespace, Namespace=, Name=%s", nsName4): 4, + }, + wantLeftOverManifests: []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: *nsWRI(2, nsName2), + }, + { + WorkResourceIdentifier: *nsWRI(4, nsName4), + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + leftOverManifests := findLeftOverManifests(tc.manifestCondsForWA, tc.existingManifestCondQIdx, tc.existingManifestConditions) + if diff := cmp.Diff(leftOverManifests, tc.wantLeftOverManifests, cmpopts.SortSlices(lessFuncAppliedResourceMeta)); diff != "" { + t.Errorf("findLeftOverManifests() mismatches (-got +want):\n%s", diff) + } + }) + } +} diff --git a/pkg/controllers/workapplier/process.go b/pkg/controllers/workapplier/process.go new file mode 100644 index 000000000..f63dd3b94 --- /dev/null +++ b/pkg/controllers/workapplier/process.go @@ -0,0 +1,490 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + "reflect" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/controller" + "go.goms.io/fleet/pkg/utils/resource" +) + +// processManifests processes all the manifests included in a Work object. +func (r *Reconciler) processManifests( + ctx context.Context, + bundles []*manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) { + workRef := klog.KObj(work) + + // Process all the manifests in parallel. + // + // This is concurrency safe as the bundles slice has been pre-allocated. + + // Prepare a child context. + // Cancel the child context anyway to avoid leaks. + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + + doWork := func(pieces int) { + bundle := bundles[pieces] + if bundle.applyErr != nil { + // Skip a manifest if it has failed pre-processing. + return + } + + r.processOneManifest(childCtx, bundle, work, expectedAppliedWorkOwnerRef) + klog.V(2).InfoS("Processed a manifest", "manifestObj", klog.KObj(bundle.manifestObj), "work", workRef) + } + r.parallelizer.ParallelizeUntil(childCtx, len(bundles), doWork, "processingManifests") +} + +// processOneManifest processes a manifest (in the JSON format) embedded in the Work object. +func (r *Reconciler) processOneManifest( + ctx context.Context, + bundle *manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) { + workRef := klog.KObj(work) + manifestObjRef := klog.KObj(bundle.manifestObj) + // Note (chenyu1): Fleet does not track references for objects in the member cluster as + // the references should be the same as those of the manifest objects, provided that Fleet + // does not support objects with generate names for now. + + // Firstly, attempt to find if an object has been created in the member cluster based on the manifest object. + if shouldSkipProcessing := r.findInMemberClusterObjectFor(ctx, bundle, work, expectedAppliedWorkOwnerRef); shouldSkipProcessing { + return + } + + // Take over the object in the member cluster that corresponds to the manifest object + // if applicable. + // + // Fleet will perform the takeover if: + // a) Fleet can find an object in the member cluster that corresponds to the manifest object, and + // it is not currently owned by Fleet (specifically the expected AppliedWork object); and + // b) takeover is allowed (i.e., the WhenToTakeOver option in the apply strategy is not set to Never). + if shouldSkipProcessing := r.takeOverInMemberClusterObjectIfApplicable(ctx, bundle, work, expectedAppliedWorkOwnerRef); shouldSkipProcessing { + return + } + + // If the ApplyStrategy is of the ReportDiff mode, Fleet would + // check for the configuration difference now; no drift detection nor apply op will be + // executed. + // + // Note that this runs even if the object is not owned by Fleet. + if shouldSkipProcessing := r.reportDiffOnlyIfApplicable(ctx, bundle, work, expectedAppliedWorkOwnerRef); shouldSkipProcessing { + return + } + + // For ClientSideApply and ServerSideApply apply strategies, ownership is a hard requirement. + // Skip the rest of the processing step if the resource has been created in the member cluster + // but Fleet is not listed as an owner of the resource in the member cluster yet (i.e., the + // takeover process does not run). This check is necessary as Fleet now supports the + // WhenToTakeOver option Never. + if !canApplyWithOwnership(bundle.inMemberClusterObj, expectedAppliedWorkOwnerRef) { + klog.V(2).InfoS("Ownership is not established yet; skip the apply op", + "manifestObj", manifestObjRef, "GVR", *bundle.gvr, "work", workRef) + bundle.applyErr = fmt.Errorf("no ownership of the object in the member cluster; takeover is needed") + bundle.applyResTyp = ManifestProcessingApplyResultTypeNotTakenOver + return + } + + // Perform a round of drift detection before running the apply op, if the ApplyStrategy + // dictates that an apply op can only be run when there are no drifts found. + if shouldSkipProcessing := r.performPreApplyDriftDetectionIfApplicable(ctx, bundle, work, expectedAppliedWorkOwnerRef); shouldSkipProcessing { + return + } + + // Perform the apply op. + appliedObj, err := r.apply(ctx, bundle.gvr, bundle.manifestObj, bundle.inMemberClusterObj, work.Spec.ApplyStrategy, expectedAppliedWorkOwnerRef) + if err != nil { + bundle.applyErr = fmt.Errorf("failed to apply the manifest: %w", err) + bundle.applyResTyp = ManifestProcessingApplyResultTypeFailedToApply + klog.ErrorS(err, "Failed to apply the manifest", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return + } + if appliedObj != nil { + // Update the bundle with the newly applied object, if an apply op has been run. + bundle.inMemberClusterObj = appliedObj + } + klog.V(2).InfoS("Apply process completed", + "manifestObj", manifestObjRef, "GVR", *bundle.gvr, "work", workRef) + + // Perform another round of drift detection after the apply op, if the ApplyStrategy dictates + // that drift detection should be done in full comparison mode. + // + // Drift detection is always enabled currently in Fleet. At this stage of execution, it is + // safe for us to assume that all the managed fields have been overwritten by the just + // completed apply op (or no apply op is necessary); consequently, no further drift + // detection is necessary if the partial comparison mode is used. However, for the full + // comparison mode, the apply op might not to able to resolve all the drifts, should there + // be any change made on the unmanaged fields; and Fleet would need to perform another + // round of drift detection. + if shouldSkipProcessing := r.performPostApplyDriftDetectionIfApplicable(ctx, bundle, work, expectedAppliedWorkOwnerRef); shouldSkipProcessing { + return + } + + // All done. + bundle.applyResTyp = ManifestProcessingApplyResultTypeApplied + klog.V(2).InfoS("Manifest processing completed", + "manifestObj", manifestObjRef, "GVR", *bundle.gvr, "work", workRef) +} + +// findInMemberClusterObjectFor attempts to find the corresponding object in the member cluster +// for a given manifest object. +// +// Note that it is possible that the object has not been created yet in the member cluster. +func (r *Reconciler) findInMemberClusterObjectFor( + ctx context.Context, + bundle *manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) (shouldSkipProcessing bool) { + inMemberClusterObj, err := r.spokeDynamicClient. + Resource(*bundle.gvr). + Namespace(bundle.manifestObj.GetNamespace()). + Get(ctx, bundle.manifestObj.GetName(), metav1.GetOptions{}) + switch { + case err == nil: + // An object derived from the manifest object has been found in the member cluster. + klog.V(2).InfoS("Found the corresponding object for the manifest object in the member cluster", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + bundle.inMemberClusterObj = inMemberClusterObj + return false + case errors.IsNotFound(err): + // The manifest object has never been applied before. + klog.V(2).InfoS("The manifest object has not been created in the member cluster yet", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + return false + default: + // An unexpected error has occurred. + bundle.applyErr = fmt.Errorf("failed to find the corresponding object for the manifest object in the member cluster: %w", err) + bundle.applyResTyp = ManifestProcessingApplyResultTypeFailedToFindObjInMemberCluster + klog.ErrorS(err, + "Failed to find the corresponding object for the manifest object in the member cluster", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true + } +} + +// takeOverInMemberClusterObjectIfApplicable attempts to take over an object in the member cluster +// as needed. +func (r *Reconciler) takeOverInMemberClusterObjectIfApplicable( + ctx context.Context, + bundle *manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) (shouldSkipProcessing bool) { + if !shouldInitiateTakeOverAttempt(bundle.inMemberClusterObj, work.Spec.ApplyStrategy, expectedAppliedWorkOwnerRef) { + // Takeover is not necessary; proceed with the processing. + klog.V(2).InfoS("Takeover is not needed; skip the step") + return false + } + + // Take over the object. Note that this steps adds only the owner reference; no other + // fields are modified (on the object from the member cluster). + takenOverInMemberClusterObj, configDiffs, err := r.takeOverPreExistingObject(ctx, + bundle.gvr, bundle.manifestObj, bundle.inMemberClusterObj, + work.Spec.ApplyStrategy, expectedAppliedWorkOwnerRef) + switch { + case err != nil: + // An unexpected error has occurred. + bundle.applyErr = fmt.Errorf("failed to take over a pre-existing object: %w", err) + bundle.applyResTyp = ManifestProcessingApplyResultTypeFailedToTakeOver + klog.ErrorS(err, "Failed to take over a pre-existing object", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true + case len(configDiffs) > 0: + // Takeover cannot be performed as configuration differences are found between the manifest + // object and the object in the member cluster. + bundle.diffs = configDiffs + bundle.applyErr = fmt.Errorf("cannot take over object: configuration differences are found between the manifest object and the corresponding object in the member cluster") + bundle.applyResTyp = ManifestProcessingApplyResultTypeFailedToTakeOver + klog.V(2).InfoS("Cannot take over object as configuration differences are found between the manifest object and the corresponding object in the member cluster", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true + } + + // Takeover process is completed; update the bundle with the newly refreshed object from the member cluster. + bundle.inMemberClusterObj = takenOverInMemberClusterObj + klog.V(2).InfoS("The corresponding object has been taken over", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + return false +} + +// shouldInitiateTakeOverAttempt checks if Fleet should initiate the takeover process for an object. +// +// A takeover process is initiated when: +// - An object that matches with the given manifest has been created; but +// - The object is not owned by Fleet (more specifically, the object is not owned by the +// expected AppliedWork object). +func shouldInitiateTakeOverAttempt(inMemberClusterObj *unstructured.Unstructured, + applyStrategy *fleetv1beta1.ApplyStrategy, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) bool { + if inMemberClusterObj == nil { + // Obviously, if the corresponding live object is not found, no takeover is + // needed. + return false + } + + // Skip the takeover process if the apply strategy forbids so. + if applyStrategy.WhenToTakeOver == fleetv1beta1.WhenToTakeOverTypeNever { + return false + } + + // Check if the live object is owned by Fleet. + curOwners := inMemberClusterObj.GetOwnerReferences() + for idx := range curOwners { + if reflect.DeepEqual(curOwners[idx], *expectedAppliedWorkOwnerRef) { + // The live object is owned by Fleet; no takeover is needed. + return false + } + } + return true +} + +// reportDiffOnlyIfApplicable checks for configuration differences between the manifest object and +// the object in the member cluster, if the ReportDiff mode is enabled. +// +// Note that if the ReportDiff mode is on, manifest processing is completed as soon as diff +// reportings are done. +func (r *Reconciler) reportDiffOnlyIfApplicable( + ctx context.Context, + bundle *manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) (shouldSkipProcessing bool) { + if work.Spec.ApplyStrategy.Type != fleetv1beta1.ApplyStrategyTypeReportDiff { + // ReportDiff mode is not enabled; proceed with the processing. + bundle.reportDiffResTyp = ManifestProcessingReportDiffResultTypeNotEnabled + klog.V(2).InfoS("ReportDiff mode is not enabled; skip the step") + return false + } + + bundle.applyResTyp = ManifestProcessingApplyResultTypeNoApplyPerformed + + if bundle.inMemberClusterObj == nil { + // The object has not created in the member cluster yet. + // + // In this case, the diff found would be the full object; for simplicity reasons, + // Fleet will use a placeholder here rather than including the full JSON representation. + bundle.reportDiffResTyp = ManifestProcessingReportDiffResultTypeFoundDiff + bundle.diffs = []fleetv1beta1.PatchDetail{ + { + // The root path. + Path: "/", + // For simplicity reason, Fleet reports a placeholder here rather than + // including the full JSON representation. + ValueInHub: "(the whole object)", + }, + } + return true + } + + // The object has been created in the member cluster; Fleet will calculate the configuration + // diffs between the manifest object and the object from the member cluster. + configDiffs, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, + bundle.gvr, + bundle.manifestObj, bundle.inMemberClusterObj, + work.Spec.ApplyStrategy.ComparisonOption) + switch { + case err != nil: + // Failed to calculate the configuration diffs. + bundle.reportDiffErr = fmt.Errorf("failed to calculate configuration diffs between the manifest object and the object from the member cluster: %w", err) + bundle.reportDiffResTyp = ManifestProcessingReportDiffResultTypeFailed + klog.ErrorS(err, + "Failed to calculate configuration diffs between the manifest object and the object from the member cluster", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + case len(configDiffs) > 0: + // Configuration diffs are found. + bundle.diffs = configDiffs + bundle.reportDiffResTyp = ManifestProcessingReportDiffResultTypeFoundDiff + default: + // No configuration diffs are found. + bundle.reportDiffResTyp = ManifestProcessingReportDiffResultTypeNoDiffFound + } + + klog.V(2).InfoS("ReportDiff process completed", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + // If the ReportDiff mode is on, no further processing is performed, regardless of whether + // diffs are found. + return true +} + +// canApplyWithOwnership checks if Fleet can perform an apply op, knowing that Fleet has +// acquired the ownership of the object, or that the object has not been created yet. +// +// Note that this function does not concern co-ownership; such checks are executed elsewhere. +func canApplyWithOwnership(inMemberClusterObj *unstructured.Unstructured, expectedAppliedWorkOwnerRef *metav1.OwnerReference) bool { + if inMemberClusterObj == nil { + // The object has not been created yet; Fleet can apply the object. + return true + } + + // Verify if the object is owned by Fleet. + curOwners := inMemberClusterObj.GetOwnerReferences() + for idx := range curOwners { + if reflect.DeepEqual(curOwners[idx], *expectedAppliedWorkOwnerRef) { + return true + } + } + return false +} + +// performPreApplyDriftDetectionIfApplicable checks if pre-apply drift detection is needed and +// runs the drift detection process if applicable. +func (r *Reconciler) performPreApplyDriftDetectionIfApplicable( + ctx context.Context, + bundle *manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) (shouldSkipProcessing bool) { + isPreApplyDriftDetectionNeeded, err := shouldPerformPreApplyDriftDetection(bundle.manifestObj, bundle.inMemberClusterObj, work.Spec.ApplyStrategy) + switch { + case err != nil: + // Fleet cannot determine if pre-apply drift detection is needed; this will only + // happen if the hash calculation process fails, specifically when Fleet cannot + // marshal the manifest object into its JSON representation. This should never happen, + // especially considering that the manifest object itself has been + // successfully decoded at this point of execution. + // + // For completion purposes, Fleet will still attempt to catch this and + // report this as an unexpected error. + _ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to determine if pre-apply drift detection is needed: %w", err)) + bundle.applyErr = fmt.Errorf("failed to determine if pre-apply drift detection is needed: %w", err) + bundle.applyResTyp = ManifestProcessingApplyResultTypeFailedToRunDriftDetection + return true + case !isPreApplyDriftDetectionNeeded: + // Drift detection is not needed; proceed with the processing. + klog.V(2).InfoS("Pre-apply drift detection is not needed; skip the step") + return false + default: + // Run the drift detection process. + drifts, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, + bundle.gvr, + bundle.manifestObj, bundle.inMemberClusterObj, + work.Spec.ApplyStrategy.ComparisonOption) + switch { + case err != nil: + // An unexpected error has occurred. + bundle.applyErr = fmt.Errorf("failed to calculate pre-apply drifts between the manifest and the object from the member cluster: %w", err) + bundle.applyResTyp = ManifestProcessingApplyResultTypeFailedToRunDriftDetection + klog.ErrorS(err, + "Failed to calculate pre-apply drifts between the manifest and the object from the member cluster", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true + case len(drifts) > 0: + // Drifts are found in the pre-apply drift detection process. + bundle.drifts = drifts + bundle.applyErr = fmt.Errorf("cannot apply manifest: drifts are found between the manifest and the object from the member cluster") + bundle.applyResTyp = ManifestProcessingApplyResultTypeFoundDrifts + klog.V(2).InfoS("Cannot apply manifest: drifts are found between the manifest and the object from the member cluster", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true + default: + // No drifts are found in the pre-apply drift detection process; carry on with the apply op. + klog.V(2).InfoS("Pre-apply drift detection completed; no drifts are found", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + return false + } + } +} + +// shouldPerformPreApplyDriftDetection checks if pre-apply drift detection should be performed. +func shouldPerformPreApplyDriftDetection(manifestObj, inMemberClusterObj *unstructured.Unstructured, applyStrategy *fleetv1beta1.ApplyStrategy) (bool, error) { + // Drift detection is performed before the apply op if (and only if): + // * Fleet reports that the manifest has been applied before (i.e., inMemberClusterObj exists); and + // * The apply strategy dictates that an apply op should only run if there is no + // detected drift; and + // * The hash of the manifest object is consistent with the last applied manifest object hash + // annotation on the corresponding resource in the member cluster (i.e., the same manifest + // object has been applied before). + if applyStrategy.WhenToApply != fleetv1beta1.WhenToApplyTypeIfNotDrifted || inMemberClusterObj == nil { + // A shortcut to save some overhead. + return false, nil + } + + cleanedManifestObj := discardFieldsIrrelevantInComparisonFrom(manifestObj) + manifestObjHash, err := resource.HashOf(cleanedManifestObj.Object) + if err != nil { + return false, err + } + + inMemberClusterObjLastAppliedManifestObjHash := inMemberClusterObj.GetAnnotations()[fleetv1beta1.ManifestHashAnnotation] + return manifestObjHash == inMemberClusterObjLastAppliedManifestObjHash, nil +} + +// performPostApplyDriftDetectionIfApplicable checks if post-apply drift detection is needed and +// runs the drift detection process if applicable. +func (r *Reconciler) performPostApplyDriftDetectionIfApplicable( + ctx context.Context, + bundle *manifestProcessingBundle, + work *fleetv1beta1.Work, + expectedAppliedWorkOwnerRef *metav1.OwnerReference, +) (shouldSkipProcessing bool) { + if !shouldPerformPostApplyDriftDetection(work.Spec.ApplyStrategy) { + // Post-apply drift detection is not needed; proceed with the processing. + klog.V(2).InfoS("Post-apply drift detection is not needed; skip the step", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + return false + } + + drifts, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, + bundle.gvr, + bundle.manifestObj, bundle.inMemberClusterObj, + work.Spec.ApplyStrategy.ComparisonOption) + switch { + case err != nil: + // An unexpected error has occurred. + bundle.applyErr = fmt.Errorf("failed to calculate post-apply drifts between the manifest object and the object from the member cluster: %w", err) + // This case counts as a partial error; the apply op has been completed, but Fleet + // cannot determine if there are any drifts. + bundle.applyResTyp = ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection + klog.ErrorS(err, + "Failed to calculate post-apply drifts between the manifest object and the object from the member cluster", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true + case len(drifts) > 0: + // Drifts are found in the post-apply drift detection process. + bundle.drifts = drifts + klog.V(2).InfoS("Post-apply drift detection completed; drifts are found", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + // The presence of such drifts are not considered as an error. + return false + default: + // No drifts are found in the post-apply drift detection process. + klog.V(2).InfoS("Post-apply drift detection completed; no drifts are found", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + return false + } +} + +// shouldPerformPostApplyDriftDetection checks if post-apply drift detection should be performed. +func shouldPerformPostApplyDriftDetection(applyStrategy *fleetv1beta1.ApplyStrategy) bool { + // Post-apply drift detection is performed if (and only if): + // * The apply strategy dictates that drift detection should run in full comparison mode. + return applyStrategy.ComparisonOption == fleetv1beta1.ComparisonOptionTypeFullComparison +} diff --git a/pkg/controllers/workapplier/process_test.go b/pkg/controllers/workapplier/process_test.go new file mode 100644 index 000000000..9f5682ab4 --- /dev/null +++ b/pkg/controllers/workapplier/process_test.go @@ -0,0 +1,99 @@ +package workapplier + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +// Note (chenyu1): The fake client Fleet uses for unit tests has trouble processing certain requests +// at the moment; affected test cases will be covered in the integration tests (w/ real clients) instead. + +// TestShouldInitiateTakeOverAttempt tests the shouldInitiateTakeOverAttempt function. +func TestShouldInitiateTakeOverAttempt(t *testing.T) { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + }, + } + nsUnstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ns) + if err != nil { + t.Fatalf("Namespace ToUnstructured() = %v, want no error", err) + } + nsUnstructured := &unstructured.Unstructured{Object: nsUnstructuredMap} + nsUnstructured.SetAPIVersion("v1") + nsUnstructured.SetKind("Namespace") + + nsWithFleetOwnerUnstructured := nsUnstructured.DeepCopy() + nsWithFleetOwnerUnstructured.SetOwnerReferences([]metav1.OwnerReference{ + *appliedWorkOwnerRef, + }) + + nsWithNonFleetOwnerUnstructured := nsUnstructured.DeepCopy() + nsWithNonFleetOwnerUnstructured.SetOwnerReferences([]metav1.OwnerReference{ + dummyOwnerRef, + }) + + testCases := []struct { + name string + inMemberClusterObj *unstructured.Unstructured + applyStrategy *fleetv1beta1.ApplyStrategy + expectedAppliedWorkOwnerRef *metav1.OwnerReference + wantShouldTakeOver bool + }{ + { + name: "no in member cluster object", + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + }, + }, + { + name: "never take over", + inMemberClusterObj: nsUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + }, + { + name: "owned by Fleet", + inMemberClusterObj: nsWithFleetOwnerUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + }, + { + name: "no owner, always take over", + inMemberClusterObj: nsUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantShouldTakeOver: true, + }, + { + name: "not owned by Fleet, take over if no diff", + inMemberClusterObj: nsWithNonFleetOwnerUnstructured, + applyStrategy: &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, + }, + expectedAppliedWorkOwnerRef: appliedWorkOwnerRef, + wantShouldTakeOver: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + shouldTakeOver := shouldInitiateTakeOverAttempt(tc.inMemberClusterObj, tc.applyStrategy, tc.expectedAppliedWorkOwnerRef) + if shouldTakeOver != tc.wantShouldTakeOver { + t.Errorf("shouldInitiateTakeOverAttempt() = %v, want %v", shouldTakeOver, tc.wantShouldTakeOver) + } + }) + } +} diff --git a/pkg/controllers/workapplier/status.go b/pkg/controllers/workapplier/status.go new file mode 100644 index 000000000..2826205e1 --- /dev/null +++ b/pkg/controllers/workapplier/status.go @@ -0,0 +1,525 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/controller" +) + +// refreshWorkStatus refreshes the status of a Work object based on the processing results of its manifests. +func (r *Reconciler) refreshWorkStatus( + ctx context.Context, + work *fleetv1beta1.Work, + bundles []*manifestProcessingBundle, +) error { + // Note (chenyu1): this method can run in parallel; however, for simplicity reasons, + // considering that in most of the time the count of manifests would be low, currently + // Fleet still does the status refresh sequentially. + + manifestCount := len(bundles) + appliedManifestsCount := 0 + availableAppliedObjectsCount := 0 + untrackableAppliedObjectsCount := 0 + diffedObjectsCount := 0 + + // Use the now timestamp as the observation time. + now := metav1.Now() + + // Rebuild the manifest conditions. + + // Pre-allocate the slice. + rebuiltManifestConds := make([]fleetv1beta1.ManifestCondition, len(bundles)) + + // Port back existing manifest conditions to the pre-allocated slice. + // + // This step is necessary at the moment primarily for two reasons: + // a) manifest condition uses metav1.Condition, the LastTransitionTime field of which requires + // that Fleet track the last known condition; + // b) part of the Fleet rollout process uses the LastTransitionTime of the Available condition + // to calculate the minimum wait period for an untrackable Work object (a Work object with + // one or more untrackable manifests). + + // Prepare an index for quicker lookup. + rebuiltManifestCondQIdx := prepareRebuiltManifestCondQIdx(bundles) + + // Port back existing manifest conditions using the index. + for idx := range work.Status.ManifestConditions { + existingManifestCond := work.Status.ManifestConditions[idx] + + existingManifestCondWRIStr, err := formatWRIString(&existingManifestCond.Identifier) + if err != nil { + // It is OK for an existing manifest condition to not have a valid identifier; this + // happens when the manifest condition was previously associated with a manifest + // that cannot be decoded. For obvious reasons Fleet does not need to port back + // such manifest conditions any way. + continue + } + + // Check if the WRI string has a match in the index. + if rebuiltManifestCondIdx, ok := rebuiltManifestCondQIdx[existingManifestCondWRIStr]; ok { + // Port back the existing manifest condition. + rebuiltManifestConds[rebuiltManifestCondIdx] = *existingManifestCond.DeepCopy() + } + } + + for idx := range bundles { + bundle := bundles[idx] + + // Update the manifest condition based on the bundle processing results. + manifestCond := &rebuiltManifestConds[idx] + manifestCond.Identifier = *bundle.id + if manifestCond.Conditions == nil { + manifestCond.Conditions = []metav1.Condition{} + } + + // Note that per API definition, the observed generation of a manifest condition is that + // of the applied resource, not that of the Work object. + inMemberClusterObjGeneration := int64(0) + if bundle.inMemberClusterObj != nil { + inMemberClusterObjGeneration = bundle.inMemberClusterObj.GetGeneration() + } + setManifestAppliedCondition(manifestCond, bundle.applyResTyp, bundle.applyErr, inMemberClusterObjGeneration) + setManifestAvailableCondition(manifestCond, bundle.availabilityResTyp, bundle.availabilityErr, inMemberClusterObjGeneration) + setManifestDiffReportedCondition(manifestCond, bundle.reportDiffResTyp, bundle.reportDiffErr, inMemberClusterObjGeneration) + + // Check if a first drifted timestamp has been set; if not, set it to the current time. + firstDriftedTimestamp := &now + if manifestCond.DriftDetails != nil && !manifestCond.DriftDetails.FirstDriftedObservedTime.IsZero() { + firstDriftedTimestamp = &manifestCond.DriftDetails.FirstDriftedObservedTime + } + // Reset the drift details (such details need no port-back). + manifestCond.DriftDetails = nil + if len(bundle.drifts) > 0 { + // Populate drift details if there are drifts found. + var observedInMemberClusterGen int64 + if bundle.inMemberClusterObj != nil { + observedInMemberClusterGen = bundle.inMemberClusterObj.GetGeneration() + } + + manifestCond.DriftDetails = &fleetv1beta1.DriftDetails{ + ObservationTime: now, + ObservedInMemberClusterGeneration: observedInMemberClusterGen, + FirstDriftedObservedTime: *firstDriftedTimestamp, + ObservedDrifts: bundle.drifts, + } + } + + // Check if a first diffed timestamp has been set; if not, set it to the current time. + firstDiffedTimestamp := &now + if manifestCond.DiffDetails != nil && !manifestCond.DiffDetails.FirstDiffedObservedTime.IsZero() { + firstDiffedTimestamp = &manifestCond.DiffDetails.FirstDiffedObservedTime + } + // Reset the diff details (such details need no port-back). + manifestCond.DiffDetails = nil + if len(bundle.diffs) > 0 { + // Populate diff details if there are diffs found. + var observedInMemberClusterGen *int64 + if bundle.inMemberClusterObj != nil { + observedInMemberClusterGen = ptr.To(bundle.inMemberClusterObj.GetGeneration()) + } + + manifestCond.DiffDetails = &fleetv1beta1.DiffDetails{ + ObservationTime: now, + ObservedInMemberClusterGeneration: observedInMemberClusterGen, + FirstDiffedObservedTime: *firstDiffedTimestamp, + ObservedDiffs: bundle.diffs, + } + + // Tally the stats. + diffedObjectsCount++ + } + + // Tally the stats. + if isManifestObjectApplied(bundle.applyResTyp) { + appliedManifestsCount++ + } + if isAppliedObjectAvailable(bundle.availabilityResTyp) { + availableAppliedObjectsCount++ + } + if bundle.availabilityResTyp == ManifestProcessingAvailabilityResultTypeNotTrackable { + untrackableAppliedObjectsCount++ + } + } + + // Refresh the Work object status conditions. + + // Do a sanity check. + if appliedManifestsCount > manifestCount || availableAppliedObjectsCount > manifestCount { + // Normally this should never happen. + return controller.NewUnexpectedBehaviorError( + fmt.Errorf("the number of applied manifests (%d) or available applied objects (%d) exceeds the total number of manifests (%d)", + appliedManifestsCount, availableAppliedObjectsCount, manifestCount)) + } + + if work.Status.Conditions == nil { + work.Status.Conditions = []metav1.Condition{} + } + setWorkAppliedCondition(work, manifestCount, appliedManifestsCount) + setWorkAvailableCondition(work, manifestCount, availableAppliedObjectsCount, untrackableAppliedObjectsCount) + setWorkDiffReportedCondition(work, manifestCount, diffedObjectsCount) + work.Status.ManifestConditions = rebuiltManifestConds + + // Update the Work object status. + if err := r.hubClient.Status().Update(ctx, work); err != nil { + return controller.NewAPIServerError(false, err) + } + return nil +} + +// refreshAppliedWorkStatus refreshes the status of an AppliedWork object based on the processing results of its manifests. +func (r *Reconciler) refreshAppliedWorkStatus( + ctx context.Context, + appliedWork *fleetv1beta1.AppliedWork, + bundles []*manifestProcessingBundle, +) error { + // Note (chenyu1): this method can run in parallel; however, for simplicity reasons, + // considering that in most of the time the count of manifests would be low, currently + // Fleet still does the status refresh sequentially. + + // Pre-allocate the slice. + // + // Manifests that failed to get applied are not included in this list, hence + // empty length. + appliedResources := make([]fleetv1beta1.AppliedResourceMeta, 0, len(bundles)) + + // Build the list of applied resources. + for idx := range bundles { + bundle := bundles[idx] + + if isManifestObjectApplied(bundle.applyResTyp) { + appliedResources = append(appliedResources, fleetv1beta1.AppliedResourceMeta{ + WorkResourceIdentifier: *bundle.id, + UID: bundle.inMemberClusterObj.GetUID(), + }) + } + } + + // Update the AppliedWork object status. + appliedWork.Status.AppliedResources = appliedResources + if err := r.spokeClient.Status().Update(ctx, appliedWork); err != nil { + return controller.NewAPIServerError(false, err) + } + klog.V(2).InfoS("Refreshed AppliedWork object status", + klog.KObj(appliedWork)) + return nil +} + +// isManifestObjectAvailable returns if an availability result type indicates that a manifest +// object in a bundle is available. +func isAppliedObjectAvailable(availabilityResTyp ManifestProcessingAvailabilityResultType) bool { + return availabilityResTyp == ManifestProcessingAvailabilityResultTypeAvailable || availabilityResTyp == ManifestProcessingAvailabilityResultTypeNotTrackable +} + +// setManifestAppliedCondition sets the Applied condition on an applied manifest. +func setManifestAppliedCondition( + manifestCond *fleetv1beta1.ManifestCondition, + appliedResTyp manifestProcessingAppliedResultType, + applyError error, + inMemberClusterObjGeneration int64, +) { + var appliedCond *metav1.Condition + switch appliedResTyp { + case ManifestProcessingApplyResultTypeApplied: + // The manifest has been successfully applied. + appliedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + Message: ManifestProcessingApplyResultTypeAppliedDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } + case ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection: + // The manifest has been successfully applied, but drift detection has failed. + // + // At this moment Fleet does not prepare a dedicated condition for drift detection + // outcomes. + appliedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection), + Message: ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetectionDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } + case ManifestProcessingApplyResultTypeNoApplyPerformed: + // ReportDiff mode is on and no apply op has been performed. In this case, Fleet + // will leave the Applied condition as it is (i.e., it might be unset, or has become + // stale). + return + default: + // The apply op fails. + appliedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(appliedResTyp), + Message: fmt.Sprintf("Failed to applied the manifest (error: %s)", applyError), + ObservedGeneration: inMemberClusterObjGeneration, + } + } + + meta.SetStatusCondition(&manifestCond.Conditions, *appliedCond) +} + +// setManifestAvailableCondition sets the Available condition on an applied manifest. +func setManifestAvailableCondition( + manifestCond *fleetv1beta1.ManifestCondition, + availabilityResTyp ManifestProcessingAvailabilityResultType, + availabilityError error, + inMemberClusterObjGeneration int64, +) { + var availableCond *metav1.Condition + switch availabilityResTyp { + case ManifestProcessingAvailabilityResultTypeSkipped: + // Availability check has been skipped for the manifest as it has not been applied yet. + // + // In this case, no availability condition is set. + case ManifestProcessingAvailabilityResultTypeFailed: + // Availability check has failed. + availableCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingAvailabilityResultTypeFailed), + Message: fmt.Sprintf(ManifestProcessingAvailabilityResultTypeFailedDescription, availabilityError), + ObservedGeneration: inMemberClusterObjGeneration, + } + case ManifestProcessingAvailabilityResultTypeNotYetAvailable: + // The manifest is not yet available. + availableCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingAvailabilityResultTypeNotYetAvailable), + Message: ManifestProcessingAvailabilityResultTypeNotYetAvailableDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } + case ManifestProcessingAvailabilityResultTypeNotTrackable: + // Fleet cannot track the availability of the manifest. + availableCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeNotTrackable), + Message: ManifestProcessingAvailabilityResultTypeNotTrackableDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } + default: + // The manifest is available. + availableCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + Message: ManifestProcessingAvailabilityResultTypeAvailableDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } + } + + if availableCond != nil { + meta.SetStatusCondition(&manifestCond.Conditions, *availableCond) + } else { + // As the conditions are port back; removal must be performed if the Available + // condition is not set. + meta.RemoveStatusCondition(&manifestCond.Conditions, fleetv1beta1.WorkConditionTypeAvailable) + } +} + +// setManifestDiffReportedCondition sets the DiffReported condition on a manifest. +func setManifestDiffReportedCondition( + manifestCond *fleetv1beta1.ManifestCondition, + reportDiffResTyp ManifestProcessingReportDiffResultType, + reportDiffError error, + inMemberClusterObjGeneration int64, +) { + var diffReportedCond *metav1.Condition + switch reportDiffResTyp { + case ManifestProcessingReportDiffResultTypeFailed: + // Diff reporting has failed. + diffReportedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingReportDiffResultTypeFailed), + Message: fmt.Sprintf(ManifestProcessingReportDiffResultTypeFailedDescription, reportDiffError), + ObservedGeneration: inMemberClusterObjGeneration, + } + case ManifestProcessingReportDiffResultTypeNotEnabled: + // Diff reporting is not enabled. + // + // For simplicity reasons, the DiffReported condition will only appear when + // the ReportDiff mode is on; in other configurations, the condition will be + // removed. + case ManifestProcessingReportDiffResultTypeNoDiffFound: + // No diff has been found. + diffReportedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound), + Message: ManifestProcessingReportDiffResultTypeNoDiffFoundDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } + case ManifestProcessingReportDiffResultTypeFoundDiff: + // Found diffs. + diffReportedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + Message: ManifestProcessingReportDiffResultTypeFoundDiffDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } + } + + if diffReportedCond != nil { + meta.SetStatusCondition(&manifestCond.Conditions, *diffReportedCond) + } else { + // As the conditions are port back; removal must be performed if the DiffReported + // condition is not set. + meta.RemoveStatusCondition(&manifestCond.Conditions, fleetv1beta1.WorkConditionTypeDiffReported) + } +} + +// setWorkAppliedCondition sets the Applied condition on a Work object. +// +// A Work object is considered to be applied if all of its manifests have been successfully applied. +func setWorkAppliedCondition( + work *fleetv1beta1.Work, + manifestCount, appliedManifestCount int, +) { + var appliedCond *metav1.Condition + switch { + case work.Spec.ApplyStrategy != nil && work.Spec.ApplyStrategy.Type == fleetv1beta1.ApplyStrategyTypeReportDiff: + // ReportDiff mode is on; no apply op has been performed, and consequently + // Fleet will not update the Applied condition. + return + case appliedManifestCount == manifestCount: + // All manifests have been successfully applied. + appliedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + // Here Fleet reuses the same reason for individual manifests. + Reason: string(ManifestProcessingApplyResultTypeApplied), + Message: allManifestsAppliedMessage, + ObservedGeneration: work.Generation, + } + default: + // Not all manifests have been successfully applied. + appliedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + Message: fmt.Sprintf(notAllManifestsAppliedMessage, appliedManifestCount, manifestCount), + ObservedGeneration: work.Generation, + } + } + meta.SetStatusCondition(&work.Status.Conditions, *appliedCond) +} + +// setWorkAvailableCondition sets the Available condition on a Work object. +// +// A Work object is considered to be available if all of its applied manifests are available. +func setWorkAvailableCondition( + work *fleetv1beta1.Work, + manifestCount, availableManifestCount, untrackableAppliedObjectsCount int, +) { + var availableCond *metav1.Condition + switch { + case work.Spec.ApplyStrategy != nil && work.Spec.ApplyStrategy.Type == fleetv1beta1.ApplyStrategyTypeReportDiff: + // ReportDiff mode is on; no apply op has been performed, and consequently + // Fleet will not update the Available condition. + return + case availableManifestCount == manifestCount && untrackableAppliedObjectsCount == 0: + // All manifests are available. + availableCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + Message: allAppliedObjectAvailableMessage, + ObservedGeneration: work.Generation, + } + case availableManifestCount == manifestCount: + // Some manifests are not trackable. + availableCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeNotTrackable), + Message: someAppliedObjectUntrackableMessage, + ObservedGeneration: work.Generation, + } + default: + // Not all manifests are available. + availableCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + Message: fmt.Sprintf(notAllAppliedObjectsAvailableMessage, availableManifestCount, manifestCount), + ObservedGeneration: work.Generation, + } + } + meta.SetStatusCondition(&work.Status.Conditions, *availableCond) +} + +// setWorkDiffReportedCondition sets the DiffReported condition on a Work object. +func setWorkDiffReportedCondition( + work *fleetv1beta1.Work, + manifestCount, diffedObjectsCount int, +) { + var diffReportedCond *metav1.Condition + switch { + case work.Spec.ApplyStrategy == nil || work.Spec.ApplyStrategy.Type != fleetv1beta1.ApplyStrategyTypeReportDiff: + // ReportDiff mode is not on; Fleet will remove DiffReported condition. + case diffedObjectsCount == 0: + // No diff has been found. + diffReportedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound), + Message: ManifestProcessingReportDiffResultTypeNoDiffFoundDescription, + ObservedGeneration: work.Generation, + } + default: + // Found diffs. + diffReportedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + Message: fmt.Sprintf(someObjectsHaveDiffs, diffedObjectsCount, manifestCount), + ObservedGeneration: work.Generation, + } + } + + if diffReportedCond != nil { + meta.SetStatusCondition(&work.Status.Conditions, *diffReportedCond) + } else { + // For simplicity reasons, Fleet will remove the DiffReported condition if the + // ReportDiff mode is not being used. + meta.RemoveStatusCondition(&work.Status.Conditions, fleetv1beta1.WorkConditionTypeDiffReported) + } +} + +// prepareRebuiltManifestCondQIdx returns a map that allows quicker look up of a manifest +// condition given a work resource identifier. +func prepareRebuiltManifestCondQIdx(bundles []*manifestProcessingBundle) map[string]int { + rebuiltManifestCondQIdx := make(map[string]int) + for idx := range bundles { + bundle := bundles[idx] + + wriStr, err := formatWRIString(bundle.id) + if err != nil { + // There might be manifest conditions without a valid identifier in the bundle set + // (e.g., decoding error has occurred when processing a bundle). + // Fleet will skip these bundles, as there is no need to port back + // information for such manifests any way for obvious reasons (manifest itself is not + // identifiable). This is not considered as an error. + continue + } + + rebuiltManifestCondQIdx[wriStr] = idx + } + return rebuiltManifestCondQIdx +} diff --git a/pkg/controllers/workapplier/status_test.go b/pkg/controllers/workapplier/status_test.go new file mode 100644 index 000000000..b83fd3a09 --- /dev/null +++ b/pkg/controllers/workapplier/status_test.go @@ -0,0 +1,853 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +// TestRefreshWorkStatus tests the refreshWorkStatus method. +func TestRefreshWorkStatus(t *testing.T) { + ctx := context.Background() + + deploy1 := deploy.DeepCopy() + deploy1.Generation = 2 + + deployName2 := "deploy-2" + deploy2 := deploy.DeepCopy() + deploy2.Name = deployName2 + + deployName3 := "deploy-3" + deploy3 := deploy.DeepCopy() + deploy3.Name = deployName3 + + workNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberReservedNSName, + }, + } + + // Round up the timestamps due to K8s API server's precision limits. + firstDriftedTime := metav1.Time{ + Time: metav1.Now().Rfc3339Copy().Time.Add(-1 * time.Hour), + } + firstDiffedTime := metav1.Time{ + Time: metav1.Now().Rfc3339Copy().Time.Add(-1 * time.Hour), + } + driftObservedTimeMustBefore := metav1.Time{ + Time: metav1.Now().Rfc3339Copy().Time.Add(-1 * time.Minute), + } + + testCases := []struct { + name string + work *fleetv1beta1.Work + bundles []*manifestProcessingBundle + wantWorkStatus *fleetv1beta1.WorkStatus + ignoreFirstDriftedDiffedTimestamps bool + }{ + { + name: "all applied, all available", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + Generation: 1, + }, + }, + bundles: []*manifestProcessingBundle{ + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy1.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeAvailable, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + }, + }, + wantWorkStatus: &fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 1, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeAvailable), + ObservedGeneration: 2, + }, + }, + }, + }, + }, + }, + { + name: "mixed applied result", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + Generation: 2, + }, + }, + bundles: []*manifestProcessingBundle{ + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy2.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeSkipped, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName3, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy3.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeFailedToTakeOver, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeSkipped, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + }, + }, + wantWorkStatus: &fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + ObservedGeneration: 2, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection), + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName3, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFailedToTakeOver), + }, + }, + }, + }, + }, + }, + { + name: "mixed availability check", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + }, + }, + bundles: []*manifestProcessingBundle{ + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeFailed, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeNotYetAvailable, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "batch", + Version: "v1", + Kind: "Job", + Name: "job", + Namespace: nsName, + Resource: "jobs", + }, + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeNotTrackable, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + }, + }, + wantWorkStatus: &fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingAvailabilityResultTypeFailed), + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingAvailabilityResultTypeNotYetAvailable), + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "batch", + Version: "v1", + Kind: "Job", + Name: "job", + Namespace: nsName, + Resource: "jobs", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingApplyResultTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingAvailabilityResultTypeNotTrackable), + }, + }, + }, + }, + }, + }, + { + name: "drift and diff details", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + Generation: 2, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + FirstDriftedObservedTime: firstDriftedTime, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFailedToTakeOver), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ptr.To(int64(0)), + FirstDiffedObservedTime: firstDiffedTime, + }, + }, + }, + }, + }, + bundles: []*manifestProcessingBundle{ + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeFoundDrifts, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeSkipped, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + drifts: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInMember: "1", + }, + }, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy2.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeFailedToTakeOver, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeSkipped, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeNotEnabled, + diffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInMember: "2", + ValueInHub: "3", + }, + }, + }, + }, + wantWorkStatus: &fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: notAllAppliedObjectsAvailableReason, + ObservedGeneration: 2, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + FirstDriftedObservedTime: firstDriftedTime, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInMember: "1", + }, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFailedToTakeOver), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + FirstDiffedObservedTime: firstDiffedTime, + ObservedInMemberClusterGeneration: ptr.To(int64(0)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInMember: "2", + ValueInHub: "3", + }, + }, + }, + }, + }, + }, + }, + { + name: "report diff mode", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName, + Generation: 2, + }, + Spec: fleetv1beta1.WorkSpec{ + ApplyStrategy: &fleetv1beta1.ApplyStrategy{ + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + }, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + FirstDriftedObservedTime: firstDriftedTime, + }, + }, + }, + }, + }, + bundles: []*manifestProcessingBundle{ + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy.DeepCopy()), + applyResTyp: ManifestProcessingApplyResultTypeNoApplyPerformed, + availabilityResTyp: ManifestProcessingAvailabilityResultTypeSkipped, + reportDiffResTyp: ManifestProcessingReportDiffResultTypeFoundDiff, + diffs: []fleetv1beta1.PatchDetail{ + { + Path: "/x", + ValueInMember: "0", + }, + }, + }, + }, + wantWorkStatus: &fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: notAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + ObservedGeneration: 2, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ManifestProcessingApplyResultTypeFoundDrifts), + }, + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ptr.To(int64(0)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/x", + ValueInMember: "0", + }, + }, + }, + }, + }, + }, + ignoreFirstDriftedDiffedTimestamps: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(workNS, tc.work). + WithStatusSubresource(tc.work). + Build() + r := &Reconciler{ + hubClient: fakeClient, + workNameSpace: memberReservedNSName, + } + + err := r.refreshWorkStatus(ctx, tc.work, tc.bundles) + if err != nil { + t.Fatalf("refreshWorkStatus() = %v, want no error", err) + } + + updatedWork := &fleetv1beta1.Work{} + if err := fakeClient.Get(ctx, types.NamespacedName{Namespace: memberReservedNSName, Name: workName}, updatedWork); err != nil { + t.Fatalf("Work Get() = %v, want no error", err) + } + opts := []cmp.Option{ + ignoreFieldConditionLTTMsg, + cmpopts.IgnoreFields(fleetv1beta1.DriftDetails{}, "ObservationTime"), + cmpopts.IgnoreFields(fleetv1beta1.DiffDetails{}, "ObservationTime"), + } + if tc.ignoreFirstDriftedDiffedTimestamps { + opts = append(opts, cmpopts.IgnoreFields(fleetv1beta1.DriftDetails{}, "FirstDriftedObservedTime")) + opts = append(opts, cmpopts.IgnoreFields(fleetv1beta1.DiffDetails{}, "FirstDiffedObservedTime")) + } + if diff := cmp.Diff( + &updatedWork.Status, tc.wantWorkStatus, + opts..., + ); diff != "" { + t.Errorf("refreshed Work status mismatches (-got, +want):\n%s", diff) + } + + for _, manifestCond := range updatedWork.Status.ManifestConditions { + if manifestCond.DriftDetails != nil && manifestCond.DriftDetails.ObservationTime.Time.Before(driftObservedTimeMustBefore.Time) { + t.Errorf("DriftDetails.ObservationTime = %v, want after %v", manifestCond.DriftDetails.ObservationTime, driftObservedTimeMustBefore) + } + + if manifestCond.DiffDetails != nil && manifestCond.DiffDetails.ObservationTime.Time.Before(driftObservedTimeMustBefore.Time) { + t.Errorf("DiffDetails.ObservationTime = %v, want after %v", manifestCond.DiffDetails.ObservationTime, driftObservedTimeMustBefore) + } + } + }) + } +} + +// TestRefreshAppliedWorkStatus tests the refreshAppliedWorkStatus method. +func TestRefreshAppliedWorkStatus(t *testing.T) { + ctx := context.Background() + + deploy1 := deploy.DeepCopy() + deploy1.UID = "123-xyz" + + deploy2 := deploy.DeepCopy() + deployName2 := "deploy-2" + deploy2.Name = deployName2 + deploy2.UID = "789-lmn" + + ns1 := ns.DeepCopy() + ns1.UID = "456-abc" + + testCases := []struct { + name string + appliedWork *fleetv1beta1.AppliedWork + bundles []*manifestProcessingBundle + wantAppliedWorkStatus *fleetv1beta1.AppliedWorkStatus + }{ + { + name: "mixed", + appliedWork: &fleetv1beta1.AppliedWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + }, + }, + bundles: []*manifestProcessingBundle{ + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy1), + applyResTyp: ManifestProcessingApplyResultTypeApplied, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "", + Version: "v1", + Kind: "Namespace", + Name: nsName, + Resource: "namespaces", + }, + inMemberClusterObj: toUnstructured(t, ns1), + applyResTyp: ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection, + }, + { + id: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName2, + Namespace: nsName, + Resource: "deployments", + }, + inMemberClusterObj: toUnstructured(t, deploy2), + applyResTyp: ManifestProcessingApplyResultTypeFailedToFindObjInMemberCluster, + }, + }, + wantAppliedWorkStatus: &fleetv1beta1.AppliedWorkStatus{ + AppliedResources: []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Name: deployName, + Namespace: nsName, + Resource: "deployments", + }, + UID: "123-xyz", + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "", + Version: "v1", + Kind: "Namespace", + Name: nsName, + Resource: "namespaces", + }, + UID: "456-abc", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(tc.appliedWork). + WithStatusSubresource(tc.appliedWork). + Build() + r := &Reconciler{ + spokeClient: fakeClient, + } + + err := r.refreshAppliedWorkStatus(ctx, tc.appliedWork, tc.bundles) + if err != nil { + t.Fatalf("refreshAppliedWorkStatus() = %v, want no error", err) + } + + updatedAppliedWork := &fleetv1beta1.AppliedWork{} + if err := fakeClient.Get(ctx, types.NamespacedName{Name: workName}, updatedAppliedWork); err != nil { + t.Fatalf("AppliedWork Get() = %v, want no error", err) + } + + if diff := cmp.Diff(&updatedAppliedWork.Status, tc.wantAppliedWorkStatus); diff != "" { + t.Errorf("refreshed AppliedWork status mismatches (-got, +want):\n%s", diff) + } + }) + } +} diff --git a/pkg/controllers/workapplier/suite_test.go b/pkg/controllers/workapplier/suite_test.go new file mode 100644 index 000000000..fc4542dd8 --- /dev/null +++ b/pkg/controllers/workapplier/suite_test.go @@ -0,0 +1,252 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "context" + "flag" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "k8s.io/klog/v2/textlogger" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + testv1alpha1 "go.goms.io/fleet/test/apis/v1alpha1" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. +var ( + hubCfg *rest.Config + memberCfg *rest.Config + hubEnv *envtest.Environment + memberEnv *envtest.Environment + hubMgr manager.Manager + hubClient client.Client + memberClient client.Client + memberDynamicClient dynamic.Interface + workApplier *Reconciler + + ctx context.Context + cancel context.CancelFunc + + // Temporary variables for migrated integration tests. + tmpEnv *envtest.Environment + tmpCfg *rest.Config + k8sClient client.Client + tmpMgr manager.Manager + workController *Reconciler + + testWorkNamespace = "test-work-namespace" +) + +const ( + // The number of max. concurrent reconciliations for the work applier controller. + maxConcurrentReconciles = 5 + // The count of workers for the work applier controller. + workerCount = 4 + + memberReservedNSName = "fleet-member-experimental" +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Work Applier Integration Test Suite") +} + +func setupResources() { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberReservedNSName, + }, + } + Expect(hubClient.Create(ctx, ns)).To(Succeed()) +} + +var _ = BeforeSuite(func() { + ctx, cancel = context.WithCancel(context.TODO()) + + By("Setup klog") + fs := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(fs) + Expect(fs.Parse([]string{"--v", "5", "-add_dir_header", "true"})).Should(Succeed()) + + klog.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("Bootstrapping test environments") + hubEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("../../../", "config", "crd", "bases"), + filepath.Join("../../../", "test", "manifests"), + }, + } + memberEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("../../../", "config", "crd", "bases"), + filepath.Join("../../../", "test", "manifests"), + }, + } + + var err error + hubCfg, err = hubEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(hubCfg).ToNot(BeNil()) + + memberCfg, err = memberEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(memberCfg).ToNot(BeNil()) + + err = fleetv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = testv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + By("Building the K8s clients") + hubClient, err = client.New(hubCfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + Expect(hubClient).ToNot(BeNil()) + + memberClient, err = client.New(memberCfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + Expect(memberClient).ToNot(BeNil()) + + // This setup also requires a client-go dynamic client for the member cluster. + memberDynamicClient, err = dynamic.NewForConfig(memberCfg) + Expect(err).ToNot(HaveOccurred()) + + By("Setting up the resources") + setupResources() + + By("Setting up the controller and the controller manager") + hubMgr, err = ctrl.NewManager(hubCfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: server.Options{ + BindAddress: "0", + }, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + memberReservedNSName: {}, + }, + }, + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4))), + }) + Expect(err).ToNot(HaveOccurred()) + + workApplier = NewReconciler( + hubClient, + memberReservedNSName, + memberDynamicClient, + memberClient, + memberClient.RESTMapper(), + hubMgr.GetEventRecorderFor("work-applier"), + maxConcurrentReconciles, + workerCount, + time.Second*5, + time.Second*5, + ) + Expect(workApplier.SetupWithManager(hubMgr)).To(Succeed()) + + go func() { + defer GinkgoRecover() + Expect(workApplier.Join(ctx)).To(Succeed()) + Expect(hubMgr.Start(ctx)).To(Succeed()) + }() + + // Temporary setup for migrated integration tests. + tmpEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("../../../", "config", "crd", "bases"), + filepath.Join("../../../", "test", "manifests"), + }, + } + + tmpCfg, err = tmpEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(tmpCfg).ToNot(BeNil()) + + k8sClient, err = client.New(tmpCfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + + workNamespace := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testWorkNamespace, + }, + } + err = k8sClient.Create(context.Background(), &workNamespace) + Expect(err).ToNot(HaveOccurred()) + + tmpMgr, err = ctrl.NewManager(tmpCfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: server.Options{ + BindAddress: "0", + }, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testWorkNamespace: {}, + }, + }, + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4))), + }) + Expect(err).ToNot(HaveOccurred()) + + tmpSpokeDynamicClient, err := dynamic.NewForConfig(tmpCfg) + Expect(err).ToNot(HaveOccurred()) + + tmpSpokeClient, err := client.New(tmpCfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + + workController = NewReconciler( + tmpMgr.GetClient(), + testWorkNamespace, + tmpSpokeDynamicClient, + tmpSpokeClient, + tmpSpokeClient.RESTMapper(), + tmpMgr.GetEventRecorderFor("work-applier"), + maxConcurrentReconciles, + workerCount, + time.Second*5, + time.Second*5, + ) + Expect(workController.SetupWithManager(tmpMgr)).To(Succeed()) + Expect(workController.Join(ctx)).To(Succeed()) + + go func() { + if err = tmpMgr.Start(ctx); err != nil { + os.Exit(1) + } + Expect(err).ToNot(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + + cancel() + By("Tearing down the test environment") + Expect(hubEnv.Stop()).To(Succeed()) + Expect(memberEnv.Stop()).To(Succeed()) + + // Temporary setup for migrated integration tests. + Expect(tmpEnv.Stop()).To(Succeed()) +}) diff --git a/pkg/controllers/workapplier/utils.go b/pkg/controllers/workapplier/utils.go new file mode 100644 index 000000000..d03a8b3d3 --- /dev/null +++ b/pkg/controllers/workapplier/utils.go @@ -0,0 +1,57 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/condition" +) + +// formatWRIString returns a string representation of a work resource identifier. +func formatWRIString(wri *fleetv1beta1.WorkResourceIdentifier) (string, error) { + switch { + case wri.Group == "" && wri.Version == "": + // The manifest object cannot be decoded, i.e., it can only be identified by its ordinal. + // + // This branch is added solely for completeness reasons; normally such objects would not + // be included in any cases that would require a WRI string formatting. + return "", fmt.Errorf("the manifest object can only be identified by its ordinal") + default: + // For a regular object, the string representation includes the actual name. + return fmt.Sprintf("GV=%s/%s, Kind=%s, Namespace=%s, Name=%s", + wri.Group, wri.Version, wri.Kind, wri.Namespace, wri.Name), nil + } +} + +// isManifestObjectApplied returns if an applied result type indicates that a manifest +// object in a bundle has been successfully applied. +func isManifestObjectApplied(appliedResTyp manifestProcessingAppliedResultType) bool { + return appliedResTyp == ManifestProcessingApplyResultTypeApplied || + appliedResTyp == ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection +} + +// isWorkObjectAvailable checks if a Work object is available. +func isWorkObjectAvailable(work *fleetv1beta1.Work) bool { + availableCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeAvailable) + return condition.IsConditionStatusTrue(availableCond, work.Generation) +} + +// isPlacedByFleetInDuplicate checks if the object has already been placed by Fleet via another +// CRP. +func isPlacedByFleetInDuplicate(ownerRefs []metav1.OwnerReference, expectedAppliedWorkOwnerRef *metav1.OwnerReference) bool { + for idx := range ownerRefs { + ownerRef := ownerRefs[idx] + if ownerRef.APIVersion == fleetv1beta1.GroupVersion.String() && ownerRef.Kind == fleetv1beta1.AppliedWorkKind && string(ownerRef.UID) != string(expectedAppliedWorkOwnerRef.UID) { + return true + } + } + return false +} diff --git a/pkg/controllers/workapplier/utils_test.go b/pkg/controllers/workapplier/utils_test.go new file mode 100644 index 000000000..af285f27f --- /dev/null +++ b/pkg/controllers/workapplier/utils_test.go @@ -0,0 +1,74 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package workapplier + +import ( + "fmt" + "testing" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +// TestFormatWRIString tests the formatWRIString function. +func TestFormatWRIString(t *testing.T) { + testCases := []struct { + name string + wri *fleetv1beta1.WorkResourceIdentifier + wantWRIString string + wantErred bool + }{ + { + name: "ordinal only", + wri: &fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + }, + wantErred: true, + }, + { + name: "regular object", + wri: deployWRI(2, nsName, deployName), + wantWRIString: fmt.Sprintf("GV=apps/v1, Kind=Deployment, Namespace=%s, Name=%s", nsName, deployName), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + wriString, err := formatWRIString(tc.wri) + if tc.wantErred { + if err == nil { + t.Errorf("formatWRIString() = nil, want error") + } + return + } + if err != nil { + t.Fatalf("formatWRIString() = %v, want no error", err) + } + + if wriString != tc.wantWRIString { + t.Errorf("formatWRIString() mismatches: got %q, want %q", wriString, tc.wantWRIString) + } + }) + } +} + +// TestIsPlacedByFleetInDuplicate tests the isPlacedByFleetInDuplicate function. +func TestIsPlacedByFleetInDuplicate(t *testing.T) { + testCases := []struct { + name string + }{ + { + name: "in duplicate", + }, + { + name: "not in duplicate", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + }) + } +} diff --git a/pkg/scheduler/framework/framework.go b/pkg/scheduler/framework/framework.go index e2349bcb6..3064c6589 100644 --- a/pkg/scheduler/framework/framework.go +++ b/pkg/scheduler/framework/framework.go @@ -29,10 +29,10 @@ import ( clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/scheduler/clustereligibilitychecker" - "go.goms.io/fleet/pkg/scheduler/framework/parallelizer" "go.goms.io/fleet/pkg/utils/annotations" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" + "go.goms.io/fleet/pkg/utils/parallelizer" ) const ( diff --git a/pkg/scheduler/framework/framework_test.go b/pkg/scheduler/framework/framework_test.go index 07ba7b1ff..7599d80d9 100644 --- a/pkg/scheduler/framework/framework_test.go +++ b/pkg/scheduler/framework/framework_test.go @@ -29,7 +29,7 @@ import ( clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/scheduler/clustereligibilitychecker" - "go.goms.io/fleet/pkg/scheduler/framework/parallelizer" + "go.goms.io/fleet/pkg/utils/parallelizer" ) const ( diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 45f6b5606..85bb5ca1f 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -329,13 +329,13 @@ var ( Kind: DeploymentKind, } - DaemonSettGVR = schema.GroupVersionResource{ + DaemonSetGVR = schema.GroupVersionResource{ Group: appv1.GroupName, Version: appv1.SchemeGroupVersion.Version, Resource: "daemonsets", } - StatefulSettGVR = schema.GroupVersionResource{ + StatefulSetGVR = schema.GroupVersionResource{ Group: appv1.GroupName, Version: appv1.SchemeGroupVersion.Version, Resource: "statefulsets", diff --git a/pkg/utils/defaulter/clusterresourceplacement.go b/pkg/utils/defaulter/clusterresourceplacement.go index 3bfe9866b..e4e5bfbcb 100644 --- a/pkg/utils/defaulter/clusterresourceplacement.go +++ b/pkg/utils/defaulter/clusterresourceplacement.go @@ -80,16 +80,31 @@ func SetDefaultsClusterResourcePlacement(obj *fleetv1beta1.ClusterResourcePlacem if obj.Spec.Strategy.ApplyStrategy == nil { obj.Spec.Strategy.ApplyStrategy = &fleetv1beta1.ApplyStrategy{} } - if obj.Spec.Strategy.ApplyStrategy.Type == "" { - obj.Spec.Strategy.ApplyStrategy.Type = fleetv1beta1.ApplyStrategyTypeClientSideApply + SetDefaultsApplyStrategy(obj.Spec.Strategy.ApplyStrategy) + + if obj.Spec.RevisionHistoryLimit == nil { + obj.Spec.RevisionHistoryLimit = ptr.To(int32(DefaultRevisionHistoryLimitValue)) + } +} + +// SetDefaultsApplyStrategy sets the default values for an ApplyStrategy object. +func SetDefaultsApplyStrategy(obj *fleetv1beta1.ApplyStrategy) { + if obj.Type == "" { + obj.Type = fleetv1beta1.ApplyStrategyTypeClientSideApply } - if obj.Spec.Strategy.ApplyStrategy.Type == fleetv1beta1.ApplyStrategyTypeServerSideApply && obj.Spec.Strategy.ApplyStrategy.ServerSideApplyConfig == nil { - obj.Spec.Strategy.ApplyStrategy.ServerSideApplyConfig = &fleetv1beta1.ServerSideApplyConfig{ + if obj.Type == fleetv1beta1.ApplyStrategyTypeServerSideApply && obj.ServerSideApplyConfig == nil { + obj.ServerSideApplyConfig = &fleetv1beta1.ServerSideApplyConfig{ ForceConflicts: false, } } - if obj.Spec.RevisionHistoryLimit == nil { - obj.Spec.RevisionHistoryLimit = ptr.To(int32(DefaultRevisionHistoryLimitValue)) + if obj.ComparisonOption == "" { + obj.ComparisonOption = fleetv1beta1.ComparisonOptionTypePartialComparison + } + if obj.WhenToApply == "" { + obj.WhenToApply = fleetv1beta1.WhenToApplyTypeAlways + } + if obj.WhenToTakeOver == "" { + obj.WhenToTakeOver = fleetv1beta1.WhenToTakeOverTypeAlways } } diff --git a/pkg/utils/defaulter/clusterresourceplacement_test.go b/pkg/utils/defaulter/clusterresourceplacement_test.go index ff18371c3..7851d5a17 100644 --- a/pkg/utils/defaulter/clusterresourceplacement_test.go +++ b/pkg/utils/defaulter/clusterresourceplacement_test.go @@ -38,7 +38,10 @@ func TestSetDefaultsClusterResourcePlacement(t *testing.T) { UnavailablePeriodSeconds: ptr.To(DefaultUnavailablePeriodSeconds), }, ApplyStrategy: &fleetv1beta1.ApplyStrategy{ - Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, + Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, }, }, RevisionHistoryLimit: ptr.To(int32(DefaultRevisionHistoryLimitValue)), @@ -69,7 +72,10 @@ func TestSetDefaultsClusterResourcePlacement(t *testing.T) { UnavailablePeriodSeconds: ptr.To(15), }, ApplyStrategy: &fleetv1beta1.ApplyStrategy{ - Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, + Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, }, }, RevisionHistoryLimit: ptr.To(int32(10)), @@ -101,7 +107,10 @@ func TestSetDefaultsClusterResourcePlacement(t *testing.T) { UnavailablePeriodSeconds: ptr.To(15), }, ApplyStrategy: &fleetv1beta1.ApplyStrategy{ - Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, + Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, }, }, RevisionHistoryLimit: ptr.To(int32(10)), @@ -131,7 +140,10 @@ func TestSetDefaultsClusterResourcePlacement(t *testing.T) { UnavailablePeriodSeconds: ptr.To(DefaultUnavailablePeriodSeconds), }, ApplyStrategy: &fleetv1beta1.ApplyStrategy{ - Type: fleetv1beta1.ApplyStrategyTypeServerSideApply, + Type: fleetv1beta1.ApplyStrategyTypeServerSideApply, + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, ServerSideApplyConfig: &fleetv1beta1.ServerSideApplyConfig{ ForceConflicts: false, }, diff --git a/pkg/utils/defaulter/work.go b/pkg/utils/defaulter/work.go index c16a8a741..3db4b75d8 100644 --- a/pkg/utils/defaulter/work.go +++ b/pkg/utils/defaulter/work.go @@ -13,14 +13,5 @@ func SetDefaultsWork(w *placementv1beta1.Work) { if w.Spec.ApplyStrategy == nil { w.Spec.ApplyStrategy = &placementv1beta1.ApplyStrategy{} } - - if w.Spec.ApplyStrategy.Type == "" { - w.Spec.ApplyStrategy.Type = placementv1beta1.ApplyStrategyTypeClientSideApply - } - - if w.Spec.ApplyStrategy.Type == placementv1beta1.ApplyStrategyTypeServerSideApply && w.Spec.ApplyStrategy.ServerSideApplyConfig == nil { - w.Spec.ApplyStrategy.ServerSideApplyConfig = &placementv1beta1.ServerSideApplyConfig{ - ForceConflicts: false, - } - } + SetDefaultsApplyStrategy(w.Spec.ApplyStrategy) } diff --git a/pkg/utils/defaulter/work_test.go b/pkg/utils/defaulter/work_test.go index a2cbca363..7bfdc0f4a 100644 --- a/pkg/utils/defaulter/work_test.go +++ b/pkg/utils/defaulter/work_test.go @@ -26,7 +26,12 @@ func TestSetDefaultsWork(t *testing.T) { }, want: placementv1beta1.Work{ Spec: placementv1beta1.WorkSpec{ - ApplyStrategy: &placementv1beta1.ApplyStrategy{Type: placementv1beta1.ApplyStrategyTypeClientSideApply}, + ApplyStrategy: &placementv1beta1.ApplyStrategy{ + Type: placementv1beta1.ApplyStrategyTypeClientSideApply, + ComparisonOption: placementv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: placementv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: placementv1beta1.WhenToTakeOverTypeAlways, + }, }, }, }, @@ -39,7 +44,12 @@ func TestSetDefaultsWork(t *testing.T) { }, want: placementv1beta1.Work{ Spec: placementv1beta1.WorkSpec{ - ApplyStrategy: &placementv1beta1.ApplyStrategy{Type: placementv1beta1.ApplyStrategyTypeClientSideApply}, + ApplyStrategy: &placementv1beta1.ApplyStrategy{ + Type: placementv1beta1.ApplyStrategyTypeClientSideApply, + ComparisonOption: placementv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: placementv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: placementv1beta1.WhenToTakeOverTypeAlways, + }, }, }, }, @@ -47,13 +57,18 @@ func TestSetDefaultsWork(t *testing.T) { name: "nil server side apply config", work: placementv1beta1.Work{ Spec: placementv1beta1.WorkSpec{ - ApplyStrategy: &placementv1beta1.ApplyStrategy{Type: placementv1beta1.ApplyStrategyTypeServerSideApply}, + ApplyStrategy: &placementv1beta1.ApplyStrategy{ + Type: placementv1beta1.ApplyStrategyTypeServerSideApply, + }, }, }, want: placementv1beta1.Work{ Spec: placementv1beta1.WorkSpec{ ApplyStrategy: &placementv1beta1.ApplyStrategy{ Type: placementv1beta1.ApplyStrategyTypeServerSideApply, + ComparisonOption: placementv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: placementv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: placementv1beta1.WhenToTakeOverTypeAlways, ServerSideApplyConfig: &placementv1beta1.ServerSideApplyConfig{ForceConflicts: false}, }, }, @@ -65,6 +80,9 @@ func TestSetDefaultsWork(t *testing.T) { Spec: placementv1beta1.WorkSpec{ ApplyStrategy: &placementv1beta1.ApplyStrategy{ Type: placementv1beta1.ApplyStrategyTypeServerSideApply, + ComparisonOption: placementv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: placementv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: placementv1beta1.WhenToTakeOverTypeAlways, ServerSideApplyConfig: &placementv1beta1.ServerSideApplyConfig{ForceConflicts: true}, }, }, @@ -73,6 +91,9 @@ func TestSetDefaultsWork(t *testing.T) { Spec: placementv1beta1.WorkSpec{ ApplyStrategy: &placementv1beta1.ApplyStrategy{ Type: placementv1beta1.ApplyStrategyTypeServerSideApply, + ComparisonOption: placementv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: placementv1beta1.WhenToApplyTypeAlways, + WhenToTakeOver: placementv1beta1.WhenToTakeOverTypeAlways, ServerSideApplyConfig: &placementv1beta1.ServerSideApplyConfig{ForceConflicts: true}, }, }, diff --git a/pkg/scheduler/framework/parallelizer/errorflag.go b/pkg/utils/parallelizer/errorflag.go similarity index 100% rename from pkg/scheduler/framework/parallelizer/errorflag.go rename to pkg/utils/parallelizer/errorflag.go diff --git a/pkg/scheduler/framework/parallelizer/errorflag_test.go b/pkg/utils/parallelizer/errorflag_test.go similarity index 100% rename from pkg/scheduler/framework/parallelizer/errorflag_test.go rename to pkg/utils/parallelizer/errorflag_test.go diff --git a/pkg/scheduler/framework/parallelizer/parallelizer.go b/pkg/utils/parallelizer/parallelizer.go similarity index 100% rename from pkg/scheduler/framework/parallelizer/parallelizer.go rename to pkg/utils/parallelizer/parallelizer.go diff --git a/pkg/scheduler/framework/parallelizer/parallelizer_test.go b/pkg/utils/parallelizer/parallelizer_test.go similarity index 100% rename from pkg/scheduler/framework/parallelizer/parallelizer_test.go rename to pkg/utils/parallelizer/parallelizer_test.go