-
Notifications
You must be signed in to change notification settings - Fork 1.3k
feat: inject NodeAffinity into DataBackup status after successful backup #6110
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -21,6 +21,7 @@ import ( | |||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||
| "github.com/fluid-cloudnative/fluid/api/v1alpha1" | ||||||||||||||||||||||||||||||||
| "github.com/fluid-cloudnative/fluid/pkg/common" | ||||||||||||||||||||||||||||||||
| "github.com/fluid-cloudnative/fluid/pkg/dataflow" | ||||||||||||||||||||||||||||||||
| "github.com/fluid-cloudnative/fluid/pkg/dataoperation" | ||||||||||||||||||||||||||||||||
| "github.com/fluid-cloudnative/fluid/pkg/runtime" | ||||||||||||||||||||||||||||||||
| "github.com/fluid-cloudnative/fluid/pkg/utils" | ||||||||||||||||||||||||||||||||
|
|
@@ -52,7 +53,13 @@ func (o *OnceHandler) GetOperationStatus(ctx runtime.ReconcileRequestContext, op | |||||||||||||||||||||||||||||||
| return | ||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||
| // TODO: inject nodeaffinity like other data operations when using job instead of pod | ||||||||||||||||||||||||||||||||
| if kubeclient.IsSucceededPod(backupPod) && result.NodeAffinity == nil { | ||||||||||||||||||||||||||||||||
| result.NodeAffinity, err = dataflow.GenerateNodeAffinityFromPod(backupPod) | ||||||||||||||||||||||||||||||||
| if err != nil { | ||||||||||||||||||||||||||||||||
| ctx.Log.V(1).Info("NodeAffinity not injected", "reason", err.Error()) | ||||||||||||||||||||||||||||||||
| err = nil | ||||||||||||||||||||||||||||||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The |
||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||
|
Comment on lines
+56
to
+62
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Reusing the named return parameter
Suggested change
|
||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||
| var finishTime time.Time | ||||||||||||||||||||||||||||||||
| if len(backupPod.Status.Conditions) != 0 { | ||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -65,3 +65,47 @@ | |
|
|
||
| return nodeAffinity, nil | ||
| } | ||
|
|
||
| // GenerateNodeAffinityFromPod generates a NodeAffinity from a Pod's annotations, | ||
| // using the same annotation-based logic as GenerateNodeAffinity for Jobs. | ||
| // This is used by DataBackup which runs as a Pod rather than a Job. | ||
| func GenerateNodeAffinityFromPod(pod *corev1.Pod) (*corev1.NodeAffinity, error) { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| if pod == nil { | ||
| return nil, nil | ||
| } | ||
| // not inject, i.e. feature gate not enabled | ||
| if v := pod.Annotations[common.AnnotationDataFlowAffinityInject]; v != "true" { | ||
|
Check warning on line 77 in pkg/dataflow/helper.go
|
||
| return nil, nil | ||
| } | ||
|
|
||
| annotations := pod.Annotations | ||
|
|
||
| nodeAffinity := &corev1.NodeAffinity{ | ||
| RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ | ||
| NodeSelectorTerms: []corev1.NodeSelectorTerm{ | ||
| { | ||
| MatchExpressions: nil, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| hasInjectedLabels := false | ||
| for key, value := range annotations { | ||
| if strings.HasPrefix(key, common.AnnotationDataFlowCustomizedAffinityPrefix) { | ||
| nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions = | ||
| append(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions, | ||
| corev1.NodeSelectorRequirement{ | ||
| Key: strings.TrimPrefix(key, common.AnnotationDataFlowCustomizedAffinityPrefix), | ||
| Operator: corev1.NodeSelectorOpIn, | ||
| Values: []string{value}, | ||
| }) | ||
| hasInjectedLabels = true | ||
| } | ||
| } | ||
| if !hasInjectedLabels { | ||
| return nil, errors.New("the affinity label is not set, wait for next reconcile") | ||
| } | ||
|
|
||
| return nodeAffinity, nil | ||
| } | ||
|
Comment on lines
+69
to
+111
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The logic in // GenerateNodeAffinityFromAnnotations generates a NodeAffinity from annotations.
func GenerateNodeAffinityFromAnnotations(annotations map[string]string) (*corev1.NodeAffinity, error) {
if v := annotations[common.AnnotationDataFlowAffinityInject]; v != "true" {
return nil, nil
}
nodeAffinity := &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: nil,
},
},
},
}
hasInjectedLabels := false
for key, value := range annotations {
if strings.HasPrefix(key, common.AnnotationDataFlowCustomizedAffinityPrefix) {
nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions =
append(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions,
corev1.NodeSelectorRequirement{
Key: strings.TrimPrefix(key, common.AnnotationDataFlowCustomizedAffinityPrefix),
Operator: corev1.NodeSelectorOpIn,
Values: []string{value},
})
hasInjectedLabels = true
}
}
if !hasInjectedLabels {
return nil, errors.New("the affinity label is not set, wait for next reconcile")
}
return nodeAffinity, nil
}
// GenerateNodeAffinityFromPod generates a NodeAffinity from a Pod's annotations,
// using the same annotation-based logic as GenerateNodeAffinity for Jobs.
// This is used by DataBackup which runs as a Pod rather than a Job.
func GenerateNodeAffinityFromPod(pod *corev1.Pod) (*corev1.NodeAffinity, error) {
if pod == nil {
return nil, nil
}
return GenerateNodeAffinityFromAnnotations(pod.Annotations)
} |
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -128,3 +128,86 @@ | |
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestGenerateNodeAffinityFromPod(t *testing.T) { | ||
|
Check failure on line 132 in pkg/dataflow/helper_test.go
|
||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a |
||
| tests := []struct { | ||
| name string | ||
| pod *v1.Pod | ||
| wantNil bool | ||
| wantErr bool | ||
| wantLen int | ||
| }{ | ||
| { | ||
| name: "nil pod returns nil", | ||
| pod: nil, | ||
| wantNil: true, | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "pod without inject annotation returns nil", | ||
| pod: &v1.Pod{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "backup-pod", | ||
| Annotations: map[string]string{}, | ||
| }, | ||
| }, | ||
| wantNil: true, | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "pod with inject annotation but no affinity labels returns error", | ||
| pod: &v1.Pod{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "backup-pod", | ||
| Annotations: map[string]string{ | ||
| common.AnnotationDataFlowAffinityInject: "true", | ||
| }, | ||
| }, | ||
| }, | ||
| wantNil: true, | ||
| wantErr: true, | ||
| }, | ||
| { | ||
| name: "pod with inject annotation and affinity labels returns NodeAffinity", | ||
| pod: &v1.Pod{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "backup-pod", | ||
| Annotations: map[string]string{ | ||
| common.AnnotationDataFlowAffinityInject: "true", | ||
| common.AnnotationDataFlowCustomizedAffinityPrefix + common.K8sNodeNameLabelKey: "node01", | ||
| common.AnnotationDataFlowCustomizedAffinityPrefix + common.K8sZoneLabelKey: "zone01", | ||
| }, | ||
| }, | ||
| }, | ||
| wantNil: false, | ||
| wantErr: false, | ||
| wantLen: 2, | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| got, err := GenerateNodeAffinityFromPod(tt.pod) | ||
| if (err != nil) != tt.wantErr { | ||
| t.Errorf("GenerateNodeAffinityFromPod() error = %v, wantErr %v", err, tt.wantErr) | ||
| return | ||
| } | ||
| if tt.wantNil { | ||
| if got != nil { | ||
| t.Errorf("GenerateNodeAffinityFromPod() expected nil, got %v", got) | ||
| } | ||
| return | ||
| } | ||
| if got == nil { | ||
| t.Fatal("GenerateNodeAffinityFromPod() expected non-nil NodeAffinity") | ||
| } | ||
| terms := got.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms | ||
| if len(terms) != 1 { | ||
| t.Fatalf("expected 1 NodeSelectorTerm, got %d", len(terms)) | ||
| } | ||
| if len(terms[0].MatchExpressions) != tt.wantLen { | ||
| t.Errorf("expected %d MatchExpressions, got %d", tt.wantLen, len(terms[0].MatchExpressions)) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I traced where the annotations this reads actually come from, and I don't think this branch ever fires in a real cluster.
GenerateNodeAffinityFromPodreturns early unless the pod carriesdataflow-affinity.fluid.io.inject=trueplus theaffinity.dataflow.fluid.io.*node-label annotations. Those annotations are only ever written byDataOpJobReconcilerinpkg/controllers/v1alpha1/fluidapp/dataflowaffinity/, whoseManagedResource()returns&batchv1.Job{}and which writes the labels onto the Job (injectPodNodeLabelsToJob). DataBackup runs as a Pod, not a Job — the pod template incharts/fluid-databackup/*/templates/databackup.yamlsets no such annotations, and there's no pod-level reconciler that backfills them. So for a real DataBackup,pod.Annotations[AnnotationDataFlowAffinityInject] != "true"and this returns(nil, nil);result.NodeAffinitystays nil. The new unit tests only pass because they hand-set those annotations on a fake pod. Where is the backup pod supposed to get the inject/affinity annotations? If there isn't a source yet, this needs the pod-side injection (equivalent of the Job reconciler) to actually deliver the feature.