Fix upgrade test case for global sync mode

Author: johscheuer

e2e/test_operator_upgrades/operator_upgrades_test.go

@@ -510,7 +510,7 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 	)
 
 	DescribeTable(
-		"one process is marked for removal",
+		"one process is marked for removal and is stuck in removal",
 		func(beforeVersion string, targetVersion string) {
 			if fixtures.VersionsAreProtocolCompatible(beforeVersion, targetVersion) {
 				Skip("this test only affects version incompatible upgrades")
@@ -561,6 +561,24 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 					}
 
 					if len(processGroup.ProcessGroupConditions) > 0 {
+						// Ignore process groups that are stuck in terminating.If the global synchronization mode is active
+						// this will be the case for all the transaction system process groups as one process groups is
+						// blocked to be removed.
+						if processGroup.GetConditionTime(fdbv1beta2.ResourcesTerminating) != nil {
+							log.Println(
+								"processGroup",
+								processGroup.ProcessGroupID,
+								"will be ignored since the process group is in terminating",
+							)
+							continue
+						}
+
+						log.Println(
+							"processGroup",
+							processGroup.ProcessGroupID,
+							"processes conditions:",
+							processGroup.ProcessGroupConditions,
+						)
 						processesToUpdate++
 					}
 				}
@@ -570,6 +588,17 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 				return processesToUpdate
 			}).WithTimeout(30 * time.Minute).WithPolling(5 * time.Second).MustPassRepeatedly(5).Should(BeNumerically("==", 0))
 
+			// Remove the buggify option and make sure that the terminating processes are removed.
+			fdbCluster.SetBuggifyBlockRemoval(nil)
+			Eventually(func(g Gomega) {
+				processGroups := fdbCluster.GetCluster().Status.ProcessGroups
+
+				for _, processGroup := range processGroups {
+					g.Expect(processGroup.GetConditionTime(fdbv1beta2.ResourcesTerminating)).
+						To(BeNil())
+				}
+			}).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())
+
 			// Make sure the cluster has no data loss.
 			fdbCluster.EnsureTeamTrackersHaveMinReplicas()
 		},

internal/coordination/coordination.go

@@ -408,7 +408,6 @@ func UpdateGlobalCoordinationState(
 			}
 
 			processes := GetProcessesFromProcessMap(processGroup.ProcessGroupID, processesMap)
-
 			var excluded bool
 			for _, process := range processes {
 				excluded = excluded || process.Excluded
@@ -418,10 +417,14 @@ func UpdateGlobalCoordinationState(
 			// exclusion timestamp set or because the processes are excluded.
 			if !(processGroup.IsExcluded() || excluded) {
 				if _, ok := pendingForExclusion[processGroup.ProcessGroupID]; !ok {
+					logger.V(1).
+						Info("Adding to pendingForExclusion", "processGroupID", processGroup.ProcessGroupID, "reason", "process group is marked for removal but not excluded")
 					updatesPendingForExclusion[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionAdd
 				}
 
 				if _, ok := pendingForInclusion[processGroup.ProcessGroupID]; !ok {
+					logger.V(1).
+						Info("Adding to pendingForInclusion", "processGroupID", processGroup.ProcessGroupID, "reason", "process group is marked for removal but not excluded")
 					updatesPendingForInclusion[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionAdd
 				}
 			} else {
@@ -469,20 +472,29 @@ func UpdateGlobalCoordinationState(
 				}
 			}
 
-			// If the process group is marked for removal and the resources are stuck in terminating or the processes are not running, we should
-			// remove them from the restart list, because there are no processes to restart.
-			if processGroup.GetConditionTime(fdbv1beta2.ResourcesTerminating) != nil ||
-				processGroup.GetConditionTime(fdbv1beta2.MissingProcesses) != nil {
-				if _, ok := pendingForRestart[processGroup.ProcessGroupID]; ok {
+			// If the process group is stuck in terminating, we can add it to the ready for inclusion list.
+			if processGroup.GetConditionTime(fdbv1beta2.ResourcesTerminating) != nil {
+				if _, ok := pendingForInclusion[processGroup.ProcessGroupID]; !ok {
 					logger.V(1).
-						Info("Removing from pendingForRestart", "processGroupID", processGroup.ProcessGroupID, "reason", "process group is marked for removal")
-					updatesPendingForRestart[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionDelete
+						Info("Adding to pendingForInclusion and readyForInclusion", "processGroupID", processGroup.ProcessGroupID, "reason", "process group is marked for removal and in terminating")
+					updatesPendingForInclusion[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionAdd
+					updatesReadyForInclusion[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionAdd
 				}
 
-				if _, ok := readyForRestart[processGroup.ProcessGroupID]; ok {
-					logger.V(1).
-						Info("Removing from readyForRestart", "processGroupID", processGroup.ProcessGroupID, "reason", "process group is marked for removal")
-					updatesReadyForRestart[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionDelete
+				// If the process group is marked for removal and the resources are stuck in terminating or the processes are not running, we should
+				// remove them from the restart list, because there are no processes to restart.
+				if processGroup.GetConditionTime(fdbv1beta2.MissingProcesses) != nil {
+					if _, ok := pendingForRestart[processGroup.ProcessGroupID]; ok {
+						logger.V(1).
+							Info("Removing from pendingForRestart", "processGroupID", processGroup.ProcessGroupID, "reason", "process group is marked for removal")
+						updatesPendingForRestart[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionDelete
+					}
+
+					if _, ok := readyForRestart[processGroup.ProcessGroupID]; ok {
+						logger.V(1).
+							Info("Removing from readyForRestart", "processGroupID", processGroup.ProcessGroupID, "reason", "process group is marked for removal")
+						updatesReadyForRestart[processGroup.ProcessGroupID] = fdbv1beta2.UpdateActionDelete
+					}
 				}
 			}