e2e: add mid-rollout image change test

Makefile

@@ -73,6 +73,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex>
 		ARTIFACTS=$(ARTIFACTS) \
 		BINK_NODE_IMAGE_DIGEST=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) \
 		BINK_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \
+		BINK_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \
 		go test -timeout 30m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) .
 
 ##@ Build
@@ -93,10 +94,13 @@ buildimg: ## Build container image.
 	$(CONTAINER_TOOL) build -t $(IMG) .
 
 .PHONY: build-update-image
-build-update-image: ## Build a derived node image for update testing and push to bink registry.
+build-update-image: ## Build derived node images for update testing and push to bink registry.
 	@printf 'FROM localhost:5000/node:latest\nRUN touch /usr/share/update-marker\n' | \
 		podman build -t localhost:5000/node:update -f - .
 	podman push --tls-verify=false localhost:5000/node:update
+	@printf 'FROM localhost:5000/node:latest\nRUN touch /usr/share/update-marker-2\n' | \
+		podman build -t localhost:5000/node:update2 -f - .
+	podman push --tls-verify=false localhost:5000/node:update2
 
 ##@ Deployment
 

test/e2e/bootcnode_test.go

@@ -7,12 +7,15 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/types"
 	corev1 "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -198,53 +201,8 @@ func TestUpdateReboot(t *testing.T) {
 	}).WithTimeout(3*time.Minute).Should(BeFalse(), "expected node to be schedulable after update")
 
 	// Phase 6: Verify update marker exists on the host via daemon pod exec.
-	g.Eventually(func() ([]corev1.Pod, error) {
-		var pods corev1.PodList
-		err := env.Client.List(ctx, &pods,
-			client.InNamespace("bootc-operator"),
-			client.MatchingLabels{
-				"app.kubernetes.io/name":      "bootc-operator",
-				"app.kubernetes.io/component": "daemon",
-			},
-		)
-		if err != nil {
-			return nil, err
-		}
-		var matched []corev1.Pod
-		for _, p := range pods.Items {
-			if p.Spec.NodeName == nodeName {
-				matched = append(matched, p)
-			}
-		}
-		return matched, nil
-	}).WithTimeout(1*time.Minute).Should(ConsistOf(
-		HaveField("Status.Phase", corev1.PodRunning),
-	), "expected running daemon pod on %s", nodeName)
-
-	// Retrieve the daemon pod for exec.
-	var daemonPods corev1.PodList
-	g.Expect(env.Client.List(ctx, &daemonPods,
-		client.InNamespace("bootc-operator"),
-		client.MatchingLabels{
-			"app.kubernetes.io/name":      "bootc-operator",
-			"app.kubernetes.io/component": "daemon",
-		},
-	)).To(Succeed())
-	var daemonPod corev1.Pod
-	for _, p := range daemonPods.Items {
-		if p.Spec.NodeName == nodeName {
-			daemonPod = p
-			break
-		}
-	}
-
-	kubeconfigPath := os.Getenv("KUBECONFIG")
-	cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfigPath,
-		"-n", "bootc-operator", "exec", daemonPod.Name, "--",
+	execOnNode(t, g, env, ctx, nodeName,
 		"stat", "/proc/1/root/usr/share/update-marker")
-	out, err := cmd.CombinedOutput()
-	g.Expect(err).NotTo(HaveOccurred(),
-		fmt.Sprintf("expected update-marker to exist on host, kubectl exec output: %s", string(out)))
 
 	t.Logf("Verified update-marker exists on host via daemon pod")
 
@@ -370,6 +328,190 @@ func TestTagResolution(t *testing.T) {
 	t.Logf("Node %q is Idle with update image", nodeName)
 }
 
+// TestMidRolloutImageChange provisions two worker nodes, starts a rollout
+// to one update image, then switches the target to a different update image
+// while one node is rebooting. It verifies both nodes converge to the final
+// image and that the non-rebooting node does not wastefully reboot into the
+// first update image.
+func TestMidRolloutImageChange(t *testing.T) {
+	g := NewWithT(t)
+	g.SetDefaultEventuallyTimeout(pollTimeout)
+	g.SetDefaultEventuallyPollingInterval(pollInterval)
+
+	env := e2eutil.New(t)
+	nodeA := env.AddNode(t)
+	nodeB := env.AddNode(t)
+
+	ctx := context.Background()
+
+	// Phase 1: Create pool with original image and wait for both nodes Idle.
+	pool := env.NewPool("mid-rollout", env.NodeImageDigestedPullSpec())
+	g.Expect(env.Client.Create(ctx, pool)).To(Succeed())
+
+	for _, nodeName := range []string{nodeA, nodeB} {
+		g.Eventually(func() (bootcv1alpha1.BootcNode, error) {
+			var bn bootcv1alpha1.BootcNode
+			err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
+			return bn, err
+		}).WithTimeout(3 * time.Minute).Should(SatisfyAll(
+			HaveField("Status.Booted", Not(BeNil())),
+			HaveField("Status.Conditions", ContainElement(And(
+				HaveField("Type", bootcv1alpha1.NodeIdle),
+				HaveField("Status", metav1.ConditionTrue),
+				HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
+			))),
+		))
+	}
+
+	t.Logf("Both nodes are Idle with original image")
+
+	// Phase 2: Record boot count for both nodes before the rollout.
+	bootCountBefore := make(map[string]string)
+	for _, nodeName := range []string{nodeA, nodeB} {
+		bootCountBefore[nodeName] = getBootCount(t, env, ctx, nodeName)
+		t.Logf("Node %q boot count before: %s", nodeName, bootCountBefore[nodeName])
+	}
+
+	// Phase 3: Patch pool to first update image.
+	updateRef1 := env.NodeImageUpdateDigestedPullSpec()
+
+	modified := pool.DeepCopy()
+	modified.Spec.Image.Ref = updateRef1
+	g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
+	*pool = *modified
+
+	t.Logf("Patched pool to first update image %s", updateRef1)
+
+	// Phase 4: Wait for any node to reach Rebooting.
+	var rebootingNode string
+	g.Eventually(func(g Gomega) string {
+		for _, name := range []string{nodeA, nodeB} {
+			var bn bootcv1alpha1.BootcNode
+			g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: name}, &bn)).To(Succeed())
+
+			cond := meta.FindStatusCondition(bn.Status.Conditions, bootcv1alpha1.NodeIdle)
+			if cond != nil &&
+				cond.Status == metav1.ConditionFalse &&
+				cond.Reason == bootcv1alpha1.NodeReasonRebooting {
+				rebootingNode = name
+				return name
+			}
+		}
+		return ""
+	}).WithTimeout(5 * time.Minute).ShouldNot(BeEmpty())
+
+	var otherNode string
+	if rebootingNode == nodeA {
+		otherNode = nodeB
+	} else {
+		otherNode = nodeA
+	}
+
+	t.Logf("Node %q is Rebooting, node %q is the other node", rebootingNode, otherNode)
+
+	// Phase 5: Immediately switch target to second update image.
+	updateRef2 := env.NodeImageUpdate2DigestedPullSpec()
+
+	modified = pool.DeepCopy()
+	modified.Spec.Image.Ref = updateRef2
+	g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
+	*pool = *modified
+
+	t.Logf("Switched pool to second update image %s", updateRef2)
+
+	// Phase 6: Wait for both nodes to be Idle with the second update image.
+	for _, nodeName := range []string{nodeA, nodeB} {
+		g.Eventually(func() (bootcv1alpha1.BootcNode, error) {
+			var bn bootcv1alpha1.BootcNode
+			err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)
+			return bn, err
+		}).WithTimeout(8*time.Minute).Should(SatisfyAll(
+			HaveField("Status.Booted", Not(BeNil())),
+			HaveField("Status.Booted.ImageDigest", Equal(env.NodeImageUpdate2Digest())),
+			HaveField("Status.Conditions", ContainElement(And(
+				HaveField("Type", bootcv1alpha1.NodeIdle),
+				HaveField("Status", metav1.ConditionTrue),
+				HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
+			))),
+		), "expected node %s to reach Idle with second update image", nodeName)
+	}
+
+	t.Logf("Both nodes are Idle with second update image")
+
+	// Phase 7: Verify the other node (the one that was NOT rebooting when
+	// we switched images) did not wastefully reboot into the first update
+	// image. It should have rebooted exactly once (into the second image).
+	bootCountAfter := getBootCount(t, env, ctx, otherNode)
+	t.Logf("Node %q boot count after: %s (before: %s)", otherNode, bootCountAfter, bootCountBefore[otherNode])
+
+	beforeCount, err := strconv.Atoi(bootCountBefore[otherNode])
+	g.Expect(err).NotTo(HaveOccurred(), "parsing before boot count")
+	afterCount, err := strconv.Atoi(bootCountAfter)
+	g.Expect(err).NotTo(HaveOccurred(), "parsing after boot count")
+
+	g.Expect(afterCount-beforeCount).To(Equal(1),
+		"expected other node %s to reboot exactly once (from %d to %d), "+
+			"an extra reboot means it wastefully booted into the first update image",
+		otherNode, beforeCount, afterCount)
+
+	t.Logf("Verified node %q rebooted exactly once (no wasteful reboot into first image)", otherNode)
+}
+
+// execOnNode finds the running daemon pod on the given node and executes
+// a command inside it via kubectl exec. It returns the command output.
+func execOnNode(t *testing.T, g Gomega, env *e2eutil.Env, ctx context.Context, nodeName string, command ...string) string {
+	t.Helper()
+
+	var daemonPod corev1.Pod
+	g.Eventually(func(g Gomega) string {
+		var pods corev1.PodList
+		g.Expect(env.Client.List(ctx, &pods,
+			client.InNamespace("bootc-operator"),
+			client.MatchingLabels{
+				"app.kubernetes.io/name":      "bootc-operator",
+				"app.kubernetes.io/component": "daemon",
+			},
+		)).To(Succeed())
+		for _, p := range pods.Items {
+			if p.Spec.NodeName == nodeName && p.Status.Phase == corev1.PodRunning {
+				daemonPod = p
+				return p.Name
+			}
+		}
+		return ""
+	}).WithTimeout(1*time.Minute).ShouldNot(BeEmpty(),
+		"expected running daemon pod on %s", nodeName)
+
+	kubeconfigPath := os.Getenv("KUBECONFIG")
+	args := append([]string{"--kubeconfig", kubeconfigPath,
+		"-n", "bootc-operator", "exec", daemonPod.Name, "--"}, command...)
+	cmd := exec.CommandContext(ctx, "kubectl", args...)
+	out, err := cmd.CombinedOutput()
+	g.Expect(err).NotTo(HaveOccurred(),
+		fmt.Sprintf("kubectl exec on %s failed: %s", nodeName, string(out)))
+
+	return strings.TrimSpace(string(out))
+}
+
+// getBootCount returns the number of boots on a node by running
+// journalctl --list-boots inside the daemon pod via kubectl exec.
+func getBootCount(t *testing.T, env *e2eutil.Env, ctx context.Context, nodeName string) string {
+	t.Helper()
+
+	g := NewWithT(t)
+
+	out := execOnNode(t, g, env, ctx, nodeName,
+		"nsenter", "-m/proc/1/ns/mnt", "--", "journalctl", "--list-boots")
+
+	count := 0
+	for _, line := range strings.Split(out, "\n") {
+		if strings.TrimSpace(line) != "" {
+			count++
+		}
+	}
+	return fmt.Sprintf("%d", count)
+}
+
 // TestPauseResume provisions a worker node, starts an update with the
 // pool paused, verifies the node stages but does not reboot, then resumes
 // and verifies the update completes.

test/e2e/e2eutil/env.go

@@ -66,6 +66,10 @@ type Env struct {
 	// nodeImageUpdateDigest is the manifest digest of the update image
 	// (e.g. "sha256:def456..."). Empty when not built.
 	nodeImageUpdateDigest string
+
+	// nodeImageUpdate2Digest is the manifest digest of the second update
+	// image (e.g. "sha256:789abc..."). Used by mid-rollout image change tests.
+	nodeImageUpdate2Digest string
 }
 
 // New connects to an existing bink cluster and returns an Env ready
@@ -96,16 +100,21 @@ func New(t *testing.T) *Env {
 	if nodeImageUpdateDigest == "" {
 		t.Fatal("BINK_NODE_IMAGE_UPDATE_DIGEST must be set")
 	}
+	nodeImageUpdate2Digest := os.Getenv("BINK_NODE_IMAGE_UPDATE2_DIGEST")
+	if nodeImageUpdate2Digest == "" {
+		t.Fatal("BINK_NODE_IMAGE_UPDATE2_DIGEST must be set")
+	}
 
 	k8sClient := buildClient(t, kubeconfigPath)
 
 	env := &Env{
-		Client:                k8sClient,
-		clusterName:           clusterName,
-		testID:                sanitizeTestName(t.Name()),
-		nodeImageDigest:       nodeImageDigest,
-		nodeImageRegistry:     nodeImageRegistry,
-		nodeImageUpdateDigest: nodeImageUpdateDigest,
+		Client:                 k8sClient,
+		clusterName:            clusterName,
+		testID:                 sanitizeTestName(t.Name()),
+		nodeImageDigest:        nodeImageDigest,
+		nodeImageRegistry:      nodeImageRegistry,
+		nodeImageUpdateDigest:  nodeImageUpdateDigest,
+		nodeImageUpdate2Digest: nodeImageUpdate2Digest,
 	}
 
 	t.Cleanup(func() {
@@ -217,13 +226,19 @@ func (e *Env) TestLabels() map[string]string {
 	return map[string]string{LabelE2ETest: e.testID}
 }
 
+// digestedPullSpec builds a digest-qualified image reference from the
+// registry and the given digest. Returns "" if either is empty.
+func (e *Env) digestedPullSpec(digest string) string {
+	if e.nodeImageRegistry == "" || digest == "" {
+		return ""
+	}
+	return e.nodeImageRegistry + "@" + digest
+}
+
 // NodeImageDigestedPullSpec returns the digest-qualified reference for the
 // seeded node image (e.g. "registry.cluster.local:5000/node@sha256:abc123").
 func (e *Env) NodeImageDigestedPullSpec() string {
-	if e.nodeImageRegistry == "" || e.nodeImageDigest == "" {
-		return ""
-	}
-	return e.nodeImageRegistry + "@" + e.nodeImageDigest
+	return e.digestedPullSpec(e.nodeImageDigest)
 }
 
 // NodeImageTagRef returns the tag-based reference for the seeded node
@@ -240,17 +255,25 @@ func (e *Env) NodeImageDigest() string {
 // NodeImageUpdateDigestedPullSpec returns the digest-qualified reference for the
 // update image (e.g. "registry.cluster.local:5000/node@sha256:def456").
 func (e *Env) NodeImageUpdateDigestedPullSpec() string {
-	if e.nodeImageRegistry == "" || e.nodeImageUpdateDigest == "" {
-		return ""
-	}
-	return e.nodeImageRegistry + "@" + e.nodeImageUpdateDigest
+	return e.digestedPullSpec(e.nodeImageUpdateDigest)
 }
 
 // NodeImageUpdateDigest returns the manifest digest of the update image.
 func (e *Env) NodeImageUpdateDigest() string {
 	return e.nodeImageUpdateDigest
 }
 
+// NodeImageUpdate2DigestedPullSpec returns the digest-qualified reference for the
+// second update image (e.g. "registry.cluster.local:5000/node@sha256:789abc").
+func (e *Env) NodeImageUpdate2DigestedPullSpec() string {
+	return e.digestedPullSpec(e.nodeImageUpdate2Digest)
+}
+
+// NodeImageUpdate2Digest returns the manifest digest of the second update image.
+func (e *Env) NodeImageUpdate2Digest() string {
+	return e.nodeImageUpdate2Digest
+}
+
 // RetagImage reads the image at srcRef from the localhost registry and
 // tags it as dstTag.
 func RetagImage(t *testing.T, srcRef, dstTag string) {