[#234] Fix data loss in Replicated cache when multiple nodes join sequentially (#235)

Author: rezigned

lib/nebulex/adapters/replicated.ex

@@ -755,20 +755,15 @@ defmodule Nebulex.Adapters.Replicated.Bootstrap do
 
       nodes ->
         # Sync process:
-        # 1. Push a new generation on all cluster nodes to make the newer one
-        #    empty.
-        # 2. Copy cached data from one of the cluster nodes; entries will be
-        #    streamed from the older generation since the newer one should be
-        #    empty.
-        # 3. Push a new generation on the current/new node to make it a mirror
-        #    of the other cluster nodes.
-        # 4. Reset GC timer for ell cluster nodes (making the generation timer
-        #    gap among cluster nodes as small as possible).
-        with :ok <- maybe_run_on_nodes(adapter_meta, nodes, :new_generation),
-             :ok <- copy_entries_from_nodes(adapter_meta, nodes),
-             :ok <- maybe_run_on_nodes(adapter_meta, [node()], :new_generation) do
-          maybe_run_on_nodes(adapter_meta, nodes, :reset_generation_timer)
-        end
+        #
+        # 1. Copy cached data from existing cluster nodes to the joining node.
+        #    At this point, both the newer and older generations are still present,
+        #    so data may be copied from both, ensuring no data is lost due to premature
+        #    generation rotation.
+        # 2. Reset the generation GC timer on all nodes to synchronize their GC intervals,
+        #    minimizing timer gaps and ensuring consistent generation rotation across the cluster.
+        :ok = copy_entries_from_nodes(adapter_meta, nodes)
+        maybe_run_on_nodes(adapter_meta, cluster_nodes, :reset_generation_timer)
     end
   end
 

test/nebulex/adapters/replicated_test.exs

@@ -219,6 +219,42 @@ defmodule Nebulex.Adapters.ReplicatedTest do
         :ok = stop_caches(node_pid_list)
       end)
     end
+
+    test "cache data is not lost when multiple nodes join sequentially" do
+      # Start with existing nodes
+      assert Replicated.put(:a, 1) == :ok
+      assert Replicated.get(:a) == 1
+
+      # Add node 3
+      node3 = [:"[email protected]"]
+      pids = start_caches(node3, [{Replicated, [name: @cache_name]}])
+
+      wait_until(fn ->
+        assert Replicated.nodes() |> :lists.usort() == :lists.usort(cluster_nodes() ++ node3)
+      end)
+
+      # Verify :a is still there on all nodes
+      wait_until(10, 1000, fn ->
+        assert_for_all_replicas(Replicated, :get, [:a], 1)
+      end)
+
+      # Add node 4
+      node4 = [:"[email protected]"]
+      pids = pids ++ start_caches(node4, [{Replicated, [name: @cache_name]}])
+
+      wait_until(fn ->
+        assert Replicated.nodes() |> :lists.usort() ==
+                 :lists.usort(cluster_nodes() ++ node3 ++ node4)
+      end)
+
+      # Now check that data is still present
+      wait_until(10, 1000, fn ->
+        assert_for_all_replicas(Replicated, :get, [:a], 1)
+      end)
+
+      # Clean up
+      :ok = stop_caches(pids)
+    end
   end
 
   describe "write-like operations locked" do