Merge pull request #119 from joaomlneto/fix-115

Add DeterministicCompletableFuture

Author: joaomlneto

docs/implementing-protocols.md

@@ -50,6 +50,11 @@ check whether safety invariants of distributed consensus are broken.
 
 - **Non-determinism**: For reproducibility we require the removal of any non-determinism! This can manifest itself in
   many ways:
-  - Avoid interfaces and collections that do not guarantee order, such as `Set` or `Map`: use the `OrderedSet` or
-    `OrderedMap` interfaces instead, and `TreeSet` or `LinkedHashMap` implementations to ensure order.
+    - Avoid interfaces and collections that do not guarantee order, such as `Set` or `Map`: use the `OrderedSet` or
+      `OrderedMap` interfaces instead, and `TreeSet` or `LinkedHashMap` implementations to ensure order.
+    - Avoid the use of Java's `CompletableFuture`: the implementation in Java is non-deterministic, as it uses a
+      `ForkJoinPool`
+      that can execute tasks in parallel - tasks that are submitted to the `CompletableFuture` can be executed in any
+      order, and this can lead to non-deterministic behavior. Use our own `DeterministicCompletableFuture`, which will
+      execute the tasks in the order they were submitted to it.
 

simulator/src/main/java/byzzbench/simulator/protocols/pbft_java/Ticket.java

@@ -4,6 +4,7 @@
 import byzzbench.simulator.protocols.pbft_java.message.PrePrepareMessage;
 import byzzbench.simulator.protocols.pbft_java.message.PrepareMessage;
 import byzzbench.simulator.protocols.pbft_java.message.RequestMessage;
+import byzzbench.simulator.utils.FIFOCompletableFuture;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 
@@ -26,7 +27,7 @@ public class Ticket<O extends Serializable, R extends Serializable> implements S
     private final Collection<Serializable> messages = new ConcurrentLinkedQueue<>();
     private final transient AtomicReference<ReplicaTicketPhase> phase = new AtomicReference<>(ReplicaTicketPhase.PRE_PREPARE);
     @Getter
-    private final transient CompletableFuture<R> result = new CompletableFuture<>();
+    private final transient CompletableFuture<R> result = new FIFOCompletableFuture<>();
     @Getter
     private volatile RequestMessage request;
 

simulator/src/main/java/byzzbench/simulator/utils/FIFOCompletableFuture.java

@@ -0,0 +1,22 @@
+package byzzbench.simulator.utils;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+/**
+ * A {@link CompletableFuture} that uses a single thread {@link Executor} to guarantee ordering of dependent tasks.
+ * Dependent tasks are executed in the order they are added (FIFO).
+ *
+ * @param <T> The result type returned by this future's {@code join} and {@code get} methods
+ */
+public class FIFOCompletableFuture<T> extends CompletableFuture<T> {
+    private static final Executor executor = Executors.newSingleThreadExecutor(
+            runnable -> new Thread(runnable, "Custom-Single-Thread")
+    );
+
+    @Override
+    public final Executor defaultExecutor() {
+        return executor;
+    }
+}