Merge pull request #128 from joaomlneto/hBFT

hbft to main

Author: joaomlneto

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "java.configuration.updateBuildConfiguration": "interactive"
+}

schedules/hbft known violation.json

@@ -0,0 +1 @@
+{"events":[{"type":"ClientRequest","eventId":2,"createdAt":1736960554.6270425,"status":"DELIVERED","recipientId":"B","senderId":"C0","timestamp":1736960554627,"payload":{"timestamp":1736960554627,"operation":"C0/1","signed":false,"signedBy":null,"type":"DefaultClientRequest"},"deliveredAt":null},{"type":"ClientRequest","eventId":9,"createdAt":1736960554.6280425,"status":"DELIVERED","recipientId":"B","senderId":"C1","timestamp":1736960554628,"payload":{"timestamp":1736960554628,"operation":"C1/1","signed":false,"signedBy":null,"type":"DefaultClientRequest"},"deliveredAt":null},{"type":"Message","eventId":16,"createdAt":1736960556.0801437,"status":"DELIVERED","recipientId":"A","senderId":"B","timestamp":0,"payload":{"viewNumber":1,"sequenceNumber":1,"digest":"CtMEMkjjICiJ0zfwSspofgpOE5M=","request":{"operation":"C0/1","timestamp":1736960554627,"clientId":"C0","signed":false,"signedBy":null,"type":"REQUEST"},"signed":true,"signedBy":"B","type":"PREPARE"},"deliveredAt":null},{"type":"Message","eventId":18,"createdAt":1736960556.0801437,"status":"DELIVERED","recipientId":"D","senderId":"B","timestamp":0,"payload":{"viewNumber":1,"sequenceNumber":1,"digest":"CtMEMkjjICiJ0zfwSspofgpOE5M=","request":{"operation":"C0/1","timestamp":1736960554627,"clientId":"C0","signed":false,"signedBy":null,"type":"REQUEST"},"signed":true,"signedBy":"B","type":"PREPARE"},"deliveredAt":null},{"type":"Message","eventId":21,"createdAt":1736960556.0811455,"status":"DELIVERED","recipientId":"D","senderId":"B","timestamp":0,"payload":{"viewNumber":1,"sequenceNumber":1,"digest":"CtMEMkjjICiJ0zfwSspofgpOE5M=","request":{"operation":"C0/1","timestamp":1736960554627,"clientId":"C0","signed":false,"signedBy":null,"type":"REQUEST"},"replicaId":"B","speculativeHistory":{"history":{},"empty":true,"greatestSeqNumber":0,"requests":{}},"signed":true,"signedBy":"B","type":"COMMIT"},"deliveredAt":null},{"type":"Message","eventId":31,"createdAt":1736960562.4083266,"status":"DELIVERED","recipientId":"D","senderId":"A","timestamp":0,"payload":{"viewNumber":1,"sequenceNumber":1,"digest":"CtMEMkjjICiJ0zfwSspofgpOE5M=","request":{"operation":"C0/1","timestamp":1736960554627,"clientId":"C0","signed":false,"signedBy":null,"type":"REQUEST"},"replicaId":"A","speculativeHistory":{"history":{"1":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"}},"empty":false,"greatestSeqNumber":1,"requests":{"1":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"}}},"signed":true,"signedBy":"A","type":"COMMIT"},"deliveredAt":null},{"type":"ClientRequest","eventId":13,"createdAt":1736960554.6280425,"status":"DELIVERED","recipientId":"D","senderId":"C1","timestamp":1736960554628,"payload":{"timestamp":1736960554628,"operation":"C1/1","signed":false,"signedBy":null,"type":"DefaultClientRequest"},"deliveredAt":null},{"type":"Timeout","eventId":42,"createdAt":1736960574.9117787,"status":"DELIVERED","description":"REQUEST1736960554628","nodeId":"D","timeout":10,"expiresAt":10.001,"task":{},"recipientId":"D","deliveredAt":null},{"type":"Timeout","eventId":22,"createdAt":1736960556.9483867,"status":"DELIVERED","description":"REQUEST1736960554628","nodeId":"B","timeout":10,"expiresAt":10.002,"task":{},"recipientId":"B","deliveredAt":null},{"type":"MutateMessage","eventId":52,"senderId":"B","recipientId":"C","payload":{"eventId":24,"mutatorId":"hbft-prepare-sec-dec"},"createdAt":1736960584.0889707,"status":"DELIVERED","deliveredAt":null},{"type":"Message","eventId":24,"createdAt":1736960556.9483867,"status":"DELIVERED","recipientId":"C","senderId":"B","timestamp":0,"payload":{"viewNumber":1,"sequenceNumber":1,"digest":"vK5bH85oOq0vaso+TSfV+PLPXFE=","request":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"},"signed":true,"signedBy":"B","type":"PREPARE"},"deliveredAt":null},{"type":"Message","eventId":47,"createdAt":1736960576.5642834,"status":"DELIVERED","recipientId":"C","senderId":"D","timestamp":0,"payload":{"newViewNumber":2,"speculativeHistoryP":null,"speculativeHistoryQ":null,"requestsR":{"1":{"operation":"C0/1","timestamp":1736960554627,"clientId":"C0","signed":false,"signedBy":null,"type":"REQUEST"}},"replicaId":"D","signed":true,"signedBy":"D","type":"VIEW-CHANGE"},"deliveredAt":null},{"type":"MutateMessage","eventId":56,"senderId":"B","recipientId":"C","payload":{"eventId":50,"mutatorId":"hbft-view-change-remove-first-request"},"createdAt":1736960613.561314,"status":"DELIVERED","deliveredAt":null},{"type":"MutateMessage","eventId":57,"senderId":"B","recipientId":"C","payload":{"eventId":50,"mutatorId":"hbft-view-change-decrement-last-request-seqNum"},"createdAt":1736960617.884014,"status":"DELIVERED","deliveredAt":null},{"type":"Message","eventId":50,"createdAt":1736960579.971162,"status":"DELIVERED","recipientId":"C","senderId":"B","timestamp":0,"payload":{"newViewNumber":2,"speculativeHistoryP":null,"speculativeHistoryQ":null,"requestsR":{"1":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"}},"replicaId":"B","signed":true,"signedBy":"B","type":"VIEW-CHANGE"},"deliveredAt":null},{"type":"Message","eventId":62,"createdAt":1736960619.5324345,"status":"DELIVERED","recipientId":"A","senderId":"C","timestamp":0,"payload":{"newViewNumber":2,"viewChangeProofs":[{"newViewNumber":2,"speculativeHistoryP":null,"speculativeHistoryQ":null,"requestsR":{"1":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"}},"replicaId":"B","signed":true,"signedBy":"B","type":"VIEW-CHANGE"},{"newViewNumber":2,"speculativeHistoryP":null,"speculativeHistoryQ":null,"requestsR":{"1":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"}},"replicaId":"C","signed":true,"signedBy":"C","type":"VIEW-CHANGE"},{"newViewNumber":2,"speculativeHistoryP":null,"speculativeHistoryQ":null,"requestsR":{"1":{"operation":"C0/1","timestamp":1736960554627,"clientId":"C0","signed":false,"signedBy":null,"type":"REQUEST"}},"replicaId":"D","signed":true,"signedBy":"D","type":"VIEW-CHANGE"}],"checkpoint":{"sequenceNumber":0,"history":null},"speculativeHistory":{"history":{"1":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"}},"empty":false,"greatestSeqNumber":1,"requests":{"1":{"operation":"C1/1","timestamp":1736960554628,"clientId":"C1","signed":false,"signedBy":null,"type":"REQUEST"}}},"signed":true,"signedBy":"C","type":"NEW-VIEW"},"deliveredAt":null}],"brokenInvariants":[],"scenarioId":"hbft","finalized":false}

schedules/hbft-bug-or-potential-violation.json

@@ -175,7 +175,7 @@ public final void setupScenario() {
             this.markReplicaFaulty(replicaIds.get(i));
         }
 
-        this.getClients().values().forEach(Client::initialize);
+        //this.getClients().values().forEach(Client::initialize);
         this.getNodes().values().forEach(Node::initialize);
         this.scheduler.initializeScenario(this);
     }

schedules/hbft-random-0-0-violation.json

@@ -1,11 +1,12 @@
 package byzzbench.simulator;
 
+import byzzbench.simulator.protocols.hbft.message.ClientRequestMessage;
 import byzzbench.simulator.transport.MessagePayload;
 import byzzbench.simulator.utils.NonNull;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.Builder;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.SuperBuilder;
 
 import java.io.Serializable;
 import java.time.Duration;
@@ -19,31 +20,31 @@
  * The client is responsible for sending requests to the replicas in the system.
  */
 @Getter
-@Builder
+@SuperBuilder
 @RequiredArgsConstructor
 public class Client implements Serializable, Node {
     /**
      * The scenario object that this client belongs to.
      */
     @JsonIgnore
     @NonNull
-    private final transient Scenario scenario;
+    protected final transient Scenario scenario;
 
     /**
      * The unique ID of the client.
      */
     @NonNull
-    private final String id;
+    protected final String id;
 
     /**
      * The sequence number of the next request to be sent by the client.
      */
-    private final AtomicLong requestSequenceNumber = new AtomicLong(0);
+    protected final AtomicLong requestSequenceNumber = new AtomicLong(0);
 
     /**
      * The maximum number of requests that can be sent by the client.
      */
-    private final long maxRequests = 3;
+    protected final long maxRequests = 100;
 
     /**
      * The replies received by the client.
@@ -63,8 +64,8 @@ public void initialize() {
      */
     public void sendRequest() {
         String recipientId = this.getScenario().getReplicas().keySet().iterator().next();
-        String requestId = String.format("%s/%d", this.id, this.requestSequenceNumber.getAndIncrement());
-        this.getScenario().getTransport().sendClientRequest(this.id, requestId, recipientId);
+        MessagePayload payload = new ClientRequestMessage(this.getCurrentTime().toEpochMilli(), recipientId);
+        this.getScenario().getTransport().sendMessage(this, payload, recipientId);
     }
 
     /**
@@ -94,7 +95,7 @@ public Instant getCurrentTime() {
      * @return the timer object
      */
     public long setTimeout(String name, Runnable r, Duration timeout) {
-        return this.scenario.getTransport().setTimeout(this, r, timeout);
+        return this.scenario.getTransport().setTimeout(this, r, timeout, "REQUEST");
     }
 
     /**

simulator/src/main/java/byzzbench/simulator/BaseScenario.java

@@ -0,0 +1,215 @@
+package byzzbench.simulator;
+
+import byzzbench.simulator.protocols.hbft.message.*;
+import byzzbench.simulator.protocols.hbft.pojo.ClientReplyKey;
+import byzzbench.simulator.transport.MessagePayload;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.experimental.SuperBuilder;
+
+import java.io.Serializable;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.util.*;
+
+
+/**
+ * Represents a client in the system. Each client has a unique identifier.
+ * The client is responsible for sending requests to the replicas in the system.
+ */
+@Getter
+@SuperBuilder
+public class HbftClient extends Client {
+    /**
+     * The message digest algorithm to use for hashing messages.
+     */
+    @JsonIgnore
+    static MessageDigest md;
+
+    static {
+        try {
+            md = MessageDigest.getInstance("SHA-1");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * The replies received by the client.
+     */
+    private final SortedMap<String, SortedMap<ClientReplyKey, Collection<MessagePayload>>> hbftreplies = new TreeMap<>();
+
+    /**
+     * The request sequence number already completed.
+     */
+    private final Set<ClientReplyKey> completedRequests = new HashSet<>();
+
+    /**
+     * The sent requests.
+     */
+    private final SortedMap<Long, RequestMessage> sentRequests = new TreeMap<>();
+
+    /**
+     * The sent requests by timestamp.
+     */
+    private final SortedMap<Long, String> sentRequestsByTimestamp = new TreeMap<>();
+
+    /**
+     * timeouts
+     */
+    private final SortedMap<Long, Long> timeouts = new TreeMap<>();
+
+    /**
+     * Timeout for client in seconds
+     */
+    private final long timeout = 8;
+
+
+    /**
+     * As of hBFT 4.1, sends a request to all replica in the system.
+     */
+    @Override
+    public void sendRequest() {
+        String requestId = String.format("%s/%d", super.id, super.requestSequenceNumber.incrementAndGet());
+        long timestamp = this.getCurrentTime().toEpochMilli();
+        RequestMessage request = new RequestMessage(requestId, timestamp, super.id);
+        this.sentRequests.put(super.requestSequenceNumber.get(), request);
+        this.sentRequestsByTimestamp.put(timestamp, requestId);
+        this.broadcastRequest(timestamp, requestId);
+
+        // Set timeout
+        Long timeoutId = this.setTimeout("REQUEST", this::retransmitOrPanic, this.timeout);
+        timeouts.put(super.requestSequenceNumber.get(), timeoutId);
+    }
+
+    public void retransmitOrPanic() {
+        long tolerance = (long) Math.floor((super.scenario.getTransport().getNodeIds().size() - 1) / 3);
+        if (this.shouldRetransmit(tolerance)) {
+            String requestId = String.format("%s/%d", super.id, super.requestSequenceNumber.get());
+            // Based on hBFT 4.1 it uses the identical request
+            // TODO: It probably should not be the same timestamp
+            long timestamp = this.sentRequests.get(super.requestSequenceNumber.get()).getTimestamp();
+            this.broadcastRequest(timestamp, requestId);
+        } else if (this.shouldPanic(tolerance)) {
+            RequestMessage message = this.sentRequests.get(super.requestSequenceNumber.get());
+            PanicMessage panic = new PanicMessage(this.digest(message), this.getCurrentTime().toEpochMilli(), super.id);
+            super.scenario.getTransport().multicast(this, super.scenario.getTransport().getNodeIds(), panic);
+        }
+        this.clearTimeout(timeouts.get(super.requestSequenceNumber.get()));
+        Long timeoutId = this.setTimeout("REQUEST", this::retransmitOrPanic, this.timeout);
+        timeouts.put(super.requestSequenceNumber.get(), timeoutId);
+    }
+
+    private void broadcastRequest(long timestamp, String requestId) {
+        MessagePayload payload = new ClientRequestMessage(timestamp, requestId);
+        SortedSet<String> replicaIds = super.scenario.getTransport().getNodeIds();
+        getScenario().getTransport().multicast(this, replicaIds, payload);
+    }
+
+    /**
+     * Handles a reply received by the client.
+     *
+     * @param senderId The ID of the sender of the reply.
+     * @param payload  The payload received by the client.
+     */
+    public void handleMessage(String senderId, MessagePayload payload) {
+        if (!(payload instanceof ClientReplyMessage clientReplyMessage)) {
+            return;
+        }
+        ReplyMessage reply = clientReplyMessage.getReply();
+        ClientReplyKey key = new ClientReplyKey(reply.getResult().toString(), reply.getSequenceNumber());
+        // Default is for testing only
+        String currRequest = this.sentRequestsByTimestamp.getOrDefault(reply.getTimestamp(), "C/0");
+        this.hbftreplies.putIfAbsent(currRequest, new TreeMap<>());
+        this.hbftreplies.get(currRequest).putIfAbsent(key, new ArrayList<>());
+        this.hbftreplies.get(currRequest).get(key).add(reply);
+
+        /**
+         * If the client received 2f + 1 correct replies,
+         * and the request has not been completed yet.
+         */
+        if (this.completedReplies(clientReplyMessage.getTolerance())
+                && !this.completedRequests.contains(key)
+                && super.requestSequenceNumber.get() <= this.maxRequests) {
+            this.completedRequests.add(key);
+            this.clearTimeout(this.timeouts.get(super.requestSequenceNumber.get()));
+            this.sendRequest();
+        }
+    }
+
+    /**
+     * Set a timeout for this replica.
+     *
+     * @param name    a name for the timeout
+     * @param r       the runnable to execute when the timeout occurs
+     * @param timeout the timeout duration
+     * @return the timer object
+     */
+    public long setTimeout(String name, Runnable r, long timeout) {
+        Duration duration = Duration.ofSeconds(timeout);
+        return super.scenario.getTransport().setTimeout(this, r, duration, name);
+    }
+
+    /**
+     * Checks whether client should retransmit the request
+     * if #replies < f + 1
+     */
+    public boolean shouldRetransmit(long tolerance) {
+        String currRequest = String.format("%s/%d", super.id, super.requestSequenceNumber.get());
+        if (!hbftreplies.containsKey(currRequest)) {
+            return true;
+        }
+        for (ClientReplyKey key : hbftreplies.get(currRequest).keySet()) {
+            return !(this.hbftreplies.get(currRequest).get(key).size() >= tolerance + 1);
+        }
+        return true;
+    }
+
+    /**
+     * Checks whether client should send PANIC
+     * if f + 1 <= #replies < 2f + 1
+     */
+    public boolean shouldPanic(long tolerance) {
+        String currRequest = String.format("%s/%d", super.id, super.requestSequenceNumber.get());
+        for (ClientReplyKey key : hbftreplies.get(currRequest).keySet()) {
+            return this.hbftreplies.get(currRequest).get(key).size() >= tolerance + 1
+                    && this.hbftreplies.get(currRequest).get(key).size() < tolerance * 2 + 1;
+        }
+        return false;
+    }
+
+    /**
+     * Checks whether it has received 2f + 1 replies
+     */
+    public boolean completedReplies(long tolerance) {
+        String currRequest = String.format("%s/%d", super.id, super.requestSequenceNumber.get());
+        if (!hbftreplies.containsKey(currRequest)) {
+            return false;
+        }
+        for (ClientReplyKey key : hbftreplies.get(currRequest).keySet()) {
+            if (this.hbftreplies.get(currRequest).get(key).size() >= 2 * tolerance + 1) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Clear all timeouts for this client.
+     */
+    // public void clearAllTimeouts() {
+    //     super.scenario.getTransport().clearClientTimeouts(super.id);
+    // }
+
+    /**
+     * Create a digest of a message.
+     *
+     * @param message the message to digest
+     * @return the digest of the message
+     */
+    public byte[] digest(Serializable message) {
+        return md.digest(message.toString().getBytes());
+    }
+
+}

simulator/src/main/java/byzzbench/simulator/Client.java

@@ -164,11 +164,12 @@ public void initialize() {
     /**
      * Handle a request received from a client.
      *
-     * @param clientId the ID of the client
-     * @param request  the request payload
+     * @param clientId  the ID of the client
+     * @param request   the request payload
+     * @param timestamp the time the request was created/sent
      * @throws Exception if an error occurs while handling the request
      */
-    public abstract void handleClientRequest(String clientId, Serializable request) throws Exception;
+    public abstract void handleClientRequest(String clientId, long timestamp, Serializable request) throws Exception;
 
     /**
      * Send a reply to a client.
@@ -186,8 +187,10 @@ public void sendReplyToClient(String clientId, Serializable reply) {
      * @param operation the operation to commit
      */
     public void commitOperation(long sequenceNumber, LogEntry operation) {
-        this.commitLog.add(sequenceNumber, operation);
-        this.notifyObserversLocalCommit(operation);
+        if (this.commitLog.get(sequenceNumber) == null) {
+            this.commitLog.add(sequenceNumber, operation);
+            this.notifyObserversLocalCommit(operation);
+        }
     }
 
     /**
@@ -214,7 +217,7 @@ public long setTimeout(String name, Runnable r, Duration timeout) {
             this.notifyObserversTimeout();
             r.run();
         };
-        return this.transport.setTimeout(this, wrapper, timeout);
+        return this.transport.setTimeout(this, wrapper, timeout, name);
     }
 
     /**
@@ -226,6 +229,13 @@ public void clearTimeout(long eventId) {
         this.transport.clearTimeout(this, eventId);
     }
 
+    /**
+     * Clear timeout based on description.
+     */
+    public void clearTimeout(String description) {
+        this.scenario.getTransport().clearTimeout(this, description);
+    }
+
     /**
      * Clear all timeouts for this replica.
      */

simulator/src/main/java/byzzbench/simulator/HbftClient.java

@@ -171,7 +171,7 @@ public final class BehaviorConfig {
     @Data
     public final class ScenarioConfig {
         private TerminationConfig termination = new TerminationConfig();
-        private String id = "pbft-java";
+        private String id = "hbft";
         private Map<String, String> params = new HashMap<>();
     }
 }