Add tests for nodes crashing right after a timeout

The tests are done by extending the existing tests in the following manner:
Once an Epoch instance times out and broadcasts an empty vote, the test splits the execution flow
into two different alternative executions:
1) If the node proceeds without crashing
2) If the node crashes and restarts

The test executes both alternatives by having the second execution clone the storage, WAL and block builder's state
and then run the same code of the first execution.

Signed-off-by: Yacov Manevich <[email protected]>

epoch.go

@@ -241,6 +241,22 @@ func (e *Epoch) restoreEmptyVoteRecord(r []byte) error {
 
 	emptyVotes := e.getOrCreateEmptyVoteSetForRound(vote.Round)
 	emptyVotes.timedOut = true
+
+	signature, err := vote.Sign(e.Signer)
+	if err != nil {
+		return err
+	}
+
+	emptyVote := &EmptyVote{
+		Signature: Signature{
+			Signer: e.ID,
+			Value: signature,
+		},
+		Vote: vote,
+	}
+
+	emptyVotes.votes[string(e.ID)] = emptyVote
+
 	return nil
 }
 
@@ -304,6 +320,24 @@ func (e *Epoch) resumeFromWal(records [][]byte) error {
 		lastMessage := Message{Notarization: &notarization}
 		e.Comm.Broadcast(&lastMessage)
 		return e.doNotarized(notarization.Vote.Round)
+	case record.EmptyVoteRecordType:
+		tbsEmptyVote, err := ParseEmptyVoteRecord(lastRecord)
+		if err != nil {
+			return err
+		}
+		signature, err := tbsEmptyVote.Sign(e.Signer)
+		if err != nil {
+			return err
+		}
+		lastMessage := Message{EmptyVoteMessage: &EmptyVote{
+			Vote: tbsEmptyVote,
+			Signature: Signature{
+				Signer: e.ID,
+				Value: signature,
+			},
+		}}
+		e.Comm.Broadcast(&lastMessage)
+		return e.startRound()
 	case record.EmptyNotarizationRecordType:
 		emptyNotarization, err := EmptyNotarizationFromRecord(lastRecord, e.QCDeserializer)
 		if err != nil {
@@ -327,7 +361,7 @@ func (e *Epoch) resumeFromWal(records [][]byte) error {
 
 		return e.startRound()
 	default:
-		return errors.New("unknown record type")
+		return fmt.Errorf("unknown record type (%d)", recordType)
 	}
 }