Add support + mission for new leader election algorithm

Part of stellar/stellar-core#4387

This change adds support for testing the new leader election algorithm
by generating configs that make use of auto quorum set configuration
where possible. In doing so, it switches many tests over to auto quorum
set configuration.

This change also adds a new set of missions that blend nodes running the
old and new leader election algorithm to assess the impact of nodes using
these different algorithms simultaneously. The good news is that in
running the test I did not see much increase in timeouts with either
majority (no ledger had more than 1 timeout).

Author: bboston7

doc/missions.md

@@ -146,3 +146,12 @@ Run a network with a mix of fast and slow nodes. This mission allows to test int
 ## MissionMaxTPSMixed
 
 Stress test a network of simulated Tier1 topology with a mix of classic and soroban traffic and report maximum achieved throughput.
+
+## MissionMixedNominationLeaderElectionWithOldMajority
+
+Run a network with a mix of nodes running the old and new nomination leader election algorithms. Contains a majority of nodes running the old algorithm.
+
+
+## MissionMixedNominationLeaderElectionWithNewMajority
+
+Run a network with a mix of nodes running the old and new nomination leader election algorithms. Contains a majority of nodes running the new algorithm.

src/FSLibrary/FSLibrary.fsproj

@@ -56,6 +56,7 @@
     <Compile Include="MissionHistoryTestnetPerformance.fs" />
     <Compile Include="MissionMixedImageLoadGeneration.fs" />
     <Compile Include="MissionMixedImageNetworkSurvey.fs" />
+    <Compile Include="MissionMixedNominationLeaderElection.fs" />
     <Compile Include="MissionVersionMixConsensus.fs" />
     <Compile Include="MissionSorobanLoadGeneration.fs" />
     <Compile Include="MissionSorobanConfigUpgrades.fs" />

src/FSLibrary/MissionMixedNominationLeaderElection.fs

@@ -0,0 +1,65 @@
+// Copyright 2024 Stellar Development Foundation and contributors. Licensed
+// under the Apache License, Version 2.0. See the COPYING file at the root
+// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
+
+// This mission runs a network with a mix of nodes running the old and new
+// nomination leader election algorithms.
+
+module MissionMixedNominationLeaderElection
+
+open StellarCoreHTTP
+open StellarCorePeer
+open StellarCoreSet
+open StellarFormation
+open StellarMissionContext
+open StellarStatefulSets
+open StellarSupercluster
+
+let mixedNominationAlgorithm (oldCount: int) (context: MissionContext) =
+    let oldNodeCount = oldCount
+    let newNodeCount = 3 - oldCount
+
+    let oldName = "core-old-leader-election"
+    let newName = "core-new-leader-election"
+
+    let oldCoreSet =
+        MakeLiveCoreSet
+            oldName
+            { CoreSetOptions.GetDefault context.image with
+                  nodeCount = oldNodeCount
+                  invariantChecks = AllInvariantsExceptBucketConsistencyChecks
+                  accelerateTime = false
+                  dumpDatabase = false
+                  forceOldStyleLeaderElection = true }
+
+    let newCoreSet =
+        MakeLiveCoreSet
+            newName
+            { CoreSetOptions.GetDefault context.image with
+                  nodeCount = newNodeCount
+                  invariantChecks = AllInvariantsExceptBucketConsistencyChecks
+                  accelerateTime = false
+                  dumpDatabase = false
+                  requireAutoQset = true }
+
+    let coreSets = [ oldCoreSet; newCoreSet ]
+
+    context.Execute
+        coreSets
+        None
+        (fun (formation: StellarFormation) ->
+            formation.WaitUntilSynced coreSets
+
+            let peer = formation.NetworkCfg.GetPeer oldCoreSet 0
+            peer.WaitForFewLedgers(3)
+            formation.UpgradeProtocolToLatest coreSets
+            peer.WaitForLatestProtocol()
+            peer.WaitForFewLedgers(60) // About 5 minutes
+
+            // Check everything is still in sync
+            formation.CheckNoErrorsAndPairwiseConsistency()
+            formation.EnsureAllNodesInSync coreSets)
+
+let mixedNominationLeaderElectionWithOldMajority (context: MissionContext) = mixedNominationAlgorithm 2 context
+
+let mixedNominationLeaderElectionWithNewMajority (context: MissionContext) = mixedNominationAlgorithm 1 context

src/FSLibrary/MissionProtocolUpgradeWithLoad.fs

@@ -18,7 +18,11 @@ let protocolUpgradeWithLoad (context: MissionContext) =
             "core"
             { CoreSetOptions.GetDefault context.image with
                   invariantChecks = AllInvariantsExceptBucketConsistencyChecks
-                  dumpDatabase = false }
+                  dumpDatabase = false
+                  // Set `requireAutoQset` to `true` as an extra check that this
+                  // mission uses the application-specific nomination leader
+                  // election protocol.
+                  requireAutoQset = true }
 
     let context =
         { context.WithSmallLoadgenOptions with

src/FSLibrary/StellarCoreCfg.fs

@@ -5,7 +5,10 @@
 module StellarCoreCfg
 
 open FSharp.Data
+open Logging
 open Nett
+open System
+open System.Collections
 open System.Text.RegularExpressions
 open StellarCoreSet
 open StellarNetworkCfg
@@ -141,6 +144,7 @@ type StellarCoreCfg =
       networkPassphrase: NetworkPassphrase
       nodeSeed: KeyPair
       nodeIsValidator: bool
+      homeDomain: string option
       runStandalone: bool
       image: string
       preferredPeers: PeerDnsName list
@@ -156,6 +160,7 @@ type StellarCoreCfg =
       unsafeQuorum: bool
       failureSafety: int
       quorumSet: QuorumSet
+      forceOldStyleLeaderElection: bool
       historyNodes: Map<PeerShortName, PeerDnsName>
       historyGetCommands: Map<PeerShortName, string>
       localHistory: bool
@@ -208,6 +213,10 @@ type StellarCoreCfg =
         t.Add("COMMANDS", logLevelCommands) |> ignore
         t.Add("CATCHUP_COMPLETE", self.catchupMode = CatchupComplete) |> ignore
 
+        match self.homeDomain with
+        | None -> ()
+        | Some hd -> t.Add("NODE_HOME_DOMAIN", hd) |> ignore
+
         match self.network.missionContext.peerReadingCapacity, self.network.missionContext.peerFloodCapacity with
         | None, None -> ()
         | Some read, Some flood ->
@@ -306,6 +315,9 @@ type StellarCoreCfg =
         t.Add("QUORUM_INTERSECTION_CHECKER", false) |> ignore
         t.Add("MANUAL_CLOSE", self.manualClose) |> ignore
 
+        if self.forceOldStyleLeaderElection then
+            t.Add("FORCE_OLD_STYLE_LEADER_ELECTION", true) |> ignore
+
         let invList =
             match self.invariantChecks with
             | AllInvariants -> [ ".*" ]
@@ -321,7 +333,7 @@ type StellarCoreCfg =
         | Some duration -> t.Add("ARTIFICIALLY_SET_SURVEY_PHASE_DURATION_FOR_TESTING", duration) |> ignore
 
         // Add tables (and subtables, recursively) for qsets.
-        let rec addQsetAt (label: string) (qs: QuorumSet) =
+        let rec addExplicitQsetAt (label: string) (qs: ExplicitQuorumSet) =
             let validators : string array =
                 Map.toArray qs.validators
                 |> Array.map (fun (n: PeerShortName, k: KeyPair) -> sprintf "%s %s" k.Address n.StringName)
@@ -333,11 +345,54 @@ type StellarCoreCfg =
             | None -> ()
             | Some (pct) -> innerTab.Add("THRESHOLD_PERCENT", pct) |> ignore
 
-            Array.iteri (fun (i: int) (qs: QuorumSet) -> addQsetAt (sprintf "%s.sub%d" label i) qs) qs.innerQuorumSets
+            Array.iteri
+                (fun (i: int) (qs: ExplicitQuorumSet) -> addExplicitQsetAt (sprintf "%s.sub%d" label i) qs)
+                qs.innerQuorumSets
+
+        let homeDomainToTable (homeDomain: HomeDomain) =
+            let ret = Toml.Create()
+            ret.Add("HOME_DOMAIN", homeDomain.name) |> ignore
+
+            ret.Add("QUALITY", homeDomain.quality.ToString() |> String.map Char.ToUpper)
+            |> ignore
+
+            ret
+
+        let autoValidatorToTable (autoValidator: AutoValidator) =
+            let ret = Toml.Create()
+            ret.Add("NAME", autoValidator.name.StringName) |> ignore
+            ret.Add("HOME_DOMAIN", autoValidator.homeDomain) |> ignore
+            ret.Add("PUBLIC_KEY", autoValidator.keys.Address) |> ignore
+
+            match Map.tryFind autoValidator.name self.historyGetCommands with
+            | Some cmd -> ret.Add("HISTORY", cmd) |> ignore
+            | None ->
+                match Map.tryFind autoValidator.name self.historyNodes with
+                | Some dnsName -> ret.Add("HISTORY", Map.find "get" (remoteHist dnsName)) |> ignore
+                | None -> ()
 
-        addQsetAt "QUORUM_SET" self.quorumSet
+            ret
 
         let localTab = t.Add("HISTORY", Toml.Create(), TomlObjectFactory.RequireTomlObject()).Added
+
+        match self.quorumSet with
+        | ExplicitQuorumSet qs ->
+            addExplicitQsetAt "QUORUM_SET" qs
+
+            for historyNode in self.historyNodes do
+                localTab.Add(historyNode.Key.StringName, remoteHist historyNode.Value) |> ignore
+
+            for historyGetCommand in self.historyGetCommands do
+                localTab.Add(historyGetCommand.Key.StringName, getHist historyGetCommand.Value)
+                |> ignore
+        | AutoQuorumSet qs ->
+            let homeDomainsTab = t.Add("HOME_DOMAINS", ([]: IDictionary list)).Added
+            List.iter (fun hd -> homeDomainsTab.Add(homeDomainToTable hd) |> ignore) qs.homeDomains
+            let validatorsTab = t.Add("VALIDATORS", ([]: IDictionary list)).Added
+            // Filter out local node
+            let validators = List.filter (fun (v: AutoValidator) -> v.keys <> self.nodeSeed) qs.validators
+            List.iter (fun v -> validatorsTab.Add(autoValidatorToTable v) |> ignore) validators
+
         // When simulateApplyWeight = Some _, stellar-core sets MODE_STORES_HISTORY
         // which is used for simulations that only test consensus.
         // In such cases, we should not pass put and mkdir commands.
@@ -351,12 +406,6 @@ type StellarCoreCfg =
             )
             |> ignore
 
-        for historyNode in self.historyNodes do
-            localTab.Add(historyNode.Key.StringName, remoteHist historyNode.Value) |> ignore
-
-        for historyGetCommand in self.historyGetCommands do
-            localTab.Add(historyGetCommand.Key.StringName, getHist historyGetCommand.Value)
-            |> ignore
 
         t
 
@@ -425,17 +474,60 @@ type NetworkCfg with
         self.CoreSetList |> List.map processCoreSet |> List.concat |> Map.ofList
 
     member self.QuorumSet(o: CoreSetOptions) : QuorumSet =
-        let ofNameKeyList (nks: (PeerShortName * KeyPair) array) (threshold: int option) : QuorumSet =
-            { thresholdPercent = threshold
-              validators = Map.ofArray nks
-              innerQuorumSets = [||] }
+        let toExplicitQSet (nks: (PeerShortName * KeyPair) array) (threshold: int option) : QuorumSet =
+            LogInfo "Using explicit quorum set configuration"
+
+            ExplicitQuorumSet
+                { thresholdPercent = threshold
+                  validators = Map.ofArray nks
+                  innerQuorumSets = [||] }
+
+        let toAutoQSet (nks: (PeerShortName * KeyPair) list) (homeDomain: string) =
+            LogInfo "Using auto quorum set configuration"
+            let homeDomains = [ { name = homeDomain; quality = High } ]
+
+            let validators =
+                List.map (fun (n: PeerShortName, k) -> { name = n; homeDomain = homeDomain; keys = k }) nks
+
+            AutoQuorumSet { homeDomains = homeDomains; validators = validators }
+
+        // Generate a QuorumSet from an array of (PeerShortName, KeyPair) pairs.
+        // Produces a simple flat qset of nodes. Uses auto quorum set
+        // configuration if possible.
+        let simpleQuorum (nks: (PeerShortName * KeyPair) array) =
+            match o.homeDomain with
+            | Some hd ->
+                if nks.Length >= 3 then
+                    // There are enough validators to use auto quorum set config
+                    toAutoQSet (List.ofArray nks) hd
+                else if o.requireAutoQset then
+                    failwith "Auto quorum set configuration requires at least 3 validators"
+                else
+                    // Fall back on manual quorum set configuration
+                    toExplicitQSet nks None
+            | None ->
+                if o.requireAutoQset then
+                    failwith "Auto quorum set configuration requires a home domain"
+                else
+                    toExplicitQSet nks None
+
+        let checkAutoQSetIncompatability (mode: string) =
+            if o.requireAutoQset then
+                failwithf "Auto quorum set configuration is incompatible with %s" mode
+            else
+                ()
 
         match o.quorumSet with
-        | AllPeersQuorum -> ofNameKeyList (self.GetNameKeyListAll()) None
-        | CoreSetQuorum (ns) -> ofNameKeyList (self.GetNameKeyList [ ns ]) None
-        | CoreSetQuorumList (q) -> ofNameKeyList (self.GetNameKeyList q) None
-        | CoreSetQuorumListWithThreshold (q, t) -> ofNameKeyList (self.GetNameKeyList q) (Some(t))
-        | ExplicitQuorum (e) -> e
+        | AllPeersQuorum -> simpleQuorum (self.GetNameKeyListAll())
+        | CoreSetQuorum (ns) -> simpleQuorum (self.GetNameKeyList [ ns ])
+        | CoreSetQuorumList (q) -> simpleQuorum (self.GetNameKeyList q)
+        | CoreSetQuorumListWithThreshold (q, t) ->
+            checkAutoQSetIncompatability "CoreSetQuorumListWithThreshold"
+            toExplicitQSet (self.GetNameKeyList q) (Some(t))
+        | ExplicitQuorum (e) ->
+            checkAutoQSetIncompatability "ExplicitQuorum"
+            ExplicitQuorumSet e
+        | AutoQuorum q -> AutoQuorumSet q
 
     member self.HistoryNodes(o: CoreSetOptions) : Map<PeerShortName, PeerDnsName> =
         match o.historyNodes, o.quorumSet with
@@ -471,6 +563,7 @@ type NetworkCfg with
           networkPassphrase = self.networkPassphrase
           nodeSeed = KeyPair.Random()
           nodeIsValidator = false
+          homeDomain = None
           runStandalone = false
           image = opts.image
           preferredPeers = self.PreferredPeers opts
@@ -486,6 +579,7 @@ type NetworkCfg with
           unsafeQuorum = opts.unsafeQuorum
           failureSafety = 0
           quorumSet = self.QuorumSet opts
+          forceOldStyleLeaderElection = opts.forceOldStyleLeaderElection
           historyNodes = self.HistoryNodes opts
           historyGetCommands = opts.historyGetCommands
           localHistory = opts.localHistory
@@ -503,6 +597,7 @@ type NetworkCfg with
           networkPassphrase = self.networkPassphrase
           nodeSeed = c.keys.[i]
           nodeIsValidator = c.options.validate
+          homeDomain = c.options.homeDomain
           runStandalone = false
           image = c.options.image
           preferredPeers =
@@ -524,6 +619,7 @@ type NetworkCfg with
           unsafeQuorum = c.options.unsafeQuorum
           failureSafety = 0
           quorumSet = self.QuorumSet c.options
+          forceOldStyleLeaderElection = c.options.forceOldStyleLeaderElection
           historyNodes = self.HistoryNodes c.options
           historyGetCommands = c.options.historyGetCommands
           localHistory = c.options.localHistory