Merge branch 'master' into leo/refactor-storage-collections-for-an

Author: zhangchiqing

.github/workflows/flaky-test-monitor.yml

@@ -55,7 +55,7 @@ jobs:
       matrix:
         targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}}
     # need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4

access/api.go

@@ -314,10 +314,17 @@ type CompatibleRange struct {
 
 // NodeVersionInfo contains information about node, such as semver, commit, sporkID, protocolVersion, etc
 type NodeVersionInfo struct {
-	Semver               string
-	Commit               string
-	SporkId              flow.Identifier
-	ProtocolVersion      uint64
+	Semver  string
+	Commit  string
+	SporkId flow.Identifier
+	// ProtocolVersion is the deprecated protocol version number.
+	// Deprecated: Previously this referred to the major software version as of the most recent spork.
+	// Replaced by protocol_state_version.
+	ProtocolVersion uint64
+	// ProtocolStateVersion is the Protocol State version as of the latest finalized block.
+	// This tracks the schema version of the Protocol State and is used to coordinate breaking changes in the Protocol.
+	// Version numbers are monotonically increasing.
+	ProtocolStateVersion uint64
 	SporkRootBlockHeight uint64
 	NodeRootBlockHeight  uint64
 	CompatibleRange      *CompatibleRange

access/handler.go

@@ -98,6 +98,7 @@ func (h *Handler) GetNodeVersionInfo(
 			Commit:               nodeVersionInfo.Commit,
 			SporkId:              nodeVersionInfo.SporkId[:],
 			ProtocolVersion:      nodeVersionInfo.ProtocolVersion,
+			ProtocolStateVersion: nodeVersionInfo.ProtocolStateVersion,
 			SporkRootBlockHeight: nodeVersionInfo.SporkRootBlockHeight,
 			NodeRootBlockHeight:  nodeVersionInfo.NodeRootBlockHeight,
 			CompatibleRange:      CompatibleRangeToMessage(nodeVersionInfo.CompatibleRange),

admin/README.md

@@ -120,3 +120,8 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"
 ```
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "backfill-tx-error-messages", "data": { "start-height": 340, "end-height": 343, "execution-node-ids":["ec7b934df29248d574ae1cc33ae77f22f0fcf96a79e009224c46374d1837824e", "8cbdc8d24a28899a33140cb68d4146cd6f2f6c18c57f54c299f26351d126919e"] }}'
 ```
+
+### Trigger chunk data pack pebble database checkpoint
+```
+curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "create-chunk-data-packs-checkpoint" }'
+```

admin/commands/storage/backfill_tx_error_messages.go

@@ -7,7 +7,6 @@ import (
 	"github.com/onflow/flow-go/admin"
 	"github.com/onflow/flow-go/admin/commands"
 	"github.com/onflow/flow-go/engine/access/ingestion/tx_error_messages"
-	commonrpc "github.com/onflow/flow-go/engine/common/rpc"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/filter"
 	"github.com/onflow/flow-go/state/protocol"
@@ -177,7 +176,7 @@ func (b *BackfillTxErrorMessagesCommand) parseExecutionNodeIds(executionNodeIdsI
 		if len(executionNodeIds) == 0 {
 			return nil, admin.NewInvalidAdminReqParameterError("execution-node-ids", "must be a non empty list of strings", executionNodeIdsIn)
 		}
-		requestedENIdentifiers, err := commonrpc.IdentifierList(executionNodeIds)
+		requestedENIdentifiers, err := flow.IdentifierListFromHex(executionNodeIds)
 		if err != nil {
 			return nil, admin.NewInvalidAdminReqParameterError("execution-node-ids", err.Error(), executionNodeIdsIn)
 		}

admin/commands/storage/chunk_data_packs_db_checkpoint.go

@@ -0,0 +1,57 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/cockroachdb/pebble"
+	"github.com/rs/zerolog/log"
+
+	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/admin/commands"
+)
+
+var _ commands.AdminCommand = (*ChunksCheckpointCommand)(nil)
+
+// ChunksCheckpointCommand creates a checkpoint for pebble database for querying the data
+// while keeping the node alive.
+type ChunksCheckpointCommand struct {
+	checkpointDir  string
+	chunkDataPacks *pebble.DB
+}
+
+func NewChunksCheckpointCommand(dir string, chunkDataPacks *pebble.DB) commands.AdminCommand {
+	return &ChunksCheckpointCommand{
+		checkpointDir:  dir,
+		chunkDataPacks: chunkDataPacks,
+	}
+}
+
+func (c *ChunksCheckpointCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) {
+	log.Info().Msgf("admintool: creating chunkDataPacks database checkpoint")
+
+	targetDir := nextTmpFolder(c.checkpointDir)
+
+	log.Info().Msgf("admintool: creating chunkDataPacks database checkpoint at: %v", targetDir)
+
+	err := c.chunkDataPacks.Checkpoint(targetDir)
+	if err != nil {
+		return nil, admin.NewInvalidAdminReqErrorf("failed to create checkpoint at %v: %w", targetDir, err)
+	}
+
+	log.Info().Msgf("admintool: successfully created chunkDataPacks database checkpoint at: %v", targetDir)
+
+	return fmt.Sprintf("successfully created chunkDataPacks db checkpoint at %v", targetDir), nil
+}
+
+func (c *ChunksCheckpointCommand) Validator(req *admin.CommandRequest) error {
+	return nil
+}
+
+func nextTmpFolder(dir string) string {
+	// use timestamp as folder name
+	folderName := time.Now().Format("2006-01-02_15-04-05")
+	return path.Join(dir, folderName)
+}

cmd/access/node_builder/access_node_builder.go

@@ -2020,12 +2020,12 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) {
 
 			}
 
-			preferredENIdentifiers, err := commonrpc.IdentifierList(backendConfig.PreferredExecutionNodeIDs)
+			preferredENIdentifiers, err := flow.IdentifierListFromHex(backendConfig.PreferredExecutionNodeIDs)
 			if err != nil {
 				return nil, fmt.Errorf("failed to convert node id string to Flow Identifier for preferred EN map: %w", err)
 			}
 
-			fixedENIdentifiers, err := commonrpc.IdentifierList(backendConfig.FixedExecutionNodeIDs)
+			fixedENIdentifiers, err := flow.IdentifierListFromHex(backendConfig.FixedExecutionNodeIDs)
 			if err != nil {
 				return nil, fmt.Errorf("failed to convert node id string to Flow Identifier for fixed EN map: %w", err)
 			}

cmd/bootstrap/cmd/block.go

@@ -42,7 +42,10 @@ func constructRootEpochEvents(
 	participants flow.IdentityList,
 	assignments flow.AssignmentList,
 	clusterQCs []*flow.QuorumCertificate,
-	dkgData dkg.DKGData) (*flow.EpochSetup, *flow.EpochCommit) {
+	dkgData dkg.ThresholdKeySet,
+	dkgIndexMap flow.DKGIndexMap,
+) (*flow.EpochSetup, *flow.EpochCommit) {
+
 	epochSetup := &flow.EpochSetup{
 		Counter:            flagEpochCounter,
 		FirstView:          firstView,
@@ -77,6 +80,7 @@ func constructRootEpochEvents(
 		ClusterQCs:         flow.ClusterQCVoteDatasFromQCs(qcsWithSignerIDs),
 		DKGGroupKey:        dkgData.PubGroupKey,
 		DKGParticipantKeys: dkgData.PubKeyShares,
+		DKGIndexMap:        dkgIndexMap,
 	}
 	return epochSetup, epochCommit
 }

cmd/bootstrap/cmd/dkg.go

@@ -10,34 +10,31 @@ import (
 	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/dkg"
 	"github.com/onflow/flow-go/model/encodable"
+	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/state/protocol/inmem"
 )
 
-func runBeaconKG(nodes []model.NodeInfo) dkg.DKGData {
+func runBeaconKG(nodes []model.NodeInfo) (dkg.ThresholdKeySet, flow.DKGIndexMap) {
 	n := len(nodes)
-
 	log.Info().Msgf("read %v node infos for DKG", n)
 
 	log.Debug().Msgf("will run DKG")
-	var dkgData dkg.DKGData
-	var err error
-	dkgData, err = bootstrapDKG.RandomBeaconKG(n, GenerateRandomSeed(crypto.KeyGenSeedMinLen))
+	randomBeaconData, err := bootstrapDKG.RandomBeaconKG(n, GenerateRandomSeed(crypto.KeyGenSeedMinLen))
 	if err != nil {
 		log.Fatal().Err(err).Msg("error running DKG")
 	}
 	log.Info().Msgf("finished running DKG")
 
-	pubKeyShares := make([]encodable.RandomBeaconPubKey, 0, len(dkgData.PubKeyShares))
-	for _, pubKey := range dkgData.PubKeyShares {
-		pubKeyShares = append(pubKeyShares, encodable.RandomBeaconPubKey{PublicKey: pubKey})
-	}
-
-	privKeyShares := make([]encodable.RandomBeaconPrivKey, 0, len(dkgData.PrivKeyShares))
-	for i, privKey := range dkgData.PrivKeyShares {
+	encodableParticipants := make([]inmem.ThresholdParticipant, 0, len(nodes))
+	for i, privKey := range randomBeaconData.PrivKeyShares {
 		nodeID := nodes[i].NodeID
 
 		encKey := encodable.RandomBeaconPrivKey{PrivateKey: privKey}
-		privKeyShares = append(privKeyShares, encKey)
+		encodableParticipants = append(encodableParticipants, inmem.ThresholdParticipant{
+			PrivKeyShare: encKey,
+			PubKeyShare:  encodable.RandomBeaconPubKey{PublicKey: randomBeaconData.PubKeyShares[i]},
+			NodeID:       nodeID,
+		})
 
 		err = common.WriteJSON(fmt.Sprintf(model.PathRandomBeaconPriv, nodeID), flagOutdir, encKey)
 		if err != nil {
@@ -46,18 +43,22 @@ func runBeaconKG(nodes []model.NodeInfo) dkg.DKGData {
 		log.Info().Msgf("wrote file %s/%s", flagOutdir, fmt.Sprintf(model.PathRandomBeaconPriv, nodeID))
 	}
 
+	indexMap := make(flow.DKGIndexMap, len(nodes))
+	for i, node := range nodes {
+		indexMap[node.NodeID] = i
+	}
+
 	// write full DKG info that will be used to construct QC
-	err = common.WriteJSON(model.PathRootDKGData, flagOutdir, inmem.EncodableFullDKG{
+	err = common.WriteJSON(model.PathRootDKGData, flagOutdir, inmem.ThresholdKeySet{
 		GroupKey: encodable.RandomBeaconPubKey{
-			PublicKey: dkgData.PubGroupKey,
+			PublicKey: randomBeaconData.PubGroupKey,
 		},
-		PubKeyShares:  pubKeyShares,
-		PrivKeyShares: privKeyShares,
+		Participants: encodableParticipants,
 	})
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
 	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathRootDKGData)
 
-	return dkgData
+	return randomBeaconData, indexMap
 }

cmd/bootstrap/cmd/finalize.go

@@ -152,7 +152,7 @@ func finalize(cmd *cobra.Command, args []string) {
 	log.Info().Msgf("received votes total: %v", len(votes))
 
 	log.Info().Msg("reading dkg data")
-	dkgData := readDKGData()
+	dkgData, _ := readRandomBeaconKeys()
 	log.Info().Msg("")
 
 	log.Info().Msg("reading intermediary bootstrapping data")
@@ -189,10 +189,9 @@ func finalize(cmd *cobra.Command, args []string) {
 		result,
 		seal,
 		rootQC,
-		intermediaryData.ProtocolVersion,
 		func(epochStateID flow.Identifier) (protocol_state.KVStoreAPI, error) {
 			return kvstore.NewDefaultKVStore(
-				intermediaryData.EpochCommitSafetyThreshold,
+				intermediaryData.FinalizationSafetyThreshold,
 				intermediaryData.EpochExtensionViewCount,
 				epochStateID)
 		},
@@ -352,29 +351,28 @@ func readRootBlock() *flow.Block {
 	return rootBlock
 }
 
-// readDKGData reads DKG data from disc, this file needs to be prepared with
-// rootblock command
-func readDKGData() dkg.DKGData {
-	encodableDKG, err := utils.ReadData[inmem.EncodableFullDKG](flagDKGDataPath)
+// readRandomBeaconKeys reads the threshold key data from disc.
+// This file needs to be prepared with rootblock command
+func readRandomBeaconKeys() (dkg.ThresholdKeySet, flow.DKGIndexMap) {
+	encodableDKG, err := utils.ReadData[inmem.ThresholdKeySet](flagDKGDataPath)
 	if err != nil {
-		log.Fatal().Err(err).Msg("could not read DKG data")
+		log.Fatal().Err(err).Msg("loading threshold key data for Random Beacon failed")
 	}
 
-	dkgData := dkg.DKGData{
+	dkgData := dkg.ThresholdKeySet{
 		PrivKeyShares: nil,
 		PubGroupKey:   encodableDKG.GroupKey.PublicKey,
 		PubKeyShares:  nil,
 	}
 
-	for _, pubKey := range encodableDKG.PubKeyShares {
-		dkgData.PubKeyShares = append(dkgData.PubKeyShares, pubKey.PublicKey)
-	}
-
-	for _, privKey := range encodableDKG.PrivKeyShares {
-		dkgData.PrivKeyShares = append(dkgData.PrivKeyShares, privKey.PrivateKey)
+	indexMap := make(flow.DKGIndexMap, len(encodableDKG.Participants))
+	for i, participant := range encodableDKG.Participants {
+		dkgData.PubKeyShares = append(dkgData.PubKeyShares, participant.PubKeyShare.PublicKey)
+		dkgData.PrivKeyShares = append(dkgData.PrivKeyShares, participant.PrivKeyShare.PrivateKey)
+		indexMap[participant.NodeID] = i
 	}
 
-	return dkgData
+	return dkgData, indexMap
 }
 
 // Validation utility methods ------------------------------------------------

cmd/bootstrap/cmd/finalize_test.go

@@ -74,7 +74,7 @@ func TestFinalize_HappyPath(t *testing.T) {
 		flagNumViewsInEpoch = 100_000
 		flagNumViewsInStakingAuction = 50_000
 		flagNumViewsInDKGPhase = 2_000
-		flagEpochCommitSafetyThreshold = 1_000
+		flagFinalizationSafetyThreshold = 1_000
 		flagEpochExtensionViewCount = 100_000
 		flagUseDefaultEpochTargetEndTime = true
 		flagEpochTimingRefCounter = 0

cmd/bootstrap/cmd/intermediary.go

@@ -18,9 +18,8 @@ type IntermediaryBootstrappingData struct {
 // like the root block).
 // This is used to pass data between the rootblock command and the finalize command.
 type IntermediaryParamsData struct {
-	ProtocolVersion            uint
-	EpochCommitSafetyThreshold uint64
-	EpochExtensionViewCount    uint64
+	FinalizationSafetyThreshold uint64
+	EpochExtensionViewCount     uint64
 }
 
 // IntermediaryEpochData stores the root epoch and the epoch config for the execution state

cmd/bootstrap/cmd/qc.go

@@ -13,10 +13,10 @@ import (
 )
 
 // constructRootQC constructs root QC based on root block, votes and dkg info
-func constructRootQC(block *flow.Block, votes []*model.Vote, allNodes, internalNodes []bootstrap.NodeInfo, dkgData dkg.DKGData) *flow.QuorumCertificate {
+func constructRootQC(block *flow.Block, votes []*model.Vote, allNodes, internalNodes []bootstrap.NodeInfo, randomBeaconData dkg.ThresholdKeySet) *flow.QuorumCertificate {
 
 	identities := bootstrap.ToIdentityList(allNodes)
-	participantData, err := run.GenerateQCParticipantData(allNodes, internalNodes, dkgData)
+	participantData, err := run.GenerateQCParticipantData(allNodes, internalNodes, randomBeaconData)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to generate QC participant data")
 	}
@@ -36,8 +36,8 @@ func constructRootQC(block *flow.Block, votes []*model.Vote, allNodes, internalN
 }
 
 // NOTE: allNodes must be in the same order as when generating the DKG
-func constructRootVotes(block *flow.Block, allNodes, internalNodes []bootstrap.NodeInfo, dkgData dkg.DKGData) {
-	participantData, err := run.GenerateQCParticipantData(allNodes, internalNodes, dkgData)
+func constructRootVotes(block *flow.Block, allNodes, internalNodes []bootstrap.NodeInfo, randomBeaconData dkg.ThresholdKeySet) {
+	participantData, err := run.GenerateQCParticipantData(allNodes, internalNodes, randomBeaconData)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to generate QC participant data")
 	}