TxDAG based Parallel Transaction Execution implementation.

.gitmodules

@@ -6,3 +6,7 @@
 	path = tests/evm-benchmarks
 	url = https://github.com/ipsilon/evm-benchmarks
 	shallow = true
+[submodule "tests-dag"]
+	path = tests-dag
+	url = https://github.com/welkin22/txdag-test-file.git
+	shallow = true

Release Note - TxDAG based parallel transaction execution (PEVM).md

@@ -0,0 +1,145 @@
+# Release Note \- TxDAG based parallel transaction execution (PEVM) \- Alpha
+
+## Version
+
+**Alpha - Not production ready**
+
+**Based on op-geth v0.5.1** 
+
+## Description
+
+This is the first release of the op-geth that implements the TxDAG based parallel transaction execution for block synchronization. The goal is to speed up the execution of transactions in blocks by exploiting parallelism in EVM, and improve the performance further with the TxDAG data guidance by reducing the re-execution caused by transaction dependencies.
+
+### Release goals
+
+* Provide the parallel transaction execution implementation that is 100% compatible with the original EVM.  
+* Provide the raw TxDAG data in file format as a POC for TxDAG guided PEVM execution  
+* Provide the op-geth binary release that is verified to successfully sync the block from genesis to block \#15000000 with TxDAG and PEVM enabled, and with performance improved in some scenarios and no obvious performance drop overall.  
+* Provide the methodology to enable and use the TxDAG base PEVM for block syncing of opBNB.  
+* Provide the preliminary performance data and analysis for further improvement work
+
+### Options Added
+
+*    --parallel                          (default: false)                   ($GETH\_PARALLEL)
+
+          Enable the experimental parallel transaction execution mode, only valid in full  
+          sync mode (default = false)  
+
+
+*    --parallel.num value                (default: 0\)                       ($GETH\_PARALLEL\_NUM)
+
+          Number of slot for transaction execution, only valid in parallel mode (runtime  
+          calculated, no fixed default value)  
+
+
+*    --parallel.txdag                    (default: false)                   ($GETH\_PARALLEL\_TXDAG)
+
+          Enable the experimental parallel TxDAG generation, only valid in full sync mode  
+          (default = false)  
+
+
+*    --parallel.txdagfile value          (default: "./parallel-txdag-output.csv") ($GETH\_PARALLEL\_TXDAGFILE)
+
+          It indicates the TxDAG file path  
+
+
+*    --parallel.unordered-merge          (default: false)                   ($GETH\_PARALLEL\_UNORDERED\_MERGE)
+
+          Enable unordered merge mode, during the parallel confirm phase, merge  
+          transaction execution results without following the transaction order.  
+
+
+### Usage
+
+* Get the TxDAG data:  
+  * User could download the TxDAG data from [https://pub-c0627345c16f47ab858c9469133073a8.r2.dev/opbnb-txdag/parallel-txdag-output\_compare\_15000000.csv](https://pub-c0627345c16f47ab858c9469133073a8.r2.dev/opbnb-txdag/parallel-txdag-output_compare_15000000.csv)  
+* Use the following options of the geth to enable TxDAG based parallel execution  
+  * '--parallel'
+  * '--parallel.txtag'
+
+		Enable the TxDAG functionality of geth, this will tell EVM to check the TxDAG data existence before block execution
+
+  * '--parallel.txdagfile=./parallel-txdag-output\_compare\_15000000.csv'  
+
+        Specify the path of the TxDAG data file 
+
+  * '--parallel.unordered-merge'  
+
+        Trust the DAG data, enable the advanced unordered-merge optimization and skip conflict check.   
+  * '--parallel.num=4'
+
+        Optional, Specify the parallel execution number, if not assigned, use the core number for the running platform   
+
+
+### Example
+```
+op-geth \  
+  --datadir="./datadir" \  
+  --verbosity=4 \  
+  --http \ 
+  --http.corsdomain="*" \  
+  --http.vhosts="*" \  
+  --http.addr=0.0.0.0 \  
+  --http.port=8545 \  
+  --http.api=net,eth,engine,debug \  
+  --pprof \  
+  --pprof.port=6070 \  
+  --ws \  
+  --ws.addr=0.0.0.0 \  
+  --ws.port=8545 \  
+  --ws.origins="*" \  
+  --ws.api=eth,engine \  
+  --syncmode=full \  
+  --networkid=$CHAIN_ID \  
+  --txpool.globalslots=20000 \  
+  --txpool.globalqueue=5000 \  
+  --txpool.accountqueue=200 \  
+  --txpool.accountslots=200 \  
+  --txpool.nolocals=true \  
+  --txpool.pricelimit=1 \   
+  --cache.preimages \  
+  --allow-insecure-unlock \  
+  --authrpc.addr="0.0.0.0" \  
+  --authrpc.port="8551" \  
+  --authrpc.vhosts="*" \  
+  --authrpc.jwtsecret=./jwt.txt \  
+  --rpc.allow-unprotected-txs \  
+  --parallel \  
+  --parallel.txdag \  
+  --parallel.txdagfile=./parallel-txdag-output.csv \  
+  --parallel.unordered-merge \  
+  --parallel.num=4 \  
+  --gcmode=full \  
+  --metrics \  
+  --metrics.port 6068 \  
+  --metrics.addr 0.0.0.0 \  
+  --rollup.sequencerhttp=$L2_RPC \  
+  --rollup.disabletxpoolgossip=false \  
+  --bootnodes=$P2P_BOOTNODES
+```
+
+### 
+
+### Performance
+
+| Scenario    | Description                                                                                                 | mgasps<br/>(Original) | mgasps<br/>(PEVM) | Improvement | Comments                                                 |
+|-------------|-------------------------------------------------------------------------------------------------------------|----------------------|--------------|-------------|----------------------------------------------------------|
+| **A**       | **Internal test chain blocks with 250k accounts transfer to another 250k account**                          | **107**              | **202**      | **88%**     | **conflict rate(avg): ~0%<br/>txs in block(avg): 3103**  |
+| **B**       | **Internal test chain blocks with 250k accounts transfer to 1 fixed account**                               | **201**              | **216**      | **7%**      | **conflict rate(avg): ~95%<br/>txs in block(avg): 3644** |
+| **C**       | **Internal test chain blocks contain random selected txs with a mix of smart contract and native transfer** | **155**              | **228**      | **47%**     | **conflict rate(avg): ~81%<br/>txs in block(avg): 1356** |
+| **Mainnet** | **opBNB mainnet block range from #9m-9.3m**                                                                 | **9.3**              | **11.2**     | **20%**     | **conflict rate(avg): ~12%<br/>txs in block(avg): 25**   |
+| **Mainnet** | **opBNB mainnet block range from #11.9m-12.1m (mostly inscription txs)**                                    | **57**               | **61.1**     | **7%**      | **conflict rate(avg): ~50%<br/>txs in block(avg): 2195** |
+
+### Additional Info
+
+Conclusions for alpha release
+
+* The performance of PEVM is highly depend on scenarios and dependencies between txs in block  
+* Generally no obvious degradation of performance observed on mainnet sync (from block#1 to #15000000)
+
+#### TxDAG data solution
+
+The current file-based TxDAG data solution will be optimized with an alternative gasless transaction based
+TxDAG transfer in the future. It will guarantee that the TxDAG data will be available with the block transactions
+as soon as the block is generated. 
+More details in [BEP-396: Accelerate Block Execution by TxDAG](https://forum.bnbchain.org/t/bep-396-accelerate-block-execution-by-txdag/2869)

cmd/evm/blockrunner.go

@@ -92,7 +92,7 @@ func blockTestCmd(ctx *cli.Context) error {
 					fmt.Println(string(state.Dump(nil)))
 				}
 			}
-		}); err != nil {
+		}, "", true, false, false); err != nil {
 			return fmt.Errorf("test %v: %w", name, err)
 		}
 	}

cmd/geth/main.go

@@ -170,8 +170,14 @@ var (
 		utils.RollupComputePendingBlock,
 		utils.RollupHaltOnIncompatibleProtocolVersionFlag,
 		utils.RollupSuperchainUpgradesFlag,
+		utils.ParallelTxFlag,
+		utils.ParallelTxUnorderedMergeFlag,
+		utils.ParallelTxParallelMergeFlag,
+		utils.ParallelTxNumFlag,
 		utils.ParallelTxDAGFlag,
+		utils.ParallelTxDAGFileFlag,
 		utils.ParallelTxDAGSenderPrivFlag,
+		utils.ParallelTxDATMaxDepthRatioFlag,
 		configFileFlag,
 		utils.LogDebugFlag,
 		utils.LogBacktraceAtFlag,

cmd/utils/flags.go

@@ -1099,12 +1099,50 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 		Category: flags.MetricsCategory,
 	}
 
+	ParallelTxFlag = &cli.BoolFlag{
+		Name:     "parallel",
+		Usage:    "Enable the experimental parallel transaction execution mode, only valid in full sync mode (default = false)",
+		Category: flags.VMCategory,
+	}
+
+	ParallelTxUnorderedMergeFlag = &cli.BoolFlag{
+		Name:     "parallel.unordered-merge",
+		Usage:    "Enable unordered merge mode, during the parallel confirm phase, merge transaction execution results without following the transaction order.",
+		Category: flags.VMCategory,
+	}
+
+	ParallelTxParallelMergeFlag = &cli.BoolFlag{
+		Name:     "parallel.parallel-merge",
+		Usage:    "Enable concurrent merge mode, during the parallel confirm phase, multiple goroutines will be used to perform concurrent merging of execution results. This option will override parallel.unordered-merge",
+		Category: flags.VMCategory,
+	}
+
+	ParallelTxNumFlag = &cli.IntFlag{
+		Name:     "parallel.num",
+		Usage:    "Number of slot for transaction execution, only valid in parallel mode (runtime calculated, no fixed default value)",
+		Category: flags.VMCategory,
+	}
+
 	ParallelTxDAGFlag = &cli.BoolFlag{
 		Name:     "parallel.txdag",
 		Usage:    "Enable the experimental parallel TxDAG generation (default = false)",
 		Category: flags.VMCategory,
 	}
 
+	ParallelTxDAGFileFlag = &cli.StringFlag{
+		Name:     "parallel.txdagfile",
+		Usage:    "It indicates the TxDAG file path",
+		Value:    "./parallel-txdag-output.csv",
+		Category: flags.VMCategory,
+	}
+
+	ParallelTxDATMaxDepthRatioFlag = &cli.Float64Flag{
+		Name:     "parallel.txdag-max-depth-ratio",
+		Usage:    "A ratio to decide whether or not to execute transactions in parallel, it will fallback to sequencial processor if the depth is larger than this value (default = 0.9)",
+		Value:    0.9,
+		Category: flags.VMCategory,
+	}
+
 	VMOpcodeOptimizeFlag = &cli.BoolFlag{
 		Name:     "vm.opcode.optimize",
 		Usage:    "enable opcode optimization",
@@ -2002,10 +2040,35 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name)
 	}
 
+	if ctx.IsSet(ParallelTxFlag.Name) {
+		cfg.ParallelTxMode = ctx.Bool(ParallelTxFlag.Name)
+	}
+
+	if ctx.IsSet(ParallelTxUnorderedMergeFlag.Name) {
+		cfg.ParallelTxUnorderedMerge = ctx.Bool(ParallelTxUnorderedMergeFlag.Name)
+	}
+
+	if ctx.IsSet(ParallelTxParallelMergeFlag.Name) {
+		cfg.ParallelTxParallelMerge = ctx.Bool(ParallelTxParallelMergeFlag.Name)
+	}
+
+	if ctx.IsSet(ParallelTxNumFlag.Name) {
+		cfg.ParallelTxNum = ctx.Int(ParallelTxNumFlag.Name)
+	}
 	if ctx.IsSet(ParallelTxDAGFlag.Name) {
 		cfg.EnableParallelTxDAG = ctx.Bool(ParallelTxDAGFlag.Name)
 	}
 
+	if ctx.IsSet(ParallelTxDAGFileFlag.Name) {
+		cfg.ParallelTxDAGFile = ctx.String(ParallelTxDAGFileFlag.Name)
+	}
+
+	if ctx.IsSet(ParallelTxDATMaxDepthRatioFlag.Name) {
+		cfg.ParallelTxDAGMaxDepthRatio = ctx.Float64(ParallelTxDATMaxDepthRatioFlag.Name)
+	} else {
+		cfg.ParallelTxDAGMaxDepthRatio = ParallelTxDATMaxDepthRatioFlag.Value
+	}
+
 	if ctx.IsSet(ParallelTxDAGSenderPrivFlag.Name) {
 		priHex := ctx.String(ParallelTxDAGSenderPrivFlag.Name)
 		if cfg.Miner.ParallelTxDAGSenderPriv, err = crypto.HexToECDSA(priHex); err != nil {

consensus/beacon/consensus.go

@@ -364,7 +364,7 @@ func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.H
 }
 
 // Finalize implements consensus.Engine and processes withdrawals on top.
-func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
+func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state state.StateDBer, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
 	if !beacon.IsPoSHeader(header) {
 		beacon.ethone.Finalize(chain, header, state, txs, uncles, nil)
 		return

consensus/beacon/oplegacy.go

@@ -60,7 +60,7 @@ func (o *OpLegacy) Prepare(chain consensus.ChainHeaderReader, header *types.Head
 	return fmt.Errorf("cannot prepare for legacy block header: %s (num %d)", header.Hash(), header.Number)
 }
 
-func (o *OpLegacy) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
+func (o *OpLegacy) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state state.StateDBer, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
 	panic(fmt.Errorf("cannot finalize legacy block header: %s (num %d)", header.Hash(), header.Number))
 }
 

consensus/clique/clique.go

@@ -30,7 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	lru "github.com/ethereum/go-ethereum/common/lru"
+	"github.com/ethereum/go-ethereum/common/lru"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
@@ -580,7 +580,7 @@ func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header
 
 // Finalize implements consensus.Engine. There is no post-transaction
 // consensus rules in clique, do nothing here.
-func (c *Clique) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
+func (c *Clique) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state state.StateDBer, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
 	// No block rewards in PoA, so the state remains as is
 }
 

consensus/consensus.go

@@ -88,7 +88,7 @@ type Engine interface {
 	//
 	// Note: The state database might be updated to reflect any consensus rules
 	// that happen at finalization (e.g. block rewards).
-	Finalize(chain ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction,
+	Finalize(chain ChainHeaderReader, header *types.Header, state state.StateDBer, txs []*types.Transaction,
 		uncles []*types.Header, withdrawals []*types.Withdrawal)
 
 	// FinalizeAndAssemble runs any post-transaction state modifications (e.g. block

consensus/ethash/consensus.go

@@ -501,7 +501,7 @@ func (ethash *Ethash) Prepare(chain consensus.ChainHeaderReader, header *types.H
 }
 
 // Finalize implements consensus.Engine, accumulating the block and uncle rewards.
-func (ethash *Ethash) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
+func (ethash *Ethash) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state state.StateDBer, txs []*types.Transaction, uncles []*types.Header, withdrawals []*types.Withdrawal) {
 	// Accumulate any block and uncle rewards
 	accumulateRewards(chain.Config(), state, header, uncles)
 }
@@ -570,7 +570,7 @@ var (
 // AccumulateRewards credits the coinbase of the given block with the mining
 // reward. The total reward consists of the static block reward and rewards for
 // included uncles. The coinbase of each uncle block is also rewarded.
-func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) {
+func accumulateRewards(config *params.ChainConfig, state state.StateDBer, header *types.Header, uncles []*types.Header) {
 	// Select the correct block reward based on chain progression
 	blockReward := FrontierBlockReward
 	if config.IsByzantium(header.Number) {

consensus/misc/dao.go

@@ -85,3 +85,16 @@ func ApplyDAOHardFork(statedb *state.StateDB) {
 		statedb.SetBalance(addr, new(uint256.Int))
 	}
 }
+
+func ApplyDAOHardFork2(statedb state.StateDBer) {
+	// Retrieve the contract to refund balances into
+	if !statedb.Exist(params.DAORefundContract) {
+		statedb.CreateAccount(params.DAORefundContract)
+	}
+
+	// Move every DAO account and extra-balance account funds into the refund contract
+	for _, addr := range params.DAODrainList() {
+		statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr))
+		statedb.SetBalance(addr, new(uint256.Int))
+	}
+}

consensus/misc/precontract.go

@@ -40,3 +40,10 @@ func ApplyPreContractHardFork(statedb *state.StateDB) {
 	statedb.SetState(WBNBContract, symbolSlot, symbolValue)
 	statedb.SelfDestruct(governanceToken)
 }
+
+// ApplyPreContractHardFork modifies the state database according to the hard-fork rules
+func ApplyPreContractHardFork2(statedb state.StateDBer) {
+	statedb.SetState(WBNBContract, nameSlot, nameValue)
+	statedb.SetState(WBNBContract, symbolSlot, symbolValue)
+	statedb.SelfDestruct(governanceToken)
+}

core/block_validator.go

@@ -162,7 +162,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 
 // ValidateState validates the various changes that happen after a state transition,
 // such as amount of used gas, the receipt roots and the state root itself.
-func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64) error {
+func (v *BlockValidator) ValidateState(block *types.Block, statedb state.StateDBer, receipts types.Receipts, usedGas uint64) error {
 	header := block.Header()
 	if block.GasUsed() != usedGas {
 		return fmt.Errorf("invalid gas used (remote: %d local: %d)", block.GasUsed(), usedGas)