Skip to content

Merkle-ize X-Chain State #3732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+1,275 −208
Open

Merkle-ize X-Chain State #3732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f2fcb3e
Merkle-ize X-Chain
joshua-kim Feb 26, 2025
3df6c46
wip
joshua-kim Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions database/memdb/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package memdb

import (
"context"
"fmt"
"slices"
"strings"
"sync"
Expand Down Expand Up @@ -38,6 +39,21 @@ func New() *Database {
return NewWithSize(DefaultSize)
}

// Copy returns a Database with the same key-value pairs as db
func Copy(db *Database) (*Database, error) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need this utility to be able to copy the memdb between VM instances, since closing a database makes it so that it's unusable afterwards.

db.lock.Lock()
defer db.lock.Unlock()

result := New()
for k, v := range db.db {
if err := result.Put([]byte(k), v); err != nil {
return nil, fmt.Errorf("failed to insert key: %w", err)
}
}

return result, nil
}

// NewWithSize returns a map pre-allocated to the provided size with the
// Database interface methods implemented.
func NewWithSize(size int) *Database {
Expand Down
6 changes: 5 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/StephenButtolph/canoto v0.17.1
github.com/antithesishq/antithesis-sdk-go v0.3.8
github.com/ava-labs/coreth v0.15.4-rc.3
github.com/ava-labs/firewood/ffi v0.0.6
github.com/ava-labs/ledger-avalanche/go v0.0.0-20241009183145-e6f90a8a1a60
github.com/ava-labs/libevm v1.13.14-0.3.0.rc.6
github.com/btcsuite/btcd/btcutil v1.1.3
Expand Down Expand Up @@ -82,6 +83,8 @@ require (
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
)

require github.com/kylelemons/godebug v1.1.0 // indirect

require (
github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e // indirect
github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec // indirect
Expand Down Expand Up @@ -146,7 +149,6 @@ require (
github.com/klauspost/compress v1.18.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand Down Expand Up @@ -201,3 +203,5 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace github.com/ava-labs/firewood-go-ethhash/ffi => /Users/joshua.kim/workspace/firewood/ffi
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/ava-labs/coreth v0.15.4-rc.3 h1:v33OOerxpGIKa1MpljXMBB3Yljy23xzsez3E/dn7TzY=
github.com/ava-labs/coreth v0.15.4-rc.3/go.mod h1:Esb0FK+KJr6co7rrhtBWsmSMXEL5JWelEsijlqAHdq0=
github.com/ava-labs/firewood-go-ethhash/ffi v0.0.12 h1:aMcrLbpJ/dyu2kZDf/Di/4JIWsUcYPyTDKymiHpejt0=
github.com/ava-labs/firewood-go-ethhash/ffi v0.0.12/go.mod h1:cq89ua3iiZ5wPBALTEQS5eG8DIZcs7ov6OiL4YR1BVY=
github.com/ava-labs/firewood/ffi v0.0.6 h1:i6ZhMrbtlmw2jwT3JxuRz2dBKiivlobB7rbhnxEKsZY=
github.com/ava-labs/firewood/ffi v0.0.6/go.mod h1:8QZT1v4oV6yBH/KsUQSYfiXm1/jjnI90ENdc1NVTpAg=
github.com/ava-labs/ledger-avalanche/go v0.0.0-20241009183145-e6f90a8a1a60 h1:EL66gtXOAwR/4KYBjOV03LTWgkEXvLePribLlJNu4g0=
github.com/ava-labs/ledger-avalanche/go v0.0.0-20241009183145-e6f90a8a1a60/go.mod h1:/7qKobTfbzBu7eSTVaXMTr56yTYk4j2Px6/8G+idxHo=
github.com/ava-labs/libevm v1.13.14-0.3.0.rc.6 h1:tyM659nDOknwTeU4A0fUVsGNIU7k0v738wYN92nqs/Y=
Expand Down
9 changes: 7 additions & 2 deletions vms/avm/block/executor/block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ func (b *Block) Verify(context.Context) error {
return nil
}

func (b *Block) Accept(context.Context) error {
func (b *Block) Accept(ctx context.Context) error {
blkID := b.ID()
defer b.manager.free(blkID)

Expand Down Expand Up @@ -248,12 +248,17 @@ func (b *Block) Accept(context.Context) error {
return err
}

checksum, err := b.manager.state.Checksum(ctx)
if err != nil {
return fmt.Errorf("failed to get checksum: %w", err)
}

b.manager.backend.Ctx.Log.Trace(
"accepted block",
zap.Stringer("blkID", blkID),
zap.Uint64("height", b.Height()),
zap.Stringer("parentID", b.Parent()),
zap.Stringer("checksum", b.manager.state.Checksum()),
zap.Stringer("checksum", checksum),
)
return nil
}
Expand Down
18 changes: 9 additions & 9 deletions vms/avm/block/executor/block_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func TestBlockVerify(t *testing.T) {
parentID := ids.GenerateTestID()
mockBlock.EXPECT().Parent().Return(parentID).AnyTimes()

mockState := statemock.NewState(ctrl)
mockState := statemock.NewMockInterface(ctrl)
mockState.EXPECT().GetBlock(parentID).Return(nil, errTest)
return &Block{
Block: mockBlock,
Expand Down Expand Up @@ -195,7 +195,7 @@ func TestBlockVerify(t *testing.T) {
parentID := ids.GenerateTestID()
mockBlock.EXPECT().Parent().Return(parentID).AnyTimes()

mockState := statemock.NewState(ctrl)
mockState := statemock.NewMockInterface(ctrl)
mockParentBlock := block.NewMockBlock(ctrl)
mockParentBlock.EXPECT().Height().Return(blockHeight) // Should be blockHeight - 1
mockState.EXPECT().GetBlock(parentID).Return(mockParentBlock, nil)
Expand Down Expand Up @@ -626,7 +626,7 @@ func TestBlockAccept(t *testing.T) {
mempool, err := mempool.New("", prometheus.NewRegistry())
require.NoError(t, err)

mockManagerState := statemock.NewState(ctrl)
mockManagerState := statemock.NewMockInterface(ctrl)
mockManagerState.EXPECT().CommitBatch().Return(nil, errTest)
mockManagerState.EXPECT().Abort()

Expand Down Expand Up @@ -660,7 +660,7 @@ func TestBlockAccept(t *testing.T) {
mempool, err := mempool.New("", prometheus.NewRegistry())
require.NoError(t, err)

mockManagerState := statemock.NewState(ctrl)
mockManagerState := statemock.NewMockInterface(ctrl)
// Note the returned batch is nil but not used
// because we mock the call to shared memory
mockManagerState.EXPECT().CommitBatch().Return(nil, nil)
Expand Down Expand Up @@ -699,7 +699,7 @@ func TestBlockAccept(t *testing.T) {
mempool, err := mempool.New("", prometheus.NewRegistry())
require.NoError(t, err)

mockManagerState := statemock.NewState(ctrl)
mockManagerState := statemock.NewMockInterface(ctrl)
// Note the returned batch is nil but not used
// because we mock the call to shared memory
mockManagerState.EXPECT().CommitBatch().Return(nil, nil)
Expand Down Expand Up @@ -744,12 +744,12 @@ func TestBlockAccept(t *testing.T) {
mempool, err := mempool.New("", prometheus.NewRegistry())
require.NoError(t, err)

mockManagerState := statemock.NewState(ctrl)
mockManagerState := statemock.NewMockInterface(ctrl)
// Note the returned batch is nil but not used
// because we mock the call to shared memory
mockManagerState.EXPECT().CommitBatch().Return(nil, nil)
mockManagerState.EXPECT().Abort()
mockManagerState.EXPECT().Checksum().Return(ids.Empty)
mockManagerState.EXPECT().Checksum(gomock.Any()).Return(ids.Empty, nil)

mockSharedMemory := atomicmock.NewSharedMemory(ctrl)
mockSharedMemory.EXPECT().Apply(gomock.Any(), gomock.Any()).Return(nil)
Expand Down Expand Up @@ -851,7 +851,7 @@ func TestBlockReject(t *testing.T) {
require.NoError(t, err)

lastAcceptedID := ids.GenerateTestID()
mockState := statemock.NewState(ctrl)
mockState := statemock.NewMockInterface(ctrl)
mockState.EXPECT().GetLastAccepted().Return(lastAcceptedID).AnyTimes()
mockState.EXPECT().GetTimestamp().Return(time.Now()).AnyTimes()

Expand Down Expand Up @@ -904,7 +904,7 @@ func TestBlockReject(t *testing.T) {
require.NoError(t, err)

lastAcceptedID := ids.GenerateTestID()
mockState := statemock.NewState(ctrl)
mockState := statemock.NewMockInterface(ctrl)
mockState.EXPECT().GetLastAccepted().Return(lastAcceptedID).AnyTimes()
mockState.EXPECT().GetTimestamp().Return(time.Now()).AnyTimes()

Expand Down
4 changes: 2 additions & 2 deletions vms/avm/block/executor/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ type Manager interface {
func NewManager(
mempool mempool.Mempool[*txs.Tx],
metrics metrics.Metrics,
state state.State,
state state.Interface,
backend *executor.Backend,
clk *mockable.Clock,
onAccept func(*txs.Tx),
Expand All @@ -72,7 +72,7 @@ func NewManager(

type manager struct {
backend *executor.Backend
state state.State
state state.Interface
metrics metrics.Metrics
mempool mempool.Mempool[*txs.Tx]
clk *mockable.Clock
Expand Down
10 changes: 5 additions & 5 deletions vms/avm/block/executor/manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func TestManagerGetStatelessBlock(t *testing.T) {
require := require.New(t)
ctrl := gomock.NewController(t)

state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
m := &manager{
state: state,
blkIDToState: map[ids.ID]*blockState{},
Expand Down Expand Up @@ -72,7 +72,7 @@ func TestManagerGetState(t *testing.T) {
require := require.New(t)
ctrl := gomock.NewController(t)

s := statemock.NewState(ctrl)
s := statemock.NewMockInterface(ctrl)
m := &manager{
state: s,
blkIDToState: map[ids.ID]*blockState{},
Expand Down Expand Up @@ -160,7 +160,7 @@ func TestManagerVerifyTx(t *testing.T) {
lastAcceptedID := ids.GenerateTestID()

// These values don't matter for this test
state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetLastAccepted().Return(lastAcceptedID)
state.EXPECT().GetTimestamp().Return(time.Time{})

Expand Down Expand Up @@ -190,7 +190,7 @@ func TestManagerVerifyTx(t *testing.T) {
lastAcceptedID := ids.GenerateTestID()

// These values don't matter for this test
state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetLastAccepted().Return(lastAcceptedID)
state.EXPECT().GetTimestamp().Return(time.Time{})

Expand Down Expand Up @@ -220,7 +220,7 @@ func TestManagerVerifyTx(t *testing.T) {
lastAcceptedID := ids.GenerateTestID()

// These values don't matter for this test
state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetLastAccepted().Return(lastAcceptedID)
state.EXPECT().GetTimestamp().Return(time.Time{})

Expand Down
3 changes: 2 additions & 1 deletion vms/avm/environment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ func setup(tb testing.TB, c *envConfig) *environment {
}

vm := &VM{
Config: vmStaticConfig,
Config: vmStaticConfig,
StateFactory: LegacyStateFactory{},
}

vmDynamicConfig := DefaultConfig
Expand Down
6 changes: 5 additions & 1 deletion vms/avm/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,9 @@ type Factory struct {
}

func (f *Factory) New(logging.Logger) (interface{}, error) {
return &VM{Config: f.Config}, nil
return &VM{
Config: f.Config,
StateFactory: &GForkStateFactory{CommitFrequency: 1_000},
},
nil
}
16 changes: 8 additions & 8 deletions vms/avm/service_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2494,7 +2494,7 @@ func TestServiceGetBlockByHeight(t *testing.T) {
{
name: "block height not found",
serviceAndExpectedBlockFunc: func(_ *testing.T, ctrl *gomock.Controller) (*Service, interface{}) {
state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetBlockIDAtHeight(blockHeight).Return(ids.Empty, database.ErrNotFound)

manager := executormock.NewManager(ctrl)
Expand All @@ -2514,7 +2514,7 @@ func TestServiceGetBlockByHeight(t *testing.T) {
{
name: "block not found",
serviceAndExpectedBlockFunc: func(_ *testing.T, ctrl *gomock.Controller) (*Service, interface{}) {
state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetBlockIDAtHeight(blockHeight).Return(blockID, nil)

manager := executormock.NewManager(ctrl)
Expand All @@ -2539,7 +2539,7 @@ func TestServiceGetBlockByHeight(t *testing.T) {
block.EXPECT().InitCtx(gomock.Any())
block.EXPECT().Txs().Return(nil)

state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetBlockIDAtHeight(blockHeight).Return(blockID, nil)

manager := executormock.NewManager(ctrl)
Expand All @@ -2564,7 +2564,7 @@ func TestServiceGetBlockByHeight(t *testing.T) {
blockBytes := []byte("hi mom")
block.EXPECT().Bytes().Return(blockBytes)

state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetBlockIDAtHeight(blockHeight).Return(blockID, nil)

expected, err := formatting.Encode(formatting.Hex, blockBytes)
Expand Down Expand Up @@ -2592,7 +2592,7 @@ func TestServiceGetBlockByHeight(t *testing.T) {
blockBytes := []byte("hi mom")
block.EXPECT().Bytes().Return(blockBytes)

state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetBlockIDAtHeight(blockHeight).Return(blockID, nil)

expected, err := formatting.Encode(formatting.HexC, blockBytes)
Expand Down Expand Up @@ -2620,7 +2620,7 @@ func TestServiceGetBlockByHeight(t *testing.T) {
blockBytes := []byte("hi mom")
block.EXPECT().Bytes().Return(blockBytes)

state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetBlockIDAtHeight(blockHeight).Return(blockID, nil)

expected, err := formatting.Encode(formatting.HexNC, blockBytes)
Expand Down Expand Up @@ -2698,7 +2698,7 @@ func TestServiceGetHeight(t *testing.T) {
{
name: "block not found",
serviceFunc: func(ctrl *gomock.Controller) *Service {
state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetLastAccepted().Return(blockID)

manager := executormock.NewManager(ctrl)
Expand All @@ -2718,7 +2718,7 @@ func TestServiceGetHeight(t *testing.T) {
{
name: "happy path",
serviceFunc: func(ctrl *gomock.Controller) *Service {
state := statemock.NewState(ctrl)
state := statemock.NewMockInterface(ctrl)
state.EXPECT().GetLastAccepted().Return(blockID)

block := block.NewMockBlock(ctrl)
Expand Down
2 changes: 1 addition & 1 deletion vms/avm/state/mocks_generate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ package state

//go:generate go run go.uber.org/mock/mockgen -package=${GOPACKAGE}mock -destination=${GOPACKAGE}mock/chain.go -mock_names=Chain=Chain . Chain
//go:generate go run go.uber.org/mock/mockgen -package=${GOPACKAGE}mock -destination=${GOPACKAGE}mock/diff.go -mock_names=Diff=Diff . Diff
//go:generate go run go.uber.org/mock/mockgen -package=${GOPACKAGE}mock -destination=${GOPACKAGE}mock/state.go -mock_names=State=State . State
//go:generate go run go.uber.org/mock/mockgen -package=${GOPACKAGE}mock -destination=${GOPACKAGE}mock/state.go -mock_names=State=Interface . Interface
Loading
Loading