Skip to content

Commit 96c625c

Browse files
zhangchiqingzhangchiqing
committed
refactor logging pebble db
1 parent 428727a commit 96c625c

File tree

20 files changed

+48
-53
lines changed

20 files changed

+48
-53
lines changed

‎cmd/access/node_builder/access_node_builder.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,6 @@ import (
121121
"github.com/onflow/flow-go/storage/operation/pebbleimpl"
122122
pstorage "github.com/onflow/flow-go/storage/pebble"
123123
"github.com/onflow/flow-go/storage/store"
124-
"github.com/onflow/flow-go/storage/util"
125124
"github.com/onflow/flow-go/utils/grpcutils"
126125
)
127126

@@ -589,7 +588,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccess
589588

590589
if executionDataDBMode == execution_data.ExecutionDataDBModePebble {
591590
builder.ExecutionDatastoreManager, err = edstorage.NewPebbleDatastoreManager(
592-
util.NewLogger(node.Logger, "pebbledb-endata"),
591+
node.Logger.With().Str("pebbledb", "endata").Logger(),
593592
datastoreDir, nil)
594593
if err != nil {
595594
return fmt.Errorf("could not create PebbleDatastoreManager for execution data: %w", err)
@@ -885,7 +884,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccess
885884
// take hours to complete.
886885

887886
pdb, err := pstorage.OpenRegisterPebbleDB(
888-
util.NewLogger(node.Logger, "pebbledb-registers"),
887+
node.Logger.With().Str("pebbledb", "registers").Logger(),
889888
builder.registersDBPath)
890889
if err != nil {
891890
return nil, fmt.Errorf("could not open registers db: %w", err)

‎cmd/execution_builder.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ import (
9696
"github.com/onflow/flow-go/storage/operation/pebbleimpl"
9797
storagepebble "github.com/onflow/flow-go/storage/pebble"
9898
"github.com/onflow/flow-go/storage/store"
99-
"github.com/onflow/flow-go/storage/util"
10099
)
101100

102101
const (
@@ -713,7 +712,7 @@ func (exeNode *ExecutionNode) LoadExecutionState(
713712
) {
714713

715714
chunkDataPackDB, err := storagepebble.OpenDefaultPebbleDB(
716-
util.NewLogger(node.Logger, "pebbledb-cdp"),
715+
node.Logger.With().Str("pebbledb", "cdp").Logger(),
717716
exeNode.exeConf.chunkDataPackDir,
718717
)
719718
if err != nil {
@@ -824,7 +823,7 @@ func (exeNode *ExecutionNode) LoadRegisterStore(
824823
Str("pebble_db_path", exeNode.exeConf.registerDir).
825824
Msg("register store enabled")
826825
pebbledb, err := storagepebble.OpenRegisterPebbleDB(
827-
util.NewLogger(node.Logger, "pebbledb-registers"),
826+
node.Logger.With().Str("pebbledb", "registers").Logger(),
828827
exeNode.exeConf.registerDir)
829828

830829
if err != nil {

‎cmd/observer/node_builder/observer_builder.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@ import (
112112
"github.com/onflow/flow-go/storage/operation/pebbleimpl"
113113
pstorage "github.com/onflow/flow-go/storage/pebble"
114114
"github.com/onflow/flow-go/storage/store"
115-
"github.com/onflow/flow-go/storage/util"
116115
"github.com/onflow/flow-go/utils/grpcutils"
117116
"github.com/onflow/flow-go/utils/io"
118117
)
@@ -1101,7 +1100,7 @@ func (builder *ObserverServiceBuilder) BuildExecutionSyncComponents() *ObserverS
11011100

11021101
if executionDataDBMode == execution_data.ExecutionDataDBModePebble {
11031102
builder.ExecutionDatastoreManager, err = edstorage.NewPebbleDatastoreManager(
1104-
util.NewLogger(node.Logger, "pebbledb-endata"),
1103+
node.Logger.With().Str("pebbledb", "endata").Logger(),
11051104
datastoreDir, nil)
11061105
if err != nil {
11071106
return fmt.Errorf("could not create PebbleDatastoreManager for execution data: %w", err)
@@ -1345,7 +1344,7 @@ func (builder *ObserverServiceBuilder) BuildExecutionSyncComponents() *ObserverS
13451344
// take hours to complete.
13461345

13471346
pdb, err := pstorage.OpenRegisterPebbleDB(
1348-
util.NewLogger(node.Logger, "pebbledb-registers"),
1347+
node.Logger.With().Str("pebbledb", "registers").Logger(),
13491348
builder.registersDBPath)
13501349
if err != nil {
13511350
return nil, fmt.Errorf("could not open registers db: %w", err)

‎cmd/scaffold.go

+5-7
Original file line numberDiff line numberDiff line change
@@ -1092,16 +1092,14 @@ func (fnb *FlowNodeBuilder) initBadgerDB() error {
10921092
return fmt.Errorf("could not create datadir (path: %s): %w", fnb.BaseConfig.datadir, err)
10931093
}
10941094

1095-
log := sutil.NewLogger(fnb.Logger, "badgerdb-protocol")
1096-
10971095
// we initialize the database with options that allow us to keep the maximum
10981096
// item size in the trie itself (up to 1MB) and where we keep all level zero
10991097
// tables in-memory as well; this slows down compaction and increases memory
11001098
// usage, but it improves overall performance and disk i/o
11011099
opts := badger.
11021100
DefaultOptions(fnb.BaseConfig.datadir).
11031101
WithKeepL0InMemory(true).
1104-
WithLogger(log).
1102+
WithLogger(sutil.NewLogger(fnb.Logger.With().Str("badgerdb", "protocol").Logger())).
11051103

11061104
// the ValueLogFileSize option specifies how big the value of a
11071105
// key-value pair is allowed to be saved into badger.
@@ -1140,7 +1138,7 @@ func (fnb *FlowNodeBuilder) initPebbleDB() error {
11401138
return nil
11411139
}
11421140

1143-
db, closer, err := scaffold.InitPebbleDB(sutil.NewLogger(fnb.Logger, "pebbledb-protocol"), fnb.BaseConfig.pebbleDir)
1141+
db, closer, err := scaffold.InitPebbleDB(fnb.Logger.With().Str("pebbledb", "protocol").Logger(), fnb.BaseConfig.pebbleDir)
11441142
if err != nil {
11451143
return err
11461144
}
@@ -1167,9 +1165,9 @@ func (fnb *FlowNodeBuilder) initSecretsDB() error {
11671165
return fmt.Errorf("could not create secrets db dir (path: %s): %w", fnb.BaseConfig.secretsdir, err)
11681166
}
11691167

1170-
log := sutil.NewLogger(fnb.Logger, "badgerdb-secret")
1171-
1172-
opts := badger.DefaultOptions(fnb.BaseConfig.secretsdir).WithLogger(log)
1168+
opts := badger.DefaultOptions(fnb.BaseConfig.secretsdir).
1169+
WithLogger(sutil.NewLogger(
1170+
fnb.Logger.With().Str("badgerdb", "secret").Logger()))
11731171

11741172
// NOTE: SN nodes need to explicitly set --insecure-secrets-db to true in order to
11751173
// disable secrets database encryption

‎cmd/scaffold/pebble_db.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@ import (
66
"os"
77

88
"github.com/cockroachdb/pebble"
9+
"github.com/rs/zerolog"
910

1011
pebblestorage "github.com/onflow/flow-go/storage/pebble"
1112
)
1213

13-
func InitPebbleDB(logger pebble.Logger, dir string) (*pebble.DB, io.Closer, error) {
14+
func InitPebbleDB(logger zerolog.Logger, dir string) (*pebble.DB, io.Closer, error) {
1415
// if the pebble DB is not set, we skip initialization
1516
// the pebble DB must be provided to initialize
1617
// since we've set an default directory for the pebble DB, this check

‎cmd/scaffold/pebble_db_test.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,19 @@ import (
77

88
"github.com/onflow/flow-go/cmd"
99
"github.com/onflow/flow-go/cmd/scaffold"
10-
"github.com/onflow/flow-go/storage/util"
1110
"github.com/onflow/flow-go/utils/unittest"
1211
)
1312

1413
func TestInitPebbleDB(t *testing.T) {
1514
unittest.RunWithTempDir(t, func(dir string) {
16-
_, closer, err := scaffold.InitPebbleDB(util.NewLogger(unittest.Logger(), "test"), dir)
15+
_, closer, err := scaffold.InitPebbleDB(unittest.Logger(), dir)
1716
require.NoError(t, err)
1817
require.NoError(t, closer.Close())
1918
})
2019
}
2120

2221
func TestInitPebbleDBDirNotSet(t *testing.T) {
23-
_, _, err := scaffold.InitPebbleDB(util.NewLogger(unittest.Logger(), "test"), cmd.NotSet)
22+
_, _, err := scaffold.InitPebbleDB(unittest.Logger(), cmd.NotSet)
2423
require.Error(t, err)
2524
require.Contains(t, err.Error(), "missing required flag")
2625
}

‎cmd/util/cmd/read-badger/cmd/storages.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212
"github.com/onflow/flow-go/storage/operation/badgerimpl"
1313
"github.com/onflow/flow-go/storage/operation/pebbleimpl"
1414
pebblestorage "github.com/onflow/flow-go/storage/pebble"
15-
"github.com/onflow/flow-go/storage/util"
1615
)
1716

1817
// InitStorages initializes the badger storages
@@ -30,7 +29,7 @@ func WithStorage(f func(storage.DB) error) error {
3029
log.Warn().Msg("both --datadir and --pebbledir are set, using --pebbledir")
3130
}
3231

33-
db, err := pebblestorage.MustOpenDefaultPebbleDB(util.NewLogger(log.Logger, "db"), flagPebbleDir)
32+
db, err := pebblestorage.MustOpenDefaultPebbleDB(log.Logger, flagPebbleDir)
3433
if err != nil {
3534
return err
3635
}
@@ -57,7 +56,8 @@ func InitBadgerAndPebble() (bdb *badger.DB, pdb *pebble.DB, err error) {
5756
if flagPebbleDir == "" {
5857
return nil, nil, fmt.Errorf("must specify --pebbledir")
5958
}
60-
pdb, err = pebblestorage.MustOpenDefaultPebbleDB(util.NewLogger(log.Logger, "pebbledb"), flagPebbleDir)
59+
pdb, err = pebblestorage.MustOpenDefaultPebbleDB(
60+
log.Logger.With().Str("pebbledb", "protocol").Logger(), flagPebbleDir)
6161
if err != nil {
6262
return nil, nil, err
6363
}

‎cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
"github.com/onflow/flow-go/storage/operation/pebbleimpl"
1717
storagepebble "github.com/onflow/flow-go/storage/pebble"
1818
"github.com/onflow/flow-go/storage/store"
19-
"github.com/onflow/flow-go/storage/util"
2019
)
2120

2221
var (
@@ -79,7 +78,8 @@ func run(*cobra.Command, []string) {
7978
serviceEvents := badger.NewServiceEvents(metrics, db)
8079

8180
// require the chunk data pack data must exist before returning the storage module
82-
chunkDataPacksPebbleDB, err := storagepebble.MustOpenDefaultPebbleDB(util.NewLogger(log.Logger, "pebbledb-cdp"), flagChunkDataPackDir)
81+
chunkDataPacksPebbleDB, err := storagepebble.MustOpenDefaultPebbleDB(
82+
log.Logger.With().Str("pebbledb", "cdp").Logger(), flagChunkDataPackDir)
8383
if err != nil {
8484
log.Fatal().Err(err).Msgf("could not open chunk data pack DB at %v", flagChunkDataPackDir)
8585
}

‎engine/testutil/nodes.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,6 @@ import (
107107
"github.com/onflow/flow-go/storage/operation/badgerimpl"
108108
storagepebble "github.com/onflow/flow-go/storage/pebble"
109109
"github.com/onflow/flow-go/storage/store"
110-
sutil "github.com/onflow/flow-go/storage/util"
111110
"github.com/onflow/flow-go/utils/unittest"
112111
)
113112

@@ -611,7 +610,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, ide
611610
require.NoError(t, err)
612611

613612
registerDir := unittest.TempPebblePath(t)
614-
pebbledb, err := storagepebble.OpenRegisterPebbleDB(sutil.NewLogger(node.Log, "pebbledb-registers"), registerDir)
613+
pebbledb, err := storagepebble.OpenRegisterPebbleDB(node.Log.With().Str("pebbledb", "registers").Logger(), registerDir)
615614
require.NoError(t, err)
616615

617616
checkpointHeight := uint64(0)

‎engine/verification/verifier/verifiers.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import (
2424
"github.com/onflow/flow-go/storage/operation/pebbleimpl"
2525
storagepebble "github.com/onflow/flow-go/storage/pebble"
2626
"github.com/onflow/flow-go/storage/store"
27-
sutil "github.com/onflow/flow-go/storage/util"
2827
)
2928

3029
// VerifyLastKHeight verifies the last k sealed blocks by verifying all chunks in the results.
@@ -225,7 +224,8 @@ func initStorages(chainID flow.ChainID, dataDir string, chunkDataPackDir string)
225224
}
226225

227226
// require the chunk data pack data must exist before returning the storage module
228-
chunkDataPackDB, err := storagepebble.MustOpenDefaultPebbleDB(sutil.NewLogger(log.Logger, "pebbledb-cdp"), chunkDataPackDir)
227+
chunkDataPackDB, err := storagepebble.MustOpenDefaultPebbleDB(
228+
log.Logger.With().Str("pebbledb", "cdp").Logger(), chunkDataPackDir)
229229
if err != nil {
230230
return nil, nil, nil, nil, nil, fmt.Errorf("could not open chunk data pack DB: %w", err)
231231
}

‎follower/database/init.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ import (
44
"io"
55

66
"github.com/cockroachdb/pebble"
7+
"github.com/rs/zerolog"
78

89
"github.com/onflow/flow-go/cmd/scaffold"
910
)
1011

1112
// InitPebbleDB is an alias for scaffold.InitPebbleDB.
12-
func InitPebbleDB(logger pebble.Logger, dir string) (*pebble.DB, io.Closer, error) {
13+
func InitPebbleDB(logger zerolog.Logger, dir string) (*pebble.DB, io.Closer, error) {
1314
return scaffold.InitPebbleDB(logger, dir)
1415
}

‎integration/testnet/network.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ import (
5656
"github.com/onflow/flow-go/state/protocol/inmem"
5757
"github.com/onflow/flow-go/state/protocol/protocol_state/kvstore"
5858
badgerstorage "github.com/onflow/flow-go/storage/badger"
59-
"github.com/onflow/flow-go/storage/util"
6059
"github.com/onflow/flow-go/utils/io"
6160
"github.com/onflow/flow-go/utils/unittest"
6261
)
@@ -686,7 +685,7 @@ func (net *FlowNetwork) addConsensusFollower(t *testing.T, rootProtocolSnapshotP
686685
dataDir := makeDir(t, tmpdir, DefaultFlowDBDir)
687686
pebbleDir := makeDir(t, tmpdir, DefaultFlowPebbleDBDir)
688687

689-
pebbleDB, _, err := database.InitPebbleDB(util.NewLogger(net.log, "pebbledb-protocol"), pebbleDir)
688+
pebbleDB, _, err := database.InitPebbleDB(net.log, pebbleDir)
690689
require.NoError(t, err)
691690

692691
// create a follower-specific directory for the bootstrap files

‎module/executiondatasync/storage/pebble_datastore_manager.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/cockroachdb/pebble"
88
ds "github.com/ipfs/go-datastore"
99
pebbleds "github.com/ipfs/go-ds-pebble"
10+
"github.com/rs/zerolog"
1011

1112
pstorage "github.com/onflow/flow-go/storage/pebble"
1213
)
@@ -29,7 +30,7 @@ type PebbleDatastoreManager struct {
2930
// options are applied.
3031
//
3132
// No errors are expected during normal operations.
32-
func NewPebbleDatastoreManager(logger pebble.Logger, path string, options *pebble.Options) (*PebbleDatastoreManager, error) {
33+
func NewPebbleDatastoreManager(logger zerolog.Logger, path string, options *pebble.Options) (*PebbleDatastoreManager, error) {
3334
if options == nil {
3435
cache := pebble.NewCache(pstorage.DefaultPebbleCacheSize)
3536
defer cache.Unref()

‎storage/pebble/bootstrap_test.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919
"github.com/onflow/flow-go/ledger/complete/mtrie/trie"
2020
"github.com/onflow/flow-go/ledger/complete/wal"
2121
"github.com/onflow/flow-go/model/flow"
22-
"github.com/onflow/flow-go/storage/util"
2322
"github.com/onflow/flow-go/utils/unittest"
2423
)
2524

@@ -31,7 +30,7 @@ func TestRegisterBootstrap_NewBootstrap(t *testing.T) {
3130
rootHeight := uint64(1)
3231
rootHash := ledger.RootHash(unittest.StateCommitmentFixture())
3332
log := zerolog.New(io.Discard)
34-
p, err := OpenRegisterPebbleDB(util.NewLogger(log, ""), dir)
33+
p, err := OpenRegisterPebbleDB(log, dir)
3534
require.NoError(t, err)
3635
// set heights
3736
require.NoError(t, initHeights(p, rootHeight))
@@ -254,7 +253,7 @@ func randomRegisterPaths(n uint16) []ledger.Path {
254253

255254
func createPebbleForTest(t *testing.T) (*pebble.DB, string) {
256255
dbDir := unittest.TempPebblePath(t)
257-
pb, err := OpenRegisterPebbleDB(util.NewLogger(unittest.Logger(), ""), dbDir)
256+
pb, err := OpenRegisterPebbleDB(unittest.Logger(), dbDir)
258257
require.NoError(t, err)
259258
return pb, dbDir
260259
}

‎storage/pebble/config.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@ package pebble
33
import (
44
"github.com/cockroachdb/pebble"
55
"github.com/cockroachdb/pebble/bloom"
6+
"github.com/rs/zerolog"
7+
8+
"github.com/onflow/flow-go/storage/util"
69
)
710

811
// DefaultPebbleOptions returns an optimized set of pebble options.
912
// This is mostly copied form pebble's nightly performance benchmark.
10-
func DefaultPebbleOptions(logger pebble.Logger, cache *pebble.Cache, comparer *pebble.Comparer) *pebble.Options {
13+
func DefaultPebbleOptions(logger zerolog.Logger, cache *pebble.Cache, comparer *pebble.Comparer) *pebble.Options {
1114
opts := &pebble.Options{
1215
Cache: cache,
1316
Comparer: comparer,
@@ -28,7 +31,7 @@ func DefaultPebbleOptions(logger pebble.Logger, cache *pebble.Cache, comparer *p
2831

2932
// The default is 1.
3033
MaxConcurrentCompactions: func() int { return 4 },
31-
Logger: logger,
34+
Logger: util.NewLogger(logger),
3235
}
3336

3437
for i := 0; i < len(opts.Levels); i++ {

‎storage/pebble/open.go

+5-4
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/cockroachdb/pebble"
1010
"github.com/hashicorp/go-multierror"
11+
"github.com/rs/zerolog"
1112

1213
"github.com/onflow/flow-go/storage"
1314
"github.com/onflow/flow-go/storage/pebble/registers"
@@ -17,7 +18,7 @@ const DefaultPebbleCacheSize = 1 << 20
1718

1819
// NewBootstrappedRegistersWithPath initializes a new Registers instance with a pebble db
1920
// if the database is not initialized, it close the database and return storage.ErrNotBootstrapped
20-
func NewBootstrappedRegistersWithPath(logger pebble.Logger, dir string) (*Registers, *pebble.DB, error) {
21+
func NewBootstrappedRegistersWithPath(logger zerolog.Logger, dir string) (*Registers, *pebble.DB, error) {
2122
db, err := OpenRegisterPebbleDB(logger, dir)
2223
if err != nil {
2324
return nil, nil, fmt.Errorf("failed to initialize pebble db: %w", err)
@@ -41,7 +42,7 @@ func NewBootstrappedRegistersWithPath(logger pebble.Logger, dir string) (*Regist
4142
// a customized comparer (NewMVCCComparer) which is needed to
4243
// implement finding register values at any given height using
4344
// pebble's SeekPrefixGE function
44-
func OpenRegisterPebbleDB(logger pebble.Logger, dir string) (*pebble.DB, error) {
45+
func OpenRegisterPebbleDB(logger zerolog.Logger, dir string) (*pebble.DB, error) {
4546
cache := pebble.NewCache(DefaultPebbleCacheSize)
4647
defer cache.Unref()
4748
// currently pebble is only used for registers
@@ -58,7 +59,7 @@ func OpenRegisterPebbleDB(logger pebble.Logger, dir string) (*pebble.DB, error)
5859
// such as cache size and comparer
5960
// If the pebbleDB is not bootstrapped at this folder, it will auto-bootstrap it,
6061
// use MustOpenDefaultPebbleDB if you want to return error instead
61-
func OpenDefaultPebbleDB(logger pebble.Logger, dir string) (*pebble.DB, error) {
62+
func OpenDefaultPebbleDB(logger zerolog.Logger, dir string) (*pebble.DB, error) {
6263
cache := pebble.NewCache(DefaultPebbleCacheSize)
6364
defer cache.Unref()
6465
opts := DefaultPebbleOptions(logger, cache, pebble.DefaultComparer)
@@ -72,7 +73,7 @@ func OpenDefaultPebbleDB(logger pebble.Logger, dir string) (*pebble.DB, error) {
7273

7374
// MustOpenDefaultPebbleDB returns error if the pebbleDB is not bootstrapped at this folder
7475
// if bootstrapped, then open the pebbleDB
75-
func MustOpenDefaultPebbleDB(logger pebble.Logger, dir string) (*pebble.DB, error) {
76+
func MustOpenDefaultPebbleDB(logger zerolog.Logger, dir string) (*pebble.DB, error) {
7677
err := IsPebbleInitialized(dir)
7778
if err != nil {
7879
return nil, fmt.Errorf("pebble db is not initialized: %w", err)

0 commit comments

Comments
 (0)