Skip to content

Commit 1ebc94d

Browse files
riptlriptl
committed
fdctl: switch tiles to topo-provided funk
1 parent e6a0803 commit 1ebc94d

24 files changed

+165
-200
lines changed

‎contrib/test/run_fd_shred_cap.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,16 @@ echo "
9090
idx_max = 8192
9191
alloc_max = 1073741824
9292
file = \"$DATA_DIR/shredcap_testnet.blockstore\"
93+
[funk]
94+
max_account_records = 150000000
95+
heap_size_gib = 100
96+
max_database_transactions = 2000
9397
[tiles]
9498
[tiles.shred]
9599
max_pending_shred_sets = 16384
96100
[tiles.replay]
97101
snapshot = \"$SNAPSHOT\"
98102
incremental = \"$INCREMENTAL\"
99-
funk_sz_gb = 100
100-
funk_rec_max = 150000000
101-
funk_txn_max = 2000
102-
funk_file = \"$DATA_DIR/shredcap_testnet.funk\"
103103
[tiles.store_int]
104104
shred_cap_replay = \"$SHREDCAP\"
105105
shred_cap_end_slot = 317018450

‎contrib/test/test_firedancer_leader.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,6 @@ echo "
5050
[tiles.replay]
5151
capture = \"firedancer-dev.solcap\"
5252
snapshot = \"$FULL_SNAPSHOT\"
53-
funk_sz_gb = 32
54-
funk_rec_max = 10000000
55-
funk_txn_max = 1024
56-
funk_file = \"/tmp/localnet.funk\"
5753
cluster_version = \"2.0.14\"
5854
[tiles.gui]
5955
enabled = false
@@ -72,6 +68,10 @@ echo "
7268
txn_max = 1024
7369
alloc_max = 10737418240
7470
file = \"/tmp/localnet.blockstore\"
71+
[funk]
72+
max_account_records = 10000000
73+
heap_size_gib = 32
74+
max_database_transactions = 1024
7575
[log]
7676
path = \"firedancer-dev.log\"
7777
level_stderr = \"INFO\"

‎src/app/firedancer-dev/commands/backtest.c

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ backtest_topo( config_t * config ) {
6666
fd_topob_wksp( topo, "replay" );
6767
fd_topo_tile_t * replay_tile = fd_topob_tile( topo, "replay", "replay", "metric_in", replay_cpu_idx, 0, 0 );
6868

69+
/* specified by [tiles.replay] */
70+
71+
fd_topob_wksp( topo, "funk" );
72+
fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
73+
config->firedancer.funk.max_account_records,
74+
config->firedancer.funk.max_database_transactions );
75+
76+
fd_topob_tile_uses( topo, replay_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
77+
6978
/**********************************************************************/
7079
/* Add the executor tiles to topo */
7180
/**********************************************************************/

‎src/app/firedancer-dev/config/default.toml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
txn_max = 1048576
1414
idx_max = 8192
1515
alloc_max = 10737418240
16+
[funk]
17+
heap_size_gib = 140
18+
max_account_records = 150000000
19+
max_database_transactions = 2000
1620
[tiles]
1721
[tiles.shred]
1822
max_pending_shred_sets = 16384
@@ -22,9 +26,6 @@
2226
[tiles.replay]
2327
snapshot_url = "http://${VALIDATOR_IP}:8899/snapshot.tar.bz2"
2428
incremental_url = "http://${VALIDATOR_IP}:8899/incremental-snapshot.tar.bz2"
25-
funk_sz_gb = 140
26-
funk_rec_max = 150000000
27-
funk_txn_max = 2000
2829
[tiles.metric]
2930
prometheus_listen_address = "0.0.0.0"
3031
prometheus_listen_port = 7999

‎src/app/firedancer-dev/config/private.toml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99
txn_max = 1024
1010
idx_max = 8192
1111
alloc_max = 10737418240
12+
[funk]
13+
heap_size_gib = 20
14+
max_account_records = 1048576
15+
max_database_transactions = 4096
1216
[tiles]
1317
[tiles.shred]
1418
max_pending_shred_sets = 16384
@@ -17,9 +21,6 @@
1721
repair_serve_listen_port = 8034
1822
[tiles.replay]
1923
snapshot_url = "http://${VALIDATOR_IP}:8899/snapshot.tar.bz2"
20-
funk_sz_gb = 20
21-
funk_txn_max = 4096
22-
funk_rec_max = 1048576
2324
[tiles.metric]
2425
prometheus_listen_address = "0.0.0.0"
2526
prometheus_listen_port = 7999

‎src/app/firedancer-dev/config/tiny.toml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
[hugetlbfs]
22
max_page_size = "huge"
33

4+
[funk]
5+
max_account_records = 1048576
6+
heap_size_gib = 20
7+
max_database_transactions = 1024
8+
49
[runtime]
510
heap_size_gib = 4
611

@@ -17,11 +22,6 @@ writer_tile_count = 1
1722
[tiles.restart]
1823
enabled = false
1924

20-
[tiles.replay]
21-
funk_sz_gb = 20
22-
funk_txn_max = 1024
23-
funk_rec_max = 1048576
24-
2525
[tiles.shred]
2626
max_pending_shred_sets = 512
2727

‎src/app/firedancer-dev/main.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ extern fd_topo_obj_callbacks_t fd_obj_cb_runtime_pub;
2222
extern fd_topo_obj_callbacks_t fd_obj_cb_blockstore;
2323
extern fd_topo_obj_callbacks_t fd_obj_cb_txncache;
2424
extern fd_topo_obj_callbacks_t fd_obj_cb_exec_spad;
25+
extern fd_topo_obj_callbacks_t fd_obj_cb_funk;
2526

2627
fd_topo_obj_callbacks_t * CALLBACKS[] = {
2728
&fd_obj_cb_mcache,
@@ -39,6 +40,7 @@ fd_topo_obj_callbacks_t * CALLBACKS[] = {
3940
&fd_obj_cb_blockstore,
4041
&fd_obj_cb_txncache,
4142
&fd_obj_cb_exec_spad,
43+
&fd_obj_cb_funk,
4244
NULL,
4345
};
4446

‎src/app/firedancer/config/default.toml

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,33 @@ user = ""
293293
snapshot_grace_period_seconds = 409
294294
max_vote_accounts = 2000000
295295

296+
# This section configures the "funk" account database. Currently, funk
297+
# stores all Solana accounts. In future versions of Firedancer, most
298+
# accounts will be offloaded to the "groove" database.
299+
[funk]
300+
# The max amount of records that the funk instance can store.
301+
# Each Solana account uses at least one record. Additional records
302+
# are used for account changes that are not yet finalized by
303+
# consensus (typically takes 13 seconds).
304+
max_account_records = 10_000_000
305+
306+
# The size of the funk heap in gibibytes. This value must be large
307+
# enough to store all Solana accounts uncompressed.
308+
heap_size_gib = 32
309+
310+
# The max amount of concurrent database transactions. These are
311+
# used to track conflicting versions of accounts until such
312+
# conflicts are resolved by the consensus algorithm. (Not to be
313+
# confused with Solana transactions).
314+
# The validator uses one database transaction for each Solana block
315+
# that is not yet finalized. It is not recommended to change this
316+
# setting.
317+
max_database_transactions = 1024
318+
319+
# This section configures the "groove" persistent account database.
320+
# [groove]
321+
# ...
322+
296323
# CPU cores in Firedancer are carefully managed. Where a typical
297324
# program lets the operating system scheduler determine which threads to
298325
# run on which cores and for how long, Firedancer overrides most of this
@@ -1030,9 +1057,6 @@ user = ""
10301057
# snapshots and frequent validator restarts are expected.
10311058
snapshot_dir = ""
10321059

1033-
funk_sz_gb = 32
1034-
funk_rec_max = 10000000
1035-
funk_txn_max = 1024
10361060
cluster_version = "1.18.0"
10371061

10381062
# The metric tile receives metrics updates published from the rest

‎src/app/firedancer/main.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ extern fd_topo_obj_callbacks_t fd_obj_cb_runtime_pub;
2121
extern fd_topo_obj_callbacks_t fd_obj_cb_blockstore;
2222
extern fd_topo_obj_callbacks_t fd_obj_cb_txncache;
2323
extern fd_topo_obj_callbacks_t fd_obj_cb_exec_spad;
24+
extern fd_topo_obj_callbacks_t fd_obj_cb_funk;
2425

2526
fd_topo_obj_callbacks_t * CALLBACKS[] = {
2627
&fd_obj_cb_mcache,
@@ -38,6 +39,7 @@ fd_topo_obj_callbacks_t * CALLBACKS[] = {
3839
&fd_obj_cb_blockstore,
3940
&fd_obj_cb_txncache,
4041
&fd_obj_cb_exec_spad,
42+
&fd_obj_cb_funk,
4143
NULL,
4244
};
4345

‎src/app/firedancer/topology.c

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ fd_topo_initialize( config_t * config ) {
326326
fd_topob_wksp( topo, "exec_spad" );
327327
fd_topob_wksp( topo, "exec_fseq" );
328328
fd_topob_wksp( topo, "writer_fseq" );
329+
fd_topob_wksp( topo, "funk" );
329330

330331
if( enable_rpc ) fd_topob_wksp( topo, "rpcsrv" );
331332

@@ -460,11 +461,25 @@ fd_topo_initialize( config_t * config ) {
460461
FOR(writer_tile_cnt) fd_topob_tile( topo, "writer", "writer", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
461462
fd_topo_tile_t * batch_tile = fd_topob_tile( topo, "batch", "batch", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
462463

463-
if( enable_rstart ) /* */ fd_topob_tile( topo, "rstart", "restart", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
464+
fd_topo_tile_t * rstart_tile = NULL;
465+
if( enable_rstart ) rstart_tile =fd_topob_tile( topo, "rstart", "restart", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
464466

465467
fd_topo_tile_t * rpcserv_tile = NULL;
466468
if( enable_rpc ) rpcserv_tile = fd_topob_tile( topo, "rpcsrv", "rpcsrv", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
467469

470+
/* Database cache */
471+
472+
fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
473+
config->firedancer.funk.max_account_records,
474+
config->firedancer.funk.max_database_transactions );
475+
476+
/* */ fd_topob_tile_uses( topo, batch_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
477+
FOR(exec_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec", i ) ], funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
478+
/* */ fd_topob_tile_uses( topo, replay_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
479+
if(rstart_tile) fd_topob_tile_uses( topo, rstart_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
480+
if(rpcserv_tile) fd_topob_tile_uses( topo, rpcserv_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
481+
FOR(writer_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "writer", i ) ], funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
482+
468483
/* Create a shared blockstore to be used by store and replay. */
469484
fd_topo_obj_t * blockstore_obj = setup_topo_blockstore( topo,
470485
"bstore",
@@ -903,10 +918,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
903918
tile->replay.tx_metadata_storage = config->rpc.extended_tx_metadata_storage;
904919
strncpy( tile->replay.capture, config->tiles.replay.capture, sizeof(tile->replay.capture) );
905920
strncpy( tile->replay.funk_checkpt, config->tiles.replay.funk_checkpt, sizeof(tile->replay.funk_checkpt) );
906-
tile->replay.funk_rec_max = config->tiles.replay.funk_rec_max;
907-
tile->replay.funk_sz_gb = config->tiles.replay.funk_sz_gb;
908-
tile->replay.funk_txn_max = config->tiles.replay.funk_txn_max;
909-
strncpy( tile->replay.funk_file, config->tiles.replay.funk_file, sizeof(tile->replay.funk_file) );
921+
tile->replay.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
910922
tile->replay.plugins_enabled = fd_topo_find_tile( &config->topo, "plugin", 0UL ) != ULONG_MAX;
911923

912924
if( FD_UNLIKELY( !strncmp( config->tiles.replay.genesis, "", 1 )
@@ -931,6 +943,8 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
931943
tile->replay.full_interval = config->tiles.batch.full_interval;
932944
tile->replay.incremental_interval = config->tiles.batch.incremental_interval;
933945

946+
FD_TEST( tile->replay.funk_obj_id == fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX ) );
947+
934948
} else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) {
935949
strncpy( tile->sign.identity_key_path, config->paths.identity_key, sizeof(tile->sign.identity_key_path) );
936950

@@ -958,19 +972,16 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
958972
strncpy( tile->eqvoc.identity_key_path, config->paths.identity_key, sizeof(tile->eqvoc.identity_key_path) );
959973
} else if( FD_UNLIKELY( !strcmp( tile->name, "rpcsrv" ) ) ) {
960974
strncpy( tile->replay.blockstore_file, config->firedancer.blockstore.file, sizeof(tile->replay.blockstore_file) );
961-
tile->replay.funk_rec_max = config->tiles.replay.funk_rec_max;
962-
tile->replay.funk_sz_gb = config->tiles.replay.funk_sz_gb;
963-
tile->replay.funk_txn_max = config->tiles.replay.funk_txn_max;
964-
strncpy( tile->replay.funk_file, config->tiles.replay.funk_file, sizeof(tile->replay.funk_file) );
975+
tile->rpcserv.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
965976
tile->rpcserv.rpc_port = config->rpc.port;
966977
tile->rpcserv.tpu_port = config->tiles.quic.regular_transaction_listen_port;
967978
tile->rpcserv.tpu_ip_addr = config->net.ip_addr;
968979
strncpy( tile->rpcserv.identity_key_path, config->paths.identity_key, sizeof(tile->rpcserv.identity_key_path) );
969980
} else if( FD_UNLIKELY( !strcmp( tile->name, "batch" ) ) ) {
981+
tile->batch.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
970982
tile->batch.full_interval = config->tiles.batch.full_interval;
971983
tile->batch.incremental_interval = config->tiles.batch.incremental_interval;
972984
strncpy( tile->batch.out_dir, config->tiles.batch.out_dir, sizeof(tile->batch.out_dir) );
973-
strncpy( tile->replay.funk_file, config->tiles.replay.funk_file, sizeof(tile->replay.funk_file) );
974985
} else if( FD_UNLIKELY( !strcmp( tile->name, "gui" ) ) ) {
975986
if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->tiles.gui.gui_listen_address, &tile->gui.listen_addr ) ) )
976987
FD_LOG_ERR(( "failed to parse gui listen address `%s`", config->tiles.gui.gui_listen_address ));
@@ -985,11 +996,11 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
985996
} else if( FD_UNLIKELY( !strcmp( tile->name, "plugin" ) ) ) {
986997

987998
} else if( FD_UNLIKELY( !strcmp( tile->name, "exec" ) ) ) {
988-
strncpy( tile->exec.funk_file, config->tiles.replay.funk_file, sizeof(tile->exec.funk_file) );
999+
tile->exec.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
9891000
} else if( FD_UNLIKELY( !strcmp( tile->name, "writer" ) ) ) {
990-
strncpy( tile->writer.funk_file, config->tiles.replay.funk_file, sizeof(tile->writer.funk_file) );
1001+
tile->writer.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
9911002
} else if( FD_UNLIKELY( !strcmp( tile->name, "rstart" ) ) ) {
992-
strncpy( tile->restart.funk_file, config->tiles.replay.funk_file, sizeof(tile->replay.funk_file) );
1003+
tile->restart.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
9931004
strncpy( tile->restart.tower_checkpt, config->tiles.replay.tower_checkpt, sizeof(tile->replay.tower_checkpt) );
9941005
strncpy( tile->restart.identity_key_path, config->paths.identity_key, sizeof(tile->restart.identity_key_path) );
9951006
fd_memcpy( tile->restart.genesis_hash, config->tiles.restart.genesis_hash, FD_BASE58_ENCODED_32_SZ );

‎src/app/shared/fd_config.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,12 @@ struct fd_configf {
118118
} limits;
119119
} runtime;
120120

121+
struct {
122+
ulong max_account_records;
123+
ulong heap_size_gib;
124+
ulong max_database_transactions;
125+
} funk;
126+
121127
struct {
122128
uint exec_tile_count; /* TODO: redundant ish with bank tile cnt */
123129
uint writer_tile_count;
@@ -365,10 +371,6 @@ struct fd_config {
365371
struct {
366372
char capture[ PATH_MAX ];
367373
char funk_checkpt[ PATH_MAX ];
368-
uint funk_rec_max;
369-
ulong funk_sz_gb;
370-
ulong funk_txn_max;
371-
char funk_file[ PATH_MAX ];
372374
char genesis[ PATH_MAX ];
373375
char incremental[ PATH_MAX ];
374376
char incremental_url[ PATH_MAX ];

‎src/app/shared/fd_config_parse.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,10 @@ fd_config_extract_podf( uchar * pod,
331331
CFG_POP ( ulong, runtime.limits.snapshot_grace_period_seconds );
332332
CFG_POP ( ulong, runtime.limits.max_vote_accounts );
333333

334+
CFG_POP ( ulong, funk.max_account_records );
335+
CFG_POP ( ulong, funk.heap_size_gib );
336+
CFG_POP ( ulong, funk.max_database_transactions );
337+
334338
return config;
335339
}
336340

@@ -442,10 +446,6 @@ fd_config_extract_pod( uchar * pod,
442446

443447
CFG_POP ( cstr, tiles.replay.capture );
444448
CFG_POP ( cstr, tiles.replay.funk_checkpt );
445-
CFG_POP ( uint, tiles.replay.funk_rec_max );
446-
CFG_POP ( ulong, tiles.replay.funk_sz_gb );
447-
CFG_POP ( ulong, tiles.replay.funk_txn_max );
448-
CFG_POP ( cstr, tiles.replay.funk_file );
449449
CFG_POP ( cstr, tiles.replay.genesis );
450450
CFG_POP ( cstr, tiles.replay.incremental );
451451
CFG_POP ( cstr, tiles.replay.incremental_url );

‎src/app/shared/fd_obj_callbacks.c

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "../../waltz/neigh/fd_neigh4_map.h"
1111
#include "../../waltz/ip/fd_fib4.h"
1212
#include "../../disco/keyguard/fd_keyswitch.h"
13+
#include "../../funk/fd_funk.h"
1314

1415
#define VAL(name) (__extension__({ \
1516
ulong __x = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.%s", obj->id, name ); \
@@ -276,6 +277,36 @@ fd_topo_obj_callbacks_t fd_obj_cb_keyswitch = {
276277
.new = keyswitch_new,
277278
};
278279

280+
static ulong
281+
funk_footprint( fd_topo_t const * topo,
282+
fd_topo_obj_t const * obj ) {
283+
(void)topo;
284+
return fd_funk_footprint( VAL("txn_max"), VAL("rec_max") );
285+
}
286+
287+
static ulong
288+
funk_align( fd_topo_t const * topo,
289+
fd_topo_obj_t const * obj ) {
290+
(void)topo; (void)obj;
291+
return fd_funk_align();
292+
}
293+
294+
static void
295+
funk_new( fd_topo_t const * topo,
296+
fd_topo_obj_t const * obj ) {
297+
(void)topo;
298+
ulong funk_seed = fd_pod_queryf_ulong( topo->props, 0UL, "obj.%lu.seed", obj->id );
299+
if( !funk_seed ) FD_TEST( fd_rng_secure( &funk_seed, sizeof(ulong) ) );
300+
FD_TEST( fd_funk_new( fd_topo_obj_laddr( topo, obj->id ), 2UL, funk_seed, VAL("txn_max"), VAL("rec_max") ) );
301+
}
302+
303+
fd_topo_obj_callbacks_t fd_obj_cb_funk = {
304+
.name = "funk",
305+
.footprint = funk_footprint,
306+
.align = funk_align,
307+
.new = funk_new,
308+
};
309+
279310
fd_topo_run_tile_t
280311
fdctl_tile_run( fd_topo_tile_t const * tile );
281312

0 commit comments

Comments
 (0)