Skip to content

Commit bc46ad6

Browse files
riptlripatel-fd
riptl
authored and
ripatel-fd
committed
quic: add SSLKEYLOGFILE support
1 parent aa5d3f7 commit bc46ad6

File tree

11 files changed

+117
-29
lines changed

11 files changed

+117
-29
lines changed

‎src/app/fdctl/config/default.toml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1101,6 +1101,11 @@ dynamic_port_range = "8900-9000"
11011101
# determines whether the feature is enabled in the validator.
11021102
retry = true
11031103

1104+
# Log TLS encryption keys to decrypt QUIC traffic to this file
1105+
# path in NSS SSLKEYLOGFILE format. An empty string (the
1106+
# default) disables key logging.
1107+
ssl_key_log_file = ""
1108+
11041109
# Verify tiles perform signature verification of incoming
11051110
# transactions, making sure that the data is well-formed, and that
11061111
# it is signed by the appropriate private key.

‎src/app/fdctl/topology.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,7 @@ fd_topo_initialize( config_t * config ) {
385385
tile->quic.idle_timeout_millis = config->tiles.quic.idle_timeout_millis;
386386
tile->quic.ack_delay_millis = config->tiles.quic.ack_delay_millis;
387387
tile->quic.retry = config->tiles.quic.retry;
388+
fd_cstr_fini( fd_cstr_append_cstr_safe( fd_cstr_init( tile->quic.key_log_path ), config->tiles.quic.ssl_key_log_file, sizeof(tile->quic.key_log_path) ) );
388389

389390
} else if( FD_UNLIKELY( !strcmp( tile->name, "bundle" ) ) ) {
390391
strncpy( tile->bundle.url, config->tiles.bundle.url, sizeof(tile->bundle.url) );

‎src/app/firedancer/config/default.toml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -950,6 +950,11 @@ user = ""
950950
# determines whether the feature is enabled in the validator.
951951
retry = true
952952

953+
# Log TLS encryption keys to decrypt QUIC traffic to this file
954+
# path in NSS SSLKEYLOGFILE format. An empty string (the
955+
# default) disables key logging.
956+
ssl_key_log_file = ""
957+
953958
# Verify tiles perform signature verification of incoming
954959
# transactions, making sure that the data is well-formed, and that
955960
# it is signed by the appropriate private key.

‎src/app/firedancer/topology.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -916,6 +916,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
916916
tile->quic.idle_timeout_millis = config->tiles.quic.idle_timeout_millis;
917917
tile->quic.ack_delay_millis = config->tiles.quic.ack_delay_millis;
918918
tile->quic.retry = config->tiles.quic.retry;
919+
fd_cstr_fini( fd_cstr_append_cstr_safe( fd_cstr_init( tile->quic.key_log_path ), config->tiles.quic.ssl_key_log_file, sizeof(tile->quic.key_log_path) ) );
919920

920921
} else if( FD_UNLIKELY( !strcmp( tile->name, "verify" ) ) ) {
921922
tile->verify.tcache_depth = config->tiles.verify.signature_cache_size;

‎src/app/shared/fd_config.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,8 @@ struct fd_config {
346346
uint idle_timeout_millis;
347347
uint ack_delay_millis;
348348
int retry;
349+
350+
char ssl_key_log_file[ PATH_MAX ];
349351
} quic;
350352

351353
struct {

‎src/app/shared/fd_config_parse.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,7 @@ fd_config_extract_pod( uchar * pod,
190190
CFG_POP ( uint, tiles.quic.idle_timeout_millis );
191191
CFG_POP ( uint, tiles.quic.ack_delay_millis );
192192
CFG_POP ( bool, tiles.quic.retry );
193+
CFG_POP ( cstr, tiles.quic.ssl_key_log_file );
193194

194195
CFG_POP ( uint, tiles.verify.signature_cache_size );
195196
CFG_POP ( uint, tiles.verify.receive_buffer_size );

‎src/disco/quic/fd_quic_tile.c

Lines changed: 78 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,19 @@
55
#include "fd_tpu.h"
66
#include "../../waltz/quic/fd_quic_private.h"
77
#include "generated/quic_seccomp.h"
8+
#include "../../util/io/fd_io.h"
89
#include "../../util/net/fd_eth.h"
910

1011
#include <errno.h>
12+
#include <fcntl.h>
1113
#include <linux/unistd.h>
1214
#include <sys/random.h>
1315

1416
#define OUT_IDX_VERIFY 0
1517
#define OUT_IDX_NET 1
1618

19+
#define FD_QUIC_KEYLOG_FLUSH_INTERVAL_NS ((long)100e6)
20+
1721
/* fd_quic_tile provides a TPU server tile.
1822
1923
This tile handles incoming transactions that clients request to be
@@ -413,10 +417,66 @@ quic_tx_aio_send( void * _ctx,
413417
return FD_AIO_SUCCESS;
414418
}
415419

420+
static void
421+
quic_tls_keylog( void * _ctx,
422+
char const * line ) {
423+
fd_quic_ctx_t * ctx = _ctx;
424+
fd_io_buffered_ostream_t * os = &ctx->keylog_stream;
425+
426+
/* Lazily flush ostream */
427+
ulong line_sz = strlen( line )+1;
428+
ulong peek_sz = fd_io_buffered_ostream_peek_sz( os );
429+
if( FD_UNLIKELY( peek_sz<line_sz ) ) {
430+
int err = fd_io_buffered_ostream_flush( os );
431+
if( FD_UNLIKELY( err ) ) {
432+
FD_LOG_ERR(( "fd_io_buffered_ostream_flush(keylog) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
433+
}
434+
peek_sz = fd_io_buffered_ostream_peek_sz( os );
435+
}
436+
if( FD_UNLIKELY( peek_sz<line_sz ) ) {
437+
FD_LOG_ERR(( "keylog buffer too small (buf_sz=%lu, line_sz=%lu)", peek_sz, line_sz ));
438+
}
439+
440+
/* Append line */
441+
char * cur = fd_io_buffered_ostream_peek( os );
442+
cur = fd_cstr_append_text( cur, line, strlen( line ) );
443+
cur = fd_cstr_append_char( cur, '\n' );
444+
fd_io_buffered_ostream_seek( os, line_sz );
445+
}
446+
447+
static void
448+
during_housekeeping( fd_quic_ctx_t * ctx ) {
449+
if( FD_UNLIKELY( ctx->keylog_stream.wbuf ) ) {
450+
long now = fd_log_wallclock();
451+
if( FD_UNLIKELY( now > ctx->keylog_next_flush ) ) {
452+
int err = fd_io_buffered_ostream_flush( &ctx->keylog_stream );
453+
if( FD_UNLIKELY( err ) ) {
454+
FD_LOG_ERR(( "fd_io_buffered_ostream_flush(keylog) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
455+
}
456+
ctx->keylog_next_flush = now + FD_QUIC_KEYLOG_FLUSH_INTERVAL_NS;
457+
}
458+
}
459+
}
460+
416461
static void
417462
privileged_init( fd_topo_t * topo,
418463
fd_topo_tile_t * tile ) {
419-
(void)topo; (void)tile;
464+
fd_quic_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
465+
if( FD_UNLIKELY( topo->objs[ tile->tile_obj_id ].footprint < scratch_footprint( tile ) ) ) {
466+
FD_LOG_ERR(( "insufficient tile scratch space" ));
467+
}
468+
fd_memset( ctx, 0, sizeof(fd_quic_ctx_t) );
469+
ctx->keylog_fd = -1;
470+
471+
if( 0!=strcmp( tile->quic.key_log_path, "" ) ) {
472+
ctx->keylog_fd = open( tile->quic.key_log_path, O_WRONLY|O_CREAT|O_APPEND, 0644 );
473+
if( FD_UNLIKELY( ctx->keylog_fd<0 ) ) {
474+
FD_LOG_ERR(( "open(%s, O_WRONLY|O_CREAT|O_APPEND, 0644) failed (%i-%s)",
475+
tile->quic.key_log_path, errno, fd_io_strerror( errno ) ));
476+
}
477+
fd_io_buffered_ostream_init( &ctx->keylog_stream, ctx->keylog_fd, ctx->keylog_buf, sizeof(ctx->keylog_buf) );
478+
FD_LOG_WARNING(( "Logging QUIC encryption keys to %s", tile->quic.key_log_path ));
479+
}
420480

421481
/* The fd_quic implementation calls fd_log_wallclock() internally
422482
which itself calls clock_gettime() which on most kernels is not a
@@ -444,9 +504,6 @@ static void
444504
unprivileged_init( fd_topo_t * topo,
445505
fd_topo_tile_t * tile ) {
446506
void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
447-
if( FD_UNLIKELY( topo->objs[ tile->tile_obj_id ].footprint < scratch_footprint( tile ) ) ) {
448-
FD_LOG_ERR(( "insufficient tile scratch space" ));
449-
}
450507

451508
if( FD_UNLIKELY( tile->in_cnt==0 ) ) {
452509
FD_LOG_ERR(( "quic tile has no input links" ));
@@ -472,7 +529,7 @@ unprivileged_init( fd_topo_t * topo,
472529

473530
FD_SCRATCH_ALLOC_INIT( l, scratch );
474531
fd_quic_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_quic_ctx_t ), sizeof( fd_quic_ctx_t ) );
475-
fd_memset( ctx, 0, sizeof(fd_quic_ctx_t) );
532+
FD_TEST( (ulong)ctx==(ulong)scratch );
476533

477534
for( ulong i=0; i<tile->in_cnt; i++ ) {
478535
fd_topo_link_t * link = &topo->links[ tile->in_link_id[ i ] ];
@@ -524,6 +581,10 @@ unprivileged_init( fd_topo_t * topo,
524581
quic->cb.now = quic_now;
525582
quic->cb.now_ctx = ctx;
526583
quic->cb.quic_ctx = ctx;
584+
if( ctx->keylog_fd>=0 ) {
585+
quic->cb.tls_keylog = quic_tls_keylog;
586+
ctx->keylog_next_flush = fd_log_wallclock() + FD_QUIC_KEYLOG_FLUSH_INTERVAL_NS;
587+
}
527588

528589
fd_quic_set_aio_net_tx( quic, quic_tx_aio );
529590
fd_quic_set_clock_tickcount( quic );
@@ -564,10 +625,8 @@ populate_allowed_seccomp( fd_topo_t const * topo,
564625
fd_topo_tile_t const * tile,
565626
ulong out_cnt,
566627
struct sock_filter * out ) {
567-
(void)topo;
568-
(void)tile;
569-
570-
populate_sock_filter_policy_quic( out_cnt, out, (uint)fd_log_private_logfile_fd() );
628+
fd_quic_ctx_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
629+
populate_sock_filter_policy_quic( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->keylog_fd );
571630
return sock_filter_policy_quic_instr_cnt;
572631
}
573632

@@ -576,15 +635,16 @@ populate_allowed_fds( fd_topo_t const * topo,
576635
fd_topo_tile_t const * tile,
577636
ulong out_fds_cnt,
578637
int * out_fds ) {
579-
(void)topo;
580-
(void)tile;
638+
fd_quic_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
581639

582-
if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
640+
if( FD_UNLIKELY( out_fds_cnt<3UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
583641

584642
ulong out_cnt = 0UL;
585643
out_fds[ out_cnt++ ] = 2; /* stderr */
586644
if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
587645
out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
646+
if( ctx->keylog_fd!=-1 )
647+
out_fds[ out_cnt++ ] = ctx->keylog_fd;
588648
return out_cnt;
589649
}
590650

@@ -594,11 +654,12 @@ populate_allowed_fds( fd_topo_t const * topo,
594654
#define STEM_CALLBACK_CONTEXT_TYPE fd_quic_ctx_t
595655
#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_quic_ctx_t)
596656

597-
#define STEM_CALLBACK_METRICS_WRITE metrics_write
598-
#define STEM_CALLBACK_BEFORE_CREDIT before_credit
599-
#define STEM_CALLBACK_BEFORE_FRAG before_frag
600-
#define STEM_CALLBACK_DURING_FRAG during_frag
601-
#define STEM_CALLBACK_AFTER_FRAG after_frag
657+
#define STEM_CALLBACK_METRICS_WRITE metrics_write
658+
#define STEM_CALLBACK_BEFORE_CREDIT before_credit
659+
#define STEM_CALLBACK_BEFORE_FRAG before_frag
660+
#define STEM_CALLBACK_DURING_FRAG during_frag
661+
#define STEM_CALLBACK_AFTER_FRAG after_frag
662+
#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
602663

603664
#include "../stem/fd_stem.c"
604665

‎src/disco/quic/fd_quic_tile.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "../topo/fd_topo.h"
77
#include "../net/fd_net_tile.h"
88
#include "../../waltz/quic/fd_quic.h"
9+
#include "../../util/io/fd_io.h"
910

1011
#define FD_QUIC_TILE_IN_MAX (8UL)
1112

@@ -39,6 +40,11 @@ typedef struct {
3940

4041
fd_wksp_t * verify_out_mem;
4142

43+
long keylog_next_flush;
44+
int keylog_fd;
45+
fd_io_buffered_ostream_t keylog_stream;
46+
char keylog_buf[ 4096 ];
47+
4248
struct {
4349
ulong txns_received_udp;
4450
ulong txns_received_quic_fast;

‎src/disco/quic/generated/quic_seccomp.h

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,32 +21,36 @@
2121
#else
2222
# error "Target architecture is unsupported by seccomp."
2323
#endif
24-
static const unsigned int sock_filter_policy_quic_instr_cnt = 15;
24+
static const unsigned int sock_filter_policy_quic_instr_cnt = 17;
2525

26-
static void populate_sock_filter_policy_quic( ulong out_cnt, struct sock_filter * out, unsigned int logfile_fd) {
27-
FD_TEST( out_cnt >= 15 );
28-
struct sock_filter filter[15] = {
26+
static void populate_sock_filter_policy_quic( ulong out_cnt, struct sock_filter * out, uint logfile_fd, uint keylog_fd) {
27+
FD_TEST( out_cnt >= 17 );
28+
struct sock_filter filter[17] = {
2929
/* Check: Jump to RET_KILL_PROCESS if the script's arch != the runtime arch */
3030
BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) ) ),
31-
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 11 ),
31+
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 13 ),
3232
/* loading syscall number in accumulator */
3333
BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, nr ) ) ),
3434
/* allow write based on expression */
3535
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_write, /* check_write */ 3, 0 ),
3636
/* allow fsync based on expression */
37-
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 6, 0 ),
37+
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 8, 0 ),
3838
/* simply allow getrandom */
39-
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_getrandom, /* RET_ALLOW */ 8, 0 ),
39+
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_getrandom, /* RET_ALLOW */ 10, 0 ),
4040
/* none of the syscalls matched */
41-
{ BPF_JMP | BPF_JA, 0, 0, /* RET_KILL_PROCESS */ 6 },
41+
{ BPF_JMP | BPF_JA, 0, 0, /* RET_KILL_PROCESS */ 8 },
4242
// check_write:
4343
/* load syscall argument 0 in accumulator */
4444
BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
45-
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 2, /* RET_ALLOW */ 5, /* lbl_1 */ 0 ),
45+
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 2, /* RET_ALLOW */ 7, /* lbl_1 */ 0 ),
4646
// lbl_1:
4747
/* load syscall argument 0 in accumulator */
4848
BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
49-
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 3, /* RET_KILL_PROCESS */ 2 ),
49+
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 5, /* lbl_2 */ 0 ),
50+
// lbl_2:
51+
/* load syscall argument 0 in accumulator */
52+
BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),
53+
BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, keylog_fd, /* RET_ALLOW */ 3, /* RET_KILL_PROCESS */ 2 ),
5054
// check_fsync:
5155
/* load syscall argument 0 in accumulator */
5256
BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])),

‎src/disco/quic/quic.seccomppolicy

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# logfile_fd: It can be disabled by configuration, but typically tiles
22
# will open a log file on boot and write all messages there.
3-
unsigned int logfile_fd
3+
uint logfile_fd, uint keylog_fd
44

55
# logging: all log messages are written to a file and/or pipe
66
#
@@ -10,7 +10,8 @@ unsigned int logfile_fd
1010
# arg 0 is the file descriptor to write to. The boot process ensures
1111
# that descriptor 2 is always STDERR.
1212
write: (or (eq (arg 0) 2)
13-
(eq (arg 0) logfile_fd))
13+
(eq (arg 0) logfile_fd)
14+
(eq (arg 0) keylog_fd))
1415

1516
# logging: 'WARNING' and above fsync the logfile to disk immediately
1617
#

0 commit comments

Comments
 (0)