fuzz, flamenco: factor out pb dumping logic to separate file

src/flamenco/runtime/Local.mk

@@ -26,6 +26,9 @@ $(call add-objs,fd_pubkey_utils,fd_flamenco)
 $(call add-hdrs,fd_txncache.h)
 $(call add-objs,fd_txncache,fd_flamenco)
 
+$(call add-hdrs, tests/fd_dump_pb.h)
+$(call add-objs, tests/fd_dump_pb,fd_flamenco)
+
 $(call add-hdrs,fd_rent_lists.h)
 
 $(call make-unit-test,test_txncache,test_txncache,fd_flamenco fd_util)

src/flamenco/runtime/fd_executor.h

@@ -33,16 +33,6 @@ get_transaction_account_lock_limit( fd_exec_slot_ctx_t const * slot_ctx ) {
   return fd_ulong_if( FD_FEATURE_ACTIVE( slot_ctx, increase_tx_account_lock_limit ), MAX_TX_ACCOUNT_LOCKS, 64UL );
 }
 
-/* Create an InstrContext protobuf struct from a given
-   transaction context and instr info.
-
-   NOTE: Calling this function requires the caller to have a scratch
-   frame ready and pushed (see dump_instr_to_protobuf) */
-void
-fd_create_instr_context_protobuf_from_instructions( fd_exec_test_instr_context_t       * instr_context,
-                                                    fd_exec_txn_ctx_t            const * txn_ctx,
-                                                    fd_instr_info_t              const * instr );
-
 /* fd_exec_instr_fn_t processes an instruction.  Returns an error code
    in FD_EXECUTOR_INSTR_{ERR_{...},SUCCESS}. */
 
@@ -151,9 +141,6 @@ fd_exec_consume_cus( fd_exec_txn_ctx_t * txn_ctx,
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
-void
-fd_dump_txn_to_protobuf( fd_exec_txn_ctx_t *txn_ctx, fd_spad_t * spad );
-
 /* We expose these only for the fuzzing harness.
    Normally you shouldn't be invoking these manually. */
 int

src/flamenco/runtime/fd_runtime.c

@@ -43,6 +43,8 @@
 #include "sysvar/fd_sysvar_slot_hashes.h"
 #include "sysvar/fd_sysvar_slot_history.h"
 
+#include "tests/fd_dump_pb.h"
+
 #include "../nanopb/pb_decode.h"
 #include "../nanopb/pb_encode.h"
 #include "../types/fd_solana_block.pb.h"

src/flamenco/runtime/tests/fd_dump_pb.h

@@ -0,0 +1,93 @@
+#ifndef HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h
+#define HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h
+
+/* fd_dump_pb.h provides APIs for dumping instructions, transactions, and slots
+   into a digestable and replayable Protobuf message. This is useful for debugging
+   ledger test mismatches, collecting seed corpora, and gathering real data to test
+   new harnesses. 
+   
+   The following arguments can be added when replaying ledger transactions: 
+      COMMON:
+        --dump-proto-output-dir <output_dir>
+            * Defines the output directory to dump Protobuf messages to
+        --dump-proto-start-slot <slot_number>
+            * Defines the starting slot to dump Protobuf messages from
+      
+      HARNESS-SPECIFIC FILTERS:
+        Transactions:
+            --dump-txn-to-pb <0/1>
+                * If enabled, transactions will be dumped to the specified output directory
+                * File name format is "txn-<base58_enc_sig>.bin" 
+                * Each file represents a single transaction as a serialized TxnContext Protobuf message
+            --dump-proto-sig-filter <base_58_enc_sig>
+                * If enabled, only transactions with the specified signature will be dumped
+
+        Instructions:
+            --dump-instr-to-pb <0/1>
+                * If enabled, instructions will be dumped to the specified output directory 
+                * File name format is "instr-<base58_enc_sig>-<instruction_idx>.bin", where instruction_idx is 1-indexed
+                * Each file represents a single instruction as a serialized InstrContext Protobuf message
+            --dump-proto-sig-filter <base_58_enc_sig>
+                * If enabled, only instructions with the specified signature will be dumped 
+
+        CPI: Currently not directly invokable with a CLI argument. See details below.
+                
+    Other notes:
+        solana-conformance (https://github.com/firedancer-io/solana-conformance)
+            * Allows decoding / executing / debugging of above Protobuf messages in an isolated environment
+            * Allows execution result(s) comparison between Firedancer and Solana / Agave
+            * See solana-conformance/README.md for functionality and use cases */
+
+#include <unistd.h>
+#include <sys/mman.h>
+#include <errno.h>
+
+#include "../../fd_flamenco.h"
+#include "../../fd_flamenco_base.h"
+#include "../fd_system_ids.h"
+#include "../fd_runtime.h"
+#include "../fd_executor.h"
+#include "../../vm/fd_vm.h"
+#include "../../../util/log/fd_log.h"
+
+#include "../../nanopb/pb_encode.h"
+#include "generated/elf.pb.h"
+#include "generated/invoke.pb.h"
+#include "generated/txn.pb.h"
+#include "generated/vm.pb.h"
+
+FD_PROTOTYPES_BEGIN
+
+#define TOSTRING(x) #x
+#define STRINGIFY(x) TOSTRING(x)
+
+void
+fd_dump_instr_to_protobuf( fd_exec_txn_ctx_t * txn_ctx,
+                           fd_instr_info_t *   instr,
+                           ushort              instruction_idx );
+
+void
+fd_dump_txn_to_protobuf( fd_exec_txn_ctx_t *txn_ctx, fd_spad_t * spad );
+
+/* Captures the state of the VM (including the instruction context).
+   Meant to be invoked at the start of the VM_SYSCALL_CPI_ENTRYPOINT like so:
+
+  ```
+   dump_vm_cpi_state(vm, STRINGIFY(FD_EXPAND_THEN_CONCAT2(sol_invoke_signed_, VM_SYSCALL_CPI_ABI)),
+                     instruction_va, acct_infos_va, acct_info_cnt, signers_seeds_va, signers_seeds_cnt);
+  ```
+
+  Assumes that a `vm_cpi_state` directory exists in the current working directory. Generates a
+  unique dump for combination of (tile_id, caller_pubkey, instr_sz). */
+void
+fd_dump_vm_cpi_state( fd_vm_t *    vm,
+                      char const * fn_name,
+                      ulong        instruction_va,
+                      ulong        acct_infos_va,
+                      ulong        acct_info_cnt,
+                      ulong        signers_seeds_va,
+                      ulong        signers_seeds_cnt );
+
+FD_PROTOTYPES_END
+
+#endif /* HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h */

src/flamenco/vm/syscall/fd_vm_syscall_cpi.c

@@ -5,129 +5,6 @@
 #include "../../runtime/fd_executor.h"
 #include "../../runtime/fd_account_old.h" /* FIXME: remove this and update to use new APIs */
 #include <stdio.h>
-#include <sys/mman.h>
-#include <unistd.h>
-#include <errno.h>
-#include "../../nanopb/pb_encode.h"
-#include "../../runtime/tests/generated/vm.pb.h"
-#include "../../runtime/tests/fd_exec_instr_test.h"
-
-#define STRINGIFY(x) TOSTRING(x)
-#define TOSTRING(x) #x
-
-/* Captures the state of the VM (including the instruction context).
-   Meant to be invoked at the start of the VM_SYSCALL_CPI_ENTRYPOINT like so:
-
-  ```
-   dump_vm_cpi_state(vm, STRINGIFY(FD_EXPAND_THEN_CONCAT2(sol_invoke_signed_, VM_SYSCALL_CPI_ABI)),
-                     instruction_va, acct_infos_va, acct_info_cnt, signers_seeds_va, signers_seeds_cnt);
-  ```
-
-  Assumes that a `vm_cp_state` directory exists in the current working directory. Generates a
-  unique dump for combination of (tile_id, caller_pubkey, instr_sz). */
-
-static FD_FN_UNUSED void
-dump_vm_cpi_state(fd_vm_t *vm,
-                  char const * fn_name,
-                  ulong   instruction_va,
-                  ulong   acct_infos_va,
-                  ulong   acct_info_cnt,
-                  ulong   signers_seeds_va,
-                  ulong   signers_seeds_cnt ) {
-  char filename[100];
-  fd_instr_info_t const *instr = vm->instr_ctx->instr;
-  sprintf(filename, "vm_cpi_state/%lu_%lu%lu_%hu.sysctx", fd_tile_id(), instr->program_id_pubkey.ul[0], instr->program_id_pubkey.ul[1], instr->data_sz);
-
-  // Check if file exists
-  if( access (filename, F_OK) != -1 ) {
-    return;
-  }
-
-  fd_exec_test_syscall_context_t sys_ctx = FD_EXEC_TEST_SYSCALL_CONTEXT_INIT_ZERO;
-  sys_ctx.has_instr_ctx = 1;
-  sys_ctx.has_vm_ctx = 1;
-  sys_ctx.has_syscall_invocation = 1;
-
-  // Copy function name
-  sys_ctx.syscall_invocation.function_name.size = fd_uint_min( (uint) strlen(fn_name), sizeof(sys_ctx.syscall_invocation.function_name.bytes) );
-  fd_memcpy( sys_ctx.syscall_invocation.function_name.bytes,
-             fn_name,
-             sys_ctx.syscall_invocation.function_name.size );
-
-  // VM Ctx integral fields
-  sys_ctx.vm_ctx.r1 = instruction_va;
-  sys_ctx.vm_ctx.r2 = acct_infos_va;
-  sys_ctx.vm_ctx.r3 = acct_info_cnt;
-  sys_ctx.vm_ctx.r4 = signers_seeds_va;
-  sys_ctx.vm_ctx.r5 = signers_seeds_cnt;
-
-  sys_ctx.vm_ctx.rodata_text_section_length = vm->text_sz;
-  sys_ctx.vm_ctx.rodata_text_section_offset = vm->text_off;
-
-  sys_ctx.vm_ctx.heap_max = vm->heap_max; /* should be equiv. to txn_ctx->heap_sz */
-
-  FD_SCRATCH_SCOPE_BEGIN{
-    sys_ctx.vm_ctx.rodata = fd_scratch_alloc( 8UL, PB_BYTES_ARRAY_T_ALLOCSIZE(vm->rodata_sz) );
-    sys_ctx.vm_ctx.rodata->size = (pb_size_t) vm->rodata_sz;
-    fd_memcpy( sys_ctx.vm_ctx.rodata->bytes, vm->rodata, vm->rodata_sz );
-
-    pb_size_t stack_sz = (pb_size_t) ( (vm->frame_cnt + 1)*FD_VM_STACK_GUARD_SZ*2 );
-    sys_ctx.syscall_invocation.stack_prefix = fd_scratch_alloc( 8UL, PB_BYTES_ARRAY_T_ALLOCSIZE(stack_sz) );
-    sys_ctx.syscall_invocation.stack_prefix->size = stack_sz;
-    fd_memcpy( sys_ctx.syscall_invocation.stack_prefix->bytes, vm->stack, stack_sz );
-
-    sys_ctx.syscall_invocation.heap_prefix = fd_scratch_alloc( 8UL, PB_BYTES_ARRAY_T_ALLOCSIZE(vm->heap_max) );
-    sys_ctx.syscall_invocation.heap_prefix->size = (pb_size_t) vm->instr_ctx->txn_ctx->heap_size;
-    fd_memcpy( sys_ctx.syscall_invocation.heap_prefix->bytes, vm->heap, vm->instr_ctx->txn_ctx->heap_size );
-
-    sys_ctx.vm_ctx.input_data_regions_count = vm->input_mem_regions_cnt;
-    sys_ctx.vm_ctx.input_data_regions = fd_scratch_alloc( 8UL, sizeof(fd_exec_test_input_data_region_t) * vm->input_mem_regions_cnt );
-    for( ulong i=0UL; i<vm->input_mem_regions_cnt; i++ ) {
-      sys_ctx.vm_ctx.input_data_regions[i].content = fd_scratch_alloc( 8UL, PB_BYTES_ARRAY_T_ALLOCSIZE(vm->input_mem_regions[i].region_sz) );
-      sys_ctx.vm_ctx.input_data_regions[i].content->size = (pb_size_t) vm->input_mem_regions[i].region_sz;
-      fd_memcpy( sys_ctx.vm_ctx.input_data_regions[i].content->bytes, (uchar *) vm->input_mem_regions[i].haddr, vm->input_mem_regions[i].region_sz );
-      sys_ctx.vm_ctx.input_data_regions[i].offset = vm->input_mem_regions[i].vaddr_offset;
-      sys_ctx.vm_ctx.input_data_regions[i].is_writable = vm->input_mem_regions[i].is_writable;
-    }
-
-    fd_create_instr_context_protobuf_from_instructions( &sys_ctx.instr_ctx,
-                                                        vm->instr_ctx->txn_ctx,
-                                                        vm->instr_ctx->instr );
-
-    // Serialize the protobuf to file (using mmap)
-    size_t pb_alloc_size = 100 * 1024 * 1024; // 100MB (largest so far is 19MB)
-    FILE *f = fopen(filename, "wb+");
-    if( ftruncate(fileno(f), (off_t) pb_alloc_size) != 0 ) {
-      FD_LOG_WARNING(("Failed to resize file %s", filename));
-      fclose(f);
-      return;
-    }
-
-    uchar *pb_alloc = mmap( NULL,
-                            pb_alloc_size,
-                            PROT_READ | PROT_WRITE,
-                            MAP_SHARED,
-                            fileno(f),
-                            0 /* offset */);
-    if( pb_alloc == MAP_FAILED ) {
-      FD_LOG_WARNING(( "Failed to mmap file %d", errno ));
-      fclose(f);
-      return;
-    }
-
-    pb_ostream_t stream = pb_ostream_from_buffer(pb_alloc, pb_alloc_size);
-    if( !pb_encode( &stream, FD_EXEC_TEST_SYSCALL_CONTEXT_FIELDS, &sys_ctx ) ) {
-      FD_LOG_WARNING(( "Failed to encode instruction context" ));
-    }
-    // resize file to actual size
-    if( ftruncate( fileno(f), (off_t) stream.bytes_written ) != 0 ) {
-      FD_LOG_WARNING(( "Failed to resize file %s", filename ));
-    }
-
-    fclose(f);
-
-  } FD_SCRATCH_SCOPE_END;
-}
 
 /* FIXME: ALGO EFFICIENCY */
 static inline int