dfindexer: pre-specify FMT length

Sometimes the FMT message describing FMT itself is not the first message
in the log. I did not know this was possible.

Author: robertlong13

DFReader.py

@@ -1273,7 +1273,6 @@ def init_arrays_fast(self):
             if f.name == 'FMT':
                 fmt_fmt = f
                 break
-        # fmt_fmt = self.formats['FMT']
         fmt_type = fmt_fmt.type
         type_offset = -1
         size_offset = -1
@@ -1292,6 +1291,7 @@ def init_arrays_fast(self):
         offsets = dfindexer.build_offsets(
             data,
             fmt_type,
+            fmt_fmt.len,
             type_offset,
             size_offset,
             self.HEAD1,

dfindexer/dfindexer.c

@@ -36,13 +36,18 @@ static void ensure_capacity(OffsetArray *arr) {
 }
 
 OffsetArray* scan_offsets(const uint8_t *data, size_t len,
-                          uint8_t fmt_type,
+                          uint8_t fmt_type, uint8_t fmt_length,
                           uint8_t type_offset, uint8_t length_offset,
                           uint8_t head1, uint8_t head2) {
     uint8_t lengths[NUM_TYPES] = {0};
     OffsetArray *results = calloc(NUM_TYPES, sizeof(OffsetArray));
     if (!results) panic("Memory allocation failed");
 
+    // Make an a-priori assumption about the FMT message length, since
+    // sometimes, somehow, the FMT for FMT ends up NOT being the first message
+    // in the file.
+    lengths[fmt_type] = fmt_length;
+
     size_t i = 0;
     size_t loop_count = 0;
     while (i + 3 < len) {

dfindexer/dfindexer.h

@@ -11,7 +11,7 @@ typedef struct OffsetArray {
 } OffsetArray;
 
 OffsetArray* scan_offsets(const uint8_t *data, size_t len,
-                          uint8_t fmt_type,
+                          uint8_t fmt_type, uint8_t fmt_length,
                           uint8_t type_offset, uint8_t length_offset,
                           uint8_t head1, uint8_t head2);
 

dfindexer/dfindexer_cy.pyx

@@ -5,22 +5,22 @@ cdef extern from "dfindexer.h":
         size_t cap
 
     OffsetArray* scan_offsets(const unsigned char* data, size_t len,
-                              unsigned char fmt_type,
+                              unsigned char fmt_type, unsigned char fmt_length,
                               unsigned char type_offset, unsigned char length_offset,
                               unsigned char head1, unsigned char head2)
 
     void free_offsets(OffsetArray* offsets)
 
 def build_offsets(const unsigned char[:] data,
-                  unsigned char fmt_type,
+                  unsigned char fmt_type, unsigned char fmt_length,
                   unsigned char type_offset, unsigned char length_offset,
                   unsigned char head1, unsigned char head2):
     cdef OffsetArray* results
     cdef size_t i, j
     cdef list py_offsets = []
 
     results = scan_offsets(&data[0], data.shape[0],
-                           fmt_type,
+                           fmt_type, fmt_length,
                            type_offset, length_offset,
                            head1, head2)
     if results == NULL: