Dictionary nulls multi page fix (tensorflow#58)

* Optimize page reading by not create array from bytes owner

* - fixes misaligned data from dictionary-encoded columns with null values (tensorflow#57)

- added test and test data file

Co-authored-by: Chirag Gupta (AZURE) <[email protected]>
Co-authored-by: Carlos Orrego <[email protected]>

Author: 3 people

src/Parquet.Test/ParquetReaderTest.cs

@@ -144,6 +144,30 @@ public void Read_multiple_data_pages()
             Assert.Equal(0.867111980015206, seq.Max(p => p.v), 5);
          }
       }
+      
+      [Fact]
+      public void Read_multi_page_dictionary_with_nulls()
+      {
+         using (var reader = new ParquetReader(OpenTestFile("/special/multi_page_dictionary_with_nulls.parquet")))
+         {
+            DataColumn[] columns = reader.ReadEntireRowGroup();
+            var rg = reader.OpenRowGroupReader(0);
+
+            // reading columns
+            var data = (string[]) columns[0].Data;
+         
+            // ground truth from spark
+            // check page boundary (first page contains 107432 rows)
+            Assert.Equal("xc3w4eudww", data[107432]);
+            Assert.Equal("bpywp4wtwk", data[107433]);
+            Assert.Equal("z6x8652rle", data[107434]);
+
+            // check near the end of the file
+            Assert.Null(data[310028]);
+            Assert.Equal("wok86kie6c", data[310029]);
+            Assert.Equal("le9i7kbbib", data[310030]);
+         }
+      }
 
       [Fact]
       public void Read_bit_packed_at_page_boundary()

src/Parquet.Test/data/special/multi_page_dictionary_with_nulls.parquet

@@ -209,7 +209,11 @@ private void ReadDataPage(Thrift.PageHeader ph, ColumnRawData cd, long maxValues
 
                   if (ph.Data_page_header == null) throw new ParquetException($"column '{_dataField.Path}' is missing data page header, file is corrupt");
 
-                  ReadColumn(reader, ph.Data_page_header.Encoding, maxValues, ph.Data_page_header.Num_values, cd);
+                  // if statistics are defined, use null count to determine the exact number of items we should read
+                  // however, I don't know if all parquet files with null values have stats defined. Maybe a better solution would
+                  // be using a count of defined values (from reading definitions?) 
+                  int maxReadCount = ph.Data_page_header.Num_values - (int)(ph.Data_page_header.Statistics?.Null_count ?? 0);
+                  ReadColumn(reader, ph.Data_page_header.Encoding, maxValues, maxReadCount, cd);
                }
             }
          }