changed data page size limit to 2mb

nikhilsinhaparseable · nikhilsinhaparseable · commit b1d6bda5885c · 2025-02-03T05:44:46.000-05:00
diff --git a/src/storage/staging.rs b/src/storage/staging.rs
@@ -327,7 +327,7 @@ pub fn parquet_writer_props(
             Encoding::DELTA_BINARY_PACKED,
         )
         .set_dictionary_enabled(false)
-        .set_data_page_size_limit(512*1024);
+        .set_data_page_size_limit(2*1024*1024);
 
     for (field, index) in custom_partition_fields {
         let field = ColumnPath::new(vec![field]);

Original file line number	Diff line number	Diff line change
`@@ -327,7 +327,7 @@ pub fn parquet_writer_props(`
`327`	`327`	`Encoding::DELTA_BINARY_PACKED,`
`328`	`328`	`)`
`329`	`329`	`.set_dictionary_enabled(false)`
`330`		`- .set_data_page_size_limit(512*1024);`
	`330`	`+ .set_data_page_size_limit(210241024);`
`331`	`331`
`332`	`332`	`for (field, index) in custom_partition_fields {`
`333`	`333`	`let field = ColumnPath::new(vec![field]);`