Update to arrow 28 (#4400)

* Update to arrow 28

* Fix avro

* Fix bad_extension_planner test

* Further tweaks

* More fixes

* Fixes

* Remove crates patch

* Review feedback

Author: tustvold

benchmarks/Cargo.toml

@@ -33,14 +33,14 @@ simd = ["datafusion/simd"]
 snmalloc = ["snmalloc-rs"]
 
 [dependencies]
-arrow = "27.0.0"
+arrow = "28.0.0"
 datafusion = { path = "../datafusion/core", version = "14.0.0", features = ["scheduler"] }
 env_logger = "0.10"
 futures = "0.3"
 mimalloc = { version = "0.1", optional = true, default-features = false }
 num_cpus = "1.13.0"
 object_store = "0.5.0"
-parquet = "27.0.0"
+parquet = "28.0.0"
 parquet-test-utils = { path = "../parquet-test-utils/", version = "0.1.0" }
 rand = "0.8.4"
 serde = { version = "1.0.136", features = ["derive"] }

datafusion-cli/Cargo.lock

@@ -29,7 +29,7 @@ rust-version = "1.62"
 readme = "README.md"
 
 [dependencies]
-arrow = "27.0.0"
+arrow = "28.0.0"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "14.0.0" }
 dirs = "4.0.0"

datafusion-cli/Cargo.toml

@@ -34,8 +34,8 @@ path = "examples/avro_sql.rs"
 required-features = ["datafusion/avro"]
 
 [dev-dependencies]
-arrow = "27.0.0"
-arrow-flight = "27.0.0"
+arrow = "28.0.0"
+arrow-flight = "28.0.0"
 async-trait = "0.1.41"
 datafusion = { path = "../datafusion/core" }
 datafusion-common = { path = "../datafusion/common" }

datafusion-examples/Cargo.toml

@@ -40,10 +40,10 @@ pyarrow = ["pyo3", "arrow/pyarrow"]
 
 [dependencies]
 apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
-arrow = { version = "27.0.0", default-features = false }
+arrow = { version = "28.0.0", default-features = false }
 chrono = { version = "0.4", default-features = false }
 cranelift-module = { version = "0.89.0", optional = true }
 object_store = { version = "0.5.0", default-features = false, optional = true }
-parquet = { version = "27.0.0", default-features = false, optional = true }
+parquet = { version = "28.0.0", default-features = false, optional = true }
 pyo3 = { version = "0.17.1", optional = true }
 sqlparser = "0.27"

datafusion/common/Cargo.toml

@@ -625,7 +625,6 @@ impl DFField {
 mod tests {
     use super::*;
     use arrow::datatypes::DataType;
-    use std::collections::BTreeMap;
 
     #[test]
     fn qualifier_in_name() -> Result<()> {
@@ -673,8 +672,8 @@ mod tests {
     fn from_qualified_schema_into_arrow_schema() -> Result<()> {
         let schema = DFSchema::try_from_qualified_schema("t1", &test_schema_1())?;
         let arrow_schema: Schema = schema.into();
-        let expected = "Field { name: \"c0\", data_type: Boolean, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: None }, \
-        Field { name: \"c1\", data_type: Boolean, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: None }";
+        let expected = "Field { name: \"c0\", data_type: Boolean, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }, \
+        Field { name: \"c1\", data_type: Boolean, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }";
         assert_eq!(expected, arrow_schema.to_string());
         Ok(())
     }
@@ -798,7 +797,7 @@ mod tests {
             field1_i16_t
                 .field()
                 .clone()
-                .with_metadata(Some(test_bmetadata_n(2))),
+                .with_metadata(test_metadata_n(2)),
         );
         let field1_i16_t_qualified =
             DFField::from_qualified("foo", field1_i16_t.field().clone());
@@ -947,12 +946,12 @@ mod tests {
     fn test_dfschema_to_schema_convertion() {
         let mut a: DFField = DFField::new(Some("table1"), "a", DataType::Int64, false);
         let mut b: DFField = DFField::new(Some("table1"), "b", DataType::Int64, false);
-        let mut a_metadata = BTreeMap::new();
+        let mut a_metadata = HashMap::new();
         a_metadata.insert("key".to_string(), "value".to_string());
-        a.field.set_metadata(Some(a_metadata));
-        let mut b_metadata = BTreeMap::new();
+        a.field.set_metadata(a_metadata);
+        let mut b_metadata = HashMap::new();
         b_metadata.insert("key".to_string(), "value".to_string());
-        b.field.set_metadata(Some(b_metadata));
+        b.field.set_metadata(b_metadata);
 
         let df_schema = Arc::new(
             DFSchema::new_with_metadata([a, b].to_vec(), HashMap::new()).unwrap(),
@@ -980,11 +979,4 @@ mod tests {
             .map(|i| (format!("k{}", i), format!("v{}", i)))
             .collect()
     }
-
-    fn test_bmetadata_n(n: usize) -> BTreeMap<String, String> {
-        (0..n)
-            .into_iter()
-            .map(|i| (format!("k{}", i), format!("v{}", i)))
-            .collect()
-    }
 }