deps: [DO NOT MERGE] test DataFusion 55.0 and Arrow/Parquet 59.1

Cargo.toml

@@ -49,15 +49,15 @@ aes-gcm = "0.10"
 anyhow = "1.0.72"
 apache-avro = { version = "0.21", features = ["snappy", "zstandard"] }
 array-init = "2"
-arrow = "58.4"
-arrow-arith = "58.4"
-arrow-array = "58.4"
-arrow-buffer = "58.4"
-arrow-cast = "58.4"
-arrow-ord = "58.4"
-arrow-schema = "58.4"
-arrow-select = "58.4"
-arrow-string = "58.4"
+arrow = "59.1"
+arrow-arith = "59.1"
+arrow-array = "59.1"
+arrow-buffer = "59.1"
+arrow-cast = "59.1"
+arrow-ord = "59.1"
+arrow-schema = "59.1"
+arrow-select = "59.1"
+arrow-string = "59.1"
 as-any = "0.3.2"
 async-trait = "0.1.89"
 aws-config = "1.8.7"
@@ -77,10 +77,10 @@ cfg-if = "1"
 chrono = "0.4.41"
 clap = { version = "4.5.48", features = ["derive", "cargo"] }
 dashmap = "6"
-datafusion = "54.1.0"
-datafusion-cli = "54.1.0"
-datafusion-ffi = "54.1.0"
-datafusion-sqllogictest = "54.1.0"
+datafusion = { git = "https://github.com/apache/datafusion.git", rev = "bc8b1a7a9323f69970fe728c95f3930cfbd5ad41" }
+datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "bc8b1a7a9323f69970fe728c95f3930cfbd5ad41" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "bc8b1a7a9323f69970fe728c95f3930cfbd5ad41" }
+datafusion-sqllogictest = { git = "https://github.com/apache/datafusion.git", rev = "bc8b1a7a9323f69970fe728c95f3930cfbd5ad41" }
 derive_builder = "0.20"
 dirs = "6"
 enum-ordinalize = "4.3.0"
@@ -122,10 +122,10 @@ murmur3 = "0.5.2"
 once_cell = "1.20"
 opendal = "0.58"
 ordered-float = "4"
-parquet = "58.4"
+parquet = "59.1"
 pilota = "0.11.10"
 pretty_assertions = "1.4"
-pyo3 = "0.28"
+pyo3 = "0.29"
 rand = "0.9.3"
 regex = "1.11.3"
 reqwest = { version = "0.12.12", default-features = false, features = ["json"] }

bindings/python/pyproject.toml

@@ -75,7 +75,7 @@ filterwarnings = [
 dev = [
   "maturin>=1.9.4,<2.0",
   "pytest>=8.3.2",
-  "datafusion>=52,<55",
+  "datafusion==54.*",
   # Set a PyIceberg lower bound; otherwise the resolver may select the
   # unrelated 0.0.2 stub.
   # Keep pyarrow direct because current pyiceberg[pyarrow] releases depend on

bindings/python/src/datafusion_table_provider.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use std::collections::HashMap;
-use std::ffi::{CStr, CString};
+use std::ffi::CStr;
 use std::sync::Arc;
 
 use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;
@@ -32,7 +32,7 @@ use pyo3::types::{PyAny, PyCapsule};
 
 use crate::runtime::runtime;
 
-// pyo3 0.28's CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`,
+// CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`,
 // so we accept &CStr to allow direct comparison without UTF-8 validation.
 pub(crate) fn validate_pycapsule(capsule: &Bound<PyCapsule>, name: &CStr) -> PyResult<()> {
     let capsule_name = capsule.name()?;
@@ -121,7 +121,7 @@ impl PyIcebergDataFusionTable {
         py: Python<'py>,
         session: Bound<'py, PyAny>,
     ) -> PyResult<Bound<'py, PyCapsule>> {
-        let capsule_name = CString::new("datafusion_table_provider").unwrap();
+        let capsule_name = c"datafusion_table_provider";
 
         let logical_codec = ffi_logical_codec_from_pycapsule(session)?;
 
@@ -132,7 +132,7 @@ impl PyIcebergDataFusionTable {
             logical_codec,
         );
 
-        PyCapsule::new(py, ffi_provider, Some(capsule_name))
+        PyCapsule::new_with_value(py, ffi_provider, capsule_name)
     }
 }
 

bindings/python/tests/test_datafusion_table_provider.py

@@ -28,9 +28,9 @@
 from pyiceberg.catalog import Catalog, load_catalog
 from pyiceberg_core.datafusion import IcebergDataFusionTable
 
-if Version(datafusion.__version__) < Version("53.0.0"):
+if Version(datafusion.__version__) < Version("55.0.0"):
     pytest.skip(
-        "Iceberg table provider requires datafusion>=53 for FFI compatibility",
+        "Iceberg table provider requires datafusion>=55 for FFI compatibility",
         allow_module_level=True,
     )
 

crates/catalog/rest/src/catalog.rs

@@ -564,7 +564,7 @@ impl RestCatalog {
 }
 
 /// All requests and expected responses are derived from the REST catalog API spec:
-/// https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml
+/// <https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml>
 #[async_trait]
 impl Catalog for RestCatalog {
     async fn list_namespaces(

crates/iceberg/Cargo.toml

@@ -92,7 +92,7 @@ mockall = { workspace = true }
 pretty_assertions = { workspace = true }
 rand = { workspace = true }
 regex = { workspace = true }
-serde_arrow = { version = "0.14", features = ["arrow-58"] }
+serde_arrow = { version = "0.14", features = ["arrow-59"] }
 tempfile = { workspace = true }
 
 [package.metadata.cargo-machete]