feat: Add ffi_enforce_no_offset feature as a workaround hack to unavailable offset support in Arrow Java

Author: viirya

.github/workflows/arrow.yml

@@ -95,6 +95,8 @@ jobs:
         run: cargo test -p arrow
       - name: Test arrow with all features except pyarrow
         run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,chrono-tz
+      - name: Test arrow with ffi_enforce_no_offset
+        run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,ffi_enforce_no_offset,chrono-tz
       - name: Run examples
         run: |
           # Test arrow examples
@@ -129,6 +131,8 @@ jobs:
         run: cargo check -p arrow --no-default-features --all-targets --features test_utils
       - name: Check compilation --no-default-features --all-targets --features ffi
         run: cargo check -p arrow --no-default-features --all-targets --features ffi
+      - name: Check compilation --no-default-features --all-targets --features ffi,ffi_enforce_no_offset
+        run: cargo check -p arrow --no-default-features --all-targets --features ffi,ffi_enforce_no_offset
       - name: Check compilation --no-default-features --all-targets --features chrono-tz
         run: cargo check -p arrow --no-default-features --all-targets --features chrono-tz
 

arrow-array/Cargo.toml

@@ -53,6 +53,20 @@ hashbrown = { version = "0.14", default-features = false }
 [features]
 ffi = ["arrow-schema/ffi", "arrow-data/ffi"]
 
+# HACKY: For offset-ed offset buffer of variable-sized binary array
+# The valid FFI output is to record the offset buffer's offset into FFI's
+# `offset` field and expose the beginning of the buffer as the pointer.
+# So the consumer can calculate correctly the length of data buffer.
+# However, due to Arrow Java's issue: https://github.com/apache/arrow/issues/42156,
+# Arrow Java does not support `offset` field in the FFI output. It causes
+# no sliced binary array can be passed between arrow-rs and Arrow Java.
+# To workaround this issue, we have to enforce offset-ed offset buffer to
+# be exposed at the sliced buffer's beginning. When calculating the length
+# of data buffer, we assume the first element of the offset buffer is always 0
+# (this is how Arrow Java and arrow-rs do). This is a hacky solution and
+# should be removed once Arrow Java supports `offset` field in the FFI output.
+ffi_enforce_no_offset = ["arrow-data/ffi_enforce_no_offset"]
+
 [dev-dependencies]
 rand = { version = "0.8", default-features = false, features = ["std", "std_rng"] }
 criterion = { version = "0.5", default-features = false }

arrow-array/src/ffi.rs

@@ -436,7 +436,16 @@ impl<'a> ImportedArrowArray<'a> {
                 let start = (unsafe { *offset_buffer.add(0) }) as usize;
                 // get last offset
                 let end = (unsafe { *offset_buffer.add(len / size_of::<i32>() - 1) }) as usize;
-                end - start
+
+                if cfg!(feature = "ffi_enforce_no_offset") {
+                    if end == start {
+                        0
+                    } else {
+                        end
+                    }
+                } else {
+                    end - start
+                }
             }
             (DataType::LargeUtf8, 2) | (DataType::LargeBinary, 2) => {
                 // the len of the data buffer (buffer 2) equals the difference between the last value
@@ -450,7 +459,16 @@ impl<'a> ImportedArrowArray<'a> {
                 let start = (unsafe { *offset_buffer.add(0) }) as usize;
                 // get last offset
                 let end = (unsafe { *offset_buffer.add(len / size_of::<i64>() - 1) }) as usize;
-                end - start
+
+                if cfg!(feature = "ffi_enforce_no_offset") {
+                    if end == start {
+                        0
+                    } else {
+                        end
+                    }
+                } else {
+                    end - start
+                }
             }
             // buffer len of primitive types
             _ => {

arrow-data/Cargo.toml

@@ -41,6 +41,20 @@ force_validate = []
 # Enable ffi support
 ffi = ["arrow-schema/ffi"]
 
+# HACKY: For offset-ed offset buffer of variable-sized binary array
+# The valid FFI output is to record the offset buffer's offset into FFI's
+# `offset` field and expose the beginning of the buffer as the pointer.
+# So the consumer can calculate correctly the length of data buffer.
+# However, due to Arrow Java's issue: https://github.com/apache/arrow/issues/42156,
+# Arrow Java does not support `offset` field in the FFI output. It causes
+# no sliced binary array can be passed between arrow-rs and Arrow Java.
+# To workaround this issue, we have to enforce offset-ed offset buffer to
+# be exposed at the sliced buffer's beginning. When calculating the length
+# of data buffer, we assume the first element of the offset buffer is always 0
+# (this is how Arrow Java and arrow-rs do). This is a hacky solution and
+# should be removed once Arrow Java supports `offset` field in the FFI output.
+ffi_enforce_no_offset = []
+
 [package.metadata.docs.rs]
 features = ["ffi"]
 

arrow-data/src/ffi.rs

@@ -152,7 +152,9 @@ impl FFI_ArrowArray {
             _ => None,
         };
 
-        let offset = if let Some(offset) = offset_offset {
+        let offset = if cfg!(feature = "ffi_enforce_no_offset") {
+            data.offset()
+        } else if let Some(offset) = offset_offset {
             if data.offset() != 0 {
                 // TODO: Adjust for data offset
                 panic!("The ArrayData of a slice offset buffer should not have offset");
@@ -184,7 +186,7 @@ impl FFI_ArrowArray {
                             | DataType::Binary
                             | DataType::LargeBinary,
                             1,
-                        ) => {
+                        ) if !cfg!(feature = "ffi_enforce_no_offset") => {
                             // For offset buffer, take original pointer without offset.
                             // Buffer offset should be handled by `FFI_ArrowArray` offset field.
                             Some(b.data_ptr().as_ptr() as *const c_void)

arrow/Cargo.toml

@@ -80,6 +80,20 @@ force_validate = ["arrow-data/force_validate"]
 ffi = ["arrow-schema/ffi", "arrow-data/ffi", "arrow-array/ffi"]
 chrono-tz = ["arrow-array/chrono-tz"]
 
+# HACKY: For offset-ed offset buffer of variable-sized binary array
+# The valid FFI output is to record the offset buffer's offset into FFI's
+# `offset` field and expose the beginning of the buffer as the pointer.
+# So the consumer can calculate correctly the length of data buffer.
+# However, due to Arrow Java's issue: https://github.com/apache/arrow/issues/42156,
+# Arrow Java does not support `offset` field in the FFI output. It causes
+# no sliced binary array can be passed between arrow-rs and Arrow Java.
+# To workaround this issue, we have to enforce offset-ed offset buffer to
+# be exposed at the sliced buffer's beginning. When calculating the length
+# of data buffer, we assume the first element of the offset buffer is always 0
+# (this is how Arrow Java and arrow-rs do). This is a hacky solution and
+# should be removed once Arrow Java supports `offset` field in the FFI output.
+ffi_enforce_no_offset = ["arrow-data/ffi_enforce_no_offset", "arrow-array/ffi_enforce_no_offset"]
+
 [dev-dependencies]
 chrono = { workspace = true }
 criterion = { version = "0.5", default-features = false }