Update

Author: viirya

common/src/main/java/org/apache/arrow/c/CometBufferImportTypeVisitor.java

@@ -260,7 +260,12 @@ public List<ArrowBuf> visit(ArrowType.LargeUtf8 type) {
           type,
           start,
           end);
-      final long len = end - start;
+      // HACK: For the issue https://github.com/apache/datafusion-comet/issues/540
+      // As Arrow Java doesn't support `offset` in C Data interface, we cannot correctly import
+      // a slice of string from arrow-rs to Java Arrow and then export it to arrow-rs again.
+      // So we add this hack to always take full length of data buffer by assuming the first offset
+      // is always 0 which is true for Arrow Java and arrow-rs.
+      final long len = end;
       offsets.getReferenceManager().retain();
       return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len));
     }
@@ -277,7 +282,12 @@ public List<ArrowBuf> visit(ArrowType.Binary type) {
           type,
           start,
           end);
-      final int len = end - start;
+      // HACK: For the issue https://github.com/apache/datafusion-comet/issues/540
+      // As Arrow Java doesn't support `offset` in C Data interface, we cannot correctly import
+      // a slice of string from arrow-rs to Java Arrow and then export it to arrow-rs again.
+      // So we add this hack to always take full length of data buffer by assuming the first offset
+      // is always 0 which is true for Arrow Java and arrow-rs.
+      final int len = end;
       offsets.getReferenceManager().retain();
       return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len));
     }
@@ -296,7 +306,12 @@ public List<ArrowBuf> visit(ArrowType.LargeBinary type) {
           type,
           start,
           end);
-      final long len = end - start;
+      // HACK: For the issue https://github.com/apache/datafusion-comet/issues/540
+      // As Arrow Java doesn't support `offset` in C Data interface, we cannot correctly import
+      // a slice of string from arrow-rs to Java Arrow and then export it to arrow-rs again.
+      // So we add this hack to always take full length of data buffer by assuming the first offset
+      // is always 0 which is true for Arrow Java and arrow-rs.
+      final long len = end;
       offsets.getReferenceManager().retain();
       return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len));
     }

core/Cargo.lock

@@ -33,13 +33,13 @@ include = [
 
 [dependencies]
 parquet-format = "4.0.0" # This must be kept in sync with that from parquet crate
-arrow = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd", features = ["prettyprint", "ffi", "chrono-tz"] }
-arrow-array = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
-arrow-buffer = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
-arrow-data = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
-arrow-schema = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
-arrow-string = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
-parquet = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd", default-features = false, features = ["experimental"] }
+arrow = { git = "https://github.com/viirya/arrow-rs.git", rev = "5aa7e7f", features = ["prettyprint", "ffi", "chrono-tz"] }
+arrow-array = { git = "https://github.com/viirya/arrow-rs.git", rev = "5aa7e7f" }
+arrow-buffer = { git = "https://github.com/viirya/arrow-rs.git", rev = "5aa7e7f" }
+arrow-data = { git = "https://github.com/viirya/arrow-rs.git", rev = "5aa7e7f" }
+arrow-schema = { git = "https://github.com/viirya/arrow-rs.git", rev = "5aa7e7f" }
+arrow-string = { git = "https://github.com/viirya/arrow-rs.git", rev = "5aa7e7f" }
+parquet = { git = "https://github.com/viirya/arrow-rs.git", rev = "5aa7e7f", default-features = false, features = ["experimental"] }
 half = { version = "2.4.1", default-features = false }
 futures = "0.3.28"
 mimalloc = { version = "*", default-features = false, optional = true }
@@ -71,12 +71,12 @@ itertools = "0.11.0"
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 chrono-tz = { version = "0.8" }
 paste = "1.0.14"
-datafusion-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1" }
-datafusion = { default-features = false, git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", features = ["unicode_expressions", "crypto_expressions"] }
-datafusion-functions = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", features = ["crypto_expressions"] }
-datafusion-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", default-features = false }
-datafusion-physical-expr-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", default-features = false }
-datafusion-physical-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", default-features = false }
+datafusion-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "bad5579" }
+datafusion = { default-features = false, git = "https://github.com/viirya/arrow-datafusion.git", rev = "bad5579", features = ["unicode_expressions", "crypto_expressions"] }
+datafusion-functions = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "bad5579", features = ["crypto_expressions"] }
+datafusion-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "bad5579", default-features = false }
+datafusion-physical-expr-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "bad5579", default-features = false }
+datafusion-physical-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "bad5579", default-features = false }
 unicode-segmentation = "^1.10.1"
 once_cell = "1.18.0"
 regex = "1.9.6"