Remove LargeUtf8|Binary, Utf8|BinaryView, and Dictionary from ScalarValue

datafusion/common/src/lib.rs

@@ -37,6 +37,7 @@ pub mod file_options;
 pub mod format;
 pub mod hash_utils;
 pub mod instant;
+pub mod logical;
 pub mod parsers;
 pub mod rounding;
 pub mod scalar;

datafusion/common/src/logical/eq.rs

@@ -0,0 +1,26 @@
+use arrow_schema::DataType;
+
+pub trait LogicallyEq<Rhs: ?Sized = Self> {
+    #[must_use]
+    fn logically_eq(&self, other: &Rhs) -> bool;
+}
+
+impl LogicallyEq for DataType {
+    fn logically_eq(&self, other: &Self) -> bool {
+        use DataType::*;
+
+        match (self, other) {
+            (Utf8 | LargeUtf8 | Utf8View, Utf8 | LargeUtf8 | Utf8View)
+            | (Binary | LargeBinary | BinaryView, Binary | LargeBinary | BinaryView) => {
+                true
+            }
+            (Dictionary(_, inner), other) | (other, Dictionary(_, inner)) => {
+                other.logically_eq(inner)
+            }
+            (RunEndEncoded(_, inner), other) | (other, RunEndEncoded(_, inner)) => {
+                other.logically_eq(inner.data_type())
+            }
+            _ => self == other,
+        }
+    }
+}

datafusion/common/src/logical/mod.rs

@@ -0,0 +1 @@
+pub mod eq;

datafusion/common/src/utils/mod.rs

@@ -378,10 +378,10 @@ pub fn array_into_list_array(arr: ArrayRef, nullable: bool) -> ListArray {
 
 /// Wrap an array into a single element `LargeListArray`.
 /// For example `[1, 2, 3]` would be converted into `[[1, 2, 3]]`
-pub fn array_into_large_list_array(arr: ArrayRef) -> LargeListArray {
+pub fn array_into_large_list_array(arr: ArrayRef, nullable: bool) -> LargeListArray {
     let offsets = OffsetBuffer::from_lengths([arr.len()]);
     LargeListArray::new(
-        Arc::new(Field::new_list_field(arr.data_type().to_owned(), true)),
+        Arc::new(Field::new_list_field(arr.data_type().to_owned(), nullable)),
         offsets,
         arr,
         None,