@@ -48,12 +48,11 @@ use datafusion::logical_expr::{expr, Between, JoinConstraint, LogicalPlan, Opera
48
48
use datafusion:: prelude:: Expr ;
49
49
use pbjson_types:: Any as ProtoAny ;
50
50
use substrait:: proto:: exchange_rel:: { ExchangeKind , RoundRobin , ScatterFields } ;
51
- use substrait:: proto:: expression:: literal:: user_defined :: Val ;
52
- use substrait :: proto :: expression :: literal :: UserDefined ;
53
- use substrait :: proto :: expression :: literal :: { List , Struct } ;
51
+ use substrait:: proto:: expression:: literal:: {
52
+ user_defined , IntervalDayToSecond , IntervalYearToMonth , List , Struct , UserDefined ,
53
+ } ;
54
54
use substrait:: proto:: expression:: subquery:: InPredicate ;
55
55
use substrait:: proto:: expression:: window_function:: BoundsType ;
56
- use substrait:: proto:: r#type:: { parameter, Parameter } ;
57
56
use substrait:: proto:: read_rel:: VirtualTable ;
58
57
use substrait:: proto:: { CrossRel , ExchangeRel } ;
59
58
use substrait:: {
@@ -95,9 +94,7 @@ use crate::variation_const::{
95
94
DATE_32_TYPE_VARIATION_REF , DATE_64_TYPE_VARIATION_REF ,
96
95
DECIMAL_128_TYPE_VARIATION_REF , DECIMAL_256_TYPE_VARIATION_REF ,
97
96
DEFAULT_CONTAINER_TYPE_VARIATION_REF , DEFAULT_TYPE_VARIATION_REF ,
98
- INTERVAL_DAY_TIME_TYPE_REF , INTERVAL_DAY_TIME_TYPE_URL ,
99
97
INTERVAL_MONTH_DAY_NANO_TYPE_REF , INTERVAL_MONTH_DAY_NANO_TYPE_URL ,
100
- INTERVAL_YEAR_MONTH_TYPE_REF , INTERVAL_YEAR_MONTH_TYPE_URL ,
101
98
LARGE_CONTAINER_TYPE_VARIATION_REF , TIMESTAMP_MICRO_TYPE_VARIATION_REF ,
102
99
TIMESTAMP_MILLI_TYPE_VARIATION_REF , TIMESTAMP_NANO_TYPE_VARIATION_REF ,
103
100
TIMESTAMP_SECOND_TYPE_VARIATION_REF , UNSIGNED_INTEGER_TYPE_VARIATION_REF ,
@@ -1534,47 +1531,31 @@ fn to_substrait_type(dt: &DataType, nullable: bool) -> Result<substrait::proto::
1534
1531
} ) ) ,
1535
1532
} ) ,
1536
1533
DataType :: Interval ( interval_unit) => {
1537
- // define two type parameters for convenience
1538
- let i32_param = Parameter {
1539
- parameter : Some ( parameter:: Parameter :: DataType ( substrait:: proto:: Type {
1540
- kind : Some ( r#type:: Kind :: I32 ( r#type:: I32 {
1534
+ match interval_unit {
1535
+ IntervalUnit :: YearMonth => Ok ( substrait:: proto:: Type {
1536
+ kind : Some ( r#type:: Kind :: IntervalYear ( r#type:: IntervalYear {
1541
1537
type_variation_reference : DEFAULT_TYPE_VARIATION_REF ,
1542
- nullability : r#type :: Nullability :: Unspecified as i32 ,
1538
+ nullability,
1543
1539
} ) ) ,
1544
- } ) ) ,
1545
- } ;
1546
- let i64_param = Parameter {
1547
- parameter : Some ( parameter:: Parameter :: DataType ( substrait:: proto:: Type {
1548
- kind : Some ( r#type:: Kind :: I64 ( r#type:: I64 {
1540
+ } ) ,
1541
+ IntervalUnit :: DayTime => Ok ( substrait:: proto:: Type {
1542
+ kind : Some ( r#type:: Kind :: IntervalDay ( r#type:: IntervalDay {
1549
1543
type_variation_reference : DEFAULT_TYPE_VARIATION_REF ,
1550
- nullability : r#type :: Nullability :: Unspecified as i32 ,
1544
+ nullability,
1551
1545
} ) ) ,
1552
- } ) ) ,
1553
- } ;
1554
-
1555
- let ( type_parameters, type_reference) = match interval_unit {
1556
- IntervalUnit :: YearMonth => {
1557
- let type_parameters = vec ! [ i32_param] ;
1558
- ( type_parameters, INTERVAL_YEAR_MONTH_TYPE_REF )
1559
- }
1560
- IntervalUnit :: DayTime => {
1561
- let type_parameters = vec ! [ i64_param] ;
1562
- ( type_parameters, INTERVAL_DAY_TIME_TYPE_REF )
1563
- }
1546
+ } ) ,
1564
1547
IntervalUnit :: MonthDayNano => {
1565
- // use 2 `i64` as `i128`
1566
- let type_parameters = vec ! [ i64_param. clone( ) , i64_param] ;
1567
- ( type_parameters, INTERVAL_MONTH_DAY_NANO_TYPE_REF )
1548
+ // Substrait doesn't currently support this type, so we represent it as a UDT
1549
+ Ok ( substrait:: proto:: Type {
1550
+ kind : Some ( r#type:: Kind :: UserDefined ( r#type:: UserDefined {
1551
+ type_reference : INTERVAL_MONTH_DAY_NANO_TYPE_REF ,
1552
+ type_variation_reference : DEFAULT_TYPE_VARIATION_REF ,
1553
+ nullability,
1554
+ type_parameters : vec ! [ ] ,
1555
+ } ) ) ,
1556
+ } )
1568
1557
}
1569
- } ;
1570
- Ok ( substrait:: proto:: Type {
1571
- kind : Some ( r#type:: Kind :: UserDefined ( r#type:: UserDefined {
1572
- type_reference,
1573
- type_variation_reference : DEFAULT_TYPE_VARIATION_REF ,
1574
- nullability,
1575
- type_parameters,
1576
- } ) ) ,
1577
- } )
1558
+ }
1578
1559
}
1579
1560
DataType :: Binary => Ok ( substrait:: proto:: Type {
1580
1561
kind : Some ( r#type:: Kind :: Binary ( r#type:: Binary {
@@ -1954,75 +1935,38 @@ fn to_substrait_literal(value: &ScalarValue) -> Result<Literal> {
1954
1935
( LiteralType :: Date ( * d) , DATE_32_TYPE_VARIATION_REF )
1955
1936
}
1956
1937
// Date64 literal is not supported in Substrait
1957
- ScalarValue :: IntervalYearMonth ( Some ( i) ) => {
1958
- let bytes = i. to_le_bytes ( ) ;
1959
- (
1960
- LiteralType :: UserDefined ( UserDefined {
1961
- type_reference : INTERVAL_YEAR_MONTH_TYPE_REF ,
1962
- type_parameters : vec ! [ Parameter {
1963
- parameter: Some ( parameter:: Parameter :: DataType (
1964
- substrait:: proto:: Type {
1965
- kind: Some ( r#type:: Kind :: I32 ( r#type:: I32 {
1966
- type_variation_reference: DEFAULT_TYPE_VARIATION_REF ,
1967
- nullability: r#type:: Nullability :: Required as i32 ,
1968
- } ) ) ,
1969
- } ,
1970
- ) ) ,
1971
- } ] ,
1972
- val : Some ( Val :: Value ( ProtoAny {
1973
- type_url : INTERVAL_YEAR_MONTH_TYPE_URL . to_string ( ) ,
1974
- value : bytes. to_vec ( ) . into ( ) ,
1975
- } ) ) ,
1976
- } ) ,
1977
- INTERVAL_YEAR_MONTH_TYPE_REF ,
1978
- )
1979
- }
1938
+ ScalarValue :: IntervalYearMonth ( Some ( i) ) => (
1939
+ LiteralType :: IntervalYearToMonth ( IntervalYearToMonth {
1940
+ // DF only tracks total months, but there should always be 12 months in a year
1941
+ years : * i / 12 ,
1942
+ months : * i % 12 ,
1943
+ } ) ,
1944
+ DEFAULT_TYPE_VARIATION_REF ,
1945
+ ) ,
1980
1946
ScalarValue :: IntervalMonthDayNano ( Some ( i) ) => {
1981
- // treat `i128` as two contiguous `i64`
1947
+ // IntervalMonthDayNano is internally represented as a 128-bit integer, containing
1948
+ // months (32bit), days (32bit), and nanoseconds (64bit)
1982
1949
let bytes = i. to_byte_slice ( ) ;
1983
- let i64_param = Parameter {
1984
- parameter : Some ( parameter:: Parameter :: DataType ( substrait:: proto:: Type {
1985
- kind : Some ( r#type:: Kind :: I64 ( r#type:: I64 {
1986
- type_variation_reference : DEFAULT_TYPE_VARIATION_REF ,
1987
- nullability : r#type:: Nullability :: Required as i32 ,
1988
- } ) ) ,
1989
- } ) ) ,
1990
- } ;
1991
1950
(
1992
1951
LiteralType :: UserDefined ( UserDefined {
1993
1952
type_reference : INTERVAL_MONTH_DAY_NANO_TYPE_REF ,
1994
- type_parameters : vec ! [ i64_param . clone ( ) , i64_param ] ,
1995
- val : Some ( Val :: Value ( ProtoAny {
1953
+ type_parameters : vec ! [ ] ,
1954
+ val : Some ( user_defined :: Val :: Value ( ProtoAny {
1996
1955
type_url : INTERVAL_MONTH_DAY_NANO_TYPE_URL . to_string ( ) ,
1997
1956
value : bytes. to_vec ( ) . into ( ) ,
1998
1957
} ) ) ,
1999
1958
} ) ,
2000
1959
INTERVAL_MONTH_DAY_NANO_TYPE_REF ,
2001
1960
)
2002
1961
}
2003
- ScalarValue :: IntervalDayTime ( Some ( i) ) => {
2004
- let bytes = i. to_byte_slice ( ) ;
2005
- (
2006
- LiteralType :: UserDefined ( UserDefined {
2007
- type_reference : INTERVAL_DAY_TIME_TYPE_REF ,
2008
- type_parameters : vec ! [ Parameter {
2009
- parameter: Some ( parameter:: Parameter :: DataType (
2010
- substrait:: proto:: Type {
2011
- kind: Some ( r#type:: Kind :: I64 ( r#type:: I64 {
2012
- type_variation_reference: DEFAULT_TYPE_VARIATION_REF ,
2013
- nullability: r#type:: Nullability :: Required as i32 ,
2014
- } ) ) ,
2015
- } ,
2016
- ) ) ,
2017
- } ] ,
2018
- val : Some ( Val :: Value ( ProtoAny {
2019
- type_url : INTERVAL_DAY_TIME_TYPE_URL . to_string ( ) ,
2020
- value : bytes. to_vec ( ) . into ( ) ,
2021
- } ) ) ,
2022
- } ) ,
2023
- INTERVAL_DAY_TIME_TYPE_REF ,
2024
- )
2025
- }
1962
+ ScalarValue :: IntervalDayTime ( Some ( i) ) => (
1963
+ LiteralType :: IntervalDayToSecond ( IntervalDayToSecond {
1964
+ days : i. days ,
1965
+ seconds : i. milliseconds / 1000 ,
1966
+ microseconds : ( i. milliseconds % 1000 ) * 1000 ,
1967
+ } ) ,
1968
+ DEFAULT_TYPE_VARIATION_REF ,
1969
+ ) ,
2026
1970
ScalarValue :: Binary ( Some ( b) ) => (
2027
1971
LiteralType :: Binary ( b. clone ( ) ) ,
2028
1972
DEFAULT_CONTAINER_TYPE_VARIATION_REF ,
0 commit comments