Skip to content

Commit 916ccc3

Browse files
waynexiawaynexia
authored
feat: include some BinaryOperator from sqlparser (#15327)
* include some BinaryOperator from sqlparser Signed-off-by: Ruihang Xia <[email protected]> * slt for new operators Signed-off-by: Ruihang Xia <[email protected]> * add document about availability Signed-off-by: Ruihang Xia <[email protected]> --------- Signed-off-by: Ruihang Xia <[email protected]>
1 parent 3f538f8 commit 916ccc3

File tree

7 files changed

+230
-17
lines changed

7 files changed

+230
-17
lines changed

‎datafusion/expr-common/src/operator.rs

Lines changed: 103 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,64 @@ pub enum Operator {
8282
BitwiseShiftLeft,
8383
/// String concat
8484
StringConcat,
85-
/// At arrow, like `@>`
85+
/// At arrow, like `@>`.
86+
///
87+
/// Currently only supported to be used with lists:
88+
/// ```sql
89+
/// select [1,3] <@ [1,2,3]
90+
/// ```
8691
AtArrow,
87-
/// Arrow at, like `<@`
92+
/// Arrow at, like `<@`.
93+
///
94+
/// Currently only supported to be used with lists:
95+
/// ```sql
96+
/// select [1,2,3] @> [1,3]
97+
/// ```
8898
ArrowAt,
99+
/// Arrow, like `->`.
100+
///
101+
/// Not implemented in DataFusion yet.
102+
Arrow,
103+
/// Long arrow, like `->>`
104+
///
105+
/// Not implemented in DataFusion yet.
106+
LongArrow,
107+
/// Hash arrow, like `#>`
108+
///
109+
/// Not implemented in DataFusion yet.
110+
HashArrow,
111+
/// Hash long arrow, like `#>>`
112+
///
113+
/// Not implemented in DataFusion yet.
114+
HashLongArrow,
115+
/// At at, like `@@`
116+
///
117+
/// Not implemented in DataFusion yet.
118+
AtAt,
119+
/// Integer division operator, like `DIV` from MySQL or `//` from DuckDB
120+
///
121+
/// Not implemented in DataFusion yet.
122+
IntegerDivide,
123+
/// Hash Minis, like `#-`
124+
///
125+
/// Not implemented in DataFusion yet.
126+
HashMinus,
127+
/// At question, like `@?`
128+
///
129+
/// Not implemented in DataFusion yet.
130+
AtQuestion,
131+
/// Question, like `?`
132+
///
133+
/// Not implemented in DataFusion yet.
134+
Question,
135+
/// Question and, like `?&`
136+
///
137+
/// Not implemented in DataFusion yet.
138+
QuestionAnd,
139+
/// Question pipe, like `?|`
140+
///
141+
/// Not implemented in DataFusion yet.
142+
QuestionPipe,
89143
}
90144

91145
impl Operator {
@@ -123,7 +177,18 @@ impl Operator {
123177
| Operator::BitwiseShiftLeft
124178
| Operator::StringConcat
125179
| Operator::AtArrow
126-
| Operator::ArrowAt => None,
180+
| Operator::ArrowAt
181+
| Operator::Arrow
182+
| Operator::LongArrow
183+
| Operator::HashArrow
184+
| Operator::HashLongArrow
185+
| Operator::AtAt
186+
| Operator::IntegerDivide
187+
| Operator::HashMinus
188+
| Operator::AtQuestion
189+
| Operator::Question
190+
| Operator::QuestionAnd
191+
| Operator::QuestionPipe => None,
127192
}
128193
}
129194

@@ -216,7 +281,18 @@ impl Operator {
216281
| Operator::BitwiseXor
217282
| Operator::BitwiseShiftRight
218283
| Operator::BitwiseShiftLeft
219-
| Operator::StringConcat => None,
284+
| Operator::StringConcat
285+
| Operator::Arrow
286+
| Operator::LongArrow
287+
| Operator::HashArrow
288+
| Operator::HashLongArrow
289+
| Operator::AtAt
290+
| Operator::IntegerDivide
291+
| Operator::HashMinus
292+
| Operator::AtQuestion
293+
| Operator::Question
294+
| Operator::QuestionAnd
295+
| Operator::QuestionPipe => None,
220296
}
221297
}
222298

@@ -245,7 +321,18 @@ impl Operator {
245321
| Operator::BitwiseXor
246322
| Operator::StringConcat
247323
| Operator::AtArrow
248-
| Operator::ArrowAt => 30,
324+
| Operator::ArrowAt
325+
| Operator::Arrow
326+
| Operator::LongArrow
327+
| Operator::HashArrow
328+
| Operator::HashLongArrow
329+
| Operator::AtAt
330+
| Operator::IntegerDivide
331+
| Operator::HashMinus
332+
| Operator::AtQuestion
333+
| Operator::Question
334+
| Operator::QuestionAnd
335+
| Operator::QuestionPipe => 30,
249336
Operator::Plus | Operator::Minus => 40,
250337
Operator::Multiply | Operator::Divide | Operator::Modulo => 45,
251338
}
@@ -286,6 +373,17 @@ impl fmt::Display for Operator {
286373
Operator::StringConcat => "||",
287374
Operator::AtArrow => "@>",
288375
Operator::ArrowAt => "<@",
376+
Operator::Arrow => "->",
377+
Operator::LongArrow => "->>",
378+
Operator::HashArrow => "#>",
379+
Operator::HashLongArrow => "#>>",
380+
Operator::AtAt => "@@",
381+
Operator::IntegerDivide => "DIV",
382+
Operator::HashMinus => "#-",
383+
Operator::AtQuestion => "@?",
384+
Operator::Question => "?",
385+
Operator::QuestionAnd => "?&",
386+
Operator::QuestionPipe => "?|",
289387
};
290388
write!(f, "{display}")
291389
}

‎datafusion/expr-common/src/type_coercion/binary.rs

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ use arrow::datatypes::{
3030
};
3131
use datafusion_common::types::NativeType;
3232
use datafusion_common::{
33-
exec_err, internal_err, plan_datafusion_err, plan_err, Diagnostic, Result, Span,
34-
Spans,
33+
exec_err, internal_err, not_impl_err, plan_datafusion_err, plan_err, Diagnostic,
34+
Result, Span, Spans,
3535
};
3636
use itertools::Itertools;
3737

@@ -182,16 +182,23 @@ impl<'a> BinaryTypeCoercer<'a> {
182182
})
183183
}
184184
AtArrow | ArrowAt => {
185-
// ArrowAt and AtArrow check for whether one array is contained in another.
186-
// The result type is boolean. Signature::comparison defines this signature.
187-
// Operation has nothing to do with comparison
188-
array_coercion(self.lhs, self.rhs).map(Signature::comparison).ok_or_else(|| {
185+
// Array contains or search (similar to LIKE) operation
186+
array_coercion(self.lhs, self.rhs)
187+
.or_else(|| like_coercion(self.lhs, self.rhs)).map(Signature::comparison).ok_or_else(|| {
188+
plan_datafusion_err!(
189+
"Cannot infer common argument type for operation {} {} {}", self.lhs, self.op, self.rhs
190+
)
191+
})
192+
}
193+
AtAt => {
194+
// text search has similar signature to LIKE
195+
like_coercion(self.lhs, self.rhs).map(Signature::comparison).ok_or_else(|| {
189196
plan_datafusion_err!(
190-
"Cannot infer common array type for arrow operation {} {} {}", self.lhs, self.op, self.rhs
197+
"Cannot infer common argument type for AtAt operation {} {} {}", self.lhs, self.op, self.rhs
191198
)
192199
})
193200
}
194-
Plus | Minus | Multiply | Divide | Modulo => {
201+
Plus | Minus | Multiply | Divide | Modulo => {
195202
let get_result = |lhs, rhs| {
196203
use arrow::compute::kernels::numeric::*;
197204
let l = new_empty_array(lhs);
@@ -248,6 +255,10 @@ impl<'a> BinaryTypeCoercer<'a> {
248255
"Cannot coerce arithmetic expression {} {} {} to valid types", self.lhs, self.op, self.rhs
249256
)
250257
}
258+
},
259+
IntegerDivide | Arrow | LongArrow | HashArrow | HashLongArrow
260+
| HashMinus | AtQuestion | Question | QuestionAnd | QuestionPipe => {
261+
not_impl_err!("Operator {} is not yet supported", self.op)
251262
}
252263
};
253264
result.map_err(|err| {

‎datafusion/physical-expr/src/expressions/binary.rs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use arrow::compute::{cast, ilike, like, nilike, nlike};
3333
use arrow::datatypes::*;
3434
use arrow::error::ArrowError;
3535
use datafusion_common::cast::as_boolean_array;
36-
use datafusion_common::{internal_err, Result, ScalarValue};
36+
use datafusion_common::{internal_err, not_impl_err, Result, ScalarValue};
3737
use datafusion_expr::binary::BinaryTypeCoercer;
3838
use datafusion_expr::interval_arithmetic::{apply_operator, Interval};
3939
use datafusion_expr::sort_properties::ExprProperties;
@@ -793,8 +793,13 @@ impl BinaryExpr {
793793
BitwiseShiftRight => bitwise_shift_right_dyn(left, right),
794794
BitwiseShiftLeft => bitwise_shift_left_dyn(left, right),
795795
StringConcat => concat_elements(left, right),
796-
AtArrow | ArrowAt => {
797-
unreachable!("ArrowAt and AtArrow should be rewritten to function")
796+
AtArrow | ArrowAt | Arrow | LongArrow | HashArrow | HashLongArrow | AtAt
797+
| HashMinus | AtQuestion | Question | QuestionAnd | QuestionPipe
798+
| IntegerDivide => {
799+
not_impl_err!(
800+
"Binary operator '{:?}' is not supported in the physical expr",
801+
self.op
802+
)
798803
}
799804
}
800805
}

‎datafusion/sql/src/expr/binary_op.rs

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,29 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
4646
BinaryOperator::PGNotILikeMatch => Ok(Operator::NotILikeMatch),
4747
BinaryOperator::BitwiseAnd => Ok(Operator::BitwiseAnd),
4848
BinaryOperator::BitwiseOr => Ok(Operator::BitwiseOr),
49+
BinaryOperator::Xor => Ok(Operator::BitwiseXor),
4950
BinaryOperator::BitwiseXor => Ok(Operator::BitwiseXor),
5051
BinaryOperator::PGBitwiseXor => Ok(Operator::BitwiseXor),
5152
BinaryOperator::PGBitwiseShiftRight => Ok(Operator::BitwiseShiftRight),
5253
BinaryOperator::PGBitwiseShiftLeft => Ok(Operator::BitwiseShiftLeft),
5354
BinaryOperator::StringConcat => Ok(Operator::StringConcat),
5455
BinaryOperator::ArrowAt => Ok(Operator::ArrowAt),
5556
BinaryOperator::AtArrow => Ok(Operator::AtArrow),
57+
BinaryOperator::Arrow => Ok(Operator::Arrow),
58+
BinaryOperator::LongArrow => Ok(Operator::LongArrow),
59+
BinaryOperator::HashArrow => Ok(Operator::HashArrow),
60+
BinaryOperator::HashLongArrow => Ok(Operator::HashLongArrow),
61+
BinaryOperator::AtAt => Ok(Operator::AtAt),
5662
BinaryOperator::Spaceship => Ok(Operator::IsNotDistinctFrom),
57-
_ => not_impl_err!("Unsupported SQL binary operator {op:?}"),
63+
BinaryOperator::DuckIntegerDivide | BinaryOperator::MyIntegerDivide => {
64+
Ok(Operator::IntegerDivide)
65+
}
66+
BinaryOperator::HashMinus => Ok(Operator::HashMinus),
67+
BinaryOperator::AtQuestion => Ok(Operator::AtQuestion),
68+
BinaryOperator::Question => Ok(Operator::Question),
69+
BinaryOperator::QuestionAnd => Ok(Operator::QuestionAnd),
70+
BinaryOperator::QuestionPipe => Ok(Operator::QuestionPipe),
71+
_ => not_impl_err!("Unsupported binary operator: {:?}", op),
5872
}
5973
}
6074
}

‎datafusion/sql/src/unparser/expr.rs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -925,6 +925,19 @@ impl Unparser<'_> {
925925
BinaryOperator::StringConcat => Ok(Operator::StringConcat),
926926
BinaryOperator::AtArrow => Ok(Operator::AtArrow),
927927
BinaryOperator::ArrowAt => Ok(Operator::ArrowAt),
928+
BinaryOperator::Arrow => Ok(Operator::Arrow),
929+
BinaryOperator::LongArrow => Ok(Operator::LongArrow),
930+
BinaryOperator::HashArrow => Ok(Operator::HashArrow),
931+
BinaryOperator::HashLongArrow => Ok(Operator::HashLongArrow),
932+
BinaryOperator::AtAt => Ok(Operator::AtAt),
933+
BinaryOperator::DuckIntegerDivide | BinaryOperator::MyIntegerDivide => {
934+
Ok(Operator::IntegerDivide)
935+
}
936+
BinaryOperator::HashMinus => Ok(Operator::HashMinus),
937+
BinaryOperator::AtQuestion => Ok(Operator::AtQuestion),
938+
BinaryOperator::Question => Ok(Operator::Question),
939+
BinaryOperator::QuestionAnd => Ok(Operator::QuestionAnd),
940+
BinaryOperator::QuestionPipe => Ok(Operator::QuestionPipe),
928941
_ => not_impl_err!("unsupported operation: {op:?}"),
929942
}
930943
}
@@ -962,6 +975,17 @@ impl Unparser<'_> {
962975
Operator::StringConcat => Ok(BinaryOperator::StringConcat),
963976
Operator::AtArrow => Ok(BinaryOperator::AtArrow),
964977
Operator::ArrowAt => Ok(BinaryOperator::ArrowAt),
978+
Operator::Arrow => Ok(BinaryOperator::Arrow),
979+
Operator::LongArrow => Ok(BinaryOperator::LongArrow),
980+
Operator::HashArrow => Ok(BinaryOperator::HashArrow),
981+
Operator::HashLongArrow => Ok(BinaryOperator::HashLongArrow),
982+
Operator::AtAt => Ok(BinaryOperator::AtAt),
983+
Operator::IntegerDivide => Ok(BinaryOperator::DuckIntegerDivide),
984+
Operator::HashMinus => Ok(BinaryOperator::HashMinus),
985+
Operator::AtQuestion => Ok(BinaryOperator::AtQuestion),
986+
Operator::Question => Ok(BinaryOperator::Question),
987+
Operator::QuestionAnd => Ok(BinaryOperator::QuestionAnd),
988+
Operator::QuestionPipe => Ok(BinaryOperator::QuestionPipe),
965989
}
966990
}
967991

‎datafusion/sqllogictest/test_files/expr.slt

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1648,6 +1648,56 @@ true
16481648
true
16491649
true
16501650

1651+
#### Other binary operators
1652+
1653+
# ArrowAt for strings
1654+
query error
1655+
select 'foo' <@ 'bar'
1656+
1657+
# AtArrow for strings
1658+
query error
1659+
select 'foo' @> 'bar'
1660+
1661+
# AtAt for strings
1662+
query error
1663+
select 'foo' @@ 'bar'
1664+
1665+
# Arrow for arrays
1666+
query error
1667+
select make_array(1,2,3) -> 2
1668+
1669+
# LongArrow for arrays
1670+
query error
1671+
select make_array(1,2,3) ->> 2
1672+
1673+
# HashArrow for structs
1674+
query error
1675+
select struct(1,2,3) #> 0
1676+
1677+
# HashLongArrow for structs
1678+
query error
1679+
select struct(1,2,3) #>> 0
1680+
1681+
# HashMinus for structs
1682+
query error
1683+
select struct(1,2,3) #- 0
1684+
1685+
# AtQuestion for JSON/structs
1686+
query error
1687+
select struct(1,2,3) @? 'a.b.c'
1688+
1689+
# Question for JSON/structs
1690+
query error
1691+
select struct(1,2,3) ? 'a.b.c'
1692+
1693+
# QuestionPipe for JSON/structs
1694+
query error
1695+
select struct(1,2,3) ?| array['a','b','c']
1696+
1697+
# QuestionAnd for JSON/structs
1698+
query error
1699+
select struct(1,2,3) ?& array['a','b','c']
1700+
16511701
#### binary_mathematical_operator_with_null_lt
16521702

16531703
# 1. Integer and NULL

‎datafusion/substrait/src/logical_plan/producer.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,17 @@ pub fn operator_to_name(op: Operator) -> &'static str {
11271127
Operator::StringConcat => "str_concat",
11281128
Operator::AtArrow => "at_arrow",
11291129
Operator::ArrowAt => "arrow_at",
1130+
Operator::Arrow => "arrow",
1131+
Operator::LongArrow => "long_arrow",
1132+
Operator::HashArrow => "hash_arrow",
1133+
Operator::HashLongArrow => "hash_long_arrow",
1134+
Operator::AtAt => "at_at",
1135+
Operator::IntegerDivide => "integer_divide",
1136+
Operator::HashMinus => "hash_minus",
1137+
Operator::AtQuestion => "at_question",
1138+
Operator::Question => "question",
1139+
Operator::QuestionAnd => "question_and",
1140+
Operator::QuestionPipe => "question_pipe",
11301141
Operator::BitwiseXor => "bitwise_xor",
11311142
Operator::BitwiseShiftRight => "bitwise_shift_right",
11321143
Operator::BitwiseShiftLeft => "bitwise_shift_left",

0 commit comments

Comments
 (0)