Skip to content

Commit 75202b5

Browse files
alambalamb
authored
Upgrade to sqlparser 0.53.0 (#13767)
* chore: Udpate to sqlparser 0.53.0 * Update for new sqlparser API * more api updates * Avoid serializing query to SQL string unless it is necessary * Box wildcard options * chore: update datafusion-cli Cargo.lock
1 parent 74480ac commit 75202b5

File tree

21 files changed

+271
-120
lines changed

21 files changed

+271
-120
lines changed

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ recursive = "0.1.1"
147147
regex = "1.8"
148148
rstest = "0.23.0"
149149
serde_json = "1"
150-
sqlparser = { version = "0.52.0", features = ["visitor"] }
150+
sqlparser = { version = "0.53.0", features = ["visitor"] }
151151
tempfile = "3"
152152
tokio = { version = "1.36", features = ["macros", "rt", "sync"] }
153153
url = "2.2"

‎datafusion-cli/Cargo.lock

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/common/src/utils/mod.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -887,10 +887,10 @@ pub fn get_available_parallelism() -> usize {
887887

888888
#[cfg(test)]
889889
mod tests {
890+
use super::*;
890891
use crate::ScalarValue::Null;
891892
use arrow::array::Float64Array;
892-
893-
use super::*;
893+
use sqlparser::tokenizer::Span;
894894

895895
#[test]
896896
fn test_bisect_linear_left_and_right() -> Result<()> {
@@ -1118,6 +1118,7 @@ mod tests {
11181118
let expected_parsed = vec![Ident {
11191119
value: identifier.to_string(),
11201120
quote_style,
1121+
span: Span::empty(),
11211122
}];
11221123

11231124
assert_eq!(

‎datafusion/core/tests/user_defined/user_defined_scalar_functions.rs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,6 @@ use arrow_array::{
2727
Array, ArrayRef, Float32Array, Float64Array, Int32Array, RecordBatch, StringArray,
2828
};
2929
use arrow_schema::{DataType, Field, Schema};
30-
use parking_lot::Mutex;
31-
use regex::Regex;
32-
use sqlparser::ast::Ident;
33-
3430
use datafusion::execution::context::{FunctionFactory, RegisterFunction, SessionState};
3531
use datafusion::prelude::*;
3632
use datafusion::{execution::registry::FunctionRegistry, test_util};
@@ -48,6 +44,10 @@ use datafusion_expr::{
4844
Volatility,
4945
};
5046
use datafusion_functions_nested::range::range_udf;
47+
use parking_lot::Mutex;
48+
use regex::Regex;
49+
use sqlparser::ast::Ident;
50+
use sqlparser::tokenizer::Span;
5151

5252
/// test that casting happens on udfs.
5353
/// c11 is f32, but `custom_sqrt` requires f64. Casting happens but the logical plan and
@@ -1187,6 +1187,7 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
11871187
name: Some(Ident {
11881188
value: "name".into(),
11891189
quote_style: None,
1190+
span: Span::empty(),
11901191
}),
11911192
data_type: DataType::Utf8,
11921193
default_expr: None,
@@ -1196,6 +1197,7 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
11961197
language: Some(Ident {
11971198
value: "plrust".into(),
11981199
quote_style: None,
1200+
span: Span::empty(),
11991201
}),
12001202
behavior: None,
12011203
function_body: Some(lit(body)),

‎datafusion/expr/src/expr.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ pub enum Expr {
313313
/// plan into physical plan.
314314
Wildcard {
315315
qualifier: Option<TableReference>,
316-
options: WildcardOptions,
316+
options: Box<WildcardOptions>,
317317
},
318318
/// List of grouping set expressions. Only valid in the context of an aggregate
319319
/// GROUP BY expression list

‎datafusion/expr/src/expr_fn.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,15 +123,15 @@ pub fn placeholder(id: impl Into<String>) -> Expr {
123123
pub fn wildcard() -> Expr {
124124
Expr::Wildcard {
125125
qualifier: None,
126-
options: WildcardOptions::default(),
126+
options: Box::new(WildcardOptions::default()),
127127
}
128128
}
129129

130130
/// Create an '*' [`Expr::Wildcard`] expression with the wildcard options
131131
pub fn wildcard_with_options(options: WildcardOptions) -> Expr {
132132
Expr::Wildcard {
133133
qualifier: None,
134-
options,
134+
options: Box::new(options),
135135
}
136136
}
137137

@@ -148,7 +148,7 @@ pub fn wildcard_with_options(options: WildcardOptions) -> Expr {
148148
pub fn qualified_wildcard(qualifier: impl Into<TableReference>) -> Expr {
149149
Expr::Wildcard {
150150
qualifier: Some(qualifier.into()),
151-
options: WildcardOptions::default(),
151+
options: Box::new(WildcardOptions::default()),
152152
}
153153
}
154154

@@ -159,7 +159,7 @@ pub fn qualified_wildcard_with_options(
159159
) -> Expr {
160160
Expr::Wildcard {
161161
qualifier: Some(qualifier.into()),
162-
options,
162+
options: Box::new(options),
163163
}
164164
}
165165

‎datafusion/optimizer/src/analyzer/inline_table_scan.rs

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@ use crate::analyzer::AnalyzerRule;
2323
use datafusion_common::config::ConfigOptions;
2424
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
2525
use datafusion_common::{Column, Result};
26-
use datafusion_expr::expr::WildcardOptions;
27-
use datafusion_expr::{logical_plan::LogicalPlan, Expr, LogicalPlanBuilder};
26+
use datafusion_expr::{logical_plan::LogicalPlan, wildcard, Expr, LogicalPlanBuilder};
2827

2928
/// Analyzed rule that inlines TableScan that provide a [`LogicalPlan`]
3029
/// (DataFrame / ViewTable)
@@ -93,10 +92,7 @@ fn generate_projection_expr(
9392
)));
9493
}
9594
} else {
96-
exprs.push(Expr::Wildcard {
97-
qualifier: None,
98-
options: WildcardOptions::default(),
99-
});
95+
exprs.push(wildcard());
10096
}
10197
Ok(exprs)
10298
}

‎datafusion/proto/src/logical_plan/from_proto.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ pub fn parse_expr(
513513
let qualifier = qualifier.to_owned().map(|x| x.try_into()).transpose()?;
514514
Ok(Expr::Wildcard {
515515
qualifier,
516-
options: WildcardOptions::default(),
516+
options: Box::new(WildcardOptions::default()),
517517
})
518518
}
519519
ExprType::ScalarUdfExpr(protobuf::ScalarUdfExprNode {

‎datafusion/proto/tests/cases/roundtrip_logical_plan.rs

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ use datafusion_common::{
6767
use datafusion_expr::dml::CopyTo;
6868
use datafusion_expr::expr::{
6969
self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, Like, ScalarFunction,
70-
Unnest, WildcardOptions,
70+
Unnest,
7171
};
7272
use datafusion_expr::logical_plan::{Extension, UserDefinedLogicalNodeCore};
7373
use datafusion_expr::{
@@ -2061,21 +2061,15 @@ fn roundtrip_unnest() {
20612061

20622062
#[test]
20632063
fn roundtrip_wildcard() {
2064-
let test_expr = Expr::Wildcard {
2065-
qualifier: None,
2066-
options: WildcardOptions::default(),
2067-
};
2064+
let test_expr = wildcard();
20682065

20692066
let ctx = SessionContext::new();
20702067
roundtrip_expr_test(test_expr, ctx);
20712068
}
20722069

20732070
#[test]
20742071
fn roundtrip_qualified_wildcard() {
2075-
let test_expr = Expr::Wildcard {
2076-
qualifier: Some("foo".into()),
2077-
options: WildcardOptions::default(),
2078-
};
2072+
let test_expr = qualified_wildcard("foo");
20792073

20802074
let ctx = SessionContext::new();
20812075
roundtrip_expr_test(test_expr, ctx);

‎datafusion/sql/src/expr/function.rs

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,11 @@ use datafusion_common::{
2222
internal_datafusion_err, internal_err, not_impl_err, plan_datafusion_err, plan_err,
2323
DFSchema, Dependency, Result,
2424
};
25-
use datafusion_expr::expr::WildcardOptions;
2625
use datafusion_expr::expr::{ScalarFunction, Unnest};
2726
use datafusion_expr::planner::PlannerResult;
2827
use datafusion_expr::{
29-
expr, Expr, ExprFunctionExt, ExprSchemable, WindowFrame, WindowFunctionDefinition,
28+
expr, qualified_wildcard, wildcard, Expr, ExprFunctionExt, ExprSchemable,
29+
WindowFrame, WindowFunctionDefinition,
3030
};
3131
use sqlparser::ast::{
3232
DuplicateTreatment, Expr as SQLExpr, Function as SQLFunction, FunctionArg,
@@ -169,6 +169,11 @@ impl FunctionArgs {
169169
"Calling {name}: SEPARATOR not supported in function arguments: {sep}"
170170
)
171171
}
172+
FunctionArgumentClause::JsonNullClause(jn) => {
173+
return not_impl_err!(
174+
"Calling {name}: JSON NULL clause not supported in function arguments: {jn}"
175+
)
176+
}
172177
}
173178
}
174179

@@ -413,28 +418,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
413418
name: _,
414419
arg: FunctionArgExpr::Wildcard,
415420
operator: _,
416-
} => Ok(Expr::Wildcard {
417-
qualifier: None,
418-
options: WildcardOptions::default(),
419-
}),
421+
} => Ok(wildcard()),
420422
FunctionArg::Unnamed(FunctionArgExpr::Expr(arg)) => {
421423
self.sql_expr_to_logical_expr(arg, schema, planner_context)
422424
}
423-
FunctionArg::Unnamed(FunctionArgExpr::Wildcard) => Ok(Expr::Wildcard {
424-
qualifier: None,
425-
options: WildcardOptions::default(),
426-
}),
425+
FunctionArg::Unnamed(FunctionArgExpr::Wildcard) => Ok(wildcard()),
427426
FunctionArg::Unnamed(FunctionArgExpr::QualifiedWildcard(object_name)) => {
428427
let qualifier = self.object_name_to_table_reference(object_name)?;
429428
// Sanity check on qualifier with schema
430429
let qualified_indices = schema.fields_indices_with_qualified(&qualifier);
431430
if qualified_indices.is_empty() {
432431
return plan_err!("Invalid qualifier {qualifier}");
433432
}
434-
Ok(Expr::Wildcard {
435-
qualifier: Some(qualifier),
436-
options: WildcardOptions::default(),
437-
})
433+
Ok(qualified_wildcard(qualifier))
438434
}
439435
_ => not_impl_err!("Unsupported qualified wildcard argument: {sql:?}"),
440436
}

‎datafusion/sql/src/expr/mod.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -593,13 +593,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
593593
}
594594
not_impl_err!("AnyOp not supported by ExprPlanner: {binary_expr:?}")
595595
}
596-
SQLExpr::Wildcard => Ok(Expr::Wildcard {
596+
SQLExpr::Wildcard(_token) => Ok(Expr::Wildcard {
597597
qualifier: None,
598-
options: WildcardOptions::default(),
598+
options: Box::new(WildcardOptions::default()),
599599
}),
600-
SQLExpr::QualifiedWildcard(object_name) => Ok(Expr::Wildcard {
600+
SQLExpr::QualifiedWildcard(object_name, _token) => Ok(Expr::Wildcard {
601601
qualifier: Some(self.object_name_to_table_reference(object_name)?),
602-
options: WildcardOptions::default(),
602+
options: Box::new(WildcardOptions::default()),
603603
}),
604604
SQLExpr::Tuple(values) => self.parse_tuple(schema, planner_context, values),
605605
_ => not_impl_err!("Unsupported ast node in sqltorel: {sql:?}"),

‎datafusion/sql/src/parser.rs

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ use std::collections::VecDeque;
2121
use std::fmt;
2222

2323
use sqlparser::ast::ExprWithAlias;
24+
use sqlparser::tokenizer::TokenWithSpan;
2425
use sqlparser::{
2526
ast::{
2627
ColumnDef, ColumnOptionDef, ObjectName, OrderByExpr, Query,
2728
Statement as SQLStatement, TableConstraint, Value,
2829
},
2930
dialect::{keywords::Keyword, Dialect, GenericDialect},
3031
parser::{Parser, ParserError},
31-
tokenizer::{Token, TokenWithLocation, Tokenizer, Word},
32+
tokenizer::{Token, Tokenizer, Word},
3233
};
3334

3435
// Use `Parser::expected` instead, if possible
@@ -338,7 +339,7 @@ impl<'a> DFParser<'a> {
338339
fn expected<T>(
339340
&self,
340341
expected: &str,
341-
found: TokenWithLocation,
342+
found: TokenWithSpan,
342343
) -> Result<T, ParserError> {
343344
parser_err!(format!("Expected {expected}, found: {found}"))
344345
}
@@ -876,6 +877,7 @@ mod tests {
876877
use super::*;
877878
use sqlparser::ast::Expr::Identifier;
878879
use sqlparser::ast::{BinaryOperator, DataType, Expr, Ident};
880+
use sqlparser::tokenizer::Span;
879881

880882
fn expect_parse_ok(sql: &str, expected: Statement) -> Result<(), ParserError> {
881883
let statements = DFParser::parse_sql(sql)?;
@@ -911,6 +913,7 @@ mod tests {
911913
name: Ident {
912914
value: name.into(),
913915
quote_style: None,
916+
span: Span::empty(),
914917
},
915918
data_type,
916919
collation: None,
@@ -1219,6 +1222,7 @@ mod tests {
12191222
expr: Identifier(Ident {
12201223
value: "c1".to_owned(),
12211224
quote_style: None,
1225+
span: Span::empty(),
12221226
}),
12231227
asc,
12241228
nulls_first,
@@ -1250,6 +1254,7 @@ mod tests {
12501254
expr: Identifier(Ident {
12511255
value: "c1".to_owned(),
12521256
quote_style: None,
1257+
span: Span::empty(),
12531258
}),
12541259
asc: Some(true),
12551260
nulls_first: None,
@@ -1259,6 +1264,7 @@ mod tests {
12591264
expr: Identifier(Ident {
12601265
value: "c2".to_owned(),
12611266
quote_style: None,
1267+
span: Span::empty(),
12621268
}),
12631269
asc: Some(false),
12641270
nulls_first: Some(true),
@@ -1290,11 +1296,13 @@ mod tests {
12901296
left: Box::new(Identifier(Ident {
12911297
value: "c1".to_owned(),
12921298
quote_style: None,
1299+
span: Span::empty(),
12931300
})),
12941301
op: BinaryOperator::Minus,
12951302
right: Box::new(Identifier(Ident {
12961303
value: "c2".to_owned(),
12971304
quote_style: None,
1305+
span: Span::empty(),
12981306
})),
12991307
},
13001308
asc: Some(true),
@@ -1335,11 +1343,13 @@ mod tests {
13351343
left: Box::new(Identifier(Ident {
13361344
value: "c1".to_owned(),
13371345
quote_style: None,
1346+
span: Span::empty(),
13381347
})),
13391348
op: BinaryOperator::Minus,
13401349
right: Box::new(Identifier(Ident {
13411350
value: "c2".to_owned(),
13421351
quote_style: None,
1352+
span: Span::empty(),
13431353
})),
13441354
},
13451355
asc: Some(true),

0 commit comments

Comments
 (0)