locations: collect locations of expressions and SelectItems

Author: lustefaniak

src/ast/dml.rs

@@ -489,7 +489,7 @@ pub struct Insert {
     pub table: bool,
     pub on: Option<OnInsert>,
     /// RETURNING
-    pub returning: Option<Vec<SelectItem>>,
+    pub returning: Option<Vec<WithSpan<SelectItem>>>,
     /// Only for mysql
     pub replace_into: bool,
     /// Only for mysql
@@ -587,7 +587,7 @@ pub struct Delete {
     /// WHERE
     pub selection: Option<Expr>,
     /// RETURNING
-    pub returning: Option<Vec<SelectItem>>,
+    pub returning: Option<Vec<WithSpan<SelectItem>>>,
     /// ORDER BY (MySQL)
     pub order_by: Vec<OrderByExpr>,
     /// LIMIT (MySQL)

src/ast/mod.rs

@@ -551,6 +551,10 @@ where
     pub fn unwrap(self) -> T {
         self.inner
     }
+
+    pub fn span_location(&self) -> Span {
+        self.span
+    }
 }
 
 pub trait SpanWrapped: Clone + Eq + Ord + std::hash::Hash + PartialOrd + PartialEq {
@@ -2442,7 +2446,7 @@ pub enum Statement {
         /// WHERE
         selection: Option<Expr>,
         /// RETURNING
-        returning: Option<Vec<SelectItem>>,
+        returning: Option<Vec<WithSpan<SelectItem>>>,
     },
     /// ```sql
     /// DELETE

src/ast/query.rs

@@ -112,7 +112,7 @@ impl fmt::Display for Query {
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub struct ProjectionSelect {
-    pub projection: Vec<SelectItem>,
+    pub projection: Vec<WithSpan<SelectItem>>,
     pub order_by: Option<OrderBy>,
     pub group_by: Option<GroupByExpr>,
 }
@@ -280,7 +280,7 @@ pub struct Select {
     /// MSSQL syntax: `TOP (<N>) [ PERCENT ] [ WITH TIES ]`
     pub top: Option<Top>,
     /// projection expressions
-    pub projection: Vec<SelectItem>,
+    pub projection: Vec<WithSpan<SelectItem>>,
     /// INTO
     pub into: Option<SelectInto>,
     /// FROM

src/parser/mod.rs

@@ -183,7 +183,7 @@ impl fmt::Display for ParserError {
 impl std::error::Error for ParserError {}
 
 // By default, allow expressions up to this deep before erroring
-const DEFAULT_REMAINING_DEPTH: usize = 50;
+const DEFAULT_REMAINING_DEPTH: usize = 48;
 
 /// Composite types declarations using angle brackets syntax can be arbitrary
 /// nested such that the following declaration is possible:
@@ -3400,7 +3400,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse a comma-separated list of 1+ SelectItem
-    pub fn parse_projection(&mut self) -> Result<Vec<SelectItem>, ParserError> {
+    pub fn parse_projection(&mut self) -> Result<Vec<WithSpan<SelectItem>>, ParserError> {
         // BigQuery and Snowflake allow trailing commas, but only in project lists
         // e.g. `SELECT 1, 2, FROM t`
         // https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#trailing_commas
@@ -8287,7 +8287,7 @@ impl<'a> Parser<'a> {
     /// use sqlparser::parser::Parser;
     ///
     /// let dialect = GenericDialect {};
-    /// let expected = vec![Ident::new("one"), Ident::new("two")];
+    /// let expected = vec![Ident::new("one").empty_span(), Ident::new("two").empty_span()];
     ///
     /// // expected usage
     /// let sql = "one.two";
@@ -11209,15 +11209,18 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse a comma-delimited list of projections after SELECT
-    pub fn parse_select_item(&mut self) -> Result<SelectItem, ParserError> {
+    pub fn parse_select_item(&mut self) -> Result<WithSpan<SelectItem>, ParserError> {
+        let start_span = self.index;
         match self.parse_wildcard_expr()? {
             Expr::QualifiedWildcard(prefix) => Ok(SelectItem::QualifiedWildcard(
                 prefix,
                 self.parse_wildcard_additional_options()?,
-            )),
-            Expr::Wildcard => Ok(SelectItem::Wildcard(
-                self.parse_wildcard_additional_options()?,
-            )),
+            )
+            .spanning(self.span_from_index(start_span))),
+            Expr::Wildcard => Ok(
+                SelectItem::Wildcard(self.parse_wildcard_additional_options()?)
+                    .spanning(self.span_from_index(start_span)),
+            ),
             Expr::Identifier(v) if v.value.to_lowercase() == "from" && v.quote_style.is_none() => {
                 parser_err!(
                     format!("Expected an expression, found: {}", v),
@@ -11240,13 +11243,17 @@ impl<'a> Parser<'a> {
                 Ok(SelectItem::ExprWithAlias {
                     expr: *right,
                     alias,
-                })
+                }
+                .spanning(self.span_from_index(start_span)))
             }
             expr => self
                 .parse_optional_alias(keywords::RESERVED_FOR_COLUMN_ALIAS)
                 .map(|alias| match alias {
-                    Some(alias) => SelectItem::ExprWithAlias { expr, alias },
-                    None => SelectItem::UnnamedExpr(expr),
+                    Some(alias) => SelectItem::ExprWithAlias { expr, alias }
+                        .spanning(self.span_from_index(start_span)),
+                    None => {
+                        SelectItem::UnnamedExpr(expr).spanning(self.span_from_index(start_span))
+                    }
                 }),
         }
     }
@@ -12249,6 +12256,57 @@ impl<'a> Parser<'a> {
         self.tokens
     }
 
+    fn span_from_index(&mut self, mut start_index: usize) -> Span {
+        let mut start_token = &self.tokens[start_index];
+        loop {
+            match start_token {
+                TokenWithLocation {
+                    token: Token::Whitespace(_),
+                    span: _,
+                } => {
+                    start_index += 1;
+                    start_token = &self.tokens[start_index];
+                    continue;
+                }
+                _ => break,
+            }
+        }
+        let start_span = start_token.span;
+
+        let mut idx = self.index.max(start_index).min(self.tokens.len() - 1);
+        loop {
+            if idx <= start_index || idx >= self.tokens.len() {
+                break;
+            }
+            let curr_token = &self.tokens[idx];
+            match curr_token {
+                TokenWithLocation {
+                    token: Token::Whitespace(_),
+                    span: _,
+                } => {
+                    idx -= 1;
+                    continue;
+                }
+                TokenWithLocation {
+                    token: Token::Comma,
+                    span: _,
+                } => {
+                    idx -= 1;
+                    continue;
+                }
+                TokenWithLocation {
+                    token: Token::Word(word),
+                    span: _,
+                } if word.keyword != Keyword::NoKeyword => {
+                    idx -= 1;
+                    continue;
+                }
+                non_whitespace => return non_whitespace.span.union(start_span),
+            }
+        }
+        start_span
+    }
+
     /// Returns true if the next keyword indicates a sub query, i.e. SELECT or WITH
     fn peek_sub_query(&mut self) -> bool {
         if self

src/tokenizer.rs

@@ -439,7 +439,7 @@ pub struct Location {
 }
 
 impl Location {
-    fn is_valid(&self) -> bool {
+    pub fn is_valid(&self) -> bool {
         self.line > 0 && self.column > 0
     }
     pub fn of(line: LineNumber, column: ColumnPosition) -> Self {

tests/sqlparser_bigquery.rs

@@ -1580,6 +1580,7 @@ fn parse_hyphenated_table_identifiers() {
                 Ident::new("x").empty_span(),
             ]))
         })
+        .empty_span()
     );
 
     let error_sql = "select foo-bar.* from foo-bar";

tests/sqlparser_clickhouse.rs

@@ -58,7 +58,8 @@ fn parse_map_access_expr() {
                     ),
                     syntax: MapAccessSyntax::Bracket
                 }],
-            })],
+            })
+            .empty_span()],
             into: None,
             from: vec![TableWithJoins {
                 relation: Table {
@@ -211,7 +212,7 @@ fn parse_delimited_identifiers() {
         }),
         expr_from_projection(&select.projection[1]),
     );
-    match &select.projection[2] {
+    match &select.projection[2].clone().unwrap() {
         SelectItem::ExprWithAlias { expr, alias } => {
             assert_eq!(
                 &Expr::Identifier(Ident::with_quote('"', "simple id").empty_span()),
@@ -319,8 +320,8 @@ fn parse_alter_table_add_projection() {
                     name: Ident::new("my_name").empty_span(),
                     select: ProjectionSelect {
                         projection: vec![
-                            UnnamedExpr(Identifier(Ident::new("a").empty_span())),
-                            UnnamedExpr(Identifier(Ident::new("b").empty_span())),
+                            UnnamedExpr(Identifier(Ident::new("a").empty_span())).empty_span(),
+                            UnnamedExpr(Identifier(Ident::new("b").empty_span())).empty_span(),
                         ],
                         group_by: Some(GroupByExpr::Expressions(
                             vec![Identifier(Ident::new("a").empty_span())],
@@ -1006,9 +1007,10 @@ fn parse_select_star_except() {
 fn parse_select_parametric_function() {
     match clickhouse_and_generic().verified_stmt("SELECT HISTOGRAM(0.5, 0.6)(x, y) FROM t") {
         Statement::Query(query) => {
-            let projection: &Vec<SelectItem> = query.body.as_select().unwrap().projection.as_ref();
+            let projection: &Vec<WithSpan<SelectItem>> =
+                query.body.as_select().unwrap().projection.as_ref();
             assert_eq!(projection.len(), 1);
-            match &projection[0] {
+            match projection[0].clone().unwrap() {
                 UnnamedExpr(Expr::Function(f)) => {
                     let args = match &f.args {
                         FunctionArguments::List(ref args) => args,
@@ -1433,10 +1435,10 @@ fn parse_create_table_on_commit_and_as_query() {
             assert_eq!(on_commit, Some(OnCommit::PreserveRows));
             assert_eq!(
                 query.unwrap().body.as_select().unwrap().projection,
-                vec![UnnamedExpr(Expr::Value(Value::Number(
-                    "1".parse().unwrap(),
-                    false
-                )))]
+                vec![
+                    UnnamedExpr(Expr::Value(Value::Number("1".parse().unwrap(), false)))
+                        .empty_span()
+                ]
             );
         }
         _ => unreachable!(),