Skip to content
This repository was archived by the owner on Dec 25, 2019. It is now read-only.

parse numbers that begin with decimals #33

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

parse numbers that begin with decimals #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 53 additions & 33 deletions src/tokenizer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,38 +386,7 @@ impl<'a> Tokenizer<'a> {
}
}
// numbers
'0'..='9' => {
let mut seen_decimal = false;
let mut s = peeking_take_while(chars, |ch| match ch {
'0'..='9' => true,
'.' if !seen_decimal => {
seen_decimal = true;
true
}
_ => false,
});
// If in e-notation, parse the e-notation with special care given to negative exponents.
match chars.peek() {
Some('e') | Some('E') => {
s.push('E');
// Consume the e-notation signifier.
chars.next();
if let Some('-') = chars.peek() {
s.push('-');
// Consume the negative sign.
chars.next();
}
let e = peeking_take_while(chars, |ch| match ch {
'0'..='9' => true,
_ => false,
});
s.push_str(&e);
}
_ => {}
}

Ok(Some(Token::Number(s)))
}
'0'..='9' => self.tokenize_number(chars),
// punctuation
'(' => self.consume_and_return(chars, Token::LParen),
')' => self.consume_and_return(chars, Token::RParen),
Expand Down Expand Up @@ -510,7 +479,22 @@ impl<'a> Tokenizer<'a> {
}
}
'=' => self.consume_and_return(chars, Token::Eq),
'.' => self.consume_and_return(chars, Token::Period),
'.' => {
chars.next(); // consume '.'
match chars.peek() {
Some('0'..='9') => {
// Add the '.' back to the chars and parse as number.
let mut chars_w_leading_zero = ".".to_string();
while let Some(token) = chars.next() {
chars_w_leading_zero.push(token);
}
let mut peekable = chars_w_leading_zero.chars().peekable();

self.tokenize_number(&mut peekable)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could be wrong—haven't actually tried—but I think this mishandles something like ..07. I'd recommend refactoring tokenize_number to take seen_decimal as a bool parameter, and then initialize s inside of tokenize_number to . if seen_decimal is true.

}
_ => Ok(Some(Token::Period)),
}
}
'!' => {
chars.next(); // consume
match chars.peek() {
Expand Down Expand Up @@ -652,6 +636,42 @@ impl<'a> Tokenizer<'a> {
Ok(Some(Token::Parameter(n)))
}

fn tokenize_number(
&self,
chars: &mut Peekable<Chars<'_>>,
) -> Result<Option<Token>, TokenizerError> {
let mut seen_decimal = false;
let mut s = peeking_take_while(chars, |ch| match ch {
'0'..='9' => true,
'.' if !seen_decimal => {
seen_decimal = true;
true
}
_ => false,
});
// If in e-notation, parse the e-notation with special care given to negative exponents.
match chars.peek() {
Some('e') | Some('E') => {
s.push('E');
// Consume the e-notation signifier.
chars.next();
if let Some('-') = chars.peek() {
s.push('-');
// Consume the negative sign.
chars.next();
}
let e = peeking_take_while(chars, |ch| match ch {
'0'..='9' => true,
_ => false,
});
s.push_str(&e);
}
_ => {}
}

Ok(Some(Token::Number(s)))
}

fn consume_and_return(
&self,
chars: &mut Peekable<Chars<'_>>,
Expand Down
14 changes: 14 additions & 0 deletions tests/sqlparser_common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,20 @@ fn parse_number() {
assert_eq!(expr, Expr::Value(Value::Number("1.0".into())));
}

#[test]
fn parse_numeric_begin_with_decimal() {
let expr = verified_expr(".1");

#[cfg(feature = "bigdecimal")]
assert_eq!(
expr,
Expr::Value(Value::Number(bigdecimal::BigDecimal::from(.1)))
);

#[cfg(not(feature = "bigdecimal"))]
assert_eq!(expr, Expr::Value(Value::Number(".1".into())));
}

#[test]
fn parse_approximate_numeric_literal() {
let expr = verified_expr("1.0E2");
Expand Down