Skip to content

Commit 7ff4133

Browse files
andygroveandygrove
committed
Refactoring
1 parent 2ddc9f7 commit 7ff4133

File tree

5 files changed

+107
-92
lines changed

5 files changed

+107
-92
lines changed

‎examples/acme_parser.rs

Lines changed: 38 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
use std::str::Chars;
2-
use std::iter::Peekable;
3-
use std::rc::Rc;
1+
use std::sync::{Arc, Mutex};
42

53
extern crate datafusion_sql;
64

75
use datafusion_sql::ansi::tokenizer::ANSISQLTokenizer;
6+
use datafusion_sql::ansi::parser::ANSISQLParser;
87
use datafusion_sql::tokenizer::*;
98
use datafusion_sql::parser::*;
109

@@ -26,100 +25,89 @@ enum AcmeExpr {
2625
}
2726

2827
struct AcmeTokenizer {
29-
generic: ANSISQLTokenizer
28+
ansi_tokenizer: Arc<Mutex<SQLTokenizer<AcmeToken>>>
3029
}
3130

3231
/// The ACME tokenizer looks for the factorial operator `!!` but delegates everything else
3332
impl SQLTokenizer<AcmeToken> for AcmeTokenizer {
3433

35-
fn precedence(&self, token: &SQLToken<AcmeToken>) -> usize {
34+
fn precedence(&self, _token: &SQLToken<AcmeToken>) -> usize {
3635
unimplemented!()
3736
}
3837

39-
fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeToken>> {
38+
fn peek_token(&mut self) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeToken>> {
4039
unimplemented!()
4140
}
4241

43-
fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeToken>> {
44-
match chars.peek() {
42+
fn next_token(&mut self) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeToken>> {
43+
let mut arc = self.ansi_tokenizer.lock().unwrap();
44+
match arc.peek_char() {
4545
Some(&ch) => match ch {
4646
'!' => {
47-
chars.next(); // consume the first `!`
48-
match chars.peek() {
47+
arc.next_char(); // consume the first `!`
48+
match arc.peek_char() {
4949
Some(&ch) => match ch {
5050
'!' => {
51-
chars.next(); // consume the second `!`
51+
arc.next_char(); // consume the second `!`
5252
Ok(Some(SQLToken::Custom(AcmeToken::Factorial)))
5353
},
5454
_ => Err(TokenizerError::UnexpectedChar(ch,Position::new(0,0)))
5555
},
5656
None => Ok(Some(SQLToken::Not))
5757
}
5858
}
59-
_ => self.generic.next_token(chars)
59+
_ => arc.next_token()
6060
}
61-
_ => self.generic.next_token(chars)
61+
_ => arc.next_token()
6262
}
6363
}
64+
65+
fn peek_char(&mut self) -> Option<&char> {
66+
unimplemented!()
67+
}
68+
69+
fn next_char(&mut self) -> Option<&char> {
70+
unimplemented!()
71+
}
6472
}
6573

6674
struct AcmeParser {
67-
tokenizer: Rc<SQLTokenizer<AcmeToken>>
75+
ansi_parser: Arc<Mutex<SQLParser<AcmeToken, AcmeExpr>>>
6876
}
69-
//
70-
//impl<'a> AcmeParser<'a> {
71-
//
72-
// pub fn new(sql: &'a str) -> Self {
73-
// AcmeParser {
74-
// chars: sql.chars().peekable()
75-
// }
76-
// }
77-
//}
7877

7978
impl SQLParser<AcmeToken, AcmeExpr> for AcmeParser {
8079

8180
fn parse_prefix(&mut self) -> Result<Box<SQLExpr<AcmeExpr>>, ParserError<AcmeToken>> {
82-
unimplemented!()
81+
//TODO: add custom overrides
82+
self.ansi_parser.lock().unwrap().parse_prefix()
8383
}
8484

85-
fn parse_infix(&mut self, left: &SQLExpr<AcmeExpr>, _precedence: usize) -> Result<Option<Box<SQLExpr<AcmeExpr>>>, ParserError<AcmeToken>> {
86-
unimplemented!()
85+
fn parse_infix(&mut self, left: &SQLExpr<AcmeExpr>, precedence: usize) -> Result<Option<Box<SQLExpr<AcmeExpr>>>, ParserError<AcmeToken>> {
86+
//TODO: add custom overrides
87+
self.ansi_parser.lock().unwrap().parse_infix(left, precedence)
8788
}
8889
}
8990

9091
fn main() {
9192

9293
let sql = "1 + !! 5 * 2";
9394

94-
// let acme_parser = AcmeParser::new(sql);
95-
96-
97-
//acme_parser
95+
// ANSI SQL tokenizer
96+
let ansi_tokenizer = Arc::new(Mutex::new(ANSISQLTokenizer { chars: sql.chars().peekable() }));
9897

99-
let mut acme_tokenizer: Rc<SQLTokenizer<AcmeToken>> = Rc::new(AcmeTokenizer {
100-
generic: ANSISQLTokenizer { }
101-
});
98+
// Custom ACME tokenizer
99+
let mut acme_tokenizer = Arc::new(Mutex::new(AcmeTokenizer {
100+
ansi_tokenizer: ansi_tokenizer.clone()
101+
}));
102102

103-
let mut acme_parser: Rc<SQLParser<AcmeToken, AcmeExpr>> = Rc::new(AcmeParser {
104-
tokenizer: acme_tokenizer.clone()
105-
});
103+
// Custom ACME parser
104+
let acme_parser: Arc<Mutex<SQLParser<AcmeToken, AcmeExpr>>> = Arc::new(Mutex::new(AcmeParser {
105+
ansi_parser: Arc::new(Mutex::new(ANSISQLParser::new(acme_tokenizer)))
106+
}));
106107

107-
// let mut pratt_parser = Rc::new(PrattParser {
108-
// chars: sql.chars().peekable(),
109-
// tokenizer: acme_tokenizer.clone(),
110-
// parser: acme_parser.clone()
111-
// });
112-
113-
let mut chars = sql.chars().peekable();
114-
115-
let expr = parse_expr(acme_tokenizer, acme_parser, &mut chars);
108+
let expr = parse_expr(acme_parser).unwrap();
116109

117110
println!("Parsed: {:?}", expr);
118-
//
119-
// let tokens = tokenize(&sql, &mut acme_tokenizer).unwrap();
120-
//
121-
// println!("tokens = {:?}", tokens);
122-
123111

124112

125113
}

‎src/ansi/parser.rs

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,30 @@
11
use std::cmp::PartialEq;
22
use std::fmt::Debug;
3-
use std::iter::Peekable;
4-
use std::str::Chars;
3+
//use std::iter::Peekable;
4+
//use std::str::Chars;
5+
6+
use std::sync::{Arc, Mutex};
57

68
use super::super::tokenizer::*;
79
use super::super::parser::*;
810

9-
pub struct ANSISQLParser<'a, TokenType> {
10-
chars: Peekable<Chars<'a>>,
11-
tokenizer: SQLTokenizer<TokenType>
11+
pub struct ANSISQLParser<TokenType> {
12+
tokenizer: Arc<Mutex<SQLTokenizer<TokenType>>>
13+
}
14+
15+
impl<TokenType> ANSISQLParser<TokenType> where TokenType: Debug + PartialEq {
16+
17+
pub fn new(tokenizer: Arc<Mutex<SQLTokenizer<TokenType>>>) -> Self {
18+
ANSISQLParser { tokenizer: tokenizer.clone() }
19+
}
1220
}
1321

14-
impl<'a, TokenType, ExprType> SQLParser<TokenType, ExprType> for ANSISQLParser<'a, TokenType>
15-
where TokenType: Debug + PartialEq, ExprType: Debug + PartialEq {
22+
impl<TokenType, ExprType> SQLParser<TokenType, ExprType> for ANSISQLParser<TokenType>
23+
where TokenType: Debug + PartialEq, ExprType: Debug {
1624

1725
fn parse_prefix(&mut self) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> {
1826

19-
match self.tokenizer.peek_token(&mut self.chars)? {
27+
match self.tokenizer.lock().unwrap().peek_token()? {
2028
Some(SQLToken::Keyword(ref k)) => match k.to_uppercase().as_ref() {
2129
"INSERT" => unimplemented!(),
2230
"UPDATE" => unimplemented!(),

‎src/ansi/tokenizer.rs

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,30 +5,32 @@ use std::str::Chars;
55

66
use super::super::tokenizer::*;
77

8-
pub struct ANSISQLTokenizer {}
8+
pub struct ANSISQLTokenizer<'a> {
9+
pub chars: Peekable<Chars<'a>>
10+
}
911

10-
impl<TokenType> SQLTokenizer<TokenType> for ANSISQLTokenizer
12+
impl<'a, TokenType> SQLTokenizer<TokenType> for ANSISQLTokenizer<'a>
1113
where TokenType: Debug + PartialEq {
1214

1315
fn precedence(&self, _token: &SQLToken<TokenType>) -> usize {
1416
unimplemented!()
1517
}
1618

17-
fn peek_token(&self, _chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>> {
19+
fn peek_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>> {
1820
unimplemented!()
1921
}
2022

21-
fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>> {
22-
match chars.next() {
23+
fn next_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>> {
24+
match self.chars.next() {
2325
Some(ch) => match ch {
2426
' ' | '\t' | '\n' => Ok(Some(SQLToken::Whitespace(ch))),
2527
'0' ... '9' => {
2628
let mut s = String::new();
2729
s.push(ch);
28-
while let Some(&ch) = chars.peek() {
30+
while let Some(&ch) = self.chars.peek() {
2931
match ch {
3032
'0' ... '9' => {
31-
chars.next(); // consume
33+
self.chars.next(); // consume
3234
s.push(ch);
3335
},
3436
_ => break
@@ -45,5 +47,13 @@ impl<TokenType> SQLTokenizer<TokenType> for ANSISQLTokenizer
4547
None => Ok(None)
4648
}
4749
}
50+
51+
fn peek_char(&mut self) -> Option<&char> {
52+
unimplemented!()
53+
}
54+
55+
fn next_char(&mut self) -> Option<&char> {
56+
unimplemented!()
57+
}
4858
}
4959

‎src/parser.rs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ use std::fmt::Debug;
33
use std::rc::Rc;
44
use std::str::Chars;
55
use std::iter::Peekable;
6+
use std::sync::{Arc, Mutex};
67

78
use super::tokenizer::*;
89

@@ -117,9 +118,13 @@ pub trait SQLParser<TokenType, ExprType>
117118
}
118119

119120

120-
pub fn parse_expr<'a, TokenType, ExprType>(_tokenizer: Rc<SQLTokenizer<TokenType>>, _parser: Rc<SQLParser<TokenType, ExprType>>, _chars: &mut Peekable<Chars<'a>>)
121+
pub fn parse_expr<'a, TokenType, ExprType>(parser: Arc<Mutex<SQLParser<TokenType, ExprType>>>)
121122
-> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> where TokenType: Debug + PartialEq, ExprType: Debug {
122-
unimplemented!()
123+
let mut guard = parser.lock().unwrap();
124+
125+
//Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>>
126+
let x = guard.parse_prefix();
127+
x
123128
}
124129

125130

‎src/tokenizer.rs

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use std::cmp::PartialEq;
22
use std::fmt::Debug;
3-
use std::iter::Peekable;
4-
use std::str::Chars;
3+
//use std::iter::Peekable;
4+
//use std::str::Chars;
55

66
#[derive(Debug)]
77
pub struct Position {
@@ -54,31 +54,35 @@ pub trait SQLTokenizer<TokenType>
5454
fn precedence(&self, token: &SQLToken<TokenType>) -> usize;
5555

5656
/// return a reference to the next token but do not advance the index
57-
fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
57+
fn peek_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
5858

5959
/// return a reference to the next token and advance the index
60-
fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
61-
}
62-
63-
64-
pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError<TokenType>>
65-
where TokenType: Debug + PartialEq
66-
{
60+
fn next_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
6761

68-
let mut peekable = sql.chars().peekable();
62+
fn peek_char(&mut self) -> Option<&char>;
6963

70-
let mut tokens : Vec<SQLToken<TokenType>> = vec![];
71-
72-
loop {
73-
match tokenizer.next_token(&mut peekable)? {
74-
Some(SQLToken::Whitespace(_)) => { /* ignore */ },
75-
Some(token) => {
76-
println!("Token: {:?}", token);
77-
tokens.push(token)
78-
},
79-
None => break
80-
}
81-
}
64+
fn next_char(&mut self) -> Option<&char>;
65+
}
8266

83-
Ok(tokens)
84-
}
67+
//
68+
//pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError<TokenType>>
69+
// where TokenType: Debug + PartialEq
70+
// {
71+
//
72+
// let mut peekable = sql.chars().peekable();
73+
//
74+
// let mut tokens : Vec<SQLToken<TokenType>> = vec![];
75+
//
76+
// loop {
77+
// match tokenizer.next_token(&mut peekable)? {
78+
// Some(SQLToken::Whitespace(_)) => { /* ignore */ },
79+
// Some(token) => {
80+
// println!("Token: {:?}", token);
81+
// tokens.push(token)
82+
// },
83+
// None => break
84+
// }
85+
// }
86+
//
87+
// Ok(tokens)
88+
//}

0 commit comments

Comments
 (0)