Optimize quote removal (#62)

Author: sethsamuel

sqllexer.go

@@ -43,7 +43,7 @@ type Token struct {
 	Value            string
 	isTableIndicator bool // true if the token is a table indicator
 	hasDigits        bool
-	quotes           []int          // private - only used by trimQuotes
+	hasQuotes        bool           // private - only used by trimQuotes
 	lastValueToken   LastValueToken // private - internal state
 }
 
@@ -88,9 +88,9 @@ type Lexer struct {
 	start            int    // the start position of the current token
 	config           *LexerConfig
 	token            *Token
-	hasDigits        bool  // true if the token has digits
-	quotes           []int // Indexes of quotes in the token
-	isTableIndicator bool  // true if the token is a table indicator
+	hasQuotes        bool // true if any quotes in token
+	hasDigits        bool // true if the token has digits
+	isTableIndicator bool // true if the token is a table indicator
 }
 
 func New(input string, opts ...lexerOption) *Lexer {
@@ -414,26 +414,23 @@ func (s *Lexer) scanDoubleQuotedIdentifier(delimiter rune) *Token {
 	}
 
 	s.start = s.cursor
-	offset := s.start                            // offset is used to calculate the indexes of quotes in the token value
-	s.quotes = append(s.quotes, s.cursor-offset) // store the opening quote position
-	ch := s.next()                               // consume the opening quote
+	s.hasQuotes = true
+	ch := s.next() // consume the opening quote
 	for {
 		// encountered the closing quote
 		// BUT if it's followed by .", then we should keep going
 		// e.g. postgres "foo"."bar"
 		// e.g. sqlserver [foo].[bar]
 		if ch == closingDelimiter {
-			s.quotes = append(s.quotes, s.cursor-offset)
 			specialCase := []rune{closingDelimiter, '.', delimiter}
 			if s.matchAt([]rune(specialCase)) {
-				s.quotes = append(s.quotes, s.cursor+2-offset)
 				ch = s.nextBy(3) // consume the "."
 				continue
 			}
 			break
 		}
 		if isEOF(ch) {
-			s.quotes = nil // if we hit EOF, we clear the quotes
+			s.hasQuotes = false // if we hit EOF, we clear the quotes
 			return s.emit(ERROR)
 		}
 		s.hasDigits = s.hasDigits || isDigit(ch)
@@ -629,16 +626,10 @@ func (s *Lexer) emit(t TokenType) *Token {
 	}
 
 	tok.hasDigits = s.hasDigits
-
-	if len(s.quotes) > 0 {
-		tok.quotes = s.quotes
-	} else {
-		tok.quotes = nil
-	}
+	tok.hasQuotes = s.hasQuotes
 
 	// Reset lexer state
 	s.start = s.cursor
-	s.quotes = nil
 	s.isTableIndicator = false
 	s.hasDigits = false
 

sqllexer_test.go

@@ -1151,35 +1151,29 @@ func TestLexerIdentifierWithQuotes(t *testing.T) {
 	tests := []struct {
 		input          string
 		expectedTokens []TokenSpec
-		expectedQuotes [][]int
+		expectedQuotes bool
 		lexerOpts      []lexerOption
 	}{
 		{
 			input: `"abc"`,
 			expectedTokens: []TokenSpec{
 				{QUOTED_IDENT, `"abc"`},
 			},
-			expectedQuotes: [][]int{
-				{0, 4},
-			},
+			expectedQuotes: true,
 		},
 		{
 			input: `"abc"."def"`,
 			expectedTokens: []TokenSpec{
 				{QUOTED_IDENT, `"abc"."def"`},
 			},
-			expectedQuotes: [][]int{
-				{0, 4, 6, 10},
-			},
+			expectedQuotes: true,
 		},
 		{
 			input: `"fóo"."bar"`,
 			expectedTokens: []TokenSpec{
 				{QUOTED_IDENT, `"fóo"."bar"`},
 			},
-			expectedQuotes: [][]int{
-				{0, 5, 7, 11},
-			},
+			expectedQuotes: true,
 		},
 		{
 			input: `SELECT "fóo"."`,
@@ -1188,7 +1182,7 @@ func TestLexerIdentifierWithQuotes(t *testing.T) {
 				{SPACE, " "},
 				{ERROR, `"fóo"."`},
 			},
-			expectedQuotes: [][]int{},
+			expectedQuotes: false,
 		},
 	}
 
@@ -1219,23 +1213,8 @@ func TestLexerIdentifierWithQuotes(t *testing.T) {
 					t.Errorf("token[%d] got value %q, want %q", i, got.Value, want.Value)
 				}
 
-				if i < len(tt.expectedQuotes) {
-					quotes := tt.expectedQuotes[i]
-					if quotes == nil {
-						if got.quotes != nil {
-							t.Errorf("token[%d] got quotes, want nil", i)
-						}
-					} else {
-						if len(got.quotes) != len(quotes) {
-							t.Errorf("token[%d] got %d quotes, want %d", i, len(got.quotes), len(quotes))
-						} else {
-							for j, quote := range quotes {
-								if got.quotes[j] != quote {
-									t.Errorf("token[%d] got quote[%d] %d, want %d", i, j, got.quotes[j], quote)
-								}
-							}
-						}
-					}
+				if got.hasQuotes != tt.expectedQuotes {
+					t.Errorf("token[%d] got %v quotes, want %v", i, got.hasQuotes, tt.expectedQuotes)
 				}
 
 				i++

sqllexer_utils.go

@@ -244,27 +244,16 @@ func replaceDigits(token *Token, placeholder string) string {
 
 func trimQuotes(token *Token) string {
 	var trimmedToken strings.Builder
-	trimmedToken.Grow(len(token.Value) - len(token.quotes))
+	trimmedToken.Grow(len(token.Value))
 
-	start := 0
-
-	// loop over token.quotes indexes, write start:token.quotes[i] to builder
-	// write start:token.End to builder
-	for i := 0; i < len(token.quotes); i++ {
-		if token.quotes[i] > len(token.Value) {
-			break
-		}
-		if token.quotes[i]-start >= 1 {
-			trimmedToken.WriteString(token.Value[start:token.quotes[i]])
+	for _, r := range token.Value {
+		if isDoubleQuote(r) || r == '[' || r == ']' || r == '`' {
+			// trimmedToken.WriteString(placeholder)
+		} else {
+			trimmedToken.WriteRune(r)
 		}
-		start = token.quotes[i] + 1
-	}
-
-	// write start:token.End to builder
-	if start < len(token.Value) {
-		trimmedToken.WriteString(token.Value[start:len(token.Value)])
 	}
-	token.quotes = nil
+	token.hasQuotes = false
 	return trimmedToken.String()
 }