Initial unicode character support for identifiers

Summary:

Test Plan:
Added a test

Reviewers:

Subscribers:

Tasks:

Tags:

Author: kaikalur

parser/cpp/prepare-javacc-grammar.sh

@@ -3,4 +3,4 @@ pwd
 GRAMMAR_DIR='../grammar'
 GEN_DIR='target/generated-sources/javacc'
 mkdir -p $GEN_DIR
-cat ./javacc-options.txt $GRAMMAR_DIR/nonreservedwords.txt $GRAMMAR_DIR/reservedwords.txt $GRAMMAR_DIR/sql-spec.txt $GRAMMAR_DIR/presto-extensions.txt $GRAMMAR_DIR/lexical-elements.txt > $GEN_DIR/parser_tmp.jjt
+cat ./javacc-options.txt $GRAMMAR_DIR/kw.txt $GRAMMAR_DIR/sql-spec.txt $GRAMMAR_DIR/presto-extensions.txt $GRAMMAR_DIR/nonreservedwords.txt $GRAMMAR_DIR/unicode-identifiers.txt $GRAMMAR_DIR/lexical-elements.txt > $GEN_DIR/parser_tmp.jjt

parser/grammar/compact_char_sets.awk

@@ -0,0 +1,38 @@
+BEGIN	{ cnt = 1; }
+/^\/\//	{ next; }
+/\"\-\"\\u/	{ 
+			gsub("\" .*$", "\"");
+			gsub("^[^\"]*", "");
+			split($0, a, "-");
+                   	gsub("\\\\u", "", a[1]);
+                   	gsub("\"", "", a[1]);
+                   	s = strtonum("0x" tolower(a[1]));
+                   	key[cnt] = s;
+			arr[s] = $0;
+                	cnt++;
+			next;
+		}
+/\"\\u/	{
+		gsub("\" .*$", ""); gsub("^.*\\u", "", $0);
+		arr[strtonum("0x" tolower($0))] = "\"\\u" $0 "\"";
+                key[cnt] = strtonum("0x" tolower($0));
+                cnt++;
+	        next;
+	}
+/}/	{
+	print("TOKEN: {");
+	print("   <#UnicodeIdentifierStart: [");
+        asort(key); 
+        i = 1;
+	do 
+	{
+		if (i > 1) printf(",");
+                printf(arr[key[i]]);
+                start = i;
+
+		while (key[i] + 1 == key[i + 1] && match(arr[key[i]], "-") == 0) { i++; }
+                if (start != i) { print("-" arr[key[i]]); } else { print ""; }
+                i++;
+	} while (i < length(key));
+	print("]> }");
+	}

parser/grammar/lexical-elements.txt

@@ -80,7 +80,7 @@ regular_identifier()
 
 | <#identifier_part: <identifier_start> | <identifier_extend> >
 
-| <#identifier_start: ["a"-"z"]  // temp
+| <#identifier_start: (<UnicodeIdentifierStart>)
 /*!! See the Syntax Rules.*/
 >
 

parser/grammar/prepare-javacc-grammar.sh

@@ -1,4 +1,6 @@
 # Concatenate all the fragments into a .jj file.
 gendir='../target/generated-sources/javacc'
 mkdir -p $gendir
-cat javacc-options-java.txt nonreservedwords.txt reservedwords.txt sql-spec.txt presto-extensions.txt lexical-elements.txt > $gendir/parser_tmp.jjt
+awk -f ./compact_char_sets.awk <  unicode-identifiers.txt > $gendir/compact-unicode-identifiers.txt
+#cp unicode-identifiers.txt  $gendir/comcpat-unicode-identifiers.txt
+cat javacc-options-java.txt kw.txt sql-spec.txt presto-extensions.txt nonreservedwords.txt $gendir/comcpat-unicode-identifiers.txt  lexical-elements.txt > $gendir/parser_tmp.jjt

parser/grammar/unicode-identifier-extend.txt

@@ -52,6 +52,7 @@ public class TestSqlParser
             "SELECT f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f(f())))))))))))))))))))))))))))));",
             "SELECT abs, 2 as abs;",
             "SELECT sqrt(x), power(y, 5), myFunction('a') FROM T;",
+            "SELECT 1 ఒకట;"
     };
 
     private AstNode parse(String sql)