Initial unicode character support for identifiers

Summary:

Test Plan:
Added a test

Reviewers:

Subscribers:

Tasks:

Tags:

Author: kaikalur

parser/cpp/prepare-javacc-grammar.sh

@@ -3,4 +3,4 @@ pwd
 GRAMMAR_DIR='../grammar'
 GEN_DIR='target/generated-sources/javacc'
 mkdir -p $GEN_DIR
-cat ./javacc-options.txt $GRAMMAR_DIR/nonreservedwords.txt $GRAMMAR_DIR/reservedwords.txt $GRAMMAR_DIR/sql-spec.txt $GRAMMAR_DIR/presto-extensions.txt $GRAMMAR_DIR/lexical-elements.txt > $GEN_DIR/parser_tmp.jjt
+cat ./javacc-options.txt $GRAMMAR_DIR/kw.txt $GRAMMAR_DIR/sql-spec.txt $GRAMMAR_DIR/presto-extensions.txt $GRAMMAR_DIR/nonreservedwords.txt $GRAMMAR_DIR/unicode-identifiers.txt $GRAMMAR_DIR/lexical-elements.txt > $GEN_DIR/parser_tmp.jjt

parser/grammar/lexical-elements.txt

@@ -80,7 +80,7 @@ regular_identifier()
 
 | <#identifier_part: <identifier_start> | <identifier_extend> >
 
-| <#identifier_start: ["a"-"z"]  // temp
+| <#identifier_start: (<UnicodeIdentifierStart>)
 /*!! See the Syntax Rules.*/
 >
 

parser/grammar/prepare-javacc-grammar.sh

@@ -1,4 +1,6 @@
 # Concatenate all the fragments into a .jj file.
 gendir='../target/generated-sources/javacc'
 mkdir -p $gendir
-cat javacc-options-java.txt nonreservedwords.txt reservedwords.txt sql-spec.txt presto-extensions.txt lexical-elements.txt > $gendir/parser_tmp.jjt
+awk -f ./compact_char_sets.awk <  unicode-identifiers.txt > $gendir/compact-unicode-identifiers.txt
+#cp unicode-identifiers.txt  $gendir/comcpat-unicode-identifiers.txt
+cat javacc-options-java.txt kw.txt sql-spec.txt presto-extensions.txt nonreservedwords.txt $gendir/comcpat-unicode-identifiers.txt  lexical-elements.txt > $gendir/parser_tmp.jjt