[GR-38748] Use original python grammar and replace source offsets with SourceRange in SST nodes

PullRequest: graalpython/2267

Author: otethal

graalpython/com.oracle.graal.python.pegparser.generator/diff_generator.py

@@ -129,6 +129,7 @@ def visit_Rule(self, node: Rule):
     def visit_Alt(self, node: Alt):
         action = re.sub(r" ([\.,\(\)\[\]]) ", r"\1", str(node.action)) # shorten action string
         self.actions[" ".join(str(item) for item in node.items)] = action
+        self.generic_visit(node)
 
 
 def main():

graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py

@@ -102,7 +102,16 @@ public void atomString7() throws Exception {
     @Test
     public void atomFString() throws Exception {
         checkTreeResult("f'a{b!r}'");
-        checkError("f'a{b!g}'", "Generic[0:4]:f-string: invalid conversion character: expected 's', 'r', or 'a'");
+        checkError("f'a{b!g}'", "Generic[2:9]:f-string: invalid conversion character: expected 's', 'r', or 'a'");
+    }
+
+    @Test
+    public void atomFStringMultiline() throws Exception {
+        checkTreeResult("f\"\"\"First line.{\n" +
+                        "   v1\n" +
+                        "+\n" +
+                        "v2}\n" +
+                        "Another line.\"\"\"");
     }
 
     @Test

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/AtomsTests.java

@@ -54,6 +54,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
 import org.junit.Rule;
 import org.junit.rules.TestName;
 
@@ -97,7 +98,7 @@ public ModTy parse(String src, String moduleName, int mode) {
 
         FExprParser fexpParser = new FExprParser() {
             @Override
-            public ExprTy parse(String code) {
+            public ExprTy parse(String code, SourceRange sourceRange) {
                 ParserTokenizer tok = new ParserTokenizer(code);
                 return (ExprTy) new Parser(tok, factory, this).parse(InputType.FSTRING);
             }
@@ -203,13 +204,13 @@ public void checkError(String source, String... expectedErrors) {
         ArrayList<String> errors = new ArrayList<>();
         ParserErrorCallback errorCb = new ParserErrorCallback() {
             @Override
-            public void onError(ParserErrorCallback.ErrorType type, int start, int end, String message) {
-                errors.add(String.format("%s[%d:%d]:%s", type.name(), start, end, message));
+            public void onError(ParserErrorCallback.ErrorType type, SourceRange sourceRange, String message) {
+                errors.add(String.format("%s[%d:%d]:%s", type.name(), sourceRange.startOffset, sourceRange.endOffset, message));
             }
         };
         FExprParser fexpParser = new FExprParser() {
             @Override
-            public ExprTy parse(String code) {
+            public ExprTy parse(String code, SourceRange range) {
                 ParserTokenizer tok = new ParserTokenizer(code);
                 return (ExprTy) new Parser(tok, factory, this, errorCb).parse(InputType.FSTRING);
             }

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/ParserTestBase.java

@@ -59,8 +59,8 @@ private String indent() {
 
     private static String addHeader(SSTNode node) {
         StringBuilder sb = new StringBuilder();
-        sb.append(node.getClass().getSimpleName()).append("[").append(node.getStartOffset());
-        sb.append(", ").append(node.getEndOffset()).append("]");
+        sb.append(node.getClass().getSimpleName()).append("[").append(node.getSourceRange().startOffset);
+        sb.append(", ").append(node.getSourceRange().endOffset).append("]");
         return sb.toString();
     }
 
@@ -224,8 +224,8 @@ public String visit(ExprTy.Compare node) {
     @Override
     public String visit(ExprTy.Constant node) {
         StringBuilder sb = new StringBuilder();
-        sb.append(node.kind).append("[").append(node.getStartOffset());
-        sb.append(", ").append(node.getEndOffset()).append("]");
+        sb.append(node.kind).append("[").append(node.getSourceRange().startOffset);
+        sb.append(", ").append(node.getSourceRange().endOffset).append("]");
         sb.append(" Value: ");
         switch (node.kind) {
             case LONG:

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/sst/SSTTreePrinterVisitor.java

@@ -470,8 +470,8 @@ private void checkTokensFromTestDataFile() throws Exception {
                     sb.append("exact_type:").append(getCPythonValueOfTokenType(token.type));
                     sb.append(" (").append(token.typeName()).append(") ");
                 }
-                sb.append("start:[").append(token.startLine).append(", ").append(token.startColumn).append("] ");
-                sb.append("end:[").append(token.endLine).append(", ").append(token.endColumn).append("] ");
+                sb.append("start:[").append(token.sourceRange.startLine).append(", ").append(token.sourceRange.startColumn).append("] ");
+                sb.append("end:[").append(token.sourceRange.endLine).append(", ").append(token.sourceRange.endColumn).append("] ");
                 sb.append("string:'").append(tokenizer.getTokenString(token)).append("'");
                 String goldenToken = goldenResult.get(goldenResultIndex);
                 assertEquals("Code: '" + line + "'", goldenToken, sb.toString());

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/tokenizer/TokenizerTest.java

@@ -0,0 +1,13 @@
+Module[0, 45]
+    JoinedStr[0, 45]
+         Values:
+            RAW[4, 15] Value: First line.
+            FormattedValue[16, 27]
+                Value: 
+                    BinOp[5, 12]
+                        Op: ADD
+                        LHS: Name[5, 7] Value: "v1"
+                        RHS: Name[10, 12] Value: "v2"
+                Conversion: NONE
+            RAW[28, 42] Value: 
+Another line.

graalpython/com.oracle.graal.python.pegparser.test/testData/parser/goldenFiles/AtomsTests/atomFStringMultiline.tast

@@ -3,4 +3,5 @@ Module[0, 12]
         Target: Name[0, 3] Value: "foo"
         Args:
             Starred[4, 11]
+                 Context: Load
                  Expr: Name[5, 11] Value: "mylist"

graalpython/com.oracle.graal.python.pegparser.test/testData/parser/goldenFiles/BasicTests/call09.tast

@@ -3,6 +3,8 @@ Module[0, 23]
         Target: Name[0, 3] Value: "foo"
         Args:
             Starred[4, 12]
+                 Context: Load
                  Expr: Name[5, 12] Value: "mylist1"
             Starred[14, 22]
+                 Context: Load
                  Expr: Name[15, 22] Value: "mylist2"

graalpython/com.oracle.graal.python.pegparser.test/testData/parser/goldenFiles/BasicTests/call10.tast

@@ -24,6 +24,7 @@ Module[0, 128]
                     Target: Name[109, 113] Value: "self"
                     Args:
                         Starred[114, 119]
+                             Context: Load
                              Expr: Name[115, 119] Value: "args"
                     KWArgs:
                         KeywordTy[121, 127]