Skip to content

Commit d2d117a

Browse files
committed
Use SourceRange instead of start/end offset
1 parent 3d5330a commit d2d117a

File tree

31 files changed

+985
-885
lines changed

31 files changed

+985
-885
lines changed

graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py

Lines changed: 132 additions & 119 deletions
Large diffs are not rendered by default.

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/AtomsTests.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,16 @@ public void atomString7() throws Exception {
102102
@Test
103103
public void atomFString() throws Exception {
104104
checkTreeResult("f'a{b!r}'");
105-
checkError("f'a{b!g}'", "Generic[0:4]:f-string: invalid conversion character: expected 's', 'r', or 'a'");
105+
checkError("f'a{b!g}'", "Generic[2:9]:f-string: invalid conversion character: expected 's', 'r', or 'a'");
106+
}
107+
108+
@Test
109+
public void atomFStringMultiline() throws Exception {
110+
checkTreeResult("f\"\"\"First line.{\n" +
111+
" v1\n" +
112+
"+\n" +
113+
"v2}\n" +
114+
"Another line.\"\"\"");
106115
}
107116

108117
@Test

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/ParserTestBase.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import java.util.Arrays;
5555
import java.util.List;
5656

57+
import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
5758
import org.junit.Rule;
5859
import org.junit.rules.TestName;
5960

@@ -97,7 +98,7 @@ public ModTy parse(String src, String moduleName, int mode) {
9798

9899
FExprParser fexpParser = new FExprParser() {
99100
@Override
100-
public ExprTy parse(String code) {
101+
public ExprTy parse(String code, SourceRange sourceRange) {
101102
ParserTokenizer tok = new ParserTokenizer(code);
102103
return (ExprTy) new Parser(tok, factory, this).parse(InputType.FSTRING);
103104
}
@@ -203,13 +204,13 @@ public void checkError(String source, String... expectedErrors) {
203204
ArrayList<String> errors = new ArrayList<>();
204205
ParserErrorCallback errorCb = new ParserErrorCallback() {
205206
@Override
206-
public void onError(ParserErrorCallback.ErrorType type, int start, int end, String message) {
207-
errors.add(String.format("%s[%d:%d]:%s", type.name(), start, end, message));
207+
public void onError(ParserErrorCallback.ErrorType type, SourceRange sourceRange, String message) {
208+
errors.add(String.format("%s[%d:%d]:%s", type.name(), sourceRange.startOffset, sourceRange.endOffset, message));
208209
}
209210
};
210211
FExprParser fexpParser = new FExprParser() {
211212
@Override
212-
public ExprTy parse(String code) {
213+
public ExprTy parse(String code, SourceRange range) {
213214
ParserTokenizer tok = new ParserTokenizer(code);
214215
return (ExprTy) new Parser(tok, factory, this, errorCb).parse(InputType.FSTRING);
215216
}

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/sst/SSTTreePrinterVisitor.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ private String indent() {
5959

6060
private static String addHeader(SSTNode node) {
6161
StringBuilder sb = new StringBuilder();
62-
sb.append(node.getClass().getSimpleName()).append("[").append(node.getStartOffset());
63-
sb.append(", ").append(node.getEndOffset()).append("]");
62+
sb.append(node.getClass().getSimpleName()).append("[").append(node.getSourceRange().startOffset);
63+
sb.append(", ").append(node.getSourceRange().endOffset).append("]");
6464
return sb.toString();
6565
}
6666

@@ -224,8 +224,8 @@ public String visit(ExprTy.Compare node) {
224224
@Override
225225
public String visit(ExprTy.Constant node) {
226226
StringBuilder sb = new StringBuilder();
227-
sb.append(node.kind).append("[").append(node.getStartOffset());
228-
sb.append(", ").append(node.getEndOffset()).append("]");
227+
sb.append(node.kind).append("[").append(node.getSourceRange().startOffset);
228+
sb.append(", ").append(node.getSourceRange().endOffset).append("]");
229229
sb.append(" Value: ");
230230
switch (node.kind) {
231231
case LONG:

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/tokenizer/TokenizerTest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -470,8 +470,8 @@ private void checkTokensFromTestDataFile() throws Exception {
470470
sb.append("exact_type:").append(getCPythonValueOfTokenType(token.type));
471471
sb.append(" (").append(token.typeName()).append(") ");
472472
}
473-
sb.append("start:[").append(token.startLine).append(", ").append(token.startColumn).append("] ");
474-
sb.append("end:[").append(token.endLine).append(", ").append(token.endColumn).append("] ");
473+
sb.append("start:[").append(token.sourceRange.startLine).append(", ").append(token.sourceRange.startColumn).append("] ");
474+
sb.append("end:[").append(token.sourceRange.endLine).append(", ").append(token.sourceRange.endColumn).append("] ");
475475
sb.append("string:'").append(tokenizer.getTokenString(token)).append("'");
476476
String goldenToken = goldenResult.get(goldenResultIndex);
477477
assertEquals("Code: '" + line + "'", goldenToken, sb.toString());
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Module[0, 45]
2+
JoinedStr[0, 45]
3+
Values:
4+
RAW[4, 15] Value: First line.
5+
FormattedValue[16, 27]
6+
Value:
7+
BinOp[5, 12]
8+
Op: ADD
9+
LHS: Name[5, 7] Value: "v1"
10+
RHS: Name[10, 12] Value: "v2"
11+
Conversion: NONE
12+
RAW[28, 42] Value:
13+
Another line.

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java

Lines changed: 26 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import com.oracle.graal.python.pegparser.sst.KeywordTy;
5353
import com.oracle.graal.python.pegparser.sst.SSTNode;
5454
import com.oracle.graal.python.pegparser.sst.StmtTy;
55+
import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
5556
import com.oracle.graal.python.pegparser.tokenizer.Token;
5657

5758
/**
@@ -170,7 +171,7 @@ private void resetParserState() {
170171

171172
/**
172173
* Get position in the tokenizer.
173-
*
174+
*
174175
* @return the position in tokenizer.
175176
*/
176177
public int mark() {
@@ -179,7 +180,7 @@ public int mark() {
179180

180181
/**
181182
* Reset position in the tokenizer
182-
*
183+
*
183184
* @param position where the tokenizer should set the current position
184185
*/
185186
public void reset(int position) {
@@ -189,7 +190,7 @@ public void reset(int position) {
189190
/**
190191
* Is the expected token on the current position in tokenizer? If there is the expected token,
191192
* then the current position in tokenizer is changed to the next token.
192-
*
193+
*
193194
* @param tokenKind - the token kind that is expected on the current position
194195
* @return The expected token or null if the token on the current position is not the expected
195196
* one.
@@ -205,7 +206,7 @@ public Token expect(int tokenKind) {
205206
/**
206207
* Is the expected token on the current position in tokenizer? If there is the expected token,
207208
* then the current position in tokenizer is changed to the next token.
208-
*
209+
*
209210
* @param text - the token on the current position has to have this text
210211
* @return The expected token or null if the token on the current position is not the expected
211212
* one.
@@ -258,7 +259,7 @@ public Token getAndInitializeToken() {
258259
Token token = tokenizer.getToken();
259260
while (token.type == Token.Kind.TYPE_IGNORE) {
260261
String tag = getText(token);
261-
comments.put(token.startLine, tag);
262+
comments.put(token.sourceRange.startLine, tag);
262263
pos++;
263264
token = tokenizer.getToken();
264265
}
@@ -288,14 +289,14 @@ public Token getLastNonWhitespaceToken() {
288289
public ExprTy.Name name_token() {
289290
Token t = expect(Token.Kind.NAME);
290291
if (t != null) {
291-
return factory.createVariable(getText(t), t.startOffset, t.endOffset);
292+
return factory.createVariable(getText(t), t.sourceRange);
292293
} else {
293294
return null;
294295
}
295296
}
296297

297298
/**
298-
*
299+
*
299300
* @return flags that influence parsing.
300301
*/
301302
public int getFlags() {
@@ -325,7 +326,7 @@ protected ExprTy.Name expect_SOFT_KEYWORD(String keyword) {
325326
Token t = tokenizer.peekToken();
326327
if (t.type == Token.Kind.NAME && getText(t).equals(keyword)) {
327328
tokenizer.getToken();
328-
return factory.createVariable(getText(t), t.startOffset, t.endOffset);
329+
return factory.createVariable(getText(t), t.sourceRange);
329330
}
330331
return null;
331332
}
@@ -350,7 +351,7 @@ public Token string_token() {
350351
public ExprTy number_token() {
351352
Token t = expect(Token.Kind.NUMBER);
352353
if (t != null) {
353-
return factory.createNumber(getText(t), t.startOffset, t.endOffset);
354+
return factory.createNumber(getText(t), t.sourceRange);
354355
} else {
355356
return null;
356357
}
@@ -374,7 +375,7 @@ public ExprTy.Name name_from_token(Token t) {
374375
return null;
375376
}
376377
String id = getText(t);
377-
return factory.createVariable(id, t.startOffset, t.endOffset);
378+
return factory.createVariable(id, t.sourceRange);
378379
}
379380

380381
/**
@@ -401,7 +402,7 @@ public ExprTy.Name dummyName(@SuppressWarnings("unused") Object... args) {
401402
if (cachedDummyName != null) {
402403
return cachedDummyName;
403404
}
404-
cachedDummyName = factory.createVariable("", 0, 0);
405+
cachedDummyName = factory.createVariable("", new SourceRange(0, 0, 0, 0, 0, 0));
405406
return cachedDummyName;
406407
}
407408

@@ -410,7 +411,7 @@ public ExprTy.Name dummyName(@SuppressWarnings("unused") Object... args) {
410411
*/
411412
public SSTNode joinNamesWithDot(ExprTy a, ExprTy b) {
412413
String id = ((ExprTy.Name) a).id + "." + ((ExprTy.Name) b).id;
413-
return factory.createVariable(id, a.getStartOffset(), b.getEndOffset());
414+
return factory.createVariable(id, a.getSourceRange().withEnd(b.getSourceRange()));
414415
}
415416

416417
/**
@@ -460,23 +461,21 @@ public ExprTy[] appendToEnd(ExprTy[] seq, ExprTy element) {
460461
public SSTNode concatenateStrings(Token[] tokens) {
461462
int n = tokens.length;
462463
String[] values = new String[n];
463-
Token t = tokens[0];
464-
int startOffset = t.startOffset;
465-
values[0] = getText(t);
466-
for (int i = 1; i < n; i++) {
467-
t = tokens[i];
464+
SourceRange[] sourceRanges = new SourceRange[n];
465+
for (int i = 0; i < n; i++) {
466+
Token t = tokens[i];
468467
values[i] = getText(t);
468+
sourceRanges[i] = t.sourceRange;
469469
}
470-
int endOffset = t.endOffset;
471-
return factory.createString(values, startOffset, endOffset, fexprParser, errorCb);
470+
return factory.createString(values, sourceRanges, fexprParser, errorCb);
472471
}
473472

474473
/**
475474
* _PyPegen_check_barry_as_flufl
476475
*/
477476
public boolean checkBarryAsFlufl(Token token) {
478477
if ((flags & PARSE_BARRY_AS_BDFL) != 0 && !getText(token).equals("<>")) {
479-
errorCb.onError(token.startOffset, token.endOffset, BARRY_AS_BDFL);
478+
errorCb.onError(token.sourceRange, BARRY_AS_BDFL);
480479
return true;
481480
}
482481
return false;
@@ -762,9 +761,9 @@ static String[] extractNames(ExprTy[] seq) {
762761
/**
763762
* _PyPegen_collect_call_seqs
764763
*/
765-
final ExprTy collectCallSequences(ExprTy[] a, KeywordOrStarred[] b, int startOffset, int endOffset) {
764+
final ExprTy collectCallSequences(ExprTy[] a, KeywordOrStarred[] b, SourceRange sourceRange) {
766765
if (b == null) {
767-
return factory.createCall(dummyName(), a, EMPTY_KWDS, startOffset, endOffset);
766+
return factory.createCall(dummyName(), a, EMPTY_KWDS, sourceRange);
768767
} else {
769768
ExprTy[] starred = extractStarredExpressions(b);
770769
ExprTy[] args;
@@ -774,7 +773,7 @@ final ExprTy collectCallSequences(ExprTy[] a, KeywordOrStarred[] b, int startOff
774773
} else {
775774
args = a;
776775
}
777-
return factory.createCall(dummyName(), args, deleteStarredExpressions(b), startOffset, endOffset);
776+
return factory.createCall(dummyName(), args, deleteStarredExpressions(b), sourceRange);
778777
}
779778
}
780779

@@ -844,7 +843,7 @@ SSTNode raiseSyntaxErrorInvalidTarget(TargetsType type, ExprTy expr) {
844843
SSTNode raiseSyntaxError(String msg, Object... arguments) {
845844
errorIndicator = true;
846845
Token errorToken = tokenizer.peekToken();
847-
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.startOffset, errorToken.endOffset, msg, arguments);
846+
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, arguments);
848847
return null;
849848
}
850849

@@ -853,7 +852,7 @@ SSTNode raiseSyntaxError(String msg, Object... arguments) {
853852
*/
854853
SSTNode raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Object... argument) {
855854
errorIndicator = true;
856-
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.startOffset, errorToken.endOffset, msg, argument);
855+
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, argument);
857856
return null;
858857
}
859858

@@ -862,7 +861,7 @@ SSTNode raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Object... ar
862861
*/
863862
SSTNode raiseSyntaxErrorKnownLocation(SSTNode where, String msg, Object... argument) {
864863
errorIndicator = true;
865-
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, where.getStartOffset(), where.getEndOffset(), msg, argument);
864+
errorCb.onError(ParserErrorCallback.ErrorType.Syntax, where.getSourceRange(), msg, argument);
866865
return null;
867866
}
868867

@@ -871,7 +870,7 @@ SSTNode raiseSyntaxErrorKnownLocation(SSTNode where, String msg, Object... argum
871870
*/
872871
SSTNode raiseErrorKnownLocation(ParserErrorCallback.ErrorType typeError, SSTNode where, String msg, Object... argument) {
873872
errorIndicator = true;
874-
errorCb.onError(typeError, where.getStartOffset(), where.getEndOffset(), msg, argument);
873+
errorCb.onError(typeError, where.getSourceRange(), msg, argument);
875874
return null;
876875
}
877876

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/DefaultParserErrorCallback.java

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -41,34 +41,30 @@
4141

4242
package com.oracle.graal.python.pegparser;
4343

44+
import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
45+
4446
import java.util.ArrayList;
4547
import java.util.List;
4648

4749
public class DefaultParserErrorCallback implements ParserErrorCallback {
4850

4951
public static class Error {
5052
private final ParserErrorCallback.ErrorType type;
51-
private int startOffset;
52-
private int endOffset;
53+
private SourceRange sourceRange;
5354
private final String message;
5455

55-
public Error(ErrorType type, int startOffset, int endOffset, String message) {
56+
public Error(ErrorType type, SourceRange sourceRange, String message) {
5657
this.type = type;
57-
this.startOffset = startOffset;
58-
this.endOffset = endOffset;
58+
this.sourceRange = sourceRange;
5959
this.message = message;
6060
}
6161

6262
public ErrorType getType() {
6363
return type;
6464
}
6565

66-
public int getStartOffset() {
67-
return startOffset;
68-
}
69-
70-
public int getEndOffset() {
71-
return endOffset;
66+
public SourceRange getSourceRange() {
67+
return sourceRange;
7268
}
7369

7470
public String getMessage() {
@@ -88,8 +84,8 @@ public boolean hasErrors() {
8884
}
8985

9086
@Override
91-
public void onError(ErrorType errorType, int startOffset, int endOffset, String message) {
92-
errors.add(new Error(errorType, startOffset, endOffset, message));
87+
public void onError(ErrorType errorType, SourceRange sourceRange, String message) {
88+
errors.add(new Error(errorType, sourceRange, message));
9389
}
9490

9591
}

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/FExprParser.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,10 @@
4141
package com.oracle.graal.python.pegparser;
4242

4343
import com.oracle.graal.python.pegparser.sst.ExprTy;
44+
import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
4445

4546
@FunctionalInterface
4647
public interface FExprParser {
47-
public abstract ExprTy parse(String code);
48+
49+
ExprTy parse(String code, SourceRange sourceRange);
4850
}

0 commit comments

Comments
 (0)