52
52
import com .oracle .graal .python .pegparser .sst .KeywordTy ;
53
53
import com .oracle .graal .python .pegparser .sst .SSTNode ;
54
54
import com .oracle .graal .python .pegparser .sst .StmtTy ;
55
+ import com .oracle .graal .python .pegparser .tokenizer .SourceRange ;
55
56
import com .oracle .graal .python .pegparser .tokenizer .Token ;
56
57
57
58
/**
@@ -170,7 +171,7 @@ private void resetParserState() {
170
171
171
172
/**
172
173
* Get position in the tokenizer.
173
- *
174
+ *
174
175
* @return the position in tokenizer.
175
176
*/
176
177
public int mark () {
@@ -179,7 +180,7 @@ public int mark() {
179
180
180
181
/**
181
182
* Reset position in the tokenizer
182
- *
183
+ *
183
184
* @param position where the tokenizer should set the current position
184
185
*/
185
186
public void reset (int position ) {
@@ -189,7 +190,7 @@ public void reset(int position) {
189
190
/**
190
191
* Is the expected token on the current position in tokenizer? If there is the expected token,
191
192
* then the current position in tokenizer is changed to the next token.
192
- *
193
+ *
193
194
* @param tokenKind - the token kind that is expected on the current position
194
195
* @return The expected token or null if the token on the current position is not the expected
195
196
* one.
@@ -205,7 +206,7 @@ public Token expect(int tokenKind) {
205
206
/**
206
207
* Is the expected token on the current position in tokenizer? If there is the expected token,
207
208
* then the current position in tokenizer is changed to the next token.
208
- *
209
+ *
209
210
* @param text - the token on the current position has to have this text
210
211
* @return The expected token or null if the token on the current position is not the expected
211
212
* one.
@@ -258,7 +259,7 @@ public Token getAndInitializeToken() {
258
259
Token token = tokenizer .getToken ();
259
260
while (token .type == Token .Kind .TYPE_IGNORE ) {
260
261
String tag = getText (token );
261
- comments .put (token .startLine , tag );
262
+ comments .put (token .sourceRange . startLine , tag );
262
263
pos ++;
263
264
token = tokenizer .getToken ();
264
265
}
@@ -288,14 +289,14 @@ public Token getLastNonWhitespaceToken() {
288
289
public ExprTy .Name name_token () {
289
290
Token t = expect (Token .Kind .NAME );
290
291
if (t != null ) {
291
- return factory .createVariable (getText (t ), t .startOffset , t . endOffset );
292
+ return factory .createVariable (getText (t ), t .sourceRange );
292
293
} else {
293
294
return null ;
294
295
}
295
296
}
296
297
297
298
/**
298
- *
299
+ *
299
300
* @return flags that influence parsing.
300
301
*/
301
302
public int getFlags () {
@@ -325,7 +326,7 @@ protected ExprTy.Name expect_SOFT_KEYWORD(String keyword) {
325
326
Token t = tokenizer .peekToken ();
326
327
if (t .type == Token .Kind .NAME && getText (t ).equals (keyword )) {
327
328
tokenizer .getToken ();
328
- return factory .createVariable (getText (t ), t .startOffset , t . endOffset );
329
+ return factory .createVariable (getText (t ), t .sourceRange );
329
330
}
330
331
return null ;
331
332
}
@@ -350,7 +351,7 @@ public Token string_token() {
350
351
public ExprTy number_token () {
351
352
Token t = expect (Token .Kind .NUMBER );
352
353
if (t != null ) {
353
- return factory .createNumber (getText (t ), t .startOffset , t . endOffset );
354
+ return factory .createNumber (getText (t ), t .sourceRange );
354
355
} else {
355
356
return null ;
356
357
}
@@ -374,7 +375,7 @@ public ExprTy.Name name_from_token(Token t) {
374
375
return null ;
375
376
}
376
377
String id = getText (t );
377
- return factory .createVariable (id , t .startOffset , t . endOffset );
378
+ return factory .createVariable (id , t .sourceRange );
378
379
}
379
380
380
381
/**
@@ -401,7 +402,7 @@ public ExprTy.Name dummyName(@SuppressWarnings("unused") Object... args) {
401
402
if (cachedDummyName != null ) {
402
403
return cachedDummyName ;
403
404
}
404
- cachedDummyName = factory .createVariable ("" , 0 , 0 );
405
+ cachedDummyName = factory .createVariable ("" , new SourceRange ( 0 , 0 , 0 , 0 , 0 , 0 ) );
405
406
return cachedDummyName ;
406
407
}
407
408
@@ -410,7 +411,7 @@ public ExprTy.Name dummyName(@SuppressWarnings("unused") Object... args) {
410
411
*/
411
412
public SSTNode joinNamesWithDot (ExprTy a , ExprTy b ) {
412
413
String id = ((ExprTy .Name ) a ).id + "." + ((ExprTy .Name ) b ).id ;
413
- return factory .createVariable (id , a .getStartOffset (), b . getEndOffset ( ));
414
+ return factory .createVariable (id , a .getSourceRange (). withEnd ( b . getSourceRange () ));
414
415
}
415
416
416
417
/**
@@ -460,23 +461,21 @@ public ExprTy[] appendToEnd(ExprTy[] seq, ExprTy element) {
460
461
public SSTNode concatenateStrings (Token [] tokens ) {
461
462
int n = tokens .length ;
462
463
String [] values = new String [n ];
463
- Token t = tokens [0 ];
464
- int startOffset = t .startOffset ;
465
- values [0 ] = getText (t );
466
- for (int i = 1 ; i < n ; i ++) {
467
- t = tokens [i ];
464
+ SourceRange [] sourceRanges = new SourceRange [n ];
465
+ for (int i = 0 ; i < n ; i ++) {
466
+ Token t = tokens [i ];
468
467
values [i ] = getText (t );
468
+ sourceRanges [i ] = t .sourceRange ;
469
469
}
470
- int endOffset = t .endOffset ;
471
- return factory .createString (values , startOffset , endOffset , fexprParser , errorCb );
470
+ return factory .createString (values , sourceRanges , fexprParser , errorCb );
472
471
}
473
472
474
473
/**
475
474
* _PyPegen_check_barry_as_flufl
476
475
*/
477
476
public boolean checkBarryAsFlufl (Token token ) {
478
477
if ((flags & PARSE_BARRY_AS_BDFL ) != 0 && !getText (token ).equals ("<>" )) {
479
- errorCb .onError (token .startOffset , token . endOffset , BARRY_AS_BDFL );
478
+ errorCb .onError (token .sourceRange , BARRY_AS_BDFL );
480
479
return true ;
481
480
}
482
481
return false ;
@@ -762,9 +761,9 @@ static String[] extractNames(ExprTy[] seq) {
762
761
/**
763
762
* _PyPegen_collect_call_seqs
764
763
*/
765
- final ExprTy collectCallSequences (ExprTy [] a , KeywordOrStarred [] b , int startOffset , int endOffset ) {
764
+ final ExprTy collectCallSequences (ExprTy [] a , KeywordOrStarred [] b , SourceRange sourceRange ) {
766
765
if (b == null ) {
767
- return factory .createCall (dummyName (), a , EMPTY_KWDS , startOffset , endOffset );
766
+ return factory .createCall (dummyName (), a , EMPTY_KWDS , sourceRange );
768
767
} else {
769
768
ExprTy [] starred = extractStarredExpressions (b );
770
769
ExprTy [] args ;
@@ -774,7 +773,7 @@ final ExprTy collectCallSequences(ExprTy[] a, KeywordOrStarred[] b, int startOff
774
773
} else {
775
774
args = a ;
776
775
}
777
- return factory .createCall (dummyName (), args , deleteStarredExpressions (b ), startOffset , endOffset );
776
+ return factory .createCall (dummyName (), args , deleteStarredExpressions (b ), sourceRange );
778
777
}
779
778
}
780
779
@@ -844,7 +843,7 @@ SSTNode raiseSyntaxErrorInvalidTarget(TargetsType type, ExprTy expr) {
844
843
SSTNode raiseSyntaxError (String msg , Object ... arguments ) {
845
844
errorIndicator = true ;
846
845
Token errorToken = tokenizer .peekToken ();
847
- errorCb .onError (ParserErrorCallback .ErrorType .Syntax , errorToken .startOffset , errorToken . endOffset , msg , arguments );
846
+ errorCb .onError (ParserErrorCallback .ErrorType .Syntax , errorToken .sourceRange , msg , arguments );
848
847
return null ;
849
848
}
850
849
@@ -853,7 +852,7 @@ SSTNode raiseSyntaxError(String msg, Object... arguments) {
853
852
*/
854
853
SSTNode raiseSyntaxErrorKnownLocation (Token errorToken , String msg , Object ... argument ) {
855
854
errorIndicator = true ;
856
- errorCb .onError (ParserErrorCallback .ErrorType .Syntax , errorToken .startOffset , errorToken . endOffset , msg , argument );
855
+ errorCb .onError (ParserErrorCallback .ErrorType .Syntax , errorToken .sourceRange , msg , argument );
857
856
return null ;
858
857
}
859
858
@@ -862,7 +861,7 @@ SSTNode raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Object... ar
862
861
*/
863
862
SSTNode raiseSyntaxErrorKnownLocation (SSTNode where , String msg , Object ... argument ) {
864
863
errorIndicator = true ;
865
- errorCb .onError (ParserErrorCallback .ErrorType .Syntax , where .getStartOffset (), where . getEndOffset (), msg , argument );
864
+ errorCb .onError (ParserErrorCallback .ErrorType .Syntax , where .getSourceRange (), msg , argument );
866
865
return null ;
867
866
}
868
867
@@ -871,7 +870,7 @@ SSTNode raiseSyntaxErrorKnownLocation(SSTNode where, String msg, Object... argum
871
870
*/
872
871
SSTNode raiseErrorKnownLocation (ParserErrorCallback .ErrorType typeError , SSTNode where , String msg , Object ... argument ) {
873
872
errorIndicator = true ;
874
- errorCb .onError (typeError , where .getStartOffset (), where . getEndOffset (), msg , argument );
873
+ errorCb .onError (typeError , where .getSourceRange (), msg , argument );
875
874
return null ;
876
875
}
877
876
0 commit comments