gh-153569: keep persistent scanner positions as source offsets

Author: pablogsal

Makefile.pre.in

@@ -394,7 +394,6 @@ PEGEN_OBJS=		\
 		Parser/peg_api.o
 
 TOKENIZER_OBJS=		\
-		Parser/lexer/buffer.o \
 		Parser/lexer/lexer.o \
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
@@ -412,7 +411,6 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/buffer.h \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \

PCbuild/_freeze_module.vcxproj

@@ -181,7 +181,6 @@
     <ClCompile Include="..\Parser\action_helpers.c" />
     <ClCompile Include="..\Parser\string_parser.c" />
     <ClCompile Include="..\Parser\token.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\lexer\state.c" />
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -469,9 +469,6 @@
     <ClCompile Include="..\Parser\lexer\string.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -423,7 +423,6 @@
     <ClInclude Include="..\Parser\lexer\state.h" />
     <ClInclude Include="..\Parser\lexer\lexer.h" />
     <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
-    <ClInclude Include="..\Parser\lexer\buffer.h" />
     <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
@@ -594,7 +593,6 @@
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -330,9 +330,6 @@
     <ClInclude Include="..\Parser\lexer\lexer_internal.h">
       <Filter>Parser</Filter>
     </ClInclude>
-    <ClInclude Include="..\Parser\lexer\buffer.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\cursor.h">
       <Filter>Parser</Filter>
     </ClInclude>
@@ -1364,9 +1361,6 @@
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Parser</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Parser</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\cursor.c">
       <Filter>Parser</Filter>
     </ClCompile>

Parser/lexer/buffer.c

@@ -35,7 +35,7 @@ _PyLexer_nextc(struct tok_state *tok)
                 return EOF;
             }
             tok->col_offset++;
-            return Py_CHARMASK(*tok->cur++); /* Fast path */
+            return Py_CHARMASK(tok->source.bytes[tok->cur++ - tok->source.base_offset]); /* Fast path */
         }
         if (tok->done != E_OK) {
             return EOF;
@@ -44,7 +44,7 @@ _PyLexer_nextc(struct tok_state *tok)
 #if defined(Py_DEBUG)
         if (tok->debug) {
             fprintf(stderr, "line[%d] = ", tok->lineno);
-            _PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
+            _PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur), tok->inp - tok->cur);
             fprintf(stderr, "  tok->done = %d\n", tok->done);
         }
 #endif
@@ -54,7 +54,7 @@ _PyLexer_nextc(struct tok_state *tok)
         }
         tok->line_start = tok->cur;
 
-        if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
+        if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start), tok->inp - tok->line_start)) {
             _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
             tok->cur = tok->inp;
             return EOF;
@@ -68,10 +68,10 @@ void
 _PyLexer_backup(struct tok_state *tok, int c)
 {
     if (c != EOF) {
-        if (--tok->cur < tok->buf) {
+        if (--tok->cur < tok->buf_offset) {
             Py_FatalError("tokenizer beginning of buffer");
         }
-        if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
+        if ((int)(unsigned char)*_PyLexer_BufferPointer(tok, tok->cur) != Py_CHARMASK(c)) {
             Py_FatalError("tok_backup: wrong character");
         }
         tok->col_offset--;
@@ -90,7 +90,7 @@ verify_identifier(struct tok_state *tok)
     PyObject *s;
     if (tok->input_error)
         return 0;
-    s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
+    s = PyUnicode_DecodeUTF8(_PyLexer_BufferPointer(tok, tok->start), tok->cur - tok->start, NULL);
     if (s == NULL) {
         if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
             tok->done = E_DECODE;
@@ -115,7 +115,7 @@ verify_identifier(struct tok_state *tok)
                 tok->done = E_ERROR;
                 return 0;
             }
-            tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
+            tok->cur = tok->start + PyBytes_GET_SIZE(s);
         }
         Py_DECREF(s);
         if (Py_UNICODE_ISPRINTABLE(ch)) {
@@ -161,10 +161,10 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     int c;
     int blankline, nonascii;
 
-    const char *p_start = NULL;
-    const char *p_end = NULL;
+    _PyTok_Off p_start = -1;
+    _PyTok_Off p_end = -1;
   nextline:
-    tok->start = NULL;
+    tok->start = -1;
     tok->starting_col_offset = -1;
     blankline = 0;
 
@@ -283,7 +283,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         }
         else {
             if (tok->tok_extra_tokens) {
-                p_start = tok->buf;
+                p_start = tok->buf_offset;
                 p_end = tok->cur;
             }
             tok->pendin--;
@@ -296,14 +296,14 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     tok_backup(tok, c);
 
  again:
-    tok->start = NULL;
+    tok->start = -1;
     /* Skip spaces */
     do {
         c = tok_nextc(tok);
     } while (c == ' ' || c == '\t' || c == '\014');
 
     /* Set start of current token */
-    tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
+    tok->start = tok->cur - 1;
     tok->starting_col_offset = tok->col_offset - 1;
 
     /* Skip comment, unless it's a type comment */
@@ -318,14 +318,14 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         }
 
         if (tok->tok_extra_tokens) {
-            p = tok->start;
+            p = _PyLexer_BufferPointer(tok, tok->start);
         }
 
         if (tok->type_comments) {
-            p = tok->start;
+            p = _PyLexer_BufferPointer(tok, tok->start);
             current_starting_col_offset = tok->starting_col_offset;
             prefix = type_comment_prefix;
-            while (*prefix && p < tok->cur) {
+            while (*prefix && p < _PyLexer_BufferPointer(tok, tok->cur)) {
                 if (*prefix == ' ') {
                     while (*p == ' ' || *p == '\t') {
                         p++;
@@ -354,24 +354,24 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
                 /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
                  * or anything ASCII and non-alphanumeric. */
                 is_type_ignore = (
-                    tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
-                    && !(tok->cur > ignore_end
+                    _PyLexer_BufferPointer(tok, tok->cur) >= ignore_end && memcmp(p, "ignore", 6) == 0
+                    && !(_PyLexer_BufferPointer(tok, tok->cur) > ignore_end
                          && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
 
                 int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
                 int start_col_offset = is_type_ignore
                     ? ignore_end_col_offset : current_starting_col_offset;
                 p_end = tok->cur;
                 if (is_type_ignore) {
-                    p_start = ignore_end;
+                    p_start = _PyLexer_BufferOffset(tok, ignore_end);
 
                     /* If this type ignore is the only thing on the line, consume the newline also. */
                     if (blankline) {
                         tok_nextc(tok);
                         tok->atbol = 1;
                     }
                 } else {
-                    p_start = type_start;
+                    p_start = _PyLexer_BufferOffset(tok, type_start);
                 }
                 _PyLexer_token_setup(tok, token, type, p_start, p_end);
                 token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
@@ -381,7 +381,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         }
         if (tok->tok_extra_tokens) {
             tok_backup(tok, c);  /* don't eat the newline or EOF */
-            p_start = p;
+            p_start = _PyLexer_BufferOffset(tok, p);
             p_end = tok->cur;
             tok->comment_newline = blankline;
             return MAKE_TOKEN(COMMENT);

Parser/lexer/buffer.h

@@ -113,8 +113,8 @@ int
 _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
                      int leading_dot)
 {
-    const char *p_start = NULL;
-    const char *p_end = NULL;
+    _PyTok_Off p_start = -1;
+    _PyTok_Off p_end = -1;
 
     if (leading_dot) {
         goto fraction;
@@ -214,7 +214,7 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
                 }
                 c = tok_nextc(tok);
             }
-            char* zeros_end = tok->cur;
+            _PyTok_Off zeros_end = tok->cur;
             if (Py_ISDIGIT(c)) {
                 nonzero = 1;
                 c = tok_decimal_tail(tok);

Parser/lexer/lexer.c

@@ -21,11 +21,12 @@ _PyTokenizer_tok_new(void)
         return NULL;
     }
 
-    tok->buf = tok->cur = tok->inp = NULL;
+    tok->cur = tok->inp = 0;
+    tok->line_start = tok->multi_line_start = -1;
     tok->fp_interactive = 0;
     tok->interactive_src_start = NULL;
     tok->interactive_src_end = NULL;
-    tok->start = NULL;
+    tok->start = -1;
     tok->done = E_OK;
     tok->fp = NULL;
     tok->tabsize = TABSIZE;
@@ -108,38 +109,19 @@ _PyToken_Init(struct token *token) {
     token->metadata = NULL;
 }
 
-static inline _PyTok_Span
-buffer_span(const struct tok_state *tok, const char *start, const char *end)
-{
-    if (start == NULL) {
-        assert(end == NULL);
-        return (_PyTok_Span){-1, -1};
-    }
-    assert(end != NULL);
-    const char *base = tok->buf;
-    assert(base != NULL);
-    assert(tok->inp >= base);
-    Py_ssize_t start_offset = start - base;
-    Py_ssize_t end_offset = end - base;
-    assert(start_offset >= 0 && start_offset <= end_offset);
-    assert(end_offset <= tok->inp - base);
-    assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
-    return _PyTok_SpanFromBounds(
-        tok->buf_offset + start_offset, tok->buf_offset + end_offset);
-}
-
 int
-_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
+_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTok_Off start, _PyTok_Off end)
 {
     token->level = tok->level;
     token->is_raw = ISSTRINGLIT(type)
         && tok->tok_mode_stack[tok->tok_mode_stack_index].raw;
-    token->span = buffer_span(tok, start, end);
+    assert((start == -1 && end == -1) || (start >= 0 && end >= start));
+    token->span = (_PyTok_Span){start, end};
     int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
     token->start_loc = (_PyTok_Loc){lineno, -1};
     token->end_loc = (_PyTok_Loc){tok->lineno, -1};
 
-    if (start != NULL && end != NULL) {
+    if (start >= 0 && end >= 0) {
         token->start_loc.byte_col = tok->starting_col_offset;
         token->end_loc.byte_col = tok->col_offset;
     }