gh-153569: use tokenizer views and remove obsolete API remnants

Author: pablogsal

Makefile.pre.in

@@ -411,7 +411,6 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
 		Parser/tokenizer/reader.h \

PCbuild/pythoncore.vcxproj

@@ -421,7 +421,6 @@
     <ClInclude Include="..\Objects\stringlib\split.h" />
     <ClInclude Include="..\Objects\unicodetype_db.h" />
     <ClInclude Include="..\Parser\lexer\state.h" />
-    <ClInclude Include="..\Parser\lexer\lexer.h" />
     <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -321,9 +321,6 @@
     <ClInclude Include="..\Objects\unicodetype_db.h">
       <Filter>Objects</Filter>
     </ClInclude>
-    <ClInclude Include="..\Parser\lexer\lexer.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
     <ClInclude Include="..\Parser\lexer\state.h">
       <Filter>Parser</Filter>
     </ClInclude>

Parser/lexer/lexer.c

@@ -89,7 +89,7 @@ verify_identifier(struct tok_state *tok)
     assert(PyUnicode_GET_LENGTH(s) > 0);
     if (invalid < PyUnicode_GET_LENGTH(s)) {
         Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
-        const char *error_cursor = tok->cur;
+        _PyTok_Off error_cursor = tok->cur;
         if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
             /* Determine the offset in UTF-8 encoded input */
             Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
@@ -105,13 +105,13 @@ verify_identifier(struct tok_state *tok)
         Py_DECREF(s);
         if (Py_UNICODE_ISPRINTABLE(ch)) {
             _PyTokenizer_syntaxerror_at(
-                tok, tok->line_start,
+                tok, _PyLexer_BufferPointer(tok, tok->line_start),
                 error_cursor - tok->line_start, tok->lineno, -1, -1,
                 "invalid character '%c' (U+%04X)", ch, ch);
         }
         else {
             _PyTokenizer_syntaxerror_at(
-                tok, tok->line_start,
+                tok, _PyLexer_BufferPointer(tok, tok->line_start),
                 error_cursor - tok->line_start, tok->lineno, -1, -1,
                 "invalid non-printable character U+%04X", ch);
         }

Parser/lexer/lexer.h

@@ -2,7 +2,7 @@
 #define _PY_LEXER_INTERNAL_H_
 
 #include "errcode.h"
-#include "lexer.h"
+#include "state.h"
 
 #define is_potential_identifier_start(c) (\
               (c >= 'a' && c <= 'z')\

Parser/lexer/lexer_internal.h

@@ -10,8 +10,6 @@
 #define MAXFTSTRINGLEVEL 150
 #define FTSTRING_STACK_INLINE_CAPACITY 1
 
-
-
 typedef enum {
     FTSTRING_MODE_MIDDLE,
     FTSTRING_MODE_EXPRESSION,
@@ -72,14 +70,6 @@ typedef struct {
     indentation_level stack[MAXINDENT];
 } lexer_layout_state;
 
-/* Supplemental source context for a terminal error. location is the reporting
-   cursor, independent of the scanner cursor; lineno == 0 means absent.
-   The text span may cover multiple physical lines. */
-typedef struct {
-    _PyTok_Loc location;
-    _PyTok_Span text_span;
-} _PyTokenizer_Diagnostic;
-
 /* Tokenizer state */
 struct tok_state {
     _PyTok_Off buf_offset;

Parser/lexer/state.h

@@ -357,7 +357,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
         if (c == EOF || (quote_size == 1 && c == '\n')) {
             int end_lineno = tok->lineno;
             _PyTok_Loc location = tok->start_loc;
-            const char *line = tok->start - location.byte_col;
+            const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col;
             Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
 
             const ftstring_state *state = _PyLexer_CurrentFTString(tok);
@@ -478,15 +478,15 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
                     tok->done = E_EOFS;
                 }
                 return string_error_token(tok, token,
-                    _PyLexer_BufferPointer(tok, current->start), location);
+                    current->start, location);
             }
             else {
                 _PyTokenizer_syntaxerror_at(
                     tok, line, cursor_offset, location.lineno, -1, -1,
                     "unterminated %c-string literal (detected at line %d)",
                     _PyLexer_StringPrefix(current->kind), end_lineno);
                 return string_error_token(tok, token,
-                    _PyLexer_BufferPointer(tok, current->start), location);
+                    current->start, location);
             }
         }
 

Parser/lexer/string.c

@@ -206,7 +206,9 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
     Py_ssize_t end_col_offset = -1;
     if (t->col_offset == -1) {
         _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
-        if (info.cursor == info.input_span.start) {
+        if (info.diagnostic.location.lineno != 0) {
+            col_offset = info.diagnostic.location.byte_col;
+        } else if (info.cursor == info.input_span.start) {
             col_offset = 0;
         } else {
             col_offset = Py_SAFE_DOWNCAST(
@@ -256,8 +258,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
     PyObject *tmp = NULL;
     p->error_indicator = 1;
     _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
-    _PyTok_Loc location = info.location;
-    _PyTok_Span text_span = info.line_span;
+    _PyTok_Loc location = info.diagnostic.location.lineno != 0
+        ? info.diagnostic.location : info.location;
+    _PyTok_Span text_span = info.diagnostic.location.lineno != 0
+        ? info.diagnostic.text_span : info.line_span;
 
     if (end_lineno == CURRENT_POS) {
         end_lineno = location.lineno;

Parser/pegen_errors.c

@@ -4,14 +4,14 @@
 
 #include "tokenizer.h"
 #include "reader.h"
-#include "reader_internal.h"
 #include "../lexer/state.h"
 
 _PyTokenizer_Info
 _PyTokenizer_GetInfo(const struct tok_state *tok)
 {
     _PyTokenizer_Info info = {
         .status = tok->done,
+        .diagnostic = tok->diagnostic,
         .location = {tok->lineno, tok->line_start < 0
             ? -1 : (int)(tok->cur - tok->line_start)},
         .cursor = tok->cur,
@@ -20,7 +20,7 @@ _PyTokenizer_GetInfo(const struct tok_state *tok)
         .level = tok->level,
         .delimiter_loc = {-1, -1},
         .in_formatted_string = tok->ftstring_depth != 0,
-        .is_interactive = tok->reader->kind == _PYTOK_READER_INTERACTIVE,
+        .is_interactive = _PyTok_ReaderIsInteractive(tok),
         .is_file = tok->fp != NULL && tok->fp != stdin,
         .filename = tok->filename,
         .module = tok->module,
@@ -81,32 +81,7 @@ const char *
 _PyTokenizer_LineView(const struct tok_state *tok, Py_ssize_t lineno,
                       Py_ssize_t *length)
 {
-    const char *line = _PyTokenizer_RetainedSource(tok);
-    if (line == NULL) {
-        line = _PyLexer_BufferPointer(tok, tok->buf_offset);
-    }
-    for (Py_ssize_t i = 1; i < lineno; i++) {
-        const char *next = strchr(line, '\n');
-        if (next == NULL) {
-            break;
-        }
-        line = next + 1;
-    }
-    const char *end = strchr(line, '\n');
-    *length = end != NULL ? end - line : (Py_ssize_t)strlen(line);
-    return line;
-}
-
-const char *
-_PyTokenizer_RetainedSource(const struct tok_state *tok)
-{
-    if (tok->reader->kind == _PYTOK_READER_PREPARED) {
-        return _PyTok_SourceData(&tok->source);
-    }
-    if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
-        return tok->source.bytes;
-    }
-    return NULL;
+    return _PyTok_SourceLineView(&tok->source, lineno, length);
 }
 
 void