gh-153569: report tokenizer diagnostics without rewinding the scanner

Author: pablogsal

Lib/test/test_codeop.py

@@ -113,6 +113,17 @@ def test_valid(self, compiler):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
+    @subTests('symbol', ('single', 'exec'))
+    @subTests('prefix', ('', 'f', 't'))
+    def test_incomplete_string_diagnostics(self, symbol, prefix):
+        opening = f'    á = {prefix}"""first\n'
+        source = 'if True:\n' + opening + 'second'
+        with self.assertRaises(_IncompleteInputError) as cm:
+            Compile()(source, '<input>', symbol)
+        text = opening + 'second' + ('\n' if symbol == 'exec' else '')
+        self.assertEqual(cm.exception.args, (
+            'incomplete input', ('<input>', 2, 9, text, 2, -1)))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Lib/test/test_source_encoding.py

@@ -3,8 +3,8 @@
 import unittest
 from test import support
 from test.support import script_helper
-from test.support.os_helper import TESTFN, unlink, rmtree
-from test.support.import_helper import unload
+from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
+from test.support.import_helper import import_module, unload
 import importlib
 import os
 import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
                 self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
 
     def test_invalid_utf8_offset_after_non_ascii(self):
+        for name in ('é', 'éé', '𝒜'):
+            with self.subTest(name=name):
+                source = ('x = ' + name).encode() + b'\xff\n'
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, '<test>', 'exec')
+                error = caught.exception
+                self.assertEqual(
+                    (error.lineno, error.offset, error.end_lineno, error.end_offset),
+                    (1, 5 + len(name), 1, 5 + len(name)),
+                )
+
+    @support.cpython_only
+    def test_invalid_utf8_file_offset_after_non_ascii(self):
+        _testcapi = import_module('_testcapi')
+        self.addCleanup(unlink, TESTFN_ASCII)
+        with open(TESTFN_ASCII, 'wb') as f:
+            f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
         with self.assertRaises(SyntaxError) as caught:
-            compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
+            _testcapi.run_file(
+                os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
         error = caught.exception
         self.assertEqual(
             (error.lineno, error.offset, error.end_lineno, error.end_offset),
-            (1, 6, 1, 6),
+            (2, 7, 2, 7),
         )
 
     def test_long_bom_conflict_message_is_not_truncated(self):

Lib/test/test_tstring.py

@@ -198,6 +198,8 @@ def test_nested_templates(self):
 
     def test_syntax_errors(self):
         for case, err in (
+            ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"),
+            ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"),
             ("t'", "unterminated t-string literal"),
             ("t'''", "unterminated triple-quoted t-string literal"),
             ("t''''", "unterminated triple-quoted t-string literal"),

Parser/lexer/lexer.c

@@ -101,6 +101,7 @@ verify_identifier(struct tok_state *tok)
     assert(PyUnicode_GET_LENGTH(s) > 0);
     if (invalid < PyUnicode_GET_LENGTH(s)) {
         Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
+        const char *error_cursor = tok->cur;
         if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
             /* Determine the offset in UTF-8 encoded input */
             Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
@@ -111,14 +112,20 @@ verify_identifier(struct tok_state *tok)
                 tok->done = E_ERROR;
                 return 0;
             }
-            tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
+            error_cursor = tok->start + PyBytes_GET_SIZE(s);
         }
         Py_DECREF(s);
         if (Py_UNICODE_ISPRINTABLE(ch)) {
-            _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
+            _PyTokenizer_syntaxerror_at(
+                tok, tok->line_start,
+                error_cursor - tok->line_start, tok->lineno, -1, -1,
+                "invalid character '%c' (U+%04X)", ch, ch);
         }
         else {
-            _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
+            _PyTokenizer_syntaxerror_at(
+                tok, tok->line_start,
+                error_cursor - tok->line_start, tok->lineno, -1, -1,
+                "invalid non-printable character U+%04X", ch);
         }
         return 0;
     }

Parser/lexer/state.h

@@ -71,6 +71,14 @@ typedef struct {
     indentation_level stack[MAXINDENT];
 } lexer_layout_state;
 
+/* Supplemental source context for a terminal error. location is the reporting
+   cursor, independent of the scanner cursor; lineno == 0 means absent.
+   The text span may cover multiple physical lines. */
+typedef struct {
+    _PyTok_Loc location;
+    _PyTok_Span text_span;
+} _PyTokenizer_Diagnostic;
+
 /* Tokenizer state */
 struct tok_state {
     /* Input state; buf <= cur <= inp */
@@ -89,6 +97,7 @@ struct tok_state {
                            expression (cf. issue 16806) */
     int starting_col_offset; /* The column offset at the beginning of a token */
     int col_offset;     /* Current col offset */
+    _PyTokenizer_Diagnostic diagnostic;
     int level;          /* () [] {} Parentheses nesting level */
             /* Used to allow free continuations inside them */
     char parenstack[MAXLEVEL];
@@ -118,6 +127,63 @@ struct tok_state {
 #endif
 };
 
+static inline _PyTok_Off
+_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
+{
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(position >= tok->buf && position <= tok->inp);
+    Py_ssize_t offset = position - tok->buf;
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset);
+    return tok->buf_offset + offset;
+}
+
+static inline char *
+_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(offset >= tok->buf_offset);
+    assert(offset - tok->buf_offset <= tok->inp - tok->buf);
+    return tok->buf + (offset - tok->buf_offset);
+}
+
+static inline const char *
+_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
+                        Py_ssize_t *length)
+{
+    assert(length != NULL);
+    assert(_PyTok_SpanIsValid(span));
+    *length = span.end - span.start;
+    (void)_PyLexer_BufferPointer(tok, span.end);
+    return _PyLexer_BufferPointer(tok, span.start);
+}
+
+static inline int
+_PyLexer_ByteColumn(const struct tok_state *tok)
+{
+    assert(tok->line_start != NULL);
+    assert(tok->cur >= tok->line_start);
+    Py_ssize_t column = tok->cur - tok->line_start;
+    assert(column <= INT_MAX);
+    return (int)column;
+}
+
+static inline _PyTok_Span
+_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
+                    const char *end)
+{
+    if (start == NULL) {
+        assert(end == NULL);
+        return (_PyTok_Span){-1, -1};
+    }
+    assert(end != NULL);
+    assert(start <= end);
+    return _PyTok_SpanFromBounds(
+        _PyLexer_BufferOffset(tok, start),
+        _PyLexer_BufferOffset(tok, end));
+}
+
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
 void _PyLexer_ImplyDedents(struct tok_state *);

Parser/lexer/string.c

@@ -7,6 +7,20 @@
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 
+static int
+string_error_token(struct tok_state *tok, struct token *token,
+                   const char *start, _PyTok_Loc location)
+{
+    tok->diagnostic = (_PyTokenizer_Diagnostic){
+        .location = {location.lineno, location.byte_col + 1},
+        .text_span = _PyLexer_BufferSpan(tok, start - location.byte_col, tok->inp),
+    };
+    int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, NULL, NULL);
+    token->start_loc = location;
+    token->end_loc = (_PyTok_Loc){location.lineno, -1};
+    return type;
+}
+
 int
 _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     assert(token != NULL);
@@ -347,55 +361,52 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
             break;
         }
         if (c == EOF || (quote_size == 1 && c == '\n')) {
-            assert(tok->multi_line_start != NULL);
-            // shift the tok_state's location into
-            // the start of string, and report the error
-            // from the initial quote character
-            tok->cur = (char *)tok->start;
-            tok->cur++;
-            tok->line_start = tok->multi_line_start;
-            int start = tok->lineno;
-            tok->lineno = tok->first_lineno;
-
-            if (INSIDE_FSTRING(tok)) {
-                /* When we are in an f-string, before raising the
-                 * unterminated string literal error, check whether
-                 * does the initial quote matches with f-strings quotes
-                 * and if it is, then this must be a missing '}' token
-                 * so raise the proper error */
-                tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
-                if (the_current_tok->quote == quote &&
-                    the_current_tok->quote_size == quote_size) {
-                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                        "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok)));
+            int end_lineno = tok->lineno;
+            _PyTok_Loc location = {tok->first_lineno, tok->starting_col_offset};
+            const char *line = tok->start - location.byte_col;
+            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
+
+            const tokenizer_mode *state = INSIDE_FSTRING(tok) ? TOK_GET_MODE(tok) : NULL;
+            if (state != NULL) {
+                /* A matching quote belongs to the surrounding formatted
+                 * string, so the expression is missing its closing brace. */
+                if (state->quote == quote && state->quote_size == quote_size) {
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
+                        "%c-string: expecting '}'",
+                        TOK_GET_STRING_PREFIX(tok));
+                    return string_error_token(tok, token, tok->start, location);
                 }
             }
 
             if (quote_size == 3) {
-                _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
-                                 " (detected at line %d)", start);
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated triple-quoted string literal"
+                    " (detected at line %d)", end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token, tok->start, location);
             }
             else {
                 if (has_escaped_quote) {
-                    _PyTokenizer_syntaxerror(
-                        tok,
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
                         "unterminated string literal (detected at line %d); "
                         "perhaps you escaped the end quote?",
-                        start
+                        end_lineno
                     );
                 } else {
-                    _PyTokenizer_syntaxerror(
-                        tok, "unterminated string literal (detected at line %d)", start
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
+                        "unterminated string literal (detected at line %d)", end_lineno
                     );
                 }
                 if (c != '\n') {
                     tok->done = E_EOLS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token, tok->start, location);
             }
         }
         if (c == quote) {
@@ -516,32 +527,31 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
                 return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
             }
 
-            assert(tok->multi_line_start != NULL);
-            // shift the tok_state's location into
-            // the start of string, and report the error
-            // from the initial quote character
-            tok->cur = (char *)current_tok->start;
-            tok->cur++;
-            tok->line_start = current_tok->multi_line_start;
-            int start = tok->lineno;
-
-            tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
-            tok->lineno = the_current_tok->first_line;
+            int end_lineno = tok->lineno;
+            _PyTok_Loc location = {current_tok->first_line,
+                (int)(current_tok->start - current_tok->multi_line_start)};
+            const char *line = current_tok->multi_line_start;
+            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
 
             if (current_tok->quote_size == 3) {
-                _PyTokenizer_syntaxerror(tok,
-                                    "unterminated triple-quoted %c-string literal"
-                                    " (detected at line %d)",
-                                    TOK_GET_STRING_PREFIX(tok), start);
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated triple-quoted %c-string literal"
+                    " (detected at line %d)",
+                    TOK_GET_STRING_PREFIX(tok), end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token,
+                    current_tok->start, location);
             }
             else {
-                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                                    "unterminated %c-string literal (detected at"
-                                    " line %d)", TOK_GET_STRING_PREFIX(tok), start));
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated %c-string literal (detected at line %d)",
+                    TOK_GET_STRING_PREFIX(tok), end_lineno);
+                return string_error_token(tok, token,
+                    current_tok->start, location);
             }
         }
 

Parser/pegen.c

@@ -218,11 +218,11 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
 
     parser_token->level = new_token->level;
     parser_token->lineno = new_token->start_loc.lineno;
-    parser_token->col_offset = p->tok->lineno == p->starting_lineno
+    parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
         ? p->starting_col_offset + new_token->start_loc.byte_col
         : new_token->start_loc.byte_col;
     parser_token->end_lineno = new_token->end_loc.lineno;
-    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+    parser_token->end_col_offset = new_token->end_loc.lineno == p->starting_lineno
         ? p->starting_col_offset + new_token->end_loc.byte_col
         : new_token->end_loc.byte_col;