Propagate error in parser

Author: StanFromIreland

Lib/test/test_type_comments.py

@@ -1,6 +1,7 @@
 import ast
 import sys
 import unittest
+from test.support import import_helper
 
 
 funcdef = """\
@@ -391,6 +392,13 @@ def check_both_ways(source):
         check_both_ways("pass  # type: ignorewhatever\n")
         check_both_ways("pass  # type: ignoreé\n")
 
+    def test_non_utf8_type_comment_with_ignore_cookie(self):
+        _testcapi = import_helper.import_module('_testcapi')
+        flags = 0x0800 | 0x1000 # PyCF_IGNORE_COOKIE | PyCF_TYPE_COMMENTS
+        with self.assertRaises(UnicodeDecodeError):
+            _testcapi.Py_CompileStringExFlags(
+                b"a=1 # type: \x80", "<test>", 256, flags)
+
     def test_func_type_input(self):
 
         def parse_func_type_input(source):

Modules/_testcapimodule.c

@@ -226,6 +226,18 @@ pycompilestring(PyObject* self, PyObject *obj) {
     return Py_CompileString(the_string, "<string>", Py_file_input);
 }
 
+static PyObject*
+pycompilestringexflags(PyObject *self, PyObject *args) {
+    const char *the_string, *filename;
+    int start, flags;
+    if (!PyArg_ParseTuple(args, "ysii", &the_string, &filename, &start, &flags)) {
+        return NULL;
+    }
+    PyCompilerFlags cf = _PyCompilerFlags_INIT;
+    cf.cf_flags = flags;
+    return Py_CompileStringExFlags(the_string, filename, start, &cf, -1);
+}
+
 static PyObject*
 test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored))
 {
@@ -2659,6 +2671,7 @@ static PyMethodDef TestMethods[] = {
     {"return_result_with_error", return_result_with_error, METH_NOARGS},
     {"getitem_with_error", getitem_with_error, METH_VARARGS},
     {"Py_CompileString",     pycompilestring, METH_O},
+    {"Py_CompileStringExFlags", pycompilestringexflags, METH_VARARGS},
     {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
     {"stack_pointer", stack_pointer, METH_NOARGS},
 #ifdef W_STOPCODE

Parser/pegen.c

@@ -966,6 +966,12 @@ _PyPegen_run_parser(Parser *p)
        return NULL;
     }
 
+    // If parsing succeeded but error_indicator was set the error was not propagated.
+    // Avoids hitting assert(!PyErr_Occurred()) in _PyAST_Validate.
+    if (p->error_indicator && PyErr_Occurred()) {
+        return NULL;
+    }
+
     if (p->start_rule == Py_single_input && bad_single_statement(p)) {
         p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
         return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");