Fix crash with yield in comprehension (#12048)

Fixes #10189

Produce an error when encountering a yield expression (both `yield` and `yield from` clauses) in comprehensions and generator expressions. The latter is a syntax error in python 3.8+; see #10189 and also [python issue 10544](https://bugs.python.org/issue10544).

Author: Alexandre Bouayad

mypy/semanal.py

@@ -3830,13 +3830,15 @@ def visit_star_expr(self, expr: StarExpr) -> None:
             expr.expr.accept(self)
 
     def visit_yield_from_expr(self, e: YieldFromExpr) -> None:
-        if not self.is_func_scope():  # not sure
+        if not self.is_func_scope():
             self.fail('"yield from" outside function', e, serious=True, blocker=True)
+        elif self.is_comprehension_stack[-1]:
+            self.fail('"yield from" inside comprehension or generator expression',
+                      e, serious=True, blocker=True)
+        elif self.function_stack[-1].is_coroutine:
+            self.fail('"yield from" in async function', e, serious=True, blocker=True)
         else:
-            if self.function_stack[-1].is_coroutine:
-                self.fail('"yield from" in async function', e, serious=True, blocker=True)
-            else:
-                self.function_stack[-1].is_generator = True
+            self.function_stack[-1].is_generator = True
         if e.expr:
             e.expr.accept(self)
 
@@ -4214,20 +4216,22 @@ def visit__promote_expr(self, expr: PromoteExpr) -> None:
         if analyzed is not None:
             expr.type = analyzed
 
-    def visit_yield_expr(self, expr: YieldExpr) -> None:
+    def visit_yield_expr(self, e: YieldExpr) -> None:
         if not self.is_func_scope():
-            self.fail('"yield" outside function', expr, serious=True, blocker=True)
-        else:
-            if self.function_stack[-1].is_coroutine:
-                if self.options.python_version < (3, 6):
-                    self.fail('"yield" in async function', expr, serious=True, blocker=True)
-                else:
-                    self.function_stack[-1].is_generator = True
-                    self.function_stack[-1].is_async_generator = True
+            self.fail('"yield" outside function', e, serious=True, blocker=True)
+        elif self.is_comprehension_stack[-1]:
+            self.fail('"yield" inside comprehension or generator expression',
+                      e, serious=True, blocker=True)
+        elif self.function_stack[-1].is_coroutine:
+            if self.options.python_version < (3, 6):
+                self.fail('"yield" in async function', e, serious=True, blocker=True)
             else:
                 self.function_stack[-1].is_generator = True
-        if expr.expr:
-            expr.expr.accept(self)
+                self.function_stack[-1].is_async_generator = True
+        else:
+            self.function_stack[-1].is_generator = True
+        if e.expr:
+            e.expr.accept(self)
 
     def visit_await_expr(self, expr: AwaitExpr) -> None:
         if not self.is_func_scope():

mypy/semanal_main.py

@@ -307,7 +307,7 @@ def semantic_analyze_target(target: str,
     Return tuple with these items:
     - list of deferred targets
     - was some definition incomplete (need to run another pass)
-    - were any new names were defined (or placeholders replaced)
+    - were any new names defined (or placeholders replaced)
     """
     state.manager.processed_targets.append(target)
     tree = state.tree

test-data/unit/check-semanal-error.test

@@ -77,10 +77,47 @@ continue # E: "continue" outside loop
 
 [case testYieldOutsideFunction]
 yield # E: "yield" outside function
-
-[case testYieldFromOutsideFunction]
 x = 1
 yield from x # E: "yield from" outside function
+[(yield 1) for _ in x] # E: "yield" inside comprehension or generator expression
+{(yield 1) for _ in x} # E: "yield" inside comprehension or generator expression
+{i: (yield 1) for i in x} # E: "yield" inside comprehension or generator expression
+((yield 1) for _ in x) # E: "yield" inside comprehension or generator expression
+y = 1
+[(yield from x) for _ in y] # E: "yield from" inside comprehension or generator expression
+{(yield from x) for _ in y} # E: "yield from" inside comprehension or generator expression
+{i: (yield from x) for i in y} # E: "yield from" inside comprehension or generator expression
+((yield from x) for _ in y) # E: "yield from" inside comprehension or generator expression
+def f(y):
+    [x for x in (yield y)]
+    {x for x in (yield y)}
+    {x: x for x in (yield y)}
+    (x for x in (yield y))
+    [x for x in (yield from y)]
+    {x for x in (yield from y)}
+    {x: x for x in (yield from y)}
+    (x for x in (yield from y))
+def g(y):
+    [(yield 1) for _ in y] # E: "yield" inside comprehension or generator expression
+    {(yield 1) for _ in y} # E: "yield" inside comprehension or generator expression
+    {i: (yield 1) for i in y} # E: "yield" inside comprehension or generator expression
+    ((yield 1) for _ in y) # E: "yield" inside comprehension or generator expression
+    lst = 1
+    [(yield from lst) for _ in y] # E: "yield from" inside comprehension or generator expression
+    {(yield from lst) for _ in y} # E: "yield from" inside comprehension or generator expression
+    {i: (yield from lst) for i in y} # E: "yield from" inside comprehension or generator expression
+    ((yield from lst) for _ in y) # E: "yield from" inside comprehension or generator expression
+def h(y):
+    lst = 1
+    [x for x in lst if (yield y)] # E: "yield" inside comprehension or generator expression
+    {x for x in lst if (yield y)} # E: "yield" inside comprehension or generator expression
+    {x: x for x in lst if (yield y)} # E: "yield" inside comprehension or generator expression
+    (x for x in lst if (yield y)) # E: "yield" inside comprehension or generator expression
+    lst = 1
+    [x for x in lst if (yield from y)] # E: "yield from" inside comprehension or generator expression
+    {x for x in lst if (yield from y)} # E: "yield from" inside comprehension or generator expression
+    {x: x for x in lst if (yield from y)} # E: "yield from" inside comprehension or generator expression
+    (x for x in lst if (yield from y)) # E: "yield from" inside comprehension or generator expression
 
 [case testImportFuncDup]