Detect and record implicit dependencies in incremental mode (#2041)

For full details see #2041

Author: Michael0x2a

mypy/build.py

@@ -29,6 +29,7 @@
                         SymbolTableNode, MODULE_REF)
 from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass
 from mypy.checker import TypeChecker
+from mypy.indirection import TypeIndirectionVisitor
 from mypy.errors import Errors, CompileError, DecodeError, report_internal_error
 from mypy import fixup
 from mypy.report import Reports
@@ -306,6 +307,7 @@ def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]:
 PRI_HIGH = 5  # top-level "from X import blah"
 PRI_MED = 10  # top-level "import X"
 PRI_LOW = 20  # either form inside a function
+PRI_INDIRECT = 30  # an indirect dependency
 PRI_ALL = 99  # include all priorities
 
 
@@ -352,6 +354,7 @@ def __init__(self, data_dir: str,
         self.modules = self.semantic_analyzer.modules
         self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors)
         self.type_checker = TypeChecker(self.errors, self.modules, options=options)
+        self.indirection_detector = TypeIndirectionVisitor()
         self.missing_modules = set()  # type: Set[str]
         self.stale_modules = set()  # type: Set[str]
         self.rechecked_modules = set()  # type: Set[str]
@@ -1422,11 +1425,40 @@ def type_check(self) -> None:
             return
         with self.wrap_context():
             manager.type_checker.visit_file(self.tree, self.xpath)
+
+            if manager.options.incremental:
+                self._patch_indirect_dependencies(manager.type_checker.module_refs)
+
             if manager.options.dump_inference_stats:
                 dump_type_stats(self.tree, self.xpath, inferred=True,
                                 typemap=manager.type_checker.type_map)
             manager.report_file(self.tree)
 
+    def _patch_indirect_dependencies(self, module_refs: Set[str]) -> None:
+        types = self.manager.type_checker.module_type_map.values()
+        valid = self.valid_references()
+
+        encountered = self.manager.indirection_detector.find_modules(types) | module_refs
+        extra = encountered - valid
+
+        for dep in sorted(extra):
+            if dep not in self.manager.modules:
+                continue
+            if dep not in self.suppressed and dep not in self.manager.missing_modules:
+                self.dependencies.append(dep)
+                self.priorities[dep] = PRI_INDIRECT
+            elif dep not in self.suppressed and dep in self.manager.missing_modules:
+                self.suppressed.append(dep)
+
+    def valid_references(self) -> Set[str]:
+        valid_refs = set(self.dependencies + self.suppressed + self.ancestors)
+        valid_refs .add(self.id)
+
+        if "os" in valid_refs:
+            valid_refs.add("os.path")
+
+        return valid_refs
+
     def write_cache(self) -> None:
         if self.path and self.manager.options.incremental and not self.manager.errors.is_errors():
             dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies]
@@ -1605,25 +1637,6 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
         else:
             process_stale_scc(graph, scc)
 
-        # TODO: This is a workaround to get around the "chaining imports" problem
-        # with the interface checks.
-        #
-        # That is, if we have a file named `module_a.py` which does:
-        #
-        #     import module_b
-        #     module_b.module_c.foo(3)
-        #
-        # ...and if the type signature of `module_c.foo(...)` were to change,
-        # module_a_ would not be rechecked since the interface of `module_b`
-        # would not be considered changed.
-        #
-        # As a workaround, this check will force a module's interface to be
-        # considered stale if anything it imports has a stale interface,
-        # which ensures these changes are caught and propagated.
-        if len(stale_deps) > 0:
-            for id in scc:
-                graph[id].mark_interface_stale()
-
 
 def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]:
     """Come up with the ideal processing order within an SCC.

mypy/checker.py

@@ -87,6 +87,8 @@ class TypeChecker(NodeVisitor[Type]):
     msg = None  # type: MessageBuilder
     # Types of type checked nodes
     type_map = None  # type: Dict[Node, Type]
+    # Types of type checked nodes within this specific module
+    module_type_map = None  # type: Dict[Node, Type]
 
     # Helper for managing conditional types
     binder = None  # type: ConditionalTypeBinder
@@ -121,6 +123,10 @@ class TypeChecker(NodeVisitor[Type]):
     suppress_none_errors = False
     options = None  # type: Options
 
+    # The set of all dependencies (suppressed or not) that this module accesses, either
+    # directly or indirectly.
+    module_refs = None  # type: Set[str]
+
     def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None:
         """Construct a type checker.
 
@@ -131,6 +137,7 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Option
         self.options = options
         self.msg = MessageBuilder(errors, modules)
         self.type_map = {}
+        self.module_type_map = {}
         self.binder = ConditionalTypeBinder()
         self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg)
         self.return_types = []
@@ -142,6 +149,7 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Option
         self.deferred_nodes = []
         self.pass_num = 0
         self.current_node_deferred = False
+        self.module_refs = set()
 
     def visit_file(self, file_node: MypyFile, path: str) -> None:
         """Type check a mypy file with the given path."""
@@ -152,6 +160,8 @@ def visit_file(self, file_node: MypyFile, path: str) -> None:
         self.weak_opts = file_node.weak_opts
         self.enter_partial_types()
         self.is_typeshed_stub = self.errors.is_typeshed_file(path)
+        self.module_type_map = {}
+        self.module_refs = set()
         if self.options.strict_optional_whitelist is None:
             self.suppress_none_errors = False
         else:
@@ -2211,6 +2221,8 @@ def check_type_equivalency(self, t1: Type, t2: Type, node: Context,
     def store_type(self, node: Node, typ: Type) -> None:
         """Store the type of a node in the type map."""
         self.type_map[node] = typ
+        if typ is not None:
+            self.module_type_map[node] = typ
 
     def typing_mode_none(self) -> bool:
         if self.is_dynamic_function() and not self.options.check_untyped_defs:

mypy/checkexpr.py

@@ -1,6 +1,6 @@
 """Expression type checker. This file is conceptually part of TypeChecker."""
 
-from typing import cast, Dict, List, Tuple, Callable, Union, Optional
+from typing import cast, Dict, Set, List, Iterable, Tuple, Callable, Union, Optional
 
 from mypy.types import (
     Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef,
@@ -16,7 +16,7 @@
     ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator,
     ConditionalExpr, ComparisonExpr, TempNode, SetComprehension,
     DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr,
-    TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2
+    TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2, MODULE_REF,
 )
 from mypy.nodes import function_type
 from mypy import nodes
@@ -36,6 +36,7 @@
 from mypy.constraints import get_actual_type
 from mypy.checkstrformat import StringFormatterChecker
 from mypy.expandtype import expand_type
+from mypy.util import split_module_names
 
 from mypy import experiments
 
@@ -45,6 +46,35 @@
                       None]
 
 
+def extract_refexpr_names(expr: RefExpr) -> Set[str]:
+    """Recursively extracts all module references from a reference expression.
+
+    Note that currently, the only two subclasses of RefExpr are NameExpr and
+    MemberExpr."""
+    output = set()  # type: Set[str]
+    while expr.kind == MODULE_REF or expr.fullname is not None:
+        if expr.kind == MODULE_REF:
+            output.add(expr.fullname)
+
+        if isinstance(expr, NameExpr):
+            is_suppressed_import = isinstance(expr.node, Var) and expr.node.is_suppressed_import
+            if isinstance(expr.node, TypeInfo):
+                # Reference to a class or a nested class
+                output.update(split_module_names(expr.node.module_name))
+            elif expr.fullname is not None and '.' in expr.fullname and not is_suppressed_import:
+                # Everything else (that is not a silenced import within a class)
+                output.add(expr.fullname.rsplit('.', 1)[0])
+            break
+        elif isinstance(expr, MemberExpr):
+            if isinstance(expr.expr, RefExpr):
+                expr = expr.expr
+            else:
+                break
+        else:
+            raise AssertionError("Unknown RefExpr subclass: {}".format(type(expr)))
+    return output
+
+
 class Finished(Exception):
     """Raised if we can terminate overload argument check early (no match)."""
 
@@ -75,6 +105,7 @@ def visit_name_expr(self, e: NameExpr) -> Type:
 
         It can be of any kind: local, member or global.
         """
+        self.chk.module_refs.update(extract_refexpr_names(e))
         result = self.analyze_ref_expr(e)
         return self.chk.narrow_type_from_binder(e, result)
 
@@ -861,6 +892,7 @@ def apply_generic_arguments2(self, overload: Overloaded, types: List[Type],
 
     def visit_member_expr(self, e: MemberExpr) -> Type:
         """Visit member expression (of form e.id)."""
+        self.chk.module_refs.update(extract_refexpr_names(e))
         result = self.analyze_ordinary_member_access(e, False)
         return self.chk.narrow_type_from_binder(e, result)
 

mypy/indirection.py

@@ -0,0 +1,103 @@
+from typing import Dict, Iterable, List, Optional, Set
+from abc import abstractmethod
+
+from mypy.visitor import NodeVisitor
+from mypy.types import TypeVisitor
+from mypy.nodes import MODULE_REF
+import mypy.nodes as nodes
+import mypy.types as types
+from mypy.util import split_module_names
+
+
+def extract_module_names(type_name: Optional[str]) -> List[str]:
+    """Returns the module names of a fully qualified type name."""
+    if type_name is not None:
+        # Discard the first one, which is just the qualified name of the type
+        possible_module_names = split_module_names(type_name)
+        return possible_module_names[1:]
+    else:
+        return []
+
+
+class TypeIndirectionVisitor(TypeVisitor[Set[str]]):
+    """Returns all module references within a particular type."""
+
+    def __init__(self) -> None:
+        self.cache = {}  # type: Dict[types.Type, Set[str]]
+
+    def find_modules(self, typs: Iterable[types.Type]) -> Set[str]:
+        return self._visit(*typs)
+
+    def _visit(self, *typs: types.Type) -> Set[str]:
+        output = set()  # type: Set[str]
+        for typ in typs:
+            if typ in self.cache:
+                modules = self.cache[typ]
+            else:
+                modules = typ.accept(self)
+                self.cache[typ] = set(modules)
+            output.update(modules)
+        return output
+
+    def visit_unbound_type(self, t: types.UnboundType) -> Set[str]:
+        return self._visit(*t.args)
+
+    def visit_type_list(self, t: types.TypeList) -> Set[str]:
+        return self._visit(*t.items)
+
+    def visit_error_type(self, t: types.ErrorType) -> Set[str]:
+        return set()
+
+    def visit_any(self, t: types.AnyType) -> Set[str]:
+        return set()
+
+    def visit_void(self, t: types.Void) -> Set[str]:
+        return set()
+
+    def visit_none_type(self, t: types.NoneTyp) -> Set[str]:
+        return set()
+
+    def visit_uninhabited_type(self, t: types.UninhabitedType) -> Set[str]:
+        return set()
+
+    def visit_erased_type(self, t: types.ErasedType) -> Set[str]:
+        return set()
+
+    def visit_deleted_type(self, t: types.DeletedType) -> Set[str]:
+        return set()
+
+    def visit_type_var(self, t: types.TypeVarType) -> Set[str]:
+        return self._visit(*t.values) | self._visit(t.upper_bound)
+
+    def visit_instance(self, t: types.Instance) -> Set[str]:
+        out = self._visit(*t.args)
+        if t.type is not None:
+            out.update(split_module_names(t.type.module_name))
+        return out
+
+    def visit_callable_type(self, t: types.CallableType) -> Set[str]:
+        out = self._visit(*t.arg_types) | self._visit(t.ret_type)
+        if t.definition is not None:
+            out.update(extract_module_names(t.definition.fullname()))
+        return out
+
+    def visit_overloaded(self, t: types.Overloaded) -> Set[str]:
+        return self._visit(*t.items()) | self._visit(t.fallback)
+
+    def visit_tuple_type(self, t: types.TupleType) -> Set[str]:
+        return self._visit(*t.items) | self._visit(t.fallback)
+
+    def visit_star_type(self, t: types.StarType) -> Set[str]:
+        return set()
+
+    def visit_union_type(self, t: types.UnionType) -> Set[str]:
+        return self._visit(*t.items)
+
+    def visit_partial_type(self, t: types.PartialType) -> Set[str]:
+        return set()
+
+    def visit_ellipsis_type(self, t: types.EllipsisType) -> Set[str]:
+        return set()
+
+    def visit_type_type(self, t: types.TypeType) -> Set[str]:
+        return self._visit(t.item)