Integrate fine-grained incremental checking with dmypy (#4447)

Fine-grained incremental mode is enabled through the `--experimental`
dmypy flag.

As a mostly unrelated change, also print daemon tracebacks to log
to make debugging crashes easier.

Author: JukkaL

mypy/dmypy_server.py

@@ -13,12 +13,14 @@
 import socket
 import sys
 import time
+import traceback
 
-from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple
 
 import mypy.build
 import mypy.errors
 import mypy.main
+import mypy.server.update
 from mypy.dmypy_util import STATUS_FILE, receive
 from mypy.gclogger import GcLogger
 
@@ -82,6 +84,12 @@ class Server:
     def __init__(self, flags: List[str]) -> None:
         """Initialize the server with the desired mypy flags."""
         self.saved_cache = {}  # type: mypy.build.SavedCache
+        if '--experimental' in flags:
+            self.fine_grained = True
+            self.fine_grained_initialized = False
+            flags.remove('--experimental')
+        else:
+            self.fine_grained = False
         sources, options = mypy.main.process_options(['-i'] + flags, False)
         if sources:
             sys.exit("dmypy: start/restart does not accept sources")
@@ -94,6 +102,10 @@ def __init__(self, flags: List[str]) -> None:
         self.options = options
         if os.path.isfile(STATUS_FILE):
             os.unlink(STATUS_FILE)
+        if self.fine_grained:
+            options.incremental = True
+            options.show_traceback = True
+            options.cache_dir = os.devnull
 
     def serve(self) -> None:
         """Serve requests, synchronously (no thread or fork)."""
@@ -128,6 +140,9 @@ def serve(self) -> None:
                 os.unlink(STATUS_FILE)
         finally:
             os.unlink(self.sockname)
+            exc_info = sys.exc_info()
+            if exc_info[0]:
+                traceback.print_exception(*exc_info)  # type: ignore
 
     def create_listening_socket(self) -> socket.socket:
         """Create the socket and set it up for listening."""
@@ -190,6 +205,14 @@ def cmd_recheck(self) -> Dict[str, object]:
 
     def check(self, sources: List[mypy.build.BuildSource],
               alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
+        if self.fine_grained:
+            return self.check_fine_grained(sources)
+        else:
+            return self.check_default(sources, alt_lib_path)
+
+    def check_default(self, sources: List[mypy.build.BuildSource],
+                      alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
+        """Check using the default (per-file) incremental mode."""
         self.last_manager = None
         with GcLogger() as gc_result:
             try:
@@ -212,6 +235,73 @@ def check(self, sources: List[mypy.build.BuildSource],
             response.update(self.last_manager.stats_summary())
         return response
 
+    def check_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+        """Check using fine-grained incremental mode."""
+        if not self.fine_grained_initialized:
+            return self.initialize_fine_grained(sources)
+        else:
+            return self.fine_grained_increment(sources)
+
+    def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+        self.file_modified = {}  # type: Dict[str, float]
+        for source in sources:
+            assert source.path
+            self.file_modified[source.path] = os.stat(source.path).st_mtime
+        try:
+            # TODO: alt_lib_path
+            result = mypy.build.build(sources=sources,
+                                      options=self.options)
+        except mypy.errors.CompileError as e:
+            output = ''.join(s + '\n' for s in e.messages)
+            if e.use_stdout:
+                out, err = output, ''
+            else:
+                out, err = '', output
+            return {'out': out, 'err': err, 'status': 2}
+        messages = result.errors
+        manager = result.manager
+        graph = result.graph
+        self.fine_grained_manager = mypy.server.update.FineGrainedBuildManager(manager, graph)
+        status = 1 if messages else 0
+        self.previous_messages = messages[:]
+        self.fine_grained_initialized = True
+        self.previous_sources = sources
+        return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
+
+    def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+        changed = self.find_changed(sources)
+        if not changed:
+            # Nothing changed -- just produce the same result as before.
+            messages = self.previous_messages
+        else:
+            messages = self.fine_grained_manager.update(changed)
+        status = 1 if messages else 0
+        self.previous_messages = messages[:]
+        self.previous_sources = sources
+        return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
+
+    def find_changed(self, sources: List[mypy.build.BuildSource]) -> List[Tuple[str, str]]:
+        changed = []
+        for source in sources:
+            path = source.path
+            assert path
+            mtime = os.stat(path).st_mtime
+            if path not in self.file_modified or self.file_modified[path] != mtime:
+                self.file_modified[path] = mtime
+                changed.append((source.module, path))
+        modules = {source.module for source in sources}
+        omitted = [source for source in self.previous_sources if source.module not in modules]
+        for source in omitted:
+            path = source.path
+            assert path
+            # Note that a file could be removed from the list of root sources but still continue
+            # to exist on the file system.
+            if not os.path.isfile(path):
+                changed.append((source.module, path))
+                if source.path in self.file_modified:
+                    del self.file_modified[source.path]
+        return changed
+
     def cmd_hang(self) -> Dict[str, object]:
         """Hang for 100 seconds, as a debug hack."""
         time.sleep(100)

mypy/server/update.py

@@ -643,7 +643,7 @@ def update_dependencies(new_modules: Mapping[str, Optional[MypyFile]],
     for id, node in new_modules.items():
         if node is None:
             continue
-        if '/typeshed/' in node.path:
+        if '/typeshed/' in node.path or node.path.startswith('typeshed/'):
             # We don't track changes to typeshed -- the assumption is that they are only changed
             # as part of mypy updates, which will invalidate everything anyway.
             #

mypy/test/testdmypy.py

@@ -28,6 +28,7 @@
         'check-enum.test',
         'check-incremental.test',
         'check-newtype.test',
+        'check-dmypy-fine-grained.test',
     ]
 else:
     dmypy_files = []  # type: List[str]
@@ -72,16 +73,8 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
         assert incremental_step >= 1
         build.find_module_clear_caches()
         original_program_text = '\n'.join(testcase.input)
-        module_data = self.parse_module(original_program_text, incremental_step)
 
-        if incremental_step == 1:
-            # In run 1, copy program text to program file.
-            for module_name, program_path, program_text in module_data:
-                if module_name == '__main__':
-                    with open(program_path, 'w') as f:
-                        f.write(program_text)
-                    break
-        elif incremental_step > 1:
+        if incremental_step > 1:
             # In runs 2+, copy *.[num] files to * files.
             for dn, dirs, files in os.walk(os.curdir):
                 for file in files:
@@ -101,10 +94,23 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
                 # Use retries to work around potential flakiness on Windows (AppVeyor).
                 retry_on_error(lambda: os.remove(path))
 
+        module_data = self.parse_module(original_program_text, incremental_step)
+
+        if incremental_step == 1:
+            # In run 1, copy program text to program file.
+            for module_name, program_path, program_text in module_data:
+                if module_name == '__main__':
+                    with open(program_path, 'w') as f:
+                        f.write(program_text)
+                    break
+
         # Parse options after moving files (in case mypy.ini is being moved).
         options = self.parse_options(original_program_text, testcase, incremental_step)
         if incremental_step == 1:
-            self.server = dmypy_server.Server([])  # TODO: Fix ugly API
+            server_options = []  # type: List[str]
+            if 'fine-grained' in testcase.file:
+                server_options.append('--experimental')
+            self.server = dmypy_server.Server(server_options)  # TODO: Fix ugly API
             self.server.options = options
 
         assert self.server is not None  # Set in step 1 and survives into next steps

test-data/unit/check-dmypy-fine-grained.test

@@ -0,0 +1,112 @@
+[case testCleanFineGrainedIncremental]
+# cmd: mypy -m a b
+[file b.py]
+import a
+x = a.f()
+
+[file a.py]
+def f() -> int:
+    return 1
+
+[file a.py.2]
+def f() -> str:
+    return ''
+[out1]
+[out2]
+
+[case testErrorFineGrainedIncremental]
+# cmd: mypy -m a b
+[file b.py]
+import a
+x = a.f()
+x = 1
+
+[file a.py]
+def f() -> int:
+    return 1
+
+[file a.py.2]
+def f() -> str:
+    return ''
+[out1]
+[out2]
+tmp/b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str")
+
+[case testAddFileFineGrainedIncremental]
+# cmd: mypy -m a
+# cmd2: mypy -m a b
+[file a.py]
+import b
+b.f(1)
+[file b.py.2]
+def f(x: str) -> None: pass
+[out1]
+tmp/a.py:1: error: Cannot find module named 'b'
+tmp/a.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+[out2]
+tmp/a.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+
+[case testDeleteFileFineGrainedIncremental]
+# cmd: mypy -m a b
+# cmd2: mypy -m a
+[file a.py]
+import b
+b.f(1)
+[file b.py]
+def f(x: str) -> None: pass
+[delete b.py.2]
+[out1]
+tmp/a.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
+[out2]
+tmp/a.py:1: error: Cannot find module named 'b'
+tmp/a.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+
+[case testInitialErrorFineGrainedIncremental]
+# cmd: mypy -m a b
+[file b.py]
+import a
+x = a.f()
+x = ''
+
+[file a.py]
+def f() -> int:
+    return 1
+
+[file a.py.2]
+def f() -> str:
+    return ''
+[out1]
+tmp/b.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int")
+[out2]
+
+[case testInitialBlockerFineGrainedIncremental]
+# cmd: mypy -m a b
+[file a.py]
+1 1
+[file b.py]
+def f() -> int:
+    return ''
+[file a.py.2]
+x = 1
+[file b.py.3]
+def f() -> int:
+    return 0
+[out1]
+tmp/a.py:1: error: invalid syntax
+[out2]
+tmp/b.py:2: error: Incompatible return value type (got "str", expected "int")
+[out3]
+
+[case testNoOpUpdateFineGrainedIncremental]
+# cmd: mypy -m a
+[file a.py]
+1()
+[file b.py.2]
+# Note: this file is not part of the build
+[file a.py.3]
+x = 1
+[out1]
+tmp/a.py:1: error: "int" not callable
+[out2]
+tmp/a.py:1: error: "int" not callable
+[out3]