Try to increase robustness of the JobOpen hack (#116)

Author: melund

anypytools/abcutils.py

@@ -193,23 +193,21 @@ def execute_anybodycon(
         subprocess_flags |= CREATE_NEW_PROCESS_GROUP
         extra_kwargs = {"creationflags": subprocess_flags}
 
-        anybodycmd = [
+        cmd = [
             str(anybodycon_path.resolve()),
             "-m",
             str(macrofile_path),
             "/deb",
             str(debug_mode),
             "/ni" if not interactive_mode else "",
         ]
-
-        proc = Popen(
-            anybodycmd,
-            stdout=logfile,
-            stderr=logfile,
-            env=env,
-            cwd=folder,
+        kwargs = {
+            "stdout": logfile,
+            "stderr": logfile,
+            "env": env,
+            "cwd": folder,
             **extra_kwargs,
-        )
+        }
 
     else:
         if os.environ.get("WINE_REDIRECT_OUTPUT", 0):
@@ -222,14 +220,13 @@ def execute_anybodycon(
                 str(debug_mode),
                 "/ni",
             ]
-            proc = Popen(
-                cmd,
-                env=env,
-                cwd=folder,
-                close_fds=False,
-                stdout=logfile,
-                stderr=logfile,
-            )
+            kwargs = {
+                "env": env,
+                "cwd": folder,
+                "close_fds": False,
+                "stdout": logfile,
+                "stderr": logfile,
+            }
         else:
             # ON Linux/Wine we use a bat file to redirect the output into a file on wine/windows
             # side. This prevents a bug with AnyBody starts it's builtin python.
@@ -248,11 +245,10 @@ def execute_anybodycon(
             macrofile_cleanup.append(batfile)
 
             cmd = ["wine", "cmd", "/c", str(batfile) + r"& exit /b %ERRORLEVEL%"]
-            proc = Popen(
-                cmd,
-                env=env,
-                cwd=folder,
-            )
+
+            kwargs = {"env": env, "cwd": folder}
+
+    proc = Popen(cmd, **kwargs)
 
     retcode = None
     _subprocess_container.add(proc.pid)
@@ -269,8 +265,10 @@ def execute_anybodycon(
         retcode = _KILLED_BY_ANYPYTOOLS
         raise e
     finally:
-        if not retcode:
+        if retcode is None:
             proc.kill()
+            if ON_WINDOWS:
+                proc._close_job_object(proc._win32_job)
         else:
             _subprocess_container.remove(proc.pid)
 
@@ -949,7 +947,7 @@ def _schedule_processes(
                 task = task_queue.get()
                 yield task
 
-            time.sleep(0.05)
+            time.sleep(0.1)
 
     def cleanup_logfiles(self, tasklist):
         for task in tasklist:

anypytools/jobpopen.py

@@ -8,10 +8,12 @@
 
 import subprocess
 from subprocess import Popen
+from threading import RLock
 
 import win32api
 import win32job
 import win32process
+import pywintypes
 
 
 class JobPopen(Popen):
@@ -41,7 +43,14 @@ def __getattr__(self, key):
 
         def CreateProcess(self, *args, **kwargs):
             hp, ht, pid, tid = self._oldapi.CreateProcess(*args, **kwargs)
-            win32job.AssignProcessToJobObject(self._job, hp)
+            try:
+                win32job.AssignProcessToJobObject(self._job, hp)
+            except BaseException as e:  # to catch pywintypes.error
+                if e.args == (6, "AssignProcessToJobObject", "The handle is invalid."):
+                    # Try to ignore an error can occur randomly sometimes.
+                    pass
+                else:
+                    raise e
             win32process.ResumeThread(ht)
             return hp, ht, pid, tid
 
@@ -56,17 +65,18 @@ def __init__(self, *args, **kwargs):
         CREATE_SUSPENDED = 0x00000004
         kwargs.setdefault("creationflags", 0)
         kwargs["creationflags"] |= CREATE_SUSPENDED
-        _winapi = subprocess._winapi  # Python 3
-        _winapi_key = "_winapi"
-        try:
-            setattr(
-                subprocess,
-                _winapi_key,
-                JobPopen._winapijobhandler(_winapi, self._win32_job),
-            )
-            super(JobPopen, self).__init__(*args, **kwargs)
-        finally:
-            setattr(subprocess, _winapi_key, _winapi)
+        with RLock():
+            _winapi = subprocess._winapi  # Python 3
+            _winapi_key = "_winapi"
+            try:
+                setattr(
+                    subprocess,
+                    _winapi_key,
+                    JobPopen._winapijobhandler(_winapi, self._win32_job),
+                )
+                super(JobPopen, self).__init__(*args, **kwargs)
+            finally:
+                setattr(subprocess, _winapi_key, _winapi)
 
     def _create_job_object(self):
         """Create a new anonymous job object"""