Skip to content

Commit 9aebece

Browse files
authored
fix: add TASKGRAPH_SERIAL and set multiprocessing context for multiprocess kind generation (#773)
This was supposed to be done in #765, but clearly I didn't push it before merging.
1 parent f849d10 commit 9aebece

File tree

1 file changed

+5
-2
lines changed

1 file changed

+5
-2
lines changed

src/taskgraph/generator.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import copy
66
import logging
7+
import multiprocessing
78
import os
89
import platform
910
from concurrent.futures import (
@@ -302,7 +303,9 @@ def _load_tasks_parallel(self, kinds, kind_graph, parameters):
302303
futures = set()
303304
edges = set(kind_graph.edges)
304305

305-
with ProcessPoolExecutor() as executor:
306+
with ProcessPoolExecutor(
307+
mp_context=multiprocessing.get_context("fork")
308+
) as executor:
306309

307310
def submit_ready_kinds():
308311
"""Create the next batch of tasks for kinds without dependencies."""
@@ -420,7 +423,7 @@ def _run(self):
420423
# redone in the new processes. Ideally this would be fixed, or we
421424
# would take another approach to parallel kind generation. In the
422425
# meantime, it's not supported outside of Linux.
423-
if platform.system() != "Linux":
426+
if platform.system() != "Linux" or os.environ.get("TASKGRAPH_SERIAL"):
424427
all_tasks = self._load_tasks_serial(kinds, kind_graph, parameters)
425428
else:
426429
all_tasks = self._load_tasks_parallel(kinds, kind_graph, parameters)

0 commit comments

Comments
 (0)