Adjust toxfile.py and --maxprocesses for coverage

Primarily, add a new plugin configuration option which customizes
the test environment with a sitecustomize.py to launch coverage
automatically. Additionally, customize `set_env` via this invocation
path. In aggregate, this is a tox-plugin version of the type of
behavior used by `pytest-cov`.

Because the `coverage.process_startup()` call adds some overhead,
highly parallel runs of the tests could start to fail under
pytest-timeout. `tox p` parallelization on top of `pytest-xdist`
parallelization became too aggressive, resulting in slow starts when
timing was bad. Scaling maxprocesses back down to 4 (no appreciable
difference in speed between 4 and 8) results in fast, successful runs.

Author: sirosen

pyproject.toml

@@ -83,7 +83,7 @@ attr = "globus_cli.version.__version__"
 profile = "black"
 
 [tool.pytest.ini_options]
-addopts = "-n auto --maxprocesses 8 --timeout 3 --color=yes"
+addopts = "-n auto --maxprocesses 4 --timeout 3 --color=yes"
 filterwarnings = ["error"]
 
 [tool.coverage.run]

tox.ini

@@ -25,6 +25,8 @@ recreate =
 dependency_groups =
     !mindeps: test
     mindeps: test-mindeps
+# invoke custom plugin code to setup coverage under pytest-xdist
+globus_cli_coverage_sitecustomize = true
 
 # the 'localsdk' factor allows CLI tests to be run against a local repo copy of globus-sdk
 # it requires that the GLOBUS_SDK_PATH env var is set

toxfile.py

@@ -25,6 +25,13 @@
 
 log = logging.getLogger(__name__)
 
+_INJECT_SITECUSTOMIZE = '''
+"""A sitecustomize.py injected by globus_cli_coverage_sitecustomize"""
+import coverage
+
+coverage.process_startup()
+'''
+
 
 @impl
 def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None:
@@ -34,6 +41,12 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None:
         default=[],
         desc="A dir tree to remove before running the environment commands",
     )
+    env_conf.add_config(
+        keys=["globus_cli_coverage_sitecustomize"],
+        of_type=bool,
+        default=[],
+        desc="Inject a sitecustomize.py to enable `coverage` under pytest-xdist",
+    )
 
 
 @impl
@@ -44,3 +57,14 @@ def tox_before_run_commands(tox_env: ToxEnv) -> None:
         if path.exists():
             log.warning(f"globus_cli_rmtree: {path}")
             shutil.rmtree(path)
+
+    if tox_env.conf.load("globus_cli_coverage_sitecustomize"):
+        site_packages_path = pathlib.Path(tox_env.conf.load("env_site_packages_dir"))
+        inject_sitecustomize_path = site_packages_path / "sitecustomize.py"
+        inject_sitecustomize_path.write_text(_INJECT_SITECUSTOMIZE)
+
+        # It is important that the tox configuration also sets
+        # COVERAGE_PROCESS_START to the coverage configuration file.
+        # If this is not done, `coverage.process_startup` will be a no-op.
+        setenv = tox_env.conf.load("set_env")
+        setenv.update({"COVERAGE_PROCESS_START": "pyproject.toml"})