[nextest-runner] make leak timeout configurable

Keep the same setting but make it configurable, both as a default
setting and via overrides.

Author: sunshowers

nextest-runner/default-config.toml

@@ -67,6 +67,13 @@ fail-fast = true
 # Example: slow-timeout = { period = "60s", terminate-after = 2 }
 slow-timeout = { period = "60s" }
 
+# Treat a test as leaky if after the process is shut down, standard output and standard error
+# aren't closed by this time.
+#
+# This usually happens in case of a test that creates a child process and lets it inherit those
+# handles, but doesn't clean the child process up (especially when it fails).
+leak-timeout = "100ms"
+
 [profile.default.junit]
 # Output a JUnit report into the given file inside 'store.dir/<profile-name>'.
 # If unspecified, JUnit is not written out.

nextest-runner/src/config.rs

@@ -204,6 +204,14 @@ impl<'cfg> NextestProfile<'cfg> {
             .unwrap_or(self.default_profile.slow_timeout)
     }
 
+    /// Returns the time after which a child process that hasn't closed its handles is marked as
+    /// leaky.
+    pub fn leak_timeout(&self) -> Duration {
+        self.custom_profile
+            .and_then(|profile| profile.leak_timeout)
+            .unwrap_or(self.default_profile.leak_timeout)
+    }
+
     /// Returns the test status level.
     pub fn status_level(&self) -> StatusLevel {
         self.custom_profile
@@ -243,6 +251,7 @@ impl<'cfg> NextestProfile<'cfg> {
     pub fn overrides_for(&self, query: &TestQuery<'_>) -> ProfileOverrides {
         let mut retries = None;
         let mut slow_timeout = None;
+        let mut leak_timeout = None;
 
         for &override_ in &self.overrides {
             if !override_.expr.matches_test(query) {
@@ -254,11 +263,15 @@ impl<'cfg> NextestProfile<'cfg> {
             if slow_timeout.is_none() && override_.data.slow_timeout.is_some() {
                 slow_timeout = override_.data.slow_timeout;
             }
+            if leak_timeout.is_none() && override_.data.leak_timeout.is_some() {
+                leak_timeout = override_.data.leak_timeout;
+            }
         }
 
         ProfileOverrides {
             retries,
             slow_timeout,
+            leak_timeout,
         }
     }
 
@@ -288,6 +301,7 @@ impl<'cfg> NextestProfile<'cfg> {
 pub struct ProfileOverrides {
     retries: Option<usize>,
     slow_timeout: Option<SlowTimeout>,
+    leak_timeout: Option<Duration>,
 }
 
 impl ProfileOverrides {
@@ -300,6 +314,11 @@ impl ProfileOverrides {
     pub fn slow_timeout(&self) -> Option<SlowTimeout> {
         self.slow_timeout
     }
+
+    /// Returns the leak timeout for this test.
+    pub fn leak_timeout(&self) -> Option<Duration> {
+        self.leak_timeout
+    }
 }
 
 /// JUnit configuration for nextest, returned by a [`NextestProfile`].
@@ -376,6 +395,8 @@ struct DefaultProfileImpl {
     fail_fast: bool,
     #[serde(deserialize_with = "require_deserialize_slow_timeout")]
     slow_timeout: SlowTimeout,
+    #[serde(with = "humantime_serde")]
+    leak_timeout: Duration,
     #[serde(default)]
     overrides: Vec<ProfileOverrideSource>,
     junit: DefaultJunitImpl,
@@ -547,6 +568,8 @@ struct CustomProfileImpl {
     fail_fast: Option<bool>,
     #[serde(default, deserialize_with = "deserialize_slow_timeout")]
     slow_timeout: Option<SlowTimeout>,
+    #[serde(default, with = "humantime_serde::option")]
+    leak_timeout: Option<Duration>,
     #[serde(default)]
     overrides: Vec<ProfileOverrideSource>,
     #[serde(default)]
@@ -571,6 +594,8 @@ struct ProfileOverrideData {
     retries: Option<usize>,
     #[serde(default, deserialize_with = "deserialize_slow_timeout")]
     slow_timeout: Option<SlowTimeout>,
+    #[serde(default)]
+    leak_timeout: Option<Duration>,
 }
 
 #[derive(Clone, Debug, Default)]

nextest-runner/src/runner.rs

@@ -93,6 +93,7 @@ impl TestRunnerBuilder {
         };
         let fail_fast = self.fail_fast.unwrap_or_else(|| profile.fail_fast());
         let slow_timeout = profile.slow_timeout();
+        let leak_timeout = profile.leak_timeout();
 
         let runtime = Runtime::new().map_err(TestRunnerBuildError::TokioRuntimeCreate)?;
         let _guard = runtime.enter();
@@ -110,6 +111,7 @@ impl TestRunnerBuilder {
                 ignore_retry_overrides,
                 fail_fast,
                 slow_timeout,
+                leak_timeout,
                 test_list,
                 target_runner,
                 runtime,
@@ -165,6 +167,7 @@ struct TestRunnerInner<'a> {
     ignore_retry_overrides: bool,
     fail_fast: bool,
     slow_timeout: crate::config::SlowTimeout,
+    leak_timeout: Duration,
     test_list: &'a TestList<'a>,
     target_runner: TargetRunner,
     runtime: Runtime,
@@ -426,6 +429,7 @@ impl<'a> TestRunnerInner<'a> {
 
         let mut status: Option<ExecutionResult> = None;
         let slow_timeout = overrides.slow_timeout().unwrap_or(self.slow_timeout);
+        let leak_timeout = overrides.leak_timeout().unwrap_or(self.leak_timeout);
         let mut is_slow = false;
 
         let mut interval = tokio::time::interval(slow_timeout.period);
@@ -558,7 +562,7 @@ impl<'a> TestRunnerInner<'a> {
             // Previously, this used to hang if spawned grandchildren inherited stdout/stderr but
             // didn't shut down properly. Now, this detects those cases and marks them as leaked.
             let leaked = loop {
-                let sleep = tokio::time::sleep(Duration::from_millis(100));
+                let sleep = tokio::time::sleep(leak_timeout);
 
                 tokio::select! {
                     res = &mut stdout_fut, if !stdout_done => {