Merge pull request #1668 from Kobzol/runtime-precise-skip

Make skipping of already benchmarked runtime benchmarks more precise

Author: Kobzol

collector/src/bin/collector.rs

@@ -88,6 +88,16 @@ struct RuntimeBenchmarkConfig {
     iterations: u32,
 }
 
+impl RuntimeBenchmarkConfig {
+    fn new(suite: BenchmarkSuite, filter: BenchmarkFilter, iterations: u32) -> Self {
+        Self {
+            runtime_suite: suite.filter(&filter),
+            filter,
+            iterations,
+        }
+    }
+}
+
 struct SharedBenchmarkConfig {
     artifact_id: ArtifactId,
     toolchain: Toolchain,
@@ -671,11 +681,11 @@ fn main_result() -> anyhow::Result<i32> {
                 artifact_id,
                 toolchain,
             };
-            let config = RuntimeBenchmarkConfig {
+            let config = RuntimeBenchmarkConfig::new(
                 runtime_suite,
-                filter: BenchmarkFilter::new(local.exclude, local.include),
+                BenchmarkFilter::new(local.exclude, local.include),
                 iterations,
-            };
+            );
             run_benchmarks(&mut rt, conn, shared, None, Some(config))?;
             Ok(0)
         }
@@ -1149,11 +1159,11 @@ fn bench_published_artifact(
             is_self_profile: false,
             bench_rustc: false,
         }),
-        Some(RuntimeBenchmarkConfig {
+        Some(RuntimeBenchmarkConfig::new(
             runtime_suite,
-            filter: BenchmarkFilter::keep_all(),
-            iterations: DEFAULT_RUNTIME_ITERATIONS,
-        }),
+            BenchmarkFilter::keep_all(),
+            DEFAULT_RUNTIME_ITERATIONS,
+        )),
     )
 }
 

collector/src/runtime/benchmark.rs

@@ -38,8 +38,29 @@ pub struct BenchmarkSuite {
 }
 
 impl BenchmarkSuite {
-    pub fn total_benchmark_count(&self) -> u64 {
-        self.benchmark_names().count() as u64
+    /// Returns a new suite containing only groups that contains at least a single benchmark
+    /// that matches the filter.
+    pub fn filter(self, filter: &BenchmarkFilter) -> Self {
+        let BenchmarkSuite {
+            groups,
+            _tmp_artifacts_dir,
+        } = self;
+
+        Self {
+            groups: groups
+                .into_iter()
+                .filter(|group| {
+                    group.benchmark_names.iter().any(|benchmark| {
+                        passes_filter(
+                            benchmark,
+                            filter.exclude.as_deref(),
+                            filter.include.as_deref(),
+                        )
+                    })
+                })
+                .collect(),
+            _tmp_artifacts_dir,
+        }
     }
 
     pub fn filtered_benchmark_count(&self, filter: &BenchmarkFilter) -> u64 {

collector/src/runtime/mod.rs

@@ -30,13 +30,8 @@ pub async fn bench_runtime(
     filter: BenchmarkFilter,
     iterations: u32,
 ) -> anyhow::Result<()> {
-    let total_benchmark_count = suite.total_benchmark_count();
     let filtered = suite.filtered_benchmark_count(&filter);
-    println!(
-        "Executing {} benchmarks ({} filtered out)\n",
-        filtered,
-        total_benchmark_count - filtered
-    );
+    println!("Executing {} benchmarks\n", filtered);
 
     let rustc_perf_version = get_rustc_perf_commit();
     let mut benchmark_index = 0;