Merge pull request #1571 from lqd/suffix-filtering

Allow excluding benchmarks with suffix

Author: Kobzol

collector/README.md

@@ -120,6 +120,11 @@ The following options alter the behaviour of the `bench_local` subcommand.
   argument is a comma-separated list of benchmark prefixes. When this option is
   specified, a benchmark is excluded from the run if its name matches one of
   the given prefixes.
+- `--exclude-suffix <EXCLUDE>`: this is used to run a subset of the benchmarks. The
+  argument is a comma-separated list of benchmark suffixes. When this option is
+  specified, a benchmark is excluded from the run if its name matches one of
+  the given suffixes. This can be useful to quickly exclude the benchmarks
+  dedicated to artifact sizes (ending with `-tiny`).
 - `--id <ID>` the identifier that will be used to identify the results in the
   database.
 - `--include <INCLUDE>`: the inverse of `--exclude`. The argument is a

collector/src/benchmark/mod.rs

@@ -371,6 +371,7 @@ pub fn get_compile_benchmarks(
     benchmark_dir: &Path,
     include: Option<&str>,
     exclude: Option<&str>,
+    exclude_suffix: Option<&str>,
 ) -> anyhow::Result<Vec<Benchmark>> {
     let mut benchmarks = Vec::new();
 
@@ -405,25 +406,23 @@ pub fn get_compile_benchmarks(
 
     let mut includes = to_hashmap(include);
     let mut excludes = to_hashmap(exclude);
+    let mut exclude_suffixes = to_hashmap(exclude_suffix);
 
     for (path, name) in paths {
         let mut skip = false;
 
-        let name_matches = |prefixes: &mut HashMap<&str, usize>| {
-            for (prefix, n) in prefixes.iter_mut() {
-                if name.as_str().starts_with(prefix) {
-                    *n += 1;
-                    return true;
-                }
-            }
-            false
+        let name_matches_prefix = |prefixes: &mut HashMap<&str, usize>| {
+            substring_matches(prefixes, |prefix| name.starts_with(prefix))
         };
 
         if let Some(includes) = includes.as_mut() {
-            skip |= !name_matches(includes);
+            skip |= !name_matches_prefix(includes);
         }
         if let Some(excludes) = excludes.as_mut() {
-            skip |= name_matches(excludes);
+            skip |= name_matches_prefix(excludes);
+        }
+        if let Some(exclude_suffixes) = exclude_suffixes.as_mut() {
+            skip |= substring_matches(exclude_suffixes, |suffix| name.ends_with(suffix));
         }
         if skip {
             continue;
@@ -433,10 +432,10 @@ pub fn get_compile_benchmarks(
         benchmarks.push(Benchmark::new(name, path)?);
     }
 
-    // All prefixes must be used at least once. This is to catch typos.
-    let check_for_unused = |option, prefixes: Option<HashMap<&str, usize>>| {
-        if let Some(prefixes) = prefixes {
-            let unused: Vec<_> = prefixes
+    // All prefixes/suffixes must be used at least once. This is to catch typos.
+    let check_for_unused = |option, substrings: Option<HashMap<&str, usize>>| {
+        if let Some(substrings) = substrings {
+            let unused: Vec<_> = substrings
                 .into_iter()
                 .filter_map(|(i, n)| if n == 0 { Some(i) } else { None })
                 .collect();
@@ -453,6 +452,7 @@ pub fn get_compile_benchmarks(
 
     check_for_unused("include", includes)?;
     check_for_unused("exclude", excludes)?;
+    check_for_unused("exclude-suffix", exclude_suffixes)?;
 
     benchmarks.sort_by_key(|benchmark| benchmark.name.clone());
 
@@ -462,3 +462,19 @@ pub fn get_compile_benchmarks(
 
     Ok(benchmarks)
 }
+
+/// Helper to verify if a benchmark name matches a given substring, like a prefix or a suffix. The
+/// `predicate` closure will be passed each substring from `substrings` until it returns true, and
+/// in that case the substring's number of uses in the map will be increased.
+fn substring_matches(
+    substrings: &mut HashMap<&str, usize>,
+    predicate: impl Fn(&str) -> bool,
+) -> bool {
+    for (substring, n) in substrings.iter_mut() {
+        if predicate(substring) {
+            *n += 1;
+            return true;
+        }
+    }
+    false
+}

collector/src/bin/collector.rs

@@ -512,6 +512,10 @@ struct LocalOptions {
     #[arg(long)]
     exclude: Option<String>,
 
+    /// Exclude all benchmarks matching a suffix in this comma-separated list
+    #[arg(long)]
+    exclude_suffix: Option<String>,
+
     /// Include only benchmarks matching a prefix in this comma-separated list
     #[arg(long)]
     include: Option<String>,
@@ -766,6 +770,7 @@ fn main_result() -> anyhow::Result<i32> {
                 &compile_benchmark_dir,
                 local.include.as_deref(),
                 local.exclude.as_deref(),
+                local.exclude_suffix.as_deref(),
             )?;
             benchmarks.retain(|b| b.category().is_primary_or_secondary());
 
@@ -832,6 +837,7 @@ fn main_result() -> anyhow::Result<i32> {
                         &compile_benchmark_dir,
                         include.as_deref(),
                         exclude.as_deref(),
+                        None,
                     )?;
                     benchmarks.retain(|b| b.category().is_primary_or_secondary());
 
@@ -892,6 +898,7 @@ fn main_result() -> anyhow::Result<i32> {
                 &compile_benchmark_dir,
                 local.include.as_deref(),
                 local.exclude.as_deref(),
+                local.exclude_suffix.as_deref(),
             )?;
             benchmarks.retain(|b| b.category().is_primary_or_secondary());
 
@@ -1051,7 +1058,7 @@ fn bench_published_artifact(
     let cargo = which("cargo")?;
 
     // Exclude benchmarks that don't work with a stable compiler.
-    let mut benchmarks = get_compile_benchmarks(&benchmark_dir, None, None)?;
+    let mut benchmarks = get_compile_benchmarks(&benchmark_dir, None, None, None)?;
     benchmarks.retain(|b| b.category().is_stable());
 
     let res = bench(