Skip to content

Commit 4a8ee35

Browse files
KobzolKobzol
committed
Propagate codegen backend through benchmark processors
1 parent 777f7d0 commit 4a8ee35

File tree

4 files changed

+109
-65
lines changed

4 files changed

+109
-65
lines changed

‎collector/src/bin/collector.rs

+10
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ struct CompileBenchmarkConfig {
8484
benchmarks: Vec<Benchmark>,
8585
profiles: Vec<Profile>,
8686
scenarios: Vec<Scenario>,
87+
backends: Vec<CodegenBackend>,
8788
iterations: Option<usize>,
8889
is_self_profile: bool,
8990
bench_rustc: bool,
@@ -185,6 +186,7 @@ fn profile_compile(
185186
benchmarks: &[Benchmark],
186187
profiles: &[Profile],
187188
scenarios: &[Scenario],
189+
backends: &[CodegenBackend],
188190
errors: &mut BenchmarkErrors,
189191
) {
190192
eprintln!("Profiling {} with {:?}", toolchain.id, profiler);
@@ -203,6 +205,7 @@ fn profile_compile(
203205
&mut processor,
204206
profiles,
205207
scenarios,
208+
backends,
206209
toolchain,
207210
Some(1),
208211
));
@@ -752,6 +755,7 @@ fn main_result() -> anyhow::Result<i32> {
752755
log_db(&db);
753756
let profiles = opts.profiles.0;
754757
let scenarios = opts.scenarios.0;
758+
let backends = opts.codegen_backends.0;
755759

756760
let pool = database::Pool::open(&db.db);
757761

@@ -787,6 +791,7 @@ fn main_result() -> anyhow::Result<i32> {
787791
benchmarks,
788792
profiles,
789793
scenarios,
794+
backends,
790795
iterations: Some(iterations),
791796
is_self_profile: self_profile.self_profile,
792797
bench_rustc: bench_rustc.bench_rustc,
@@ -865,6 +870,7 @@ fn main_result() -> anyhow::Result<i32> {
865870
benchmarks,
866871
profiles: vec![Profile::Check, Profile::Debug, Profile::Doc, Profile::Opt],
867872
scenarios: Scenario::all(),
873+
backends: vec![CodegenBackend::Llvm],
868874
iterations: runs.map(|v| v as usize),
869875
is_self_profile: self_profile.self_profile,
870876
bench_rustc: bench_rustc.bench_rustc,
@@ -932,6 +938,7 @@ fn main_result() -> anyhow::Result<i32> {
932938

933939
let profiles = &opts.profiles.0;
934940
let scenarios = &opts.scenarios.0;
941+
let backends = &opts.codegen_backends.0;
935942

936943
let mut benchmarks = get_compile_benchmarks(
937944
&compile_benchmark_dir,
@@ -970,6 +977,7 @@ fn main_result() -> anyhow::Result<i32> {
970977
&benchmarks,
971978
profiles,
972979
scenarios,
980+
backends,
973981
&mut errors,
974982
);
975983
Ok(id)
@@ -1266,6 +1274,7 @@ fn bench_published_artifact(
12661274
benchmarks: compile_benchmarks,
12671275
profiles,
12681276
scenarios,
1277+
backends: vec![CodegenBackend::Llvm],
12691278
iterations: Some(3),
12701279
is_self_profile: false,
12711280
bench_rustc: false,
@@ -1373,6 +1382,7 @@ fn bench_compile(
13731382
processor,
13741383
&config.profiles,
13751384
&config.scenarios,
1385+
&config.backends,
13761386
&shared.toolchain,
13771387
config.iterations,
13781388
)))

‎collector/src/compile/benchmark/mod.rs

+81-60
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use crate::compile::benchmark::category::Category;
2+
use crate::compile::benchmark::codegen_backend::CodegenBackend;
23
use crate::compile::benchmark::patch::Patch;
34
use crate::compile::benchmark::profile::Profile;
45
use crate::compile::benchmark::scenario::Scenario;
@@ -178,6 +179,7 @@ impl Benchmark {
178179
toolchain: &'a Toolchain,
179180
cwd: &'a Path,
180181
profile: Profile,
182+
backend: CodegenBackend,
181183
) -> CargoProcess<'a> {
182184
let mut cargo_args = self
183185
.config
@@ -199,6 +201,7 @@ impl Benchmark {
199201
processor_name: self.name.clone(),
200202
cwd,
201203
profile,
204+
backend,
202205
incremental: false,
203206
processor_etc: None,
204207
manifest_path: self
@@ -226,6 +229,7 @@ impl Benchmark {
226229
processor: &mut dyn Processor,
227230
profiles: &[Profile],
228231
scenarios: &[Scenario],
232+
backends: &[CodegenBackend],
229233
toolchain: &Toolchain,
230234
iterations: Option<usize>,
231235
) -> anyhow::Result<()> {
@@ -293,11 +297,17 @@ impl Benchmark {
293297
for (profile, prep_dir) in &profile_dirs {
294298
let server = server.clone();
295299
let thread = s.spawn::<_, anyhow::Result<()>>(move || {
296-
wait_for_future(
297-
self.mk_cargo_process(toolchain, prep_dir.path(), *profile)
298-
.jobserver(server)
299-
.run_rustc(false),
300-
)?;
300+
wait_for_future(async move {
301+
// Prepare all backend artifacts into the same target directory
302+
for backend in backends {
303+
let server = server.clone();
304+
self.mk_cargo_process(toolchain, prep_dir.path(), *profile, *backend)
305+
.jobserver(server)
306+
.run_rustc(false)
307+
.await?;
308+
}
309+
Ok::<(), anyhow::Error>(())
310+
})?;
301311
Ok(())
302312
});
303313
threads.push(thread);
@@ -320,77 +330,88 @@ impl Benchmark {
320330
preparation_start.elapsed().as_secs()
321331
);
322332

323-
for (profile, prep_dir) in profile_dirs {
324-
eprintln!("Running {}: {:?} + {:?}", self.name, profile, scenarios);
325-
326-
// We want at least two runs for all benchmarks (since we run
327-
// self-profile separately).
328-
processor.start_first_collection();
329-
for i in 0..std::cmp::max(iterations, 2) {
330-
if i == 1 {
331-
let different = processor.finished_first_collection();
332-
if iterations == 1 && !different {
333-
// Don't run twice if this processor doesn't need it and
334-
// we've only been asked to run once.
335-
break;
336-
}
337-
}
338-
log::debug!("Benchmark iteration {}/{}", i + 1, iterations);
339-
// Don't delete the directory on error.
340-
let timing_dir = ManuallyDrop::new(self.make_temp_dir(prep_dir.path())?);
341-
let cwd = timing_dir.path();
342-
343-
// A full non-incremental build.
344-
if scenarios.contains(&Scenario::Full) {
345-
self.mk_cargo_process(toolchain, cwd, profile)
346-
.processor(processor, Scenario::Full, "Full", None)
347-
.run_rustc(true)
348-
.await?;
349-
}
333+
for &backend in backends {
334+
for (profile, prep_dir) in &profile_dirs {
335+
let profile = *profile;
336+
eprintln!(
337+
"Running {}: {:?} + {:?} + {:?}",
338+
self.name, profile, scenarios, backend
339+
);
350340

351-
// Rustdoc does not support incremental compilation
352-
if profile != Profile::Doc {
353-
// An incremental from scratch (slowest incremental case).
354-
// This is required for any subsequent incremental builds.
355-
if scenarios.iter().any(|s| s.is_incr()) {
356-
self.mk_cargo_process(toolchain, cwd, profile)
357-
.incremental(true)
358-
.processor(processor, Scenario::IncrFull, "IncrFull", None)
359-
.run_rustc(true)
360-
.await?;
341+
// We want at least two runs for all benchmarks (since we run
342+
// self-profile separately).
343+
processor.start_first_collection();
344+
for i in 0..std::cmp::max(iterations, 2) {
345+
if i == 1 {
346+
let different = processor.finished_first_collection();
347+
if iterations == 1 && !different {
348+
// Don't run twice if this processor doesn't need it and
349+
// we've only been asked to run once.
350+
break;
351+
}
361352
}
362-
363-
// An incremental build with no changes (fastest incremental case).
364-
if scenarios.contains(&Scenario::IncrUnchanged) {
365-
self.mk_cargo_process(toolchain, cwd, profile)
366-
.incremental(true)
367-
.processor(processor, Scenario::IncrUnchanged, "IncrUnchanged", None)
353+
log::debug!("Benchmark iteration {}/{}", i + 1, iterations);
354+
// Don't delete the directory on error.
355+
let timing_dir = ManuallyDrop::new(self.make_temp_dir(prep_dir.path())?);
356+
let cwd = timing_dir.path();
357+
358+
// A full non-incremental build.
359+
if scenarios.contains(&Scenario::Full) {
360+
self.mk_cargo_process(toolchain, cwd, profile, backend)
361+
.processor(processor, Scenario::Full, "Full", None)
368362
.run_rustc(true)
369363
.await?;
370364
}
371365

372-
if scenarios.contains(&Scenario::IncrPatched) {
373-
for (i, patch) in self.patches.iter().enumerate() {
374-
log::debug!("applying patch {}", patch.name);
375-
patch.apply(cwd).map_err(|s| anyhow::anyhow!("{}", s))?;
366+
// Rustdoc does not support incremental compilation
367+
if profile != Profile::Doc {
368+
// An incremental from scratch (slowest incremental case).
369+
// This is required for any subsequent incremental builds.
370+
if scenarios.iter().any(|s| s.is_incr()) {
371+
self.mk_cargo_process(toolchain, cwd, profile, backend)
372+
.incremental(true)
373+
.processor(processor, Scenario::IncrFull, "IncrFull", None)
374+
.run_rustc(true)
375+
.await?;
376+
}
376377

377-
// An incremental build with some changes (realistic
378-
// incremental case).
379-
let scenario_str = format!("IncrPatched{}", i);
380-
self.mk_cargo_process(toolchain, cwd, profile)
378+
// An incremental build with no changes (fastest incremental case).
379+
if scenarios.contains(&Scenario::IncrUnchanged) {
380+
self.mk_cargo_process(toolchain, cwd, profile, backend)
381381
.incremental(true)
382382
.processor(
383383
processor,
384-
Scenario::IncrPatched,
385-
&scenario_str,
386-
Some(patch),
384+
Scenario::IncrUnchanged,
385+
"IncrUnchanged",
386+
None,
387387
)
388388
.run_rustc(true)
389389
.await?;
390390
}
391+
392+
if scenarios.contains(&Scenario::IncrPatched) {
393+
for (i, patch) in self.patches.iter().enumerate() {
394+
log::debug!("applying patch {}", patch.name);
395+
patch.apply(cwd).map_err(|s| anyhow::anyhow!("{}", s))?;
396+
397+
// An incremental build with some changes (realistic
398+
// incremental case).
399+
let scenario_str = format!("IncrPatched{}", i);
400+
self.mk_cargo_process(toolchain, cwd, profile, backend)
401+
.incremental(true)
402+
.processor(
403+
processor,
404+
Scenario::IncrPatched,
405+
&scenario_str,
406+
Some(patch),
407+
)
408+
.run_rustc(true)
409+
.await?;
410+
}
411+
}
391412
}
413+
drop(ManuallyDrop::into_inner(timing_dir));
392414
}
393-
drop(ManuallyDrop::into_inner(timing_dir));
394415
}
395416
}
396417

‎collector/src/compile/execute/bencher.rs

+11-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::compile::benchmark::codegen_backend::CodegenBackend;
12
use crate::compile::benchmark::profile::Profile;
23
use crate::compile::benchmark::scenario::Scenario;
34
use crate::compile::benchmark::BenchmarkName;
@@ -77,6 +78,7 @@ impl<'a> BenchProcessor<'a> {
7778
&mut self,
7879
scenario: database::Scenario,
7980
profile: Profile,
81+
backend: CodegenBackend,
8082
stats: (Stats, Option<SelfProfile>, Option<SelfProfileFiles>),
8183
) {
8284
let version = get_rustc_perf_commit();
@@ -187,24 +189,30 @@ impl<'a> Processor for BenchProcessor<'a> {
187189
}
188190

189191
let fut = match data.scenario {
190-
Scenario::Full => {
191-
self.insert_stats(database::Scenario::Empty, data.profile, res)
192-
}
192+
Scenario::Full => self.insert_stats(
193+
database::Scenario::Empty,
194+
data.profile,
195+
data.backend,
196+
res,
197+
),
193198
Scenario::IncrFull => self.insert_stats(
194199
database::Scenario::IncrementalEmpty,
195200
data.profile,
201+
data.backend,
196202
res,
197203
),
198204
Scenario::IncrUnchanged => self.insert_stats(
199205
database::Scenario::IncrementalFresh,
200206
data.profile,
207+
data.backend,
201208
res,
202209
),
203210
Scenario::IncrPatched => {
204211
let patch = data.patch.unwrap();
205212
self.insert_stats(
206213
database::Scenario::IncrementalPatch(patch.name),
207214
data.profile,
215+
data.backend,
208216
res,
209217
)
210218
}

‎collector/src/compile/execute/mod.rs

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
//! Execute benchmarks.
22
3+
use crate::compile::benchmark::codegen_backend::CodegenBackend;
34
use crate::compile::benchmark::patch::Patch;
45
use crate::compile::benchmark::profile::Profile;
56
use crate::compile::benchmark::scenario::Scenario;
@@ -119,6 +120,7 @@ pub struct CargoProcess<'a> {
119120
pub toolchain: &'a Toolchain,
120121
pub cwd: &'a Path,
121122
pub profile: Profile,
123+
pub backend: CodegenBackend,
122124
pub incremental: bool,
123125
pub processor_etc: Option<(&'a mut dyn Processor, Scenario, &'a str, Option<&'a Patch>)>,
124126
pub processor_name: BenchmarkName,
@@ -196,11 +198,12 @@ impl<'a> CargoProcess<'a> {
196198
// really.
197199
pub async fn run_rustc(&mut self, needs_final: bool) -> anyhow::Result<()> {
198200
log::info!(
199-
"run_rustc with incremental={}, profile={:?}, scenario={:?}, patch={:?}",
201+
"run_rustc with incremental={}, profile={:?}, scenario={:?}, patch={:?}, backend={:?}",
200202
self.incremental,
201203
self.profile,
202204
self.processor_etc.as_ref().map(|v| v.1),
203-
self.processor_etc.as_ref().and_then(|v| v.3)
205+
self.processor_etc.as_ref().and_then(|v| v.3),
206+
self.backend
204207
);
205208

206209
loop {
@@ -332,6 +335,7 @@ impl<'a> CargoProcess<'a> {
332335
scenario,
333336
scenario_str,
334337
patch,
338+
backend: self.backend,
335339
};
336340
match processor.process_output(&data, output).await {
337341
Ok(Retry::No) => return Ok(()),
@@ -397,6 +401,7 @@ pub struct ProcessOutputData<'a> {
397401
scenario: Scenario,
398402
scenario_str: &'a str,
399403
patch: Option<&'a Patch>,
404+
backend: CodegenBackend,
400405
}
401406

402407
/// Trait used by `Benchmark::measure()` to provide different kinds of

0 commit comments

Comments
 (0)