Skip to content

Commit ef64e68

Browse files
KobzolKobzol
committed
Store compile benchmark codegen backend into the database
Since we're already modifying the pstat_series table, this commit also renames the cache column to scenario, and the statistic column to metric, to resolve a long-standing mismatch in the terminology.
1 parent 4a8ee35 commit ef64e68

File tree

10 files changed

+128
-35
lines changed

10 files changed

+128
-35
lines changed

‎collector/src/compile/execute/bencher.rs

+7
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,14 @@ impl<'a> BenchProcessor<'a> {
9292
Profile::Clippy => database::Profile::Clippy,
9393
};
9494

95+
let backend = match backend {
96+
CodegenBackend::Llvm => database::CodegenBackend::Llvm,
97+
};
98+
9599
if let Some(files) = stats.2 {
96100
if env::var_os("RUSTC_PERF_UPLOAD_TO_S3").is_some() {
101+
// FIXME: Record codegen backend in the self profile name
102+
97103
// We can afford to have the uploads run concurrently with
98104
// rustc. Generally speaking, they take up almost no CPU time
99105
// (just copying data into the network). Plus, during
@@ -131,6 +137,7 @@ impl<'a> BenchProcessor<'a> {
131137
self.benchmark.0.as_str(),
132138
profile,
133139
scenario,
140+
backend,
134141
stat,
135142
value,
136143
));

‎database/schema.md

+12-10
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,16 @@ Here is the diagram for compile-time benchmarks:
2626
│ └────────────┘ └───────────────┘ │└────────────┘ │
2727
│ │ │
2828
│ │ │
29-
│ ┌───────────────┐ ┌──────────┐ | │
29+
│ ┌───────────────┐ ┌──────────┐ │ │
3030
│ │ pstat_series │ │ pstat │ │ │
3131
│ ├───────────────┤ ├──────────┤ │ │
3232
│ │ id * │◄┐│ id * │ │ │
3333
└─┤ crate │ └┤ series │ │ │
3434
│ profile │ │ aid ├───┼───────────────┘
35-
│ cache │ │ cid ├───┘
36-
│ statistic │ │ value │
37-
└───────────────┘ └──────────┘
35+
│ scenario │ │ cid │ │
36+
│ backend │ │ value ├───┘
37+
│ metric │ └──────────┘
38+
└───────────────┘
3839
```
3940

4041
For runtime benchmarks the schema very similar, but there are different table names:
@@ -140,19 +141,20 @@ of a crate, profile, scenario and the metric being collected.
140141

141142
* crate (aka `benchmark`): the benchmarked crate which might be a crate from crates.io or a crate made specifically to stress some part of the compiler.
142143
* profile: what type of compilation is happening - check build, optimized build (a.k.a. release build), debug build, or doc build.
143-
* cache (aka `scenario`): describes how much of the incremental cache is full. An empty incremental cache means that the compiler must do a full build.
144-
* statistic (aka `metric`): the type of metric being collected
144+
* scenario: describes how much of the incremental cache is full. An empty incremental cache means that the compiler must do a full build.
145+
* backend: codegen backend used for compilation.
146+
* metric: the type of metric being collected.
145147

146148
This corresponds to a [`statistic description`](../docs/glossary.md).
147149

148-
There is a separate table for this collection to avoid duplicating crates, prfiles, scenarios etc.
150+
There is a separate table for this collection to avoid duplicating crates, profiles, scenarios etc.
149151
many times in the `pstat` table.
150152

151153
```
152154
sqlite> select * from pstat_series limit 1;
153-
id crate profile cache statistic
154-
---------- ---------- ---------- ---------- ------------
155-
1 helloworld check full task-clock:u
155+
id crate profile scenario backend metric
156+
---------- ---------- ---------- ---------- ------- ------------
157+
1 helloworld check full llvm task-clock:u
156158
```
157159

158160
### pstat

‎database/src/bin/import-sqlite.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ async fn main() {
4545
let sqlite_aid = sqlite_conn.artifact_id(&aid).await;
4646
let postgres_aid = postgres_conn.artifact_id(&aid).await;
4747

48-
for (&(benchmark, profile, scenario, metric), id) in
48+
for (&(benchmark, profile, scenario, backend, metric), id) in
4949
sqlite_idx.compile_statistic_descriptions()
5050
{
5151
if benchmarks.insert(benchmark) {
@@ -73,6 +73,7 @@ async fn main() {
7373
&benchmark,
7474
profile,
7575
scenario,
76+
backend,
7677
metric.as_str(),
7778
stat,
7879
)

‎database/src/lib.rs

+39-4
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,39 @@ impl PartialOrd for Scenario {
360360
}
361361
}
362362

363+
/// The codegen backend used for compilation.
364+
#[derive(
365+
Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
366+
)]
367+
pub enum CodegenBackend {
368+
/// The default LLVM backend
369+
Llvm,
370+
}
371+
372+
impl CodegenBackend {
373+
pub fn as_str(self) -> &'static str {
374+
match self {
375+
CodegenBackend::Llvm => "llvm",
376+
}
377+
}
378+
}
379+
380+
impl FromStr for CodegenBackend {
381+
type Err = String;
382+
fn from_str(s: &str) -> Result<Self, Self::Err> {
383+
Ok(match s.to_ascii_lowercase().as_str() {
384+
"llvm" => CodegenBackend::Llvm,
385+
_ => return Err(format!("{} is not a codegen backend", s)),
386+
})
387+
}
388+
}
389+
390+
impl fmt::Display for CodegenBackend {
391+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
392+
write!(f, "{}", self.as_str())
393+
}
394+
}
395+
363396
/// An identifier for a built version of the compiler
364397
#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
365398
pub enum ArtifactId {
@@ -427,7 +460,7 @@ pub struct Index {
427460
artifacts: Indexed<Box<str>>,
428461
/// Id lookup of compile stat description ids
429462
/// For legacy reasons called `pstat_series` in the database, and so the name is kept here.
430-
pstat_series: Indexed<(Benchmark, Profile, Scenario, Metric)>,
463+
pstat_series: Indexed<(Benchmark, Profile, Scenario, CodegenBackend, Metric)>,
431464
/// Id lookup of runtime stat description ids
432465
runtime_pstat_series: Indexed<(Benchmark, Metric)>,
433466
}
@@ -547,6 +580,7 @@ pub enum DbLabel {
547580
benchmark: Benchmark,
548581
profile: Profile,
549582
scenario: Scenario,
583+
backend: CodegenBackend,
550584
metric: Metric,
551585
},
552586
}
@@ -564,10 +598,11 @@ impl Lookup for DbLabel {
564598
benchmark,
565599
profile,
566600
scenario,
601+
backend,
567602
metric,
568603
} => index
569604
.pstat_series
570-
.get(&(*benchmark, *profile, *scenario, *metric)),
605+
.get(&(*benchmark, *profile, *scenario, *backend, *metric)),
571606
}
572607
}
573608
}
@@ -617,7 +652,7 @@ impl Index {
617652
self.pstat_series
618653
.map
619654
.keys()
620-
.map(|(_, _, _, metric)| metric)
655+
.map(|(_, _, _, _, metric)| metric)
621656
.collect::<std::collections::HashSet<_>>()
622657
.into_iter()
623658
.map(|s| s.to_string())
@@ -643,7 +678,7 @@ impl Index {
643678
&self,
644679
) -> impl Iterator<
645680
Item = (
646-
&(Benchmark, Profile, Scenario, Metric),
681+
&(Benchmark, Profile, Scenario, CodegenBackend, Metric),
647682
StatisticalDescriptionId,
648683
),
649684
> + '_ {

‎database/src/pool.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::{ArtifactCollection, ArtifactId, ArtifactIdNumber, CompileBenchmark};
1+
use crate::{ArtifactCollection, ArtifactId, ArtifactIdNumber, CodegenBackend, CompileBenchmark};
22
use crate::{CollectionId, Index, Profile, QueuedCommit, Scenario, Step};
33
use chrono::{DateTime, Utc};
44
use hashbrown::HashMap;
@@ -44,6 +44,7 @@ pub trait Connection: Send + Sync {
4444
benchmark: &str,
4545
profile: Profile,
4646
scenario: Scenario,
47+
backend: CodegenBackend,
4748
metric: &str,
4849
value: f64,
4950
);

‎database/src/pool/postgres.rs

+21-9
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction};
22
use crate::{
3-
ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, CollectionId, Commit, CommitType,
4-
CompileBenchmark, Date, Index, Profile, QueuedCommit, Scenario,
3+
ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, CodegenBackend, CollectionId,
4+
Commit, CommitType, CompileBenchmark, Date, Index, Profile, QueuedCommit, Scenario,
55
};
66
use anyhow::Context as _;
77
use chrono::{DateTime, TimeZone, Utc};
@@ -254,6 +254,15 @@ static MIGRATIONS: &[&str] = &[
254254
UNIQUE(aid, component)
255255
);
256256
"#,
257+
// Add codegen backend column and add it to the unique constraint.
258+
// Also rename cache to scenario and statistic to metric, while we're at it.
259+
r#"
260+
alter table pstat_series rename column cache to scenario;
261+
alter table pstat_series rename column statistic to metric;
262+
alter table pstat_series add backend text not null default 'llvm';
263+
alter table pstat_series drop constraint pstat_series_crate_profile_cache_statistic_key;
264+
alter table pstat_series add constraint test_case UNIQUE(crate, profile, scenario, backend, metric);
265+
"#,
257266
];
258267

259268
#[async_trait::async_trait]
@@ -441,8 +450,8 @@ impl PostgresConnection {
441450
.await
442451
.unwrap(),
443452
get_error: conn.prepare("select benchmark, error from error where aid = $1").await.unwrap(),
444-
insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, cache, statistic) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
445-
select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and cache = $3 and statistic = $4").await.unwrap(),
453+
insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, scenario, backend, metric) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
454+
select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and scenario = $3 and backend = $4 and metric = $5").await.unwrap(),
446455
collection_id: conn.prepare("insert into collection (perf_commit) VALUES ($1) returning id").await.unwrap(),
447456
record_duration: conn.prepare("
448457
insert into artifact_collection_duration (
@@ -592,7 +601,7 @@ where
592601
pstat_series: self
593602
.conn()
594603
.query(
595-
"select id, crate, profile, cache, statistic from pstat_series;",
604+
"select id, crate, profile, scenario, backend, metric from pstat_series;",
596605
&[],
597606
)
598607
.await
@@ -605,7 +614,8 @@ where
605614
Benchmark::from(row.get::<_, String>(1).as_str()),
606615
Profile::from_str(row.get::<_, String>(2).as_str()).unwrap(),
607616
row.get::<_, String>(3).as_str().parse().unwrap(),
608-
row.get::<_, String>(4).as_str().into(),
617+
CodegenBackend::from_str(row.get::<_, String>(4).as_str()).unwrap(),
618+
row.get::<_, String>(5).as_str().into(),
609619
),
610620
)
611621
})
@@ -799,16 +809,18 @@ where
799809
benchmark: &str,
800810
profile: Profile,
801811
scenario: Scenario,
812+
backend: CodegenBackend,
802813
metric: &str,
803814
stat: f64,
804815
) {
805816
let profile = profile.to_string();
806817
let scenario = scenario.to_string();
818+
let backend = backend.to_string();
807819
let sid = self
808820
.conn()
809821
.query_opt(
810822
&self.statements().select_pstat_series,
811-
&[&benchmark, &profile, &scenario, &metric],
823+
&[&benchmark, &profile, &scenario, &backend, &metric],
812824
)
813825
.await
814826
.unwrap();
@@ -818,14 +830,14 @@ where
818830
self.conn()
819831
.query_opt(
820832
&self.statements().insert_pstat_series,
821-
&[&benchmark, &profile, &scenario, &metric],
833+
&[&benchmark, &profile, &scenario, &backend, &metric],
822834
)
823835
.await
824836
.unwrap();
825837
self.conn()
826838
.query_one(
827839
&self.statements().select_pstat_series,
828-
&[&benchmark, &profile, &scenario, &metric],
840+
&[&benchmark, &profile, &scenario, &backend, &metric],
829841
)
830842
.await
831843
.unwrap()

‎database/src/pool/sqlite.rs

+29-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction};
22
use crate::{
3-
ArtifactCollection, ArtifactId, Benchmark, CollectionId, Commit, CommitType, CompileBenchmark,
4-
Date, Profile,
3+
ArtifactCollection, ArtifactId, Benchmark, CodegenBackend, CollectionId, Commit, CommitType,
4+
CompileBenchmark, Date, Profile,
55
};
66
use crate::{ArtifactIdNumber, Index, QueuedCommit};
77
use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
@@ -367,6 +367,24 @@ static MIGRATIONS: &[Migration] = &[
367367
);
368368
"#,
369369
),
370+
// Add codegen backend column and add it to the unique constraint.
371+
// Also rename cache to scenario and statistic to metric, while we're at it.
372+
Migration::without_foreign_key_constraints(
373+
r#"
374+
create table pstat_series_new(
375+
id integer primary key not null,
376+
crate text not null references benchmark(name) on delete cascade on update cascade,
377+
profile text not null,
378+
scenario text not null,
379+
backend text not null,
380+
metric text not null,
381+
UNIQUE(crate, profile, scenario, backend, metric)
382+
);
383+
insert into pstat_series_new select id, crate, profile, cache, 'llvm', statistic from pstat_series;
384+
drop table pstat_series;
385+
alter table pstat_series_new rename to pstat_series;
386+
"#,
387+
),
370388
];
371389

372390
#[async_trait::async_trait]
@@ -482,7 +500,7 @@ impl Connection for SqliteConnection {
482500
.collect();
483501
let pstat_series = self
484502
.raw()
485-
.prepare("select id, crate, profile, cache, statistic from pstat_series;")
503+
.prepare("select id, crate, profile, scenario, backend, metric from pstat_series;")
486504
.unwrap()
487505
.query_map(params![], |row| {
488506
Ok((
@@ -491,7 +509,8 @@ impl Connection for SqliteConnection {
491509
Benchmark::from(row.get::<_, String>(1)?.as_str()),
492510
Profile::from_str(row.get::<_, String>(2)?.as_str()).unwrap(),
493511
row.get::<_, String>(3)?.as_str().parse().unwrap(),
494-
row.get::<_, String>(4)?.as_str().into(),
512+
CodegenBackend::from_str(row.get::<_, String>(4)?.as_str()).unwrap(),
513+
row.get::<_, String>(5)?.as_str().into(),
495514
),
496515
))
497516
})
@@ -633,21 +652,25 @@ impl Connection for SqliteConnection {
633652
benchmark: &str,
634653
profile: Profile,
635654
scenario: crate::Scenario,
655+
backend: CodegenBackend,
636656
metric: &str,
637657
value: f64,
638658
) {
639659
let profile = profile.to_string();
640660
let scenario = scenario.to_string();
641-
self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, cache, statistic) VALUES (?, ?, ?, ?)", params![
661+
let backend = backend.to_string();
662+
self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, scenario, backend, metric) VALUES (?, ?, ?, ?, ?)", params![
642663
&benchmark,
643664
&profile,
644665
&scenario,
666+
&backend,
645667
&metric,
646668
]).unwrap();
647-
let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and cache = ? and statistic = ?", params![
669+
let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and scenario = ? and backend = ? and metric = ?", params![
648670
&benchmark,
649671
&profile,
650672
&scenario,
673+
&backend,
651674
&metric,
652675
], |r| r.get(0)).unwrap();
653676
self.raw_ref()

‎docs/glossary.md

+2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ The following is a glossary of domain specific terminology. Although benchmarks
2121
- `incr-full`: incremental compilation is used, with an empty incremental cache.
2222
- `incr-unchanged`: incremental compilation is used, with a full incremental cache and no code changes made.
2323
- `incr-patched`: incremental compilation is used, with a full incremental cache and some code changes made.
24+
* **backend**: the codegen backend used for compiling Rust code.
25+
- `llvm`: the default codegen backend
2426
* **category**: a high-level group of benchmarks. Currently, there are three categories, primary (mostly real-world crates), secondary (mostly stress tests), and stable (old real-world crates, only used for the dashboard).
2527
* **artifact type**: describes what kind of artifact does the benchmark build. Either `library` or `binary`.
2628

0 commit comments

Comments
 (0)