Skip to content

Commit 004db47

Browse files
ZalatharZalathar
committedSep 5, 2023
Add test suite coverage-map to test coverage mappings emitted by LLVM
We compile each test file to LLVM IR assembly, and then pass that IR to a dedicated program that can decode LLVM coverage maps and print them in a more human-readable format. We can then check that output against known-good snapshots. This test suite has some advantages over the existing `run-coverage` tests: - We can test coverage instrumentation without needing to run target binaries. - We can observe subtle improvements/regressions in the underlying coverage mappings that don't make a visible difference to coverage reports.
·
1.87.01.74.0
1 parent 1367104 commit 004db47

File tree

11 files changed

+306
-5
lines changed

11 files changed

+306
-5
lines changed
 

‎src/bootstrap/builder.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,7 @@ impl<'a> Builder<'a> {
726726
test::Tidy,
727727
test::Ui,
728728
test::RunPassValgrind,
729+
test::CoverageMap,
729730
test::RunCoverage,
730731
test::MirOpt,
731732
test::Codegen,

‎src/bootstrap/test.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1340,6 +1340,12 @@ host_test!(RunMakeFullDeps {
13401340

13411341
default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" });
13421342

1343+
default_test!(CoverageMap {
1344+
path: "tests/coverage-map",
1345+
mode: "coverage-map",
1346+
suite: "coverage-map"
1347+
});
1348+
13431349
host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" });
13441350
host_test!(RunCoverageRustdoc {
13451351
path: "tests/run-coverage-rustdoc",
@@ -1545,6 +1551,14 @@ note: if you're sure you want to do this, please open an issue as to why. In the
15451551
.arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
15461552
}
15471553

1554+
if mode == "coverage-map" {
1555+
let coverage_dump = builder.ensure(tool::CoverageDump {
1556+
compiler: compiler.with_stage(0),
1557+
target: compiler.host,
1558+
});
1559+
cmd.arg("--coverage-dump-path").arg(coverage_dump);
1560+
}
1561+
15481562
if mode == "run-make" || mode == "run-coverage" {
15491563
let rust_demangler = builder
15501564
.ensure(tool::RustDemangler {

‎src/tools/compiletest/src/common.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ string_enum! {
6666
JsDocTest => "js-doc-test",
6767
MirOpt => "mir-opt",
6868
Assembly => "assembly",
69+
CoverageMap => "coverage-map",
6970
RunCoverage => "run-coverage",
7071
}
7172
}
@@ -161,6 +162,9 @@ pub struct Config {
161162
/// The rust-demangler executable.
162163
pub rust_demangler_path: Option<PathBuf>,
163164

165+
/// The coverage-dump executable.
166+
pub coverage_dump_path: Option<PathBuf>,
167+
164168
/// The Python executable to use for LLDB and htmldocck.
165169
pub python: String,
166170

@@ -639,6 +643,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
639643
UI_STDERR_32,
640644
UI_STDERR_16,
641645
UI_COVERAGE,
646+
UI_COVERAGE_MAP,
642647
];
643648
pub const UI_STDERR: &str = "stderr";
644649
pub const UI_STDOUT: &str = "stdout";
@@ -649,6 +654,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr";
649654
pub const UI_STDERR_32: &str = "32bit.stderr";
650655
pub const UI_STDERR_16: &str = "16bit.stderr";
651656
pub const UI_COVERAGE: &str = "coverage";
657+
pub const UI_COVERAGE_MAP: &str = "cov-map";
652658

653659
/// Absolute path to the directory where all output for all tests in the given
654660
/// `relative_dir` group should reside. Example:

‎src/tools/compiletest/src/lib.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
4848
.reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
4949
.optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
5050
.optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
51+
.optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
5152
.reqopt("", "python", "path to python to use for doc tests", "PATH")
5253
.optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
5354
.optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
@@ -218,6 +219,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
218219
rustc_path: opt_path(matches, "rustc-path"),
219220
rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
220221
rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
222+
coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
221223
python: matches.opt_str("python").unwrap(),
222224
jsondocck_path: matches.opt_str("jsondocck-path"),
223225
jsondoclint_path: matches.opt_str("jsondoclint-path"),

‎src/tools/compiletest/src/runtest.rs

Lines changed: 66 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJs
66
use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
77
use crate::common::{CompareMode, FailMode, PassMode};
88
use crate::common::{Config, TestPaths};
9-
use crate::common::{Pretty, RunCoverage, RunPassValgrind};
10-
use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT};
9+
use crate::common::{CoverageMap, Pretty, RunCoverage, RunPassValgrind};
10+
use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
1111
use crate::compute_diff::{write_diff, write_filtered_diff};
1212
use crate::errors::{self, Error, ErrorKind};
1313
use crate::header::TestProps;
@@ -254,6 +254,7 @@ impl<'test> TestCx<'test> {
254254
MirOpt => self.run_mir_opt_test(),
255255
Assembly => self.run_assembly_test(),
256256
JsDocTest => self.run_js_doc_test(),
257+
CoverageMap => self.run_coverage_map_test(),
257258
RunCoverage => self.run_coverage_test(),
258259
}
259260
}
@@ -467,6 +468,46 @@ impl<'test> TestCx<'test> {
467468
}
468469
}
469470

471+
fn run_coverage_map_test(&self) {
472+
let Some(coverage_dump_path) = &self.config.coverage_dump_path else {
473+
self.fatal("missing --coverage-dump");
474+
};
475+
476+
let proc_res = self.compile_test_and_save_ir();
477+
if !proc_res.status.success() {
478+
self.fatal_proc_rec("compilation failed!", &proc_res);
479+
}
480+
drop(proc_res);
481+
482+
let llvm_ir_path = self.output_base_name().with_extension("ll");
483+
484+
let mut dump_command = Command::new(coverage_dump_path);
485+
dump_command.arg(llvm_ir_path);
486+
let proc_res = self.run_command_to_procres(&mut dump_command);
487+
if !proc_res.status.success() {
488+
self.fatal_proc_rec("coverage-dump failed!", &proc_res);
489+
}
490+
491+
let kind = UI_COVERAGE_MAP;
492+
493+
let expected_coverage_dump = self.load_expected_output(kind);
494+
let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]);
495+
496+
let coverage_dump_errors = self.compare_output(
497+
kind,
498+
&actual_coverage_dump,
499+
&expected_coverage_dump,
500+
self.props.compare_output_lines_by_subset,
501+
);
502+
503+
if coverage_dump_errors > 0 {
504+
self.fatal_proc_rec(
505+
&format!("{coverage_dump_errors} errors occurred comparing coverage output."),
506+
&proc_res,
507+
);
508+
}
509+
}
510+
470511
fn run_coverage_test(&self) {
471512
let should_run = self.run_if_enabled();
472513
let proc_res = self.compile_test(should_run, Emit::None);
@@ -650,6 +691,10 @@ impl<'test> TestCx<'test> {
650691
let mut cmd = Command::new(tool_path);
651692
configure_cmd_fn(&mut cmd);
652693

694+
self.run_command_to_procres(&mut cmd)
695+
}
696+
697+
fn run_command_to_procres(&self, cmd: &mut Command) -> ProcRes {
653698
let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`"));
654699

655700
let proc_res = ProcRes {
@@ -2321,9 +2366,11 @@ impl<'test> TestCx<'test> {
23212366
}
23222367
}
23232368
DebugInfo => { /* debuginfo tests must be unoptimized */ }
2324-
RunCoverage => {
2325-
// Coverage reports are affected by optimization level, and
2326-
// the current snapshots assume no optimization by default.
2369+
CoverageMap | RunCoverage => {
2370+
// Coverage mappings and coverage reports are affected by
2371+
// optimization level, so they ignore the optimize-tests
2372+
// setting and set an optimization level in their mode's
2373+
// compile flags (below) or in per-test `compile-flags`.
23272374
}
23282375
_ => {
23292376
rustc.arg("-O");
@@ -2392,8 +2439,22 @@ impl<'test> TestCx<'test> {
23922439

23932440
rustc.arg(dir_opt);
23942441
}
2442+
CoverageMap => {
2443+
rustc.arg("-Cinstrument-coverage");
2444+
// These tests only compile to MIR, so they don't need the
2445+
// profiler runtime to be present.
2446+
rustc.arg("-Zno-profiler-runtime");
2447+
// Coverage mappings are sensitive to MIR optimizations, and
2448+
// the current snapshots assume `opt-level=2` unless overridden
2449+
// by `compile-flags`.
2450+
rustc.arg("-Copt-level=2");
2451+
}
23952452
RunCoverage => {
23962453
rustc.arg("-Cinstrument-coverage");
2454+
// Coverage reports are sometimes sensitive to optimizations,
2455+
// and the current snapshots assume no optimization unless
2456+
// overridden by `compile-flags`.
2457+
rustc.arg("-Copt-level=0");
23972458
}
23982459
RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake
23992460
| CodegenUnits | JsDocTest | Assembly => {

‎tests/coverage-map/if.cov-map

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
Function name: if::main
2+
Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 03, 01, 02, 0c, 05, 02, 0d, 02, 06, 02, 02, 06, 00, 07, 07, 01, 05, 01, 02]
3+
Number of files: 1
4+
- file 0 => global file 1
5+
Number of expressions: 2
6+
- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
7+
- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
8+
Number of file 0 mappings: 4
9+
- Code(Counter(0)) at (prev + 3, 1) to (start + 2, 12)
10+
- Code(Counter(1)) at (prev + 2, 13) to (start + 2, 6)
11+
- Code(Expression(0, Sub)) at (prev + 2, 6) to (start + 0, 7)
12+
= (c0 - c1)
13+
- Code(Expression(1, Add)) at (prev + 1, 5) to (start + 1, 2)
14+
= (c1 + (c0 - c1))
15+

‎tests/coverage-map/if.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// compile-flags: --edition=2021
2+
3+
fn main() {
4+
let cond = std::env::args().len() == 1;
5+
if cond {
6+
println!("true");
7+
}
8+
println!("done");
9+
}

‎tests/coverage-map/long_and_wide.cov-map

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
Function name: long_and_wide::far_function
2+
Raw bytes (10): 0x[01, 01, 00, 01, 01, 96, 01, 01, 00, 15]
3+
Number of files: 1
4+
- file 0 => global file 1
5+
Number of expressions: 0
6+
Number of file 0 mappings: 1
7+
- Code(Counter(0)) at (prev + 150, 1) to (start + 0, 21)
8+
9+
Function name: long_and_wide::long_function
10+
Raw bytes (10): 0x[01, 01, 00, 01, 01, 10, 01, 84, 01, 02]
11+
Number of files: 1
12+
- file 0 => global file 1
13+
Number of expressions: 0
14+
Number of file 0 mappings: 1
15+
- Code(Counter(0)) at (prev + 16, 1) to (start + 132, 2)
16+
17+
Function name: long_and_wide::main
18+
Raw bytes (9): 0x[01, 01, 00, 01, 01, 07, 01, 04, 02]
19+
Number of files: 1
20+
- file 0 => global file 1
21+
Number of expressions: 0
22+
Number of file 0 mappings: 1
23+
- Code(Counter(0)) at (prev + 7, 1) to (start + 4, 2)
24+
25+
Function name: long_and_wide::wide_function
26+
Raw bytes (10): 0x[01, 01, 00, 01, 01, 0e, 01, 00, 8b, 01]
27+
Number of files: 1
28+
- file 0 => global file 1
29+
Number of expressions: 0
30+
Number of file 0 mappings: 1
31+
- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 139)
32+

‎tests/coverage-map/long_and_wide.rs

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
// compile-flags: --edition=2021
2+
// ignore-tidy-linelength
3+
4+
// This file deliberately contains line and column numbers larger than 127,
5+
// to verify that `coverage-dump`'s ULEB128 parser can handle them.
6+
7+
fn main() {
8+
wide_function();
9+
long_function();
10+
far_function();
11+
}
12+
13+
#[rustfmt::skip]
14+
fn wide_function() { /* */ (); }
15+
16+
fn long_function() {
17+
//
18+
//
19+
//
20+
//
21+
//
22+
//
23+
//
24+
//
25+
//
26+
//
27+
//
28+
//
29+
//
30+
//
31+
//
32+
//
33+
//
34+
//
35+
//
36+
//
37+
//
38+
//
39+
//
40+
//
41+
//
42+
//
43+
//
44+
//
45+
//
46+
//
47+
//
48+
//
49+
//
50+
//
51+
//
52+
//
53+
//
54+
//
55+
//
56+
//
57+
//
58+
//
59+
//
60+
//
61+
//
62+
//
63+
//
64+
//
65+
//
66+
//
67+
//
68+
//
69+
//
70+
//
71+
//
72+
//
73+
//
74+
//
75+
//
76+
//
77+
//
78+
//
79+
//
80+
//
81+
//
82+
//
83+
//
84+
//
85+
//
86+
//
87+
//
88+
//
89+
//
90+
//
91+
//
92+
//
93+
//
94+
//
95+
//
96+
//
97+
//
98+
//
99+
//
100+
//
101+
//
102+
//
103+
//
104+
//
105+
//
106+
//
107+
//
108+
//
109+
//
110+
//
111+
//
112+
//
113+
//
114+
//
115+
//
116+
//
117+
//
118+
//
119+
//
120+
//
121+
//
122+
//
123+
//
124+
//
125+
//
126+
//
127+
//
128+
//
129+
//
130+
//
131+
//
132+
//
133+
//
134+
//
135+
//
136+
//
137+
//
138+
//
139+
//
140+
//
141+
//
142+
//
143+
//
144+
//
145+
//
146+
//
147+
//
148+
}
149+
150+
fn far_function() {}

‎tests/coverage-map/trivial.cov-map

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Function name: trivial::main
2+
Raw bytes (9): 0x[01, 01, 00, 01, 01, 03, 01, 00, 0d]
3+
Number of files: 1
4+
- file 0 => global file 1
5+
Number of expressions: 0
6+
Number of file 0 mappings: 1
7+
- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 13)
8+

‎tests/coverage-map/trivial.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
// compile-flags: --edition=2021
2+
3+
fn main() {}

0 commit comments

Comments
 (0)
Please sign in to comment.