Skip to content

Commit 33778c9

Browse files
author
Jonathan Woollett-Light
committed
Update code coverage
Updated code coverage to use grcov. Signed-off-by: Jonathan Woollett-Light <[email protected]>
1 parent 41a386b commit 33778c9

File tree

2 files changed

+75
-99
lines changed

2 files changed

+75
-99
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,5 @@ __pycache__
99
.vscode
1010
test_results/*
1111
*.core
12+
*.profraw
13+
grcov

‎tests/integration_tests/build/test_coverage.py

+73-99
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,10 @@
88
target should be put in `s3://spec.firecracker` and automatically updated.
99
"""
1010

11-
1211
import os
13-
import platform
14-
import re
15-
import shutil
1612
import pytest
1713

1814
from framework import utils
19-
import host_tools.cargo_build as host # pylint: disable=import-error
2015
from host_tools import proc
2116

2217
# We have different coverages based on the host kernel version. This is
@@ -29,114 +24,93 @@
2924
# Checkout the cpuid crate. In the future other
3025
# differences may appear.
3126
if utils.is_io_uring_supported():
32-
COVERAGE_DICT = {"Intel": 82.99, "AMD": 82.31, "ARM": 82.41}
27+
COVERAGE_DICT = {"Intel": 90.84, "AMD": 90.11, "ARM": 82.41}
3328
else:
34-
COVERAGE_DICT = {"Intel": 80.15, "AMD": 79.48, "ARM": 79.59}
29+
COVERAGE_DICT = {"Intel": 88.39, "AMD": 87.67, "ARM": 88.39}
3530

3631
PROC_MODEL = proc.proc_type()
3732

38-
COVERAGE_MAX_DELTA = 0.05
39-
40-
CARGO_KCOV_REL_PATH = os.path.join(host.CARGO_BUILD_REL_PATH, "kcov")
41-
42-
KCOV_COVERAGE_FILE = "index.js"
43-
"""kcov will aggregate coverage data in this file."""
33+
# Toolchain target architecture.
34+
if ("Intel" in PROC_MODEL) or ("AMD" in PROC_MODEL):
35+
ARCH = "x86_64"
36+
elif "ARM" in PROC_MODEL:
37+
ARCH = "aarch64"
38+
else:
39+
raise Exception(f"Unsupported processor model ({PROC_MODEL})")
4440

45-
KCOV_COVERED_LINES_REGEX = r'"covered_lines":"(\d+)"'
46-
"""Regex for extracting number of total covered lines found by kcov."""
41+
# Toolchain target.
42+
TARGET = f"{ARCH}-unknown-linux-musl"
4743

48-
KCOV_TOTAL_LINES_REGEX = r'"total_lines" : "(\d+)"'
49-
"""Regex for extracting number of total executable lines found by kcov."""
44+
# We allow coverage to have a max difference of `COVERAGE_MAX_DELTA` as percentage before failing
45+
# the test.
46+
COVERAGE_MAX_DELTA = 0.05
5047

51-
SECCOMPILER_BUILD_DIR = "../build/seccompiler"
48+
# grcov 0.8.* requires GLIBC >2.27, this is not present in ubuntu 18.04, when we update the docker
49+
# container with a newer version of ubuntu we can also update this.
50+
GRCOV_VERSION = "0.7.1"
5251

5352

5453
@pytest.mark.timeout(400)
55-
def test_coverage(test_fc_session_root_path, test_session_tmp_path):
56-
"""Test line coverage for rust tests is within bounds.
57-
58-
The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov
59-
after a coverage run.
54+
def test_coverage():
55+
"""Test code coverage
6056
6157
@type: build
6258
"""
63-
proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
64-
assert len(proc_model) == 1, "Could not get processor model!"
65-
coverage_target_pct = COVERAGE_DICT[proc_model[0]]
66-
exclude_pattern = (
67-
"${CARGO_HOME:-$HOME/.cargo/},"
68-
"build/,"
69-
"tests/,"
70-
"usr/lib/gcc,"
71-
"lib/x86_64-linux-gnu/,"
72-
"test_utils.rs,"
73-
# The following files/directories are auto-generated
74-
"bootparam.rs,"
75-
"elf.rs,"
76-
"mpspec.rs,"
77-
"msr_index.rs,"
78-
"bindings.rs,"
79-
"_gen"
59+
# Get coverage target.
60+
processor_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
61+
assert len(processor_model) == 1, "Could not get processor model!"
62+
coverage_target = COVERAGE_DICT[processor_model[0]]
63+
64+
# Re-direct to repository root.
65+
os.chdir("..")
66+
67+
# Add `llvm-tools-preview` component
68+
utils.run_cmd("rustup component add llvm-tools-preview")
69+
70+
# Generate test profiles.
71+
utils.run_cmd(
72+
f'\
73+
export RUSTFLAGS="-Cinstrument-coverage" \
74+
&& export LLVM_PROFILE_FILE="your_name-%p-%m.profraw" \
75+
&& cargo test --all --target={TARGET} -- --test-threads=1 \
76+
'
8077
)
81-
exclude_region = "'mod tests {'"
82-
target = "{}-unknown-linux-musl".format(platform.machine())
83-
84-
cmd = (
85-
'CARGO_WRAPPER="kcov" RUSTFLAGS="{}" CARGO_TARGET_DIR={} '
86-
"cargo kcov --all "
87-
"--target {} --output {} -- "
88-
"--exclude-pattern={} "
89-
"--exclude-region={} --verify"
90-
).format(
91-
host.get_rustflags(),
92-
os.path.join(test_fc_session_root_path, CARGO_KCOV_REL_PATH),
93-
target,
94-
test_session_tmp_path,
95-
exclude_pattern,
96-
exclude_region,
97-
)
98-
# We remove the seccompiler custom build directory, created by the
99-
# vmm-level `build.rs`.
100-
# If we don't delete it before and after running the kcov command, we will
101-
# run into linker errors.
102-
shutil.rmtree(SECCOMPILER_BUILD_DIR, ignore_errors=True)
103-
# By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify`
104-
# to kcov. To pass others arguments, we need to include the defaults.
105-
utils.run_cmd(cmd)
106-
107-
shutil.rmtree(SECCOMPILER_BUILD_DIR)
108-
109-
coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE)
110-
with open(coverage_file, encoding="utf-8") as cov_output:
111-
contents = cov_output.read()
112-
covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0])
113-
total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0])
114-
coverage = covered_lines / total_lines * 100
115-
print("Number of executable lines: {}".format(total_lines))
116-
print("Number of covered lines: {}".format(covered_lines))
117-
print("Thus, coverage is: {:.2f}%".format(coverage))
118-
119-
coverage_low_msg = (
120-
"Current code coverage ({:.2f}%) is >{:.2f}% below the target ({}%).".format(
121-
coverage, COVERAGE_MAX_DELTA, coverage_target_pct
122-
)
123-
)
124-
125-
assert coverage >= coverage_target_pct - COVERAGE_MAX_DELTA, coverage_low_msg
12678

127-
# Get the name of the variable that needs updating.
128-
namespace = globals()
129-
cov_target_name = [name for name in namespace if namespace[name] is COVERAGE_DICT][
130-
0
131-
]
132-
133-
coverage_high_msg = (
134-
"Current code coverage ({:.2f}%) is >{:.2f}% above the target ({}%).\n"
135-
"Please update the value of {}.".format(
136-
coverage, COVERAGE_MAX_DELTA, coverage_target_pct, cov_target_name
137-
)
79+
# Generate coverage report.
80+
utils.run_cmd(
81+
f'\
82+
cargo install --version {GRCOV_VERSION} grcov \
83+
&& grcov . \
84+
-s . \
85+
--binary-path ./build/cargo_target/{TARGET}/debug/ \
86+
--ignore "build/*" \
87+
-t html \
88+
--branch \
89+
--ignore-not-existing \
90+
-o ./build/cargo_target/{TARGET}/debug/coverage \
91+
'
13892
)
13993

140-
assert coverage <= coverage_target_pct + COVERAGE_MAX_DELTA, coverage_high_msg
141-
142-
return (f"{coverage}%", f"{coverage_target_pct}% +/- {COVERAGE_MAX_DELTA * 100}%")
94+
# Extract coverage from html report.
95+
# When we update grcov to 0.8.* we can update this to pull the coverage from a generated .json
96+
# file.
97+
index = open(
98+
f"./build/cargo_target/{TARGET}/debug/coverage/index.html", encoding="utf-8"
99+
)
100+
index_contents = index.read()
101+
end = index_contents.find(" %</abbr></p>")
102+
start = index_contents[:end].rfind(">")
103+
coverage_str = index_contents[start + 1 : end]
104+
coverage = float(coverage_str)
105+
106+
# Compare coverage.
107+
high = coverage_target * (1.0 + COVERAGE_MAX_DELTA)
108+
low = coverage_target * (1.0 - COVERAGE_MAX_DELTA)
109+
assert (
110+
low <= coverage
111+
), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% below \
112+
the target ({coverage_target:.2f}%)"
113+
assert (
114+
coverage <= high
115+
), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% above \
116+
the target ({coverage_target:.2f}%)"

0 commit comments

Comments
 (0)