Skip to content

Commit 26f5412

Browse files
Jonathan Woollett-LightJonathanWoollett-Light
Jonathan Woollett-Light
authored and
JonathanWoollett-Light
committed
Update code coverage
Updated code coverage to use grcov. Signed-off-by: Jonathan Woollett-Light <[email protected]>
1 parent 62de50c commit 26f5412

File tree

3 files changed

+82
-106
lines changed

3 files changed

+82
-106
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ __pycache__
99
.vscode
1010
test_results/*
1111
*.core
12+
*.profraw

‎tests/integration_tests/build/test_coverage.py

+76-99
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,10 @@
88
target should be put in `s3://spec.firecracker` and automatically updated.
99
"""
1010

11-
1211
import os
13-
import platform
14-
import re
15-
import shutil
1612
import pytest
1713

1814
from framework import utils
19-
import host_tools.cargo_build as host # pylint: disable=import-error
2015
from host_tools import proc
2116

2217
# We have different coverages based on the host kernel version. This is
@@ -29,114 +24,96 @@
2924
# Checkout the cpuid crate. In the future other
3025
# differences may appear.
3126
if utils.is_io_uring_supported():
32-
COVERAGE_DICT = {"Intel": 82.99, "AMD": 82.31, "ARM": 82.41}
27+
COVERAGE_DICT = {"Intel": 90.84, "AMD": 90.11, "ARM": 90.51}
3328
else:
34-
COVERAGE_DICT = {"Intel": 80.15, "AMD": 79.48, "ARM": 79.59}
29+
COVERAGE_DICT = {"Intel": 88.39, "AMD": 87.67, "ARM": 87.95}
3530

3631
PROC_MODEL = proc.proc_type()
3732

38-
COVERAGE_MAX_DELTA = 0.05
39-
40-
CARGO_KCOV_REL_PATH = os.path.join(host.CARGO_BUILD_REL_PATH, "kcov")
41-
42-
KCOV_COVERAGE_FILE = "index.js"
43-
"""kcov will aggregate coverage data in this file."""
33+
# Toolchain target architecture.
34+
if ("Intel" in PROC_MODEL) or ("AMD" in PROC_MODEL):
35+
ARCH = "x86_64"
36+
elif "ARM" in PROC_MODEL:
37+
ARCH = "aarch64"
38+
else:
39+
raise Exception(f"Unsupported processor model ({PROC_MODEL})")
4440

45-
KCOV_COVERED_LINES_REGEX = r'"covered_lines":"(\d+)"'
46-
"""Regex for extracting number of total covered lines found by kcov."""
41+
# Toolchain target.
42+
# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
43+
# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
44+
# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
45+
TARGET = f"{ARCH}-unknown-linux-gnu"
4746

48-
KCOV_TOTAL_LINES_REGEX = r'"total_lines" : "(\d+)"'
49-
"""Regex for extracting number of total executable lines found by kcov."""
47+
# We allow coverage to have a max difference of `COVERAGE_MAX_DELTA` as percentage before failing
48+
# the test.
49+
COVERAGE_MAX_DELTA = 0.05
5050

51-
SECCOMPILER_BUILD_DIR = "../build/seccompiler"
51+
# grcov 0.8.* requires GLIBC >2.27, this is not present in ubuntu 18.04, when we update the docker
52+
# container with a newer version of ubuntu we can also update this.
53+
GRCOV_VERSION = "0.7.1"
5254

5355

5456
@pytest.mark.timeout(400)
55-
def test_coverage(test_fc_session_root_path, test_session_tmp_path):
56-
"""Test line coverage for rust tests is within bounds.
57-
58-
The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov
59-
after a coverage run.
57+
def test_coverage():
58+
"""Test code coverage
6059
6160
@type: build
6261
"""
63-
proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
64-
assert len(proc_model) == 1, "Could not get processor model!"
65-
coverage_target_pct = COVERAGE_DICT[proc_model[0]]
66-
exclude_pattern = (
67-
"${CARGO_HOME:-$HOME/.cargo/},"
68-
"build/,"
69-
"tests/,"
70-
"usr/lib/gcc,"
71-
"lib/x86_64-linux-gnu/,"
72-
"test_utils.rs,"
73-
# The following files/directories are auto-generated
74-
"bootparam.rs,"
75-
"elf.rs,"
76-
"mpspec.rs,"
77-
"msr_index.rs,"
78-
"bindings.rs,"
79-
"_gen"
62+
# Get coverage target.
63+
processor_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
64+
assert len(processor_model) == 1, "Could not get processor model!"
65+
coverage_target = COVERAGE_DICT[processor_model[0]]
66+
67+
# Re-direct to repository root.
68+
os.chdir("..")
69+
70+
# Add `llvm-tools-preview` component
71+
utils.run_cmd("rustup component add llvm-tools-preview")
72+
73+
# Generate test profiles.
74+
utils.run_cmd(
75+
f'\
76+
export RUSTFLAGS="-Cinstrument-coverage" \
77+
&& export LLVM_PROFILE_FILE="your_name-%p-%m.profraw" \
78+
&& cargo test --all --target={TARGET} -- --test-threads=1 \
79+
'
8080
)
81-
exclude_region = "'mod tests {'"
82-
target = "{}-unknown-linux-musl".format(platform.machine())
83-
84-
cmd = (
85-
'CARGO_WRAPPER="kcov" RUSTFLAGS="{}" CARGO_TARGET_DIR={} '
86-
"cargo kcov --all "
87-
"--target {} --output {} -- "
88-
"--exclude-pattern={} "
89-
"--exclude-region={} --verify"
90-
).format(
91-
host.get_rustflags(),
92-
os.path.join(test_fc_session_root_path, CARGO_KCOV_REL_PATH),
93-
target,
94-
test_session_tmp_path,
95-
exclude_pattern,
96-
exclude_region,
97-
)
98-
# We remove the seccompiler custom build directory, created by the
99-
# vmm-level `build.rs`.
100-
# If we don't delete it before and after running the kcov command, we will
101-
# run into linker errors.
102-
shutil.rmtree(SECCOMPILER_BUILD_DIR, ignore_errors=True)
103-
# By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify`
104-
# to kcov. To pass others arguments, we need to include the defaults.
105-
utils.run_cmd(cmd)
106-
107-
shutil.rmtree(SECCOMPILER_BUILD_DIR)
108-
109-
coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE)
110-
with open(coverage_file, encoding="utf-8") as cov_output:
111-
contents = cov_output.read()
112-
covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0])
113-
total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0])
114-
coverage = covered_lines / total_lines * 100
115-
print("Number of executable lines: {}".format(total_lines))
116-
print("Number of covered lines: {}".format(covered_lines))
117-
print("Thus, coverage is: {:.2f}%".format(coverage))
118-
119-
coverage_low_msg = (
120-
"Current code coverage ({:.2f}%) is >{:.2f}% below the target ({}%).".format(
121-
coverage, COVERAGE_MAX_DELTA, coverage_target_pct
122-
)
123-
)
124-
125-
assert coverage >= coverage_target_pct - COVERAGE_MAX_DELTA, coverage_low_msg
12681

127-
# Get the name of the variable that needs updating.
128-
namespace = globals()
129-
cov_target_name = [name for name in namespace if namespace[name] is COVERAGE_DICT][
130-
0
131-
]
132-
133-
coverage_high_msg = (
134-
"Current code coverage ({:.2f}%) is >{:.2f}% above the target ({}%).\n"
135-
"Please update the value of {}.".format(
136-
coverage, COVERAGE_MAX_DELTA, coverage_target_pct, cov_target_name
137-
)
82+
# Generate coverage report.
83+
utils.run_cmd(
84+
f'\
85+
cargo install --version {GRCOV_VERSION} grcov \
86+
&& grcov . \
87+
-s . \
88+
--binary-path ./build/cargo_target/{TARGET}/debug/ \
89+
--ignore "build/*" \
90+
-t html \
91+
--branch \
92+
--ignore-not-existing \
93+
-o ./build/cargo_target/{TARGET}/debug/coverage \
94+
'
13895
)
13996

140-
assert coverage <= coverage_target_pct + COVERAGE_MAX_DELTA, coverage_high_msg
141-
142-
return (f"{coverage}%", f"{coverage_target_pct}% +/- {COVERAGE_MAX_DELTA * 100}%")
97+
# Extract coverage from html report.
98+
# When we update grcov to 0.8.* we can update this to pull the coverage from a generated .json
99+
# file.
100+
index = open(
101+
f"./build/cargo_target/{TARGET}/debug/coverage/index.html", encoding="utf-8"
102+
)
103+
index_contents = index.read()
104+
end = index_contents.find(" %</abbr></p>")
105+
start = index_contents[:end].rfind(">")
106+
coverage_str = index_contents[start + 1 : end]
107+
coverage = float(coverage_str)
108+
109+
# Compare coverage.
110+
high = coverage_target * (1.0 + COVERAGE_MAX_DELTA)
111+
low = coverage_target * (1.0 - COVERAGE_MAX_DELTA)
112+
assert (
113+
coverage >= low
114+
), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% below \
115+
the target ({coverage_target:.2f}%)"
116+
assert (
117+
coverage <= high
118+
), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% above \
119+
the target ({coverage_target:.2f}%)"

‎tests/integration_tests/build/test_unittests.py

+5-7
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
import host_tools.cargo_build as host # pylint:disable=import-error
88

99
MACHINE = platform.machine()
10-
# No need to run unittests for musl since
11-
# we run coverage with musl for all platforms.
12-
TARGET = "{}-unknown-linux-gnu".format(MACHINE)
10+
# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
11+
# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
12+
# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
13+
TARGET = "{}-unknown-linux-musl".format(MACHINE)
1314

1415

1516
def test_unittests(test_fc_session_root_path):
@@ -20,7 +21,4 @@ def test_unittests(test_fc_session_root_path):
2021
"""
2122
extra_args = "--release --target {} ".format(TARGET)
2223

23-
host.cargo_test(
24-
test_fc_session_root_path,
25-
extra_args=extra_args
26-
)
24+
host.cargo_test(test_fc_session_root_path, extra_args=extra_args)

0 commit comments

Comments
 (0)