Skip to content

Commit 0a21983

Browse files
Support clang build coverage execution for unit tests (project-chip#37563)
* Make coverage seemingly work: set per test output raw profile, add scripting logic * Adding scripts and fixes * Restyle * use llvm cov to merge things * Attempt for better coverage merging. -object seems to do the trick * make lcov work on my machine as well * Restyled by gn --------- Co-authored-by: Restyled.io <commits@restyled.io>
1 parent 6d1f573 commit 0a21983

File tree

6 files changed

+185
-2
lines changed

6 files changed

+185
-2
lines changed

build/chip/chip_test_suite.gni

+27
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@ import("//build_overrides/googletest.gni")
1818
import("//build_overrides/pigweed.gni")
1919

2020
import("${chip_root}/build/chip/tests.gni")
21+
import("${dir_pw_build}/python.gni")
2122
import("${dir_pw_unit_test}/test.gni")
2223

24+
# Need access to build options (specifically build_coverage)
25+
import("${build_root}/config/compiler/compiler.gni")
2326
assert(chip_build_tests)
2427

2528
declare_args() {
@@ -112,6 +115,26 @@ template("chip_test_suite") {
112115
_test_output_dir = invoker.output_dir
113116
}
114117

118+
if (use_coverage && is_clang) {
119+
# Generates clang coverage to "<TestName>.profraw" instead of "deafault.profraw"
120+
_clang_coverage_setup = "${root_build_dir}/clang_static_coverage_config/${_test_name}ClangCoverageConfig.cpp"
121+
pw_python_action("${_test_name}-clang-coverage") {
122+
script = "${chip_root}/scripts/build/clang_coverage_wrapper.py"
123+
outputs = [ _clang_coverage_setup ]
124+
args = [
125+
"--output",
126+
rebase_path(_clang_coverage_setup),
127+
"--raw-profile-filename",
128+
"coverage/${_test_name}.profraw",
129+
]
130+
}
131+
132+
source_set("${_test_name}-clang-coverage-src") {
133+
sources = [ _clang_coverage_setup ]
134+
deps = [ ":${_test_name}-clang-coverage" ]
135+
}
136+
}
137+
115138
pw_test(_test_name) {
116139
# Forward certain variables from the invoker.
117140
forward_variables_from(invoker,
@@ -125,6 +148,10 @@ template("chip_test_suite") {
125148
# Link to the common lib for this suite so we get its `sources`.
126149
public_deps += [ ":${_suite_name}.lib" ]
127150

151+
if (use_coverage && is_clang) {
152+
public_deps += [ ":${_test_name}-clang-coverage-src" ]
153+
}
154+
128155
if (pw_unit_test_BACKEND == "$dir_pw_unit_test:googletest") {
129156
test_main = "$dir_pigweed/third_party/googletest:gmock_main"
130157
}

build/config/compiler/BUILD.gn

+9
Original file line numberDiff line numberDiff line change
@@ -549,6 +549,15 @@ config("coverage") {
549549
cflags = [ "--coverage" ]
550550
}
551551
ldflags = cflags
552+
553+
if (is_clang) {
554+
# Looking to add buildid which _could_ be used for coverage
555+
# file format using `%b` (see
556+
# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program)
557+
# however at the time of writing this, linux clang used during bootstrap
558+
# does not seem to support this.
559+
ldflags += [ "-Wl,--build-id" ]
560+
}
552561
}
553562

554563
config("coverage_default") {

scripts/build/build/targets.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,7 @@ def BuildHostFakeTarget():
7979
"-clang").ExceptIfRe('-libfuzzer')
8080
target.AppendModifier("pw-fuzztest", fuzzing_type=HostFuzzingType.PW_FUZZTEST).OnlyIfRe(
8181
"-clang").ExceptIfRe('-(libfuzzer|ossfuzz|asan)')
82-
target.AppendModifier('coverage', use_coverage=True).OnlyIfRe(
83-
'-(chip-tool|all-clusters)').ExceptIfRe('-clang')
82+
target.AppendModifier('coverage', use_coverage=True)
8483
target.AppendModifier('dmalloc', use_dmalloc=True)
8584
target.AppendModifier('clang', use_clang=True)
8685

scripts/build/builders/host.py

+53
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414

1515
import os
16+
import shlex
1617
from enum import Enum, auto
1718
from platform import uname
1819
from typing import Optional
@@ -547,6 +548,13 @@ def GnBuildEnv(self):
547548
if self.board == HostBoard.ARM64:
548549
self.build_env['PKG_CONFIG_PATH'] = os.path.join(
549550
self.SysRootPath('SYSROOT_AARCH64'), 'lib/aarch64-linux-gnu/pkgconfig')
551+
if self.app == HostApp.TESTS and self.use_coverage and self.use_clang:
552+
# Every test is expected to have a distinct build ID, so `%m` will be
553+
# distinct.
554+
#
555+
# Output is relative to "oputput_dir" since that is where GN executs
556+
self.build_env['LLVM_PROFILE_FILE'] = os.path.join("coverage", "profiles", "run_%b.profraw")
557+
550558
return self.build_env
551559

552560
def SysRootPath(self, name):
@@ -621,6 +629,51 @@ def PostBuildCommand(self):
621629
self._Execute(['genhtml', os.path.join(self.coverage_dir, 'lcov_final.info'), '--output-directory',
622630
os.path.join(self.coverage_dir, 'html')], title="HTML coverage")
623631

632+
# coverage for clang works by having perfdata for every test run, which are in "*.profraw" files
633+
if self.app == HostApp.TESTS and self.use_coverage and self.use_clang:
634+
# Clang coverage config generates "coverage/{name}.profraw" for each test indivdually
635+
# Here we are merging ALL raw profiles into a single indexed file
636+
637+
_indexed_instrumentation = shlex.quote(os.path.join(self.coverage_dir, "merged.profdata"))
638+
639+
self._Execute([
640+
"bash",
641+
"-c",
642+
f'find {shlex.quote(self.coverage_dir)} -name "*.profraw"'
643+
+ f' | xargs -n 10240 llvm-profdata merge -sparse -o {_indexed_instrumentation}'
644+
],
645+
title="Generating merged coverage data")
646+
647+
_lcov_data = os.path.join(self.coverage_dir, "merged.lcov")
648+
649+
self._Execute([
650+
"bash",
651+
"-c",
652+
f'find {shlex.quote(self.coverage_dir)} -name "*.profraw"'
653+
+ ' | xargs -n1 basename | sed "s/\\.profraw//" '
654+
+ f' | xargs -I @ echo -object {shlex.quote(os.path.join(self.output_dir, "tests", "@"))}'
655+
+ f' | xargs -n 10240 llvm-cov export -format=lcov --instr-profile {_indexed_instrumentation} '
656+
+ ' --ignore-filename-regex "/third_party/"'
657+
+ ' --ignore-filename-regex "/tests/"'
658+
+ ' --ignore-filename-regex "/usr/include/"'
659+
+ ' --ignore-filename-regex "/usr/lib/"'
660+
+ f' | cat >{shlex.quote(_lcov_data)}'
661+
],
662+
title="Generating lcov data")
663+
664+
self._Execute([
665+
"genhtml",
666+
"--ignore-errors",
667+
"inconsistent",
668+
"--ignore-errors",
669+
"range",
670+
# "--hierarchical" <- this may be interesting
671+
"--output",
672+
os.path.join(self.output_dir, "html"),
673+
os.path.join(self.coverage_dir, "merged.lcov"),
674+
],
675+
title="Generating HTML coverage report")
676+
624677
if self.app == HostApp.JAVA_MATTER_CONTROLLER:
625678
self.createJavaExecutable("java-matter-controller")
626679

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/usr/bin/env -S python3 -B
2+
3+
# Copyright (c) 2025 Project CHIP Authors
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
import logging
17+
18+
import click
19+
import coloredlogs
20+
import jinja2
21+
22+
# Supported log levels, mapping string values required for argument
23+
# parsing into logging constants
24+
__LOG_LEVELS__ = {
25+
"debug": logging.DEBUG,
26+
"info": logging.INFO,
27+
"warn": logging.WARN,
28+
"fatal": logging.FATAL,
29+
}
30+
31+
# This repesents the code that is generated according to
32+
# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#using-the-profiling-runtime-without-static-initializers
33+
#
34+
# So that the output of coverage is customized and does not go to `default.profraw`
35+
_CPP_TEMPLATE = """\
36+
extern "C" {
37+
38+
int __llvm_profile_runtime = 0;
39+
void __llvm_profile_initialize_file();
40+
void __llvm_profile_write_file();
41+
void __llvm_profile_set_filename(const char *);
42+
43+
} // extern "C"
44+
45+
struct StaticInitLLVMProfile {
46+
StaticInitLLVMProfile() {
47+
__llvm_profile_set_filename("{{raw_profile_filename}}");
48+
}
49+
~StaticInitLLVMProfile() {
50+
__llvm_profile_write_file();
51+
}
52+
} gInitLLVMProfilingPaths;
53+
"""
54+
55+
56+
@click.command()
57+
@click.option(
58+
"--log-level",
59+
default="INFO",
60+
type=click.Choice([k for k in __LOG_LEVELS__.keys()], case_sensitive=False),
61+
help="Determines the verbosity of script output.",
62+
)
63+
@click.option(
64+
"--no-log-timestamps",
65+
default=False,
66+
is_flag=True,
67+
help="Skip timestaps in log output",
68+
)
69+
@click.option(
70+
"--output",
71+
help="What file to output when runnning under clang profiling",
72+
)
73+
@click.option(
74+
"--raw-profile-filename",
75+
help="Filename to use for output",
76+
)
77+
def main(log_level, no_log_timestamps, output, raw_profile_filename):
78+
log_fmt = "%(asctime)s %(levelname)-7s %(message)s"
79+
if no_log_timestamps:
80+
log_fmt = "%(levelname)-7s %(message)s"
81+
coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)
82+
83+
logging.info("Writing output to %s (profile name: %s)", output, raw_profile_filename)
84+
with open(output, "wt") as f:
85+
f.write(jinja2.Template(_CPP_TEMPLATE).render(raw_profile_filename=raw_profile_filename))
86+
87+
logging.debug("Writing completed")
88+
89+
90+
if __name__ == "__main__":
91+
main(auto_envvar_prefix="CHIP")

scripts/build_coverage.sh

+4
Original file line numberDiff line numberDiff line change
@@ -218,22 +218,26 @@ fi
218218
mkdir -p "$COVERAGE_ROOT"
219219

220220
lcov --initial --capture --directory "$OUTPUT_ROOT/obj/src" \
221+
--ignore-errors inconsistent \
221222
--exclude="$PWD"/zzz_generated/* \
222223
--exclude="$PWD"/third_party/* \
223224
--exclude=/usr/include/* \
224225
--output-file "$COVERAGE_ROOT/lcov_base.info"
225226

226227
lcov --capture --directory "$OUTPUT_ROOT/obj/src" \
228+
--ignore-errors inconsistent \
227229
--exclude="$PWD"/zzz_generated/* \
228230
--exclude="$PWD"/third_party/* \
229231
--exclude=/usr/include/* \
230232
--output-file "$COVERAGE_ROOT/lcov_test.info"
231233

232234
lcov --add-tracefile "$COVERAGE_ROOT/lcov_base.info" \
233235
--add-tracefile "$COVERAGE_ROOT/lcov_test.info" \
236+
--ignore-errors inconsistent \
234237
--output-file "$COVERAGE_ROOT/lcov_final.info"
235238

236239
genhtml "$COVERAGE_ROOT/lcov_final.info" \
240+
--ignore-errors inconsistent \
237241
--output-directory "$COVERAGE_ROOT/html" \
238242
--title "SHA:$(git rev-parse HEAD)" \
239243
--header-title "Matter SDK Coverage Report"

0 commit comments

Comments
 (0)