Add --coverage option for local.py, make clang coverage builds use -fcoverage-mapping/-fprofile-instr-generate (project-chip#37457)

* Add coverage support flags (but no save yet for the flag)

* Minor cleanup, save config options when loading variants, so build flags transcend running

* Fix yaml content

* Restyled by isort

* Switch coverage to clang, ensure out dir for config exists

* Slight doc update, allow local.py to use ccache

* Start adding filter support and comma separation support to local.py

* Fix the runner termination logic

* gen-coverage works

* Restyle

* make linter happy

* Make it clearer what errors we ignore, add more error logic to make this run on more machines

* Restyle

* Add more ignores for cleaner results

* Restyle

* One more ignore

* Better organization, make paths consistent

* Fix logic. Merged reports now work!

* Restyle

* fix stderr logging

* fix fail dir creation

* Make mismatched FIFO pid much easier to investigate

* Add hierarchical view by default - it seems easier for us to narrow down coverage

* Add coverage support for yaml

* Use exec to execute the sub-program

* Fix up arguments in local.py to support propper quoting

* For chip tool tests also run chip-tool with coverage support

* Fix the path-variance runs of unit test,  so I can run unit tests with coverage

* Updated comment a bit

* Increase timeout for test running to cover slow tests, skip slow tests or tests that we know will fail

* Also exclude manual tests by default from local runs

* match default exclusions ... also flaky excluded

* Support keep going for yaml tests, add more ignores

* Multiprocessing coverage, coverage now works

* Add missing assignment

* Restyled by autopep8

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>
Co-authored-by: Restyled.io <commits@restyled.io>

Author: 3 people

build/config/compiler/BUILD.gn

@@ -536,7 +536,18 @@ config("fuzzing_default") {
 }
 
 config("coverage") {
-  cflags = [ "--coverage" ]
+  if (is_clang) {
+    cflags = [
+      # CLANG builds use the clang-specific coverage, so that versions of lcov/gcov
+      # do not have to match and `llvm-cov export -format=lcov` can be used
+      #
+      # Documentation: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
+      "-fprofile-instr-generate",
+      "-fcoverage-mapping",
+    ]
+  } else {
+    cflags = [ "--coverage" ]
+  }
   ldflags = cflags
 }
 

scripts/build/build/targets.py

@@ -80,7 +80,7 @@ def BuildHostFakeTarget():
     target.AppendModifier("pw-fuzztest", fuzzing_type=HostFuzzingType.PW_FUZZTEST).OnlyIfRe(
         "-clang").ExceptIfRe('-(libfuzzer|ossfuzz|asan)')
     target.AppendModifier('coverage', use_coverage=True).OnlyIfRe(
-        '-(chip-tool|all-clusters)')
+        '-(chip-tool|all-clusters)').ExceptIfRe('-clang')
     target.AppendModifier('dmalloc', use_dmalloc=True)
     target.AppendModifier('clang', use_clang=True)
 
@@ -186,8 +186,7 @@ def BuildHostTarget():
         "-clang").ExceptIfRe('-libfuzzer')
     target.AppendModifier("pw-fuzztest", fuzzing_type=HostFuzzingType.PW_FUZZTEST).OnlyIfRe(
         "-clang").ExceptIfRe('-(libfuzzer|ossfuzz|asan)')
-    target.AppendModifier('coverage', use_coverage=True).OnlyIfRe(
-        '-(chip-tool|all-clusters|tests)')
+    target.AppendModifier('coverage', use_coverage=True)
     target.AppendModifier('dmalloc', use_dmalloc=True)
     target.AppendModifier('clang', use_clang=True)
     target.AppendModifier('test', extra_tests=True)

scripts/build/builders/host.py

@@ -411,6 +411,7 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE,
         if use_coverage:
             self.extra_gn_options.append('use_coverage=true')
 
+        self.use_clang = use_clang  # for usage in other commands
         if use_clang:
             self.extra_gn_options.append('is_clang=true')
 
@@ -586,7 +587,7 @@ def generate(self):
             self._Execute(['mkdir', '-p', self.coverage_dir], title="Create coverage output location")
 
     def PreBuildCommand(self):
-        if self.app == HostApp.TESTS and self.use_coverage:
+        if self.app == HostApp.TESTS and self.use_coverage and not self.use_clang:
             cmd = ['ninja', '-C', self.output_dir]
 
             if self.ninja_jobs is not None:
@@ -604,7 +605,8 @@ def PreBuildCommand(self):
                            '--output-file', os.path.join(self.coverage_dir, 'lcov_base.info')], title="Initial coverage baseline")
 
     def PostBuildCommand(self):
-        if self.app == HostApp.TESTS and self.use_coverage:
+        # TODO: CLANG coverage is not yet implemented, requires different tooling
+        if self.app == HostApp.TESTS and self.use_coverage and not self.use_clang:
             self._Execute(['lcov', '--capture', '--directory', os.path.join(self.output_dir, 'obj'),
                            '--exclude', os.path.join(self.chip_dir, '**/tests/*'),
                            '--exclude', os.path.join(self.chip_dir, 'zzz_generated/*'),

scripts/tests/chiptest/test_definition.py

@@ -210,6 +210,36 @@ def items(self):
                 self.microwave_oven_app, self.chip_repl_yaml_tester_cmd,
                 self.chip_tool_with_python_cmd, self.rvc_app, self.network_manager_app]
 
+    def items_with_key(self):
+        """
+        Returns all path items and also the corresponding "Application Key" which
+        is the typical application name.
+
+        This is to provide scripts a consistent way to reference a path, even if
+        the paths used for individual appplications contain different names
+        (e.g. they could be wrapper scripts).
+        """
+        return [
+            (self.chip_tool, "chip-tool"),
+            (self.all_clusters_app, "chip-all-clusters-app"),
+            (self.lock_app, "chip-lock-app"),
+            (self.fabric_bridge_app, "fabric-bridge-app"),
+            (self.ota_provider_app, "chip-ota-provider-app"),
+            (self.ota_requestor_app, "chip-ota-requestor-app"),
+            (self.tv_app, "chip-tv-app"),
+            (self.bridge_app, "chip-bridge-app"),
+            (self.lit_icd_app, "lit-icd-app"),
+            (self.microwave_oven_app, "chip-microwave-oven-app"),
+            (self.chip_repl_yaml_tester_cmd, "yamltest_with_chip_repl_tester.py"),
+            (
+                # This path varies, however it is a fixed python tool so it may be ok
+                self.chip_tool_with_python_cmd,
+                os.path.basename(self.chip_tool_with_python_cmd[-1]),
+            ),
+            (self.rvc_app, "chip-rvc-app"),
+            (self.network_manager_app, "matter-network-manager-app"),
+        ]
+
 
 @dataclass
 class CaptureLine:
@@ -330,7 +360,7 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
                                 "don't know which application to run")
 
             if not dry_run:
-                for path in paths.items():
+                for path, key in paths.items_with_key():
                     # Do not add chip-tool or chip-repl-yaml-tester-cmd to the register
                     if path == paths.chip_tool or path == paths.chip_repl_yaml_tester_cmd or path == paths.chip_tool_with_python_cmd:
                         continue
@@ -345,8 +375,6 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
                     # For the app indicated by self.target, give it the 'default' key to add to the register
                     if path == target_app:
                         key = 'default'
-                    else:
-                        key = os.path.basename(path[-1])
 
                     app = App(runner, path)
                     # Add the App to the register immediately, so if it fails during