[Tizen] Add code coverage for tests on QEMU (project-chip#36245)

* Add -coverage target

* Allow to specify runner

* Fix the .gcda files output location

* Allow runner in interactive mode

* Remove wrap

* Fix expected test output

* Collect common lcov args

* Update scripts/build/builders/tizen.py

Co-authored-by: Arkadiusz Bokowy <arkadiusz.bokowy@gmail.com>

---------

Co-authored-by: Andrei Litvin <andy314@gmail.com>
Co-authored-by: Arkadiusz Bokowy <arkadiusz.bokowy@gmail.com>

Author: 3 people

scripts/build/build/targets.py

@@ -692,6 +692,8 @@ def BuildTizenTarget():
     target.AppendModifier("no-wifi", enable_wifi=False)
     target.AppendModifier("asan", use_asan=True)
     target.AppendModifier("ubsan", use_ubsan=True)
+    target.AppendModifier('coverage', use_coverage=True).OnlyIfRe(
+        '-tests')
     target.AppendModifier('with-ui', with_ui=True)
 
     return target

scripts/build/builders/tizen.py

@@ -89,6 +89,7 @@ def __init__(self,
                  use_asan: bool = False,
                  use_tsan: bool = False,
                  use_ubsan: bool = False,
+                 use_coverage: bool = False,
                  with_ui: bool = False,
                  ):
         super(TizenBuilder, self).__init__(
@@ -130,9 +131,59 @@ def __init__(self,
             raise Exception("TSAN sanitizer not supported by Tizen toolchain")
         if use_ubsan:
             self.extra_gn_options.append('is_ubsan=true')
+        self.use_coverage = use_coverage
+        if use_coverage:
+            self.extra_gn_options.append('use_coverage=true')
         if with_ui:
             self.extra_gn_options.append('chip_examples_enable_ui=true')
 
+    def generate(self):
+        super(TizenBuilder, self).generate()
+        if self.app == TizenApp.TESTS and self.use_coverage:
+            self.coverage_dir = os.path.join(self.output_dir, 'coverage')
+            self._Execute(['mkdir', '-p', self.coverage_dir], title="Create coverage output location")
+
+    def lcov_args(self):
+        gcov = os.path.join(os.environ['TIZEN_SDK_TOOLCHAIN'], 'bin/arm-linux-gnueabi-gcov')
+        return [
+            'lcov', '--gcov-tool', gcov, '--ignore-errors', 'unused,mismatch', '--capture', '--directory', os.path.join(
+                self.output_dir, 'obj'),
+            '--exclude', '**/src/controller/*',
+            '--exclude', '**/connectedhomeip/zzz_generated/*',
+            '--exclude', '**/connectedhomeip/third_party/*',
+            '--exclude', '/opt/*',
+        ]
+
+    def PreBuildCommand(self):
+        if self.app == TizenApp.TESTS and self.use_coverage:
+            cmd = ['ninja', '-C', self.output_dir]
+
+            if self.ninja_jobs is not None:
+                cmd.append('-j' + str(self.ninja_jobs))
+
+            cmd.append('Tizen')
+
+            self._Execute(cmd, title="Build-only")
+
+            self._Execute(self.lcov_args() + [
+                '--initial',
+                '--output-file', os.path.join(self.coverage_dir, 'lcov_base.info')
+            ], title="Initial coverage baseline")
+
+    def PostBuildCommand(self):
+        if self.app == TizenApp.TESTS and self.use_coverage:
+
+            self._Execute(self.lcov_args() + ['--output-file', os.path.join(self.coverage_dir,
+                          'lcov_test.info')], title="Update coverage")
+
+            gcov = os.path.join(os.environ['TIZEN_SDK_TOOLCHAIN'], 'bin/arm-linux-gnueabi-gcov')
+            self._Execute(['lcov', '--gcov-tool', gcov, '--add-tracefile', os.path.join(self.coverage_dir, 'lcov_base.info'),
+                           '--add-tracefile', os.path.join(self.coverage_dir, 'lcov_test.info'),
+                           '--output-file', os.path.join(self.coverage_dir, 'lcov_final.info')
+                           ], title="Final coverage info")
+            self._Execute(['genhtml', os.path.join(self.coverage_dir, 'lcov_final.info'), '--output-directory',
+                           os.path.join(self.coverage_dir, 'html')], title="HTML coverage")
+
     def GnBuildArgs(self):
         # Make sure that required ENV variables are defined
         for env in ('TIZEN_SDK_ROOT', 'TIZEN_SDK_SYSROOT'):

scripts/build/testdata/all_targets_linux_x64.txt

@@ -21,6 +21,6 @@ nrf-native-posix-64-tests
 nuttx-x64-light
 qpg-qpg6105-{lock,light,shell,persistent-storage,light-switch,thermostat}[-updateimage][-data-model-disabled][-data-model-enabled]
 stm32-stm32wb5mm-dk-light
-tizen-arm-{all-clusters,chip-tool,light,tests}[-no-ble][-no-thread][-no-wifi][-asan][-ubsan][-with-ui]
+tizen-arm-{all-clusters,chip-tool,light,tests}[-no-ble][-no-thread][-no-wifi][-asan][-ubsan][-coverage][-with-ui]
 telink-{tlsr9118bdk40d,tlsr9518adk80d,tlsr9528a,tlsr9528a_retention,tlsr9258a,tlsr9258a_retention}-{air-quality-sensor,all-clusters,all-clusters-minimal,bridge,contact-sensor,light,light-switch,lock,ota-requestor,pump,pump-controller,shell,smoke-co-alarm,temperature-measurement,thermostat,window-covering}[-ota][-dfu][-shell][-rpc][-factory-data][-4mb][-mars][-usb][-data-model-disabled][-data-model-enabled]
 openiotsdk-{shell,lock}[-mbedtls][-psa]

src/test_driver/tizen/chip_tests/BUILD.gn

@@ -24,12 +24,15 @@ tizen_qemu_run("chip-tests") {
   virtio_net = true
 
   # Share tests directory.
-  share = "${root_out_dir}/tests"
+  share = "${root_out_dir}"
+
+  # Runner script file name.
+  runner = "runner-${target_name}.sh"
 
   # Copy runner script to shared directory.
   copy("${target_name}:runner") {
     sources = [ "runner.sh" ]
-    outputs = [ share + "/runner.sh" ]
+    outputs = [ share + "/" + runner ]
   }
 
   # Build CHIP unit tests.

src/test_driver/tizen/chip_tests/runner.sh

@@ -21,6 +21,10 @@ set -e
 # Print CHIP logs on stdout
 dlogutil CHIP &
 
+# Set the correct path for .gcda files
+export GCOV_PREFIX=/mnt/chip
+export GCOV_PREFIX_STRIP=5
+
 FAILED=()
 STATUS=0
 
@@ -43,7 +47,7 @@ while IFS= read -r TEST; do
         echo -e "DONE: \e[31mFAIL\e[0m"
     fi
 
-done < <(find /mnt/chip -type f -executable ! -name runner.sh)
+done < <(find /mnt/chip/tests -type f -executable ! -name runner.sh)
 
 if [ ! "$STATUS" -eq 0 ]; then
     echo

src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn

@@ -26,10 +26,13 @@ tizen_qemu_run("lighting-app") {
   # Share output directory.
   share = "${root_out_dir}"
 
+  # Runner script file name.
+  runner = "runner-${target_name}.sh"
+
   # Copy runner script to shared directory.
   copy("${target_name}:runner") {
     sources = [ "runner.sh" ]
-    outputs = [ share + "/runner.sh" ]
+    outputs = [ share + "/" + runner ]
   }
 
   # Build applications used in the test.

src/test_driver/tizen/integration_tests/lighting-app/runner.sh

@@ -21,6 +21,10 @@ set -e
 # Print CHIP logs on stdout
 dlogutil CHIP &
 
+# Set the correct path for .gcda files
+export GCOV_PREFIX=/mnt/chip
+export GCOV_PREFIX_STRIP=5
+
 # Install lighting Matter app
 pkgcmd -i -t tpk -p /mnt/chip/org.tizen.matter.*/out/org.tizen.matter.*.tpk
 # Launch lighting Matter app

third_party/tizen/tizen_qemu.py

@@ -66,9 +66,10 @@
           "default: $TIZEN_SDK_ROOT/iot-sysdata.img"))
 parser.add_argument(
     '--share', type=str,
-    help=("host directory to share with the guest; if file named 'runner.sh' "
-          "is present at the root of that directory, it will be executed "
-          "automatically after boot"))
+    help=("host directory to share with the guest"))
+parser.add_argument(
+    '--runner', type=str,
+    help=("path to the runner script which will run automatically after boot. path should be relative to shared directory"))
 parser.add_argument(
     '--output', metavar='FILE', default="/dev/null",
     help="store the QEMU output in a FILE")
@@ -136,6 +137,9 @@ def whereis(binary_name):
     # Run root shell instead of the runner script.
     kernel_args += " rootshell"
 
+if args.runner:
+    kernel_args += " runner=/mnt/chip/%s" % args.runner
+
 qemu_args += [
     '-kernel', args.kernel,
     '-append', kernel_args,
@@ -153,7 +157,7 @@ def whereis(binary_name):
         for line in iter(proc.stdout.readline, b''):
 
             # Forward the output to the stdout and the log file.
-            sys.stdout.write(line.decode(sys.stdout.encoding))
+            sys.stdout.write(line.decode(sys.stdout.encoding, errors='ignore'))
             sys.stdout.flush()
             output.write(line)
 

third_party/tizen/tizen_sdk.gni

@@ -175,6 +175,7 @@ template("tizen_qemu_run") {
   testonly = true
 
   assert(defined(invoker.share), "It is required to specify path to share.")
+  assert(defined(invoker.runner), "It is required to specify runner script.")
 
   # Store QEMU output in a dedicated log file.
   output_log_file = "${root_out_dir}/tizen-qemu-" + target_name + ".log"
@@ -186,6 +187,7 @@ template("tizen_qemu_run") {
 
     args = [
       "--share=" + invoker.share,
+      "--runner=" + invoker.runner,
       "--output=" + rebase_path(output_log_file),
     ]
     if (defined(invoker.virtio_net) && invoker.virtio_net) {