Updated efr32 test runner to support running multiple test binaries

scripts/build/builders/efr32.py

@@ -273,11 +273,10 @@ def build_outputs(self):
 
         if self.app == Efr32App.UNIT_TEST:
             # Include test runner python wheels
-            for root, dirs, files in os.walk(os.path.join(self.output_dir, 'chip_nl_test_runner_wheels')):
+            for root, dirs, files in os.walk(os.path.join(self.output_dir, 'chip_pw_test_runner_wheels')):
                 for file in files:
-                    yield BuilderOutput(
-                        os.path.join(root, file),
-                        os.path.join("chip_nl_test_runner_wheels", file))
+                    items["chip_pw_test_runner_wheels/" +
+                          file] = os.path.join(root, file)
 
         # Figure out flash bundle files and build accordingly
         with open(os.path.join(self.output_dir, self.app.FlashBundleName())) as f:

scripts/build/builders/host.py

@@ -216,7 +216,7 @@ def OutputNames(self):
         elif self == HostApp.PYTHON_BINDINGS:
             yield 'controller/python'  # Directory containing WHL files
         elif self == HostApp.EFR32_TEST_RUNNER:
-            yield 'chip_nl_test_runner_wheels'
+            yield 'chip_pw_test_runner_wheels'
         elif self == HostApp.TV_CASTING:
             yield 'chip-tv-casting-app'
             yield 'chip-tv-casting-app.map'

src/BUILD.gn

@@ -60,17 +60,23 @@ if (chip_build_tests) {
       "${chip_root}/src/access/tests",
       "${chip_root}/src/crypto/tests",
       "${chip_root}/src/inet/tests",
-      "${chip_root}/src/lib/address_resolve/tests",
-      "${chip_root}/src/lib/asn1/tests",
-      "${chip_root}/src/lib/core/tests",
-      "${chip_root}/src/messaging/tests",
-      "${chip_root}/src/protocols/bdx/tests",
-      "${chip_root}/src/protocols/interaction_model/tests",
-      "${chip_root}/src/protocols/user_directed_commissioning/tests",
-      "${chip_root}/src/transport/retransmit/tests",
-      "${chip_root}/src/app/icd/server/tests",
     ]
 
+    # Skip on efr32 due to flash limitations.
+    if (chip_device_platform != "efr32") {
+      tests += [
+        "${chip_root}/src/lib/address_resolve/tests",
+        "${chip_root}/src/lib/asn1/tests",
+        "${chip_root}/src/lib/core/tests",
+        "${chip_root}/src/messaging/tests",
+        "${chip_root}/src/protocols/bdx/tests",
+        "${chip_root}/src/protocols/interaction_model/tests",
+        "${chip_root}/src/protocols/user_directed_commissioning/tests",
+        "${chip_root}/src/transport/retransmit/tests",
+        "${chip_root}/src/app/icd/server/tests",
+      ]
+    }
+
     # Skip DNSSD tests for Mbed platform due to flash memory size limitations
     if (current_os != "mbed") {
       tests += [

src/test_driver/efr32/BUILD.gn

@@ -130,8 +130,8 @@ group("efr32") {
 
 group("runner") {
   deps = [
-    "${efr32_project_dir}/py:nl_test_runner.install",
-    "${efr32_project_dir}/py:nl_test_runner_wheel",
+    "${efr32_project_dir}/py:pw_test_runner.install",
+    "${efr32_project_dir}/py:pw_test_runner_wheel",
   ]
 }
 

src/test_driver/efr32/README.md

@@ -1,6 +1,6 @@
 #CHIP EFR32 Test Driver
 
-This builds and runs the NLUnitTest on the efr32 device
+This builds and runs the PW Unit Tests on the efr32 device
 
 <hr>
 
@@ -14,7 +14,7 @@ This builds and runs the NLUnitTest on the efr32 device
 
 ## Introduction
 
-This builds a test binary which contains the NLUnitTests and can be flashed onto
+This builds a test binary which contains the pw_unit_test and can be flashed onto
 a device. The device is controlled using the included RPCs, through the python
 test runner.
 
@@ -99,7 +99,7 @@ Or build using build script from the root
 
     ```
     cd <connectedhomeip>
-    ./scripts/build/build_examples.py --target linux-x64-nl-test-runner build
+    ./scripts/build/build_examples.py --target linux-x64-pw-test-runner build
     ```
 
 The runner will be installed into the venv and python wheels will be packaged in
@@ -108,7 +108,7 @@ the output folder for deploying.
 Then the python wheels need to installed using pip3.
 
     ```
-    pip3 install out/debug/chip_nl_test_runner_wheels/*.whl
+    pip3 install out/debug/chip_pw_test_runner_wheels/*.whl
     ```
 
 Other python libraries may need to be installed such as
@@ -117,8 +117,8 @@ Other python libraries may need to be installed such as
     pip3 install pyserial
     ```
 
--   To run the tests:
+To run all tests:
 
     ```
-    python -m nl_test_runner.nl_test_runner -d /dev/ttyACM1 -f out/debug/matter-silabs-device_tests.s37 -o out.log
+    python -m pw_test_runner.pw_test_runner -d /dev/ttyACM1 -f out/debug -o out.log
     ```

src/test_driver/efr32/py/BUILD.gn

@@ -19,39 +19,13 @@ import("$dir_pw_build/python.gni")
 import("$dir_pw_build/python_dist.gni")
 import("${chip_root}/examples/common/pigweed/pigweed_rpcs.gni")
 
-# TODO [PW_MIGRATION]: remove nl test runner script once transition away from nlunit-test is completed
-pw_python_package("nl_test_runner") {
+pw_python_package("pw_test_runner") {
   setup = [
     "pyproject.toml",
     "setup.cfg",
     "setup.py",
   ]
 
-  sources = [
-    "nl_test_runner/__init__.py",
-    "nl_test_runner/nl_test_runner.py",
-  ]
-
-  python_deps = [
-    "$dir_pw_hdlc/py",
-    "$dir_pw_protobuf_compiler/py",
-    "$dir_pw_rpc/py",
-    "${chip_root}/src/test_driver/efr32:nl_test_service.python",
-  ]
-}
-
-pw_python_wheels("nl_test_runner_wheel") {
-  packages = [ ":nl_test_runner" ]
-  directory = "$root_out_dir/chip_nl_test_runner_wheels"
-}
-
-pw_python_package("pw_test_runner") {
-  setup = [
-    "pw_test_runner/pyproject.toml",
-    "pw_test_runner/setup.cfg",
-    "pw_test_runner/setup.py",
-  ]
-
   sources = [
     "pw_test_runner/__init__.py",
     "pw_test_runner/pw_test_runner.py",
@@ -61,6 +35,7 @@ pw_python_package("pw_test_runner") {
     "$dir_pw_hdlc/py",
     "$dir_pw_protobuf_compiler/py",
     "$dir_pw_rpc/py",
+    #"${chip_root}/src/test_driver/efr32:nl_test_service.python",   #++++ is this still needed?
   ]
 }
 

src/test_driver/efr32/py/build/lib/pw_test_runner/pw_test_runner.py

@@ -0,0 +1,138 @@
+#
+#    Copyright (c) 2024 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+import argparse
+import glob
+import logging
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+import serial  # type: ignore
+from pw_hdlc import rpc
+from pw_unit_test.rpc import run_tests
+
+PW_LOG = logging.getLogger(__name__)
+
+PROTO = Path(os.environ["PW_ROOT"],
+             "pw_unit_test/pw_unit_test_proto/unit_test.proto")
+
+
+class colors:
+    HEADER = "\033[95m"
+    OKBLUE = "\033[94m"
+    OKCYAN = "\033[96m"
+    OKGREEN = "\033[92m"
+    WARNING = "\033[93m"
+    FAIL = "\033[91m"
+    ENDC = "\033[0m"
+    BOLD = "\033[1m"
+
+
+PASS_STRING = colors.OKGREEN + "\N{check mark}" + colors.ENDC
+FAIL_STRING = colors.FAIL + "FAILED" + colors.ENDC
+
+
+def _parse_args():
+    """Parses and returns the command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="CHIP on device unit test runner.")
+    parser.add_argument("-d", "--device", help="the serial port to use")
+    parser.add_argument(
+        "-b", "--baudrate", type=int, default=115200, help="the baud rate to use"
+    )
+    parser.add_argument(
+        "-f",
+        "--flash_image",
+        help="A firmware image which will be flashed berfore runnning the test.  Or a directory containing firmware images, each of which will be flashed and then run.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=argparse.FileType("wb"),
+        default=sys.stdout.buffer,
+        help=(
+            "The file to which to write device output (HDLC channel 1); "
+            "provide - or omit for stdout."
+        ),
+    )
+    return parser.parse_args()
+
+
+def flash_device(device: str, flash_image: str):
+    """flashes the EFR32 device using commander"""
+    err = subprocess.call(
+        ["commander", "flash", "--device", "EFR32", flash_image])
+    if err:
+        raise Exception("flash failed")
+
+
+def get_hdlc_rpc_client(device: str, baudrate: int, output: Any, **kwargs):
+    """Get the HdlcRpcClient based on arguments."""
+    serial_device = serial.Serial(device, baudrate, timeout=1)
+    reader = rpc.SerialReader(serial_device, 8192)
+    write = serial_device.write
+    return rpc.HdlcRpcClient(
+        reader,
+        PROTO,
+        rpc.default_channels(write),
+        lambda data: rpc.write_to_file(data, output),
+    )
+
+
+def run(args) -> int:
+    """Run the tests. Return the number of failed tests."""
+    with get_hdlc_rpc_client(**vars(args)) as client:
+        test_records = run_tests(client.rpcs())
+        return len(test_records.failing_tests)
+
+
+def list_images(flash_directory: str) -> list[str]:
+    filenames: list[str] = glob.glob(os.path.join(flash_directory, "*.s37"))
+    return list(map(lambda x: os.path.join(flash_directory, x), filenames))
+
+
+def main() -> int:
+    args = _parse_args()
+
+    failures = 0
+    if args.flash_image:
+        if os.path.isdir(args.flash_image):
+            images = list_images(args.flash_image)
+            if not images:
+                raise Exception(f"No images found in `{args.flash_image}`")
+        elif os.path.isdir(args.flash_image):
+            images = [args.flash_image]
+        else:
+            raise Exception(f"File or directory not found `{args.flash_image}`")
+
+        for image in images:
+            flash_device(args.device, image)
+            time.sleep(1)  # Give time for device to boot
+
+            failures += run(args)
+    else:  # No image provided. Just run what's on the device.
+        failures += run(args)
+
+    return failures
+
+
+if __name__ == "__main__":
+    sys.exit(main())