Create CHIP-specific backends for pigweed assert and logs (project-chip#36297)

* Add a matter backend for assertions, to not depend on basic assertions in pigweed itself

* No nullptr pass

* Add a log backend, update copyrights

* Restyle

* Switch all backends for log and assert to the new chip backends instead of previous built-in/basic ones. Remove old log backend

* Update public to fix linter for gn references

* Fix dependencies (this should fix boufalolab compile at least

* Remove pw_log_basic as we do not use it anymore

* Restyle

* Remove one more log_basic usage

* Fix typo in log type - application is not defined

* Avoid infinite recursion on logging now that PW-logging is not used

* Remove echo_pb2 from the console as this is already imported by the pw_system.console it seems

* Make PWConsole work again

* make linter happy

* Clean up all usages of CHIP_USE_PW_LOGGING

* Mass rename backends to pw_backends

* One more rename

* Remove some more includes

* Allow to keep going and report failed tetsts in local.py

* Do not force fabric admin on BLE ... that seems odd

* Switch to no wifi on fabric admin/bridge too

* More cleanup for sizes for subprocess

* More updates, generally switching to textio did not seem to work well, so just set bufsize to 0

* Update logic: apparently readlines is NOT lazy

* Remove unused import

* Fix unit tests

* Even better display and avoid crashes in pw rpc. Sill need to fix mcore test

* Fix flag logic

* Make MCOREFS_1_5 less racy: window status is updated several times it seems, so wait for the right status (it starts as not open, then switches to open)

* Restyled by autopep8

* Ensure elapsed and time remaining is always updated

* Make local.py have a less noisy argument since on failures we generally want to read the files not have them in stdout (except for CI where we only see stdout right now)

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>
Co-authored-by: Restyled.io <commits@restyled.io>

Author: 3 people

.github/workflows/tests.yaml

@@ -494,7 +494,7 @@ jobs:
                       --target linux-x64-microwave-oven-ipv6only-no-ble-no-wifi-tsan-clang-test \
                       --target linux-x64-rvc-ipv6only-no-ble-no-wifi-tsan-clang-test \
                       --target linux-x64-network-manager-ipv6only-no-ble-no-wifi-tsan-clang-test \
-                      --target linux-x64-fabric-admin-rpc-ipv6only-clang \
+                      --target linux-x64-fabric-admin-rpc-ipv6only-no-ble-no-wifi-clang \
                       --target linux-x64-fabric-bridge-rpc-ipv6only-no-ble-no-wifi-clang \
                       --target linux-x64-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang \
                       --target linux-x64-python-bindings \
@@ -511,7 +511,7 @@ jobs:
                   echo "CHIP_MICROWAVE_OVEN_APP: out/linux-x64-microwave-oven-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-microwave-oven-app" >> /tmp/test_env.yaml
                   echo "CHIP_RVC_APP: out/linux-x64-rvc-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-rvc-app" >> /tmp/test_env.yaml
                   echo "NETWORK_MANAGEMENT_APP: out/linux-x64-network-manager-ipv6only-no-ble-no-wifi-tsan-clang-test/matter-network-manager-app" >> /tmp/test_env.yaml
-                  echo "FABRIC_ADMIN_APP: out/linux-x64-fabric-admin-rpc-ipv6only-clang/fabric-admin" >> /tmp/test_env.yaml
+                  echo "FABRIC_ADMIN_APP: out/linux-x64-fabric-admin-rpc-ipv6only-no-ble-no-wifi-clang/fabric-admin" >> /tmp/test_env.yaml
                   echo "FABRIC_BRIDGE_APP: out/linux-x64-fabric-bridge-rpc-ipv6only-no-ble-no-wifi-clang/fabric-bridge-app" >> /tmp/test_env.yaml
                   echo "LIGHTING_APP_NO_UNIQUE_ID: out/linux-x64-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang/chip-lighting-app" >> /tmp/test_env.yaml
                   echo "TRACE_APP: out/trace_data/app-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml

.gn

@@ -35,8 +35,8 @@ default_args = {
 
   # Required for pw_unit_test
   pw_sys_io_BACKEND = "$dir_pw_sys_io_stdio"
-  pw_assert_BACKEND = "$dir_pw_assert_log"
-  pw_log_BACKEND = "$dir_pw_log_basic"
+  pw_assert_BACKEND = "//src/pw_backends/assert"
+  pw_log_BACKEND = "//src/pw_backends/log"
 
   # TODO: Make sure only unit tests link against this
   pw_build_LINK_DEPS = [

config/bouffalolab/common/lib/pw_rpc/pw_rpc.gni

@@ -15,8 +15,8 @@
 import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND =
     "${chip_root}/examples/platform/bouffalolab/common/rpc/pw_sys_io:pw_sys_io"
 

config/efr32/lib/pw_rpc/pw_rpc.gni

@@ -15,8 +15,8 @@
 import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_rpc_CONFIG = "$dir_pw_rpc:disable_global_mutex"
 pw_sys_io_BACKEND =
     "${chip_root}/examples/platform/silabs/pw_sys_io:pw_sys_io_silabs"

config/mbed/chip-gn/lib/pw_rpc/pw_rpc.gni

@@ -15,8 +15,8 @@
 import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND =
     "${chip_root}/examples/platform/mbed/pw_sys_io:pw_sys_io_mbed"
 pw_rpc_system_server_BACKEND =

config/nxp/lib/pw_rpc/pw_rpc.gni

@@ -15,8 +15,8 @@
 import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_rpc_CONFIG = "$dir_pw_rpc:disable_global_mutex"
 pw_sys_io_BACKEND = "${chip_root}/examples/platform/nxp/pw_sys_io:pw_sys_io_nxp"
 

config/qpg/lib/pw_rpc/pw_rpc.gni

@@ -15,8 +15,8 @@
 import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND = "${chip_root}/examples/platform/qpg/pw_sys_io:pw_sys_io_qpg"
 
 pw_build_LINK_DEPS = [

examples/all-clusters-app/linux/with_pw_rpc.gni

@@ -21,8 +21,8 @@ import("${chip_root}/config/standalone/args.gni")
 
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND = "$dir_pw_sys_io_stdio"
 pw_trace_BACKEND = "$dir_pw_trace_tokenized"
 pw_unit_test_MAIN = "$dir_pw_unit_test:logging_main"
@@ -39,4 +39,3 @@ pw_build_LINK_DEPS = [
 ]
 
 chip_enable_pw_rpc = true
-chip_use_pw_logging = true

examples/android/CHIPTest/args.gni

@@ -28,8 +28,8 @@ chip_monolithic_tests = false
 
 pw_sys_io_BACKEND = "$dir_pw_sys_io_stdio"
 
-pw_assert_BACKEND = "$dir_pw_assert_log"
-pw_log_BACKEND = "$dir_pw_log_basic"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
 
 pw_unit_test_BACKEND = "$dir_pw_unit_test:light"
 

examples/chef/linux/with_pw_rpc.gni

@@ -21,8 +21,8 @@ import("${chip_root}/config/standalone/args.gni")
 
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND = "$dir_pw_sys_io_stdio"
 pw_trace_BACKEND = "$dir_pw_trace_tokenized"
 pw_unit_test_MAIN = "$dir_pw_unit_test:logging_main"
@@ -39,4 +39,3 @@ pw_build_LINK_DEPS = [
 ]
 
 chip_enable_pw_rpc = true
-chip_use_pw_logging = true

examples/common/pigweed/RpcService.cpp

@@ -25,7 +25,6 @@
 #include "pw_hdlc/decoder.h"
 #include "pw_hdlc/default_addresses.h"
 #include "pw_hdlc/rpc_channel.h"
-#include "pw_log/log.h"
 #include "pw_rpc/channel.h"
 #include "pw_status/status.h"
 #include "pw_stream/sys_io_stream.h"
@@ -96,20 +95,6 @@ void Start(void (*RegisterServices)(pw::rpc::Server &), ::chip::rpc::Mutex * uar
     PW_DASSERT(RegisterServices != nullptr);
     uart_mutex = uart_mutex_;
 
-    // Send log messages to HDLC address 1. This prevents logs from interfering
-    // with pw_rpc communications.
-    pw::log_basic::SetOutput([](std::string_view log) {
-        if (uart_mutex)
-        {
-            uart_mutex->Lock();
-        }
-        pw::hdlc::WriteUIFrame(1, pw::as_bytes(pw::span(log)), sysIoWriter);
-        if (uart_mutex)
-        {
-            uart_mutex->Unlock();
-        }
-    });
-
     // Set up the server and start processing data.
     RegisterServices(server);
 

examples/common/pigweed/rpc_console/py/chip_rpc/console.py

@@ -39,7 +39,7 @@
 import argparse
 import sys
 from pathlib import Path
-from typing import Any, Collection
+from typing import Collection
 
 import pw_system.console
 from pw_hdlc import rpc
@@ -52,7 +52,6 @@
 from button_service import button_service_pb2
 from descriptor_service import descriptor_service_pb2
 from device_service import device_service_pb2
-from echo_service import echo_pb2
 from fabric_admin_service import fabric_admin_service_pb2
 from fabric_bridge_service import fabric_bridge_service_pb2
 from lighting_service import lighting_service_pb2
@@ -72,13 +71,6 @@ def _parse_args():
                         type=int,
                         default=115200,
                         help='the baud rate to use')
-    parser.add_argument(
-        '-o',
-        '--output',
-        type=argparse.FileType('wb'),
-        default=sys.stdout.buffer,
-        help=('The file to which to write device output (HDLC channel 1); '
-              'provide - or omit for stdout.'))
     parser.add_argument(
         '-r',
         '--raw_serial',
@@ -99,7 +91,7 @@ def _parse_args():
 
 def show_console(device: str, baudrate: int,
                  token_databases: Collection[Path],
-                 socket_addr: str, output: Any, raw_serial: bool) -> int:
+                 socket_addr: str, raw_serial: bool) -> int:
 
     # TODO: this shows a default console with little customization
     #       Ideally we should at least customize the default messages
@@ -114,30 +106,24 @@ def show_console(device: str, baudrate: int,
         device=device,
         baudrate=baudrate,
         socket_addr=socket_addr,
-        output=output,
         hdlc_encoding=not raw_serial,
         token_databases=token_databases,
         logfile="",
         device_logfile="",
         channel_id=rpc.DEFAULT_CHANNEL_ID,
 
         # Defaults beyond the original console
-        proto_globs=[],
         ticks_per_second=None,
         host_logfile="",
         json_logfile="",
         rpc_logging=False,
-        # the pt-python based console seems to break on python 3.1 with
-        # "set_wakeup_fd only works in main thread of the main interpreter"
-        use_ipython=True,
         compiled_protos=[
                 actions_service_pb2,
                 attributes_service_pb2,
                 boolean_state_service_pb2,
                 button_service_pb2,
                 descriptor_service_pb2,
                 device_service_pb2,
-                echo_pb2,
                 fabric_admin_service_pb2,
                 fabric_bridge_service_pb2,
                 lighting_service_pb2,

examples/common/pigweed/system_rpc_server.cc

@@ -20,7 +20,6 @@
 #include "pw_hdlc/decoder.h"
 #include "pw_hdlc/default_addresses.h"
 #include "pw_hdlc/rpc_channel.h"
-#include "pw_log/log.h"
 #include "pw_rpc_system_server/rpc_server.h"
 #include "pw_stream/sys_io_stream.h"
 
@@ -40,12 +39,7 @@ rpc::Server server(channels);
 
 } // namespace
 
-void Init()
-{
-    // Send log messages to HDLC address 1. This prevents logs from interfering
-    // with pw_rpc communications.
-    pw::log_basic::SetOutput([](std::string_view log) { pw::hdlc::WriteUIFrame(1, pw::as_bytes(pw::span(log)), writer); });
-}
+void Init() {}
 
 rpc::Server & Server()
 {

examples/fabric-admin/args.gni

@@ -32,4 +32,3 @@ matter_log_json_payload_decode_full = true
 # make fabric-admin very strict by default
 chip_tlv_validate_char_string_on_read = true
 chip_tlv_validate_char_string_on_write = true
-chip_enable_ble = true

examples/fabric-admin/scripts/fabric-sync-app.py

@@ -32,6 +32,7 @@ async def forward_f(prefix: bytes, f_in: asyncio.StreamReader,
 
     This function can optionally feed received lines to a callback function.
     """
+
     while line := await f_in.readline():
         if cb is not None:
             cb(line)

examples/fabric-admin/with_pw_rpc.gni

@@ -21,8 +21,8 @@ import("${chip_root}/config/standalone/args.gni")
 
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND = "$dir_pw_sys_io_stdio"
 pw_trace_BACKEND = "$dir_pw_trace_tokenized"
 pw_unit_test_MAIN = "$dir_pw_unit_test:logging_main"
@@ -39,4 +39,3 @@ pw_build_LINK_DEPS = [
 ]
 
 chip_enable_pw_rpc = true
-chip_use_pw_logging = true

examples/fabric-bridge-app/linux/with_pw_rpc.gni

@@ -21,8 +21,8 @@ import("${chip_root}/config/standalone/args.gni")
 
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND = "$dir_pw_sys_io_stdio"
 pw_trace_BACKEND = "$dir_pw_trace_tokenized"
 pw_unit_test_MAIN = "$dir_pw_unit_test:logging_main"
@@ -38,5 +38,4 @@ pw_build_LINK_DEPS = [
   "$dir_pw_log:impl",
 ]
 
-chip_use_pw_logging = true
 bridge_enable_pw_rpc = true

examples/light-switch-app/genio/args.gni

@@ -18,8 +18,8 @@ import("${chip_root}/src/platform/mt793x/args.gni")
 
 mt793x_sdk_target = get_label_info(":sdk", "label_no_toolchain")
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 
 chip_enable_ble = true
 

examples/light-switch-app/genio/build_for_wifi_args.gni

@@ -18,5 +18,5 @@ mt793x_sdk_target = get_label_info(":sdk", "label_no_toolchain")
 chip_enable_openthread = false
 import("${chip_root}/src/platform/MT793X/wifi_args.gni")
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"

examples/light-switch-app/qpg/args.gni

@@ -32,5 +32,5 @@ chip_stack_lock_tracking = "none"
 matter_device_vid = "0xFFF1"
 matter_device_pid = "0x8004"
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"

examples/lighting-app-data-mode-no-unique-id/linux/with_pw_rpc.gni

@@ -21,8 +21,8 @@ import("${chip_root}/config/standalone/args.gni")
 
 import("//build_overrides/pigweed.gni")
 
-pw_log_BACKEND = "$dir_pw_log_basic"
-pw_assert_BACKEND = "$dir_pw_assert_log:check_backend"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_sys_io_BACKEND = "$dir_pw_sys_io_stdio"
 pw_trace_BACKEND = "$dir_pw_trace_tokenized"
 pw_unit_test_MAIN = "$dir_pw_unit_test:logging_main"
@@ -39,4 +39,3 @@ pw_build_LINK_DEPS = [
 ]
 
 chip_enable_pw_rpc = true
-chip_use_pw_logging = true

examples/lighting-app/bouffalolab/bl602/args.gni

@@ -19,8 +19,8 @@ import("${chip_root}/src/platform/bouffalolab/BL602/args.gni")
 
 bl_iot_sdk_target = get_label_info(":sdk", "label_no_toolchain")
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_rpc_CONFIG = "$dir_pw_rpc:disable_global_mutex"
 
 chip_detail_logging = false

examples/lighting-app/bouffalolab/bl702/args.gni

@@ -19,8 +19,8 @@ import("${chip_root}/src/platform/bouffalolab/BL702/args.gni")
 
 bl_iot_sdk_target = get_label_info(":sdk", "label_no_toolchain")
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_rpc_CONFIG = "$dir_pw_rpc:disable_global_mutex"
 
 chip_detail_logging = false

examples/lighting-app/bouffalolab/bl702l/args.gni

@@ -19,8 +19,8 @@ import("${chip_root}/src/platform/bouffalolab/BL702L/args.gni")
 
 bl_iot_sdk_target = get_label_info(":sdk", "label_no_toolchain")
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 pw_rpc_CONFIG = "$dir_pw_rpc:disable_global_mutex"
 
 chip_detail_logging = false

examples/lighting-app/genio/args.gni

@@ -19,8 +19,8 @@ import("${chip_root}/src/platform/mt793x/args.gni")
 
 mt793x_sdk_target = get_label_info(":sdk", "label_no_toolchain")
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"
 
 chip_enable_ble = true
 

examples/lighting-app/genio/build_for_wifi_args.gni

@@ -18,5 +18,5 @@ mt793x_sdk_target = get_label_info(":sdk", "label_no_toolchain")
 chip_enable_openthread = false
 import("${chip_root}/src/platform/MT793X/wifi_args.gni")
 
-pw_log_BACKEND = "${chip_root}/src/lib/support/pw_log_chip"
-pw_assert_BACKEND = "$dir_pw_assert_log"
+pw_log_BACKEND = "${chip_root}/src/pw_backends/log"
+pw_assert_BACKEND = "${chip_root}/src/pw_backends/assert"