Update python tests to maintain the test metadata as a local file (project-chip#36359)

andy31415 · web-flow · commit 62a4a9758ec9 · 2024-11-04T23:02:29.000Z
* Add metadata that is shared for python tests

* More updates

* Fix typo

* More metadata updates
diff --git a/scripts/tests/local.py b/scripts/tests/local.py
@@ -33,6 +33,7 @@
 import click
 import coloredlogs
 import tabulate
+import yaml
 
 # We compile for the local architecture. Figure out what platform we need
 
@@ -43,12 +44,12 @@ def _get_native_machine_target():
     """
     current_system_info = platform.uname()
     arch = current_system_info.machine
-    if arch == 'x86_64':
-        arch = 'x64'
-    elif arch == 'i386' or arch == 'i686':
-        arch = 'x86'
-    elif arch in ('aarch64', 'aarch64_be', 'armv8b', 'armv8l'):
-        arch = 'arm64'
+    if arch == "x86_64":
+        arch = "x64"
+    elif arch == "i386" or arch == "i686":
+        arch = "x86"
+    elif arch in ("aarch64", "aarch64_be", "armv8b", "armv8l"):
+        arch = "arm64"
 
     return f"{current_system_info.system.lower()}-{arch}"
 
@@ -165,14 +166,17 @@ def _do_build_apps():
         f"{target_prefix}-all-clusters-no-ble-clang-boringssl",
         f"{target_prefix}-bridge-no-ble-clang-boringssl",
         f"{target_prefix}-energy-management-no-ble-clang-boringssl",
+        f"{target_prefix}-fabric-admin-rpc-ipv6only-clang-boringssl",
+        f"{target_prefix}-fabric-bridge-rpc-ipv6only-clang-boringssl",
+        f"{target_prefix}-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang",
         f"{target_prefix}-lit-icd-no-ble-clang-boringssl",
         f"{target_prefix}-lock-no-ble-clang-boringssl",
         f"{target_prefix}-microwave-oven-no-ble-clang-boringssl",
+        f"{target_prefix}-network-manager-ipv6only-no-ble-clang-boringssl",
         f"{target_prefix}-ota-provider-no-ble-clang-boringssl",
         f"{target_prefix}-ota-requestor-no-ble-clang-boringssl",
         f"{target_prefix}-rvc-no-ble-clang-boringssl",
         f"{target_prefix}-tv-app-no-ble-clang-boringssl",
-        f"{target_prefix}-network-manager-ipv6only-no-ble-clang-boringssl",
     ]
 
     cmd = ["./scripts/build/build_examples.py"]
@@ -354,6 +358,11 @@ def as_runner(path):
             CHIP_RVC_APP: {as_runner(f'out/{target_prefix}-rvc-no-ble-clang-boringssl/chip-rvc-app')}
             NETWORK_MANAGEMENT_APP: {
                 as_runner(f'out/{target_prefix}-network-manager-ipv6only-no-ble-clang-boringssl/matter-network-manager-app')}
+            FABRIC_ADMIN_APP: {
+                as_runner(f'out/{target_prefix}-fabric-admin-rpc-ipv6only-clang-boringssl/fabric-admin')}
+            FABRIC_BRIDGE_APP: {
+                as_runner(f'out/{target_prefix}-fabric-bridge-rpc-ipv6only-clang-boringssl/fabric-bridge-app')}
+            LIGHTING_APP_NO_UNIQUE_ID: {as_runner(f'out/{target_prefix}-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang/chip-lighting-app')}
             TRACE_APP: out/trace_data/app-{{SCRIPT_BASE_NAME}}
             TRACE_TEST_JSON: out/trace_data/test-{{SCRIPT_BASE_NAME}}
             TRACE_TEST_PERFETTO: out/trace_data/test-{{SCRIPT_BASE_NAME}}
@@ -382,58 +391,11 @@ def as_runner(path):
     if not os.path.exists("out/trace_data"):
         os.mkdir("out/trace_data")
 
-    # IGNORES are taken out of `src/python_testing/execute_python_tests.py` in the SDK
-    excluded_patterns = {
-        "MinimalRepresentation.py",
-        "TC_CNET_4_4.py",
-        "TC_CCTRL_2_1.py",
-        "TC_CCTRL_2_2.py",
-        "TC_CCTRL_2_3.py",
-        "TC_DGGEN_3_2.py",
-        "TC_EEVSE_Utils.py",
-        "TC_ECOINFO_2_1.py",
-        "TC_ECOINFO_2_2.py",
-        "TC_EWATERHTRBase.py",
-        "TC_EWATERHTR_2_1.py",
-        "TC_EWATERHTR_2_2.py",
-        "TC_EWATERHTR_2_3.py",
-        "TC_EnergyReporting_Utils.py",
-        "TC_OpstateCommon.py",
-        "TC_pics_checker.py",
-        "TC_TMP_2_1.py",
-        "TC_MCORE_FS_1_1.py",
-        "TC_MCORE_FS_1_2.py",
-        "TC_MCORE_FS_1_3.py",
-        "TC_MCORE_FS_1_4.py",
-        "TC_MCORE_FS_1_5.py",
-        "TC_OCC_3_1.py",
-        "TC_OCC_3_2.py",
-        "TC_BRBINFO_4_1.py",
-        "TestCommissioningTimeSync.py",
-        "TestConformanceSupport.py",
-        "TestChoiceConformanceSupport.py",
-        "TC_DEMTestBase.py",
-        "choice_conformance_support.py",
-        "TestConformanceTest.py",  # Unit test of the conformance test (TC_DeviceConformance) - does not run against an app.
-        "TestIdChecks.py",
-        "TestSpecParsingDeviceType.py",
-        "TestMatterTestingSupport.py",
-        "TestSpecParsingSupport.py",
-        "TestTimeSyncTrustedTimeSource.py",
-        "basic_composition_support.py",
-        "conformance_support.py",
-        "drlk_2_x_common.py",
-        "execute_python_tests.py",
-        "global_attribute_ids.py",
-        "hello_external_runner.py",
-        "hello_test.py",
-        "matter_testing_support.py",
-        "pics_support.py",
-        "spec_parsing_support.py",
-        "taglist_and_topology_test_support.py",
-        "test_plan_support.py",
-        "test_plan_table_generator.py",
-    }
+    metadata = yaml.full_load(open("src/python_testing/test_metadata.yaml"))
+    excluded_patterns = set([item["name"] for item in metadata["not_automated"]])
+
+    # NOTE: for slow tests. we add logs to not get impatient
+    slow_test_duration = dict([(item["name"], item["duration"]) for item in metadata["slow_tests"]])
 
     if not os.path.isdir("src/python_testing"):
         raise Exception(
@@ -448,31 +410,6 @@ def as_runner(path):
     test_scripts.append("src/controller/python/test/test_scripts/mobile-device-test.py")
     test_scripts.sort()  # order consistent
 
-    # NOTE: VERY slow tests. we add logs to not get impatient
-    slow_test_duration = {
-        "mobile-device-test.py": "3 minutes",
-        "TC_AccessChecker.py": "1.5 minutes",
-        "TC_CADMIN_1_9.py": "40 seconds",
-        "TC_CC_2_2.py": "1.5 minutes",
-        "TC_DEM_2_10.py": "40 seconds",
-        "TC_DeviceBasicComposition.py": "25 seconds",
-        "TC_DRLK_2_12.py": "30 seconds",
-        "TC_DRLK_2_3.py": "30 seconds",
-        "TC_EEVSE_2_6.py": "30 seconds",
-        "TC_FAN_3_1.py": "15 seconds",
-        "TC_OPSTATE_2_5.py": "1.25 minutes",
-        "TC_OPSTATE_2_6.py": "35 seconds",
-        "TC_PS_2_3.py": "30 seconds",
-        "TC_RR_1_1.py": "25 seconds",
-        "TC_SWTCH.py": "1 minute",
-        "TC_TIMESYNC_2_10.py": "20 seconds",
-        "TC_TIMESYNC_2_11.py": "30 seconds",
-        "TC_TIMESYNC_2_12.py": "20 seconds",
-        "TC_TIMESYNC_2_7.py": "20 seconds",
-        "TC_TIMESYNC_2_8.py": "1.5 minutes",
-        "TC_ICDM_5_1.py": "TODO",
-    }
-
     execution_times = []
     try:
         to_run = []
@@ -698,7 +635,10 @@ def chip_tool_tests(target, target_glob, include_tags, expected_failures, runner
 
     target_prefix = _get_native_machine_target()
     cmd.extend(
-        ["--chip-tool", f"./out/{target_prefix}-chip-tool-no-ble-clang-boringssl/chip-tool"]
+        [
+            "--chip-tool",
+            f"./out/{target_prefix}-chip-tool-no-ble-clang-boringssl/chip-tool",
+        ]
     )
 
     if target is not None:
diff --git a/src/python_testing/execute_python_tests.py b/src/python_testing/execute_python_tests.py
@@ -20,6 +20,8 @@
 import os
 import subprocess
 
+import yaml
+
 # Function to load --app argument environment variables from a file
 
 
@@ -36,11 +38,8 @@ def load_env_from_yaml(file_path):
     Args:
         file_path (str): The path to the YAML file containing the environment variables.
     """
-    with open(file_path, 'r') as file:
-        for line in file:
-            if line.strip():  # Skip empty lines
-                key, value = line.strip().split(': ', 1)
-                os.environ[key] = value
+    for key, value in yaml.full_load(open(file_path, 'r')).items():
+        os.environ[key] = value
 
 
 def main(search_directory, env_file):
@@ -53,38 +52,8 @@ def main(search_directory, env_file):
     # Define the base command to run tests
     base_command = os.path.join(chip_root, "scripts/tests/run_python_test.py")
 
-    # Define the test python script files and patterns to exclude
-    excluded_patterns = {
-        "MinimalRepresentation.py",  # Code/Test not being used or not shared code for any other tests
-        "TC_CNET_4_4.py",  # It has no CI execution block, is not executed in CI
-        "TC_DGGEN_3_2.py",  # src/python_testing/test_testing/test_TC_DGGEN_3_2.py is the Unit test of this test
-        "TC_EEVSE_Utils.py",  # Shared code for TC_EEVSE, not a standalone test
-        "TC_EWATERHTRBase.py",  # Shared code for TC_EWATERHTR, not a standalone test
-        "TC_EnergyReporting_Utils.py",  # Shared code for TC_EEM and TC_EPM, not a standalone test
-        "TC_OpstateCommon.py",  # Shared code for TC_OPSTATE, not a standalone test
-        "TC_pics_checker.py",  # Currently isn't enabled because we don't have any examples with conformant PICS
-        "TC_TMP_2_1.py",  # src/python_testing/test_testing/test_TC_TMP_2_1.py is the Unit test of this test
-        "TC_OCC_3_1.py",  # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change
-        "TC_OCC_3_2.py",  # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change
-        "TestCommissioningTimeSync.py",  # Code/Test not being used or not shared code for any other tests
-        "TestConformanceSupport.py",  # Unit test - does not run against an app
-        "TestChoiceConformanceSupport.py",  # Unit test - does not run against an app
-        "TC_DEMTestBase.py",  # Shared code for TC_DEM, not a standalone test
-        "TestConformanceTest.py",  # Unit test of the conformance test (TC_DeviceConformance) - does not run against an app
-        "TestIdChecks.py",  # Unit test - does not run against an app
-        "TestSpecParsingDeviceType.py",  # Unit test - does not run against an app
-        "TestConformanceTest.py",  # Unit test - does not run against an app
-        "TestMatterTestingSupport.py",  # Unit test - does not run against an app
-        "TestSpecParsingSupport.py",  # Unit test - does not run against an app
-        "TestTimeSyncTrustedTimeSource.py",  # Unit test and shared code for scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py
-        "drlk_2_x_common.py",  # Shared code for TC_DRLK, not a standalone test
-        # Python code that runs all the python tests from src/python_testing (This code itself run via tests.yaml)
-        "execute_python_tests.py",
-        "hello_external_runner.py",  # Code/Test not being used or not shared code for any other tests
-        "hello_test.py",  # Is a template for tests
-        "test_plan_support.py",  # Shared code for TC_*, not a standalone test
-        "test_plan_table_generator.py",  # Code/Test not being used or not shared code for any other tests
-    }
+    metadata = yaml.full_load(open(os.path.join(chip_root, "src/python_testing/test_metadata.yaml")))
+    excluded_patterns = set([item["name"] for item in metadata["not_automated"]])
 
     # Get all .py files in the directory
     all_python_files = glob.glob(os.path.join(search_directory, "*.py"))
diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/tasks.py b/src/python_testing/matter_testing_infrastructure/chip/testing/tasks.py
@@ -123,7 +123,7 @@ def start(self,
                 # hang on the join call in our thread entry point in case of
                 # Python process termination (not-caught exception).
                 self.p.terminate()
-                raise TimeoutError("Expected output not found")
+                raise TimeoutError("Expected output '%r' not found within %s seconds" % (expected_output, timeout))
             self.expected_output = None
 
     def send(self, message: str, end: str = "\n",
diff --git a/src/python_testing/test_metadata.yaml b/src/python_testing/test_metadata.yaml
@@ -0,0 +1,102 @@
+# Tests that are NOT executed in CI, along with the reason on
+# why they are not.
+not_automated:
+    - name: MinimalRepresentation.py
+      reason: Code/Test not being used or not shared code for any other tests
+    - name: TC_CNET_4_4.py
+      reason: It has no CI execution block, is not executed in CI
+    - name: TC_DGGEN_3_2.py
+      reason:
+          src/python_testing/test_testing/test_TC_DGGEN_3_2.py is the Unit test
+          of this test
+    - name: TC_EEVSE_Utils.py
+      reason: Shared code for TC_EEVSE, not a standalone test
+    - name: TC_EWATERHTRBase.py
+      reason: Shared code for TC_EWATERHTR, not a standalone test
+    - name: TC_EnergyReporting_Utils.py
+      reason: Shared code for TC_EEM and TC_EPM, not a standalone test
+    - name: TC_OpstateCommon.py
+      reason: Shared code for TC_OPSTATE, not a standalone test
+    - name: TC_pics_checker.py
+      reason:
+          Currently isn't enabled because we don't have any examples with
+          conformant PICS
+    - name: TC_TMP_2_1.py
+      reason:
+          src/python_testing/test_testing/test_TC_TMP_2_1.py is the Unit test of
+          this test
+    - name: TC_OCC_3_1.py
+      reason:
+          There are CI issues for the test cases that implements manually
+          controlling sensor device for the occupancy state ON/OFF change
+    - name: TC_OCC_3_2.py
+      reason:
+          There are CI issues for the test cases that implements manually
+          controlling sensor device for the occupancy state ON/OFF change
+    - name: TestCommissioningTimeSync.py
+      reason: Code/Test not being used or not shared code for any other tests
+    - name: TestConformanceSupport.py
+      reason: Unit test - does not run against an app
+    - name: TestChoiceConformanceSupport.py
+      reason: Unit test - does not run against an app
+    - name: TC_DEMTestBase.py
+      reason: Shared code for TC_DEM, not a standalone test
+    - name: TestConformanceTest.py
+      reason:
+          Unit test of the conformance test (TC_DeviceConformance) - does not
+          run against an app
+    - name: TestIdChecks.py
+      reason: Unit test - does not run against an app
+    - name: TestSpecParsingDeviceType.py
+      reason: Unit test - does not run against an app
+    - name: TestConformanceTest.py
+      reason: Unit test - does not run against an app
+    - name: TestMatterTestingSupport.py
+      reason: Unit test - does not run against an app
+    - name: TestSpecParsingSupport.py
+      reason: Unit test - does not run against an app
+    - name: TestTimeSyncTrustedTimeSource.py
+      reason:
+          Unit test and shared code for
+          scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py
+    - name: drlk_2_x_common.py
+      reason: Shared code for TC_DRLK, not a standalone test
+    - name: execute_python_tests.py
+      reason:
+          Python code that runs all the python tests from src/python_testing
+          (This code itself run via tests.yaml)
+    - name: hello_external_runner.py
+      reason: Code/Test not being used or not shared code for any other tests
+    - name: hello_test.py
+      reason: Is a template for tests
+    - name: test_plan_support.py
+      reason: Shared code for TC_*, not a standalone test
+    - name: test_plan_table_generator.py
+      reason: Code/Test not being used or not shared code for any other tests
+
+# This is a list of slow tests (just arbitrarely picked around 20 seconds)
+# used in some script reporting for "be patient" messages as well as potentially
+# to consider improving. May not be exhaustive
+slow_tests:
+    - { name: mobile-device-test.py, duration: 3 minutes }
+    - { name: TC_AccessChecker.py, duration: 1.5 minutes }
+    - { name: TC_BRBINFO_4_1.py, duration: 2 minutes }
+    - { name: TC_CADMIN_1_9.py, duration: 40 seconds }
+    - { name: TC_CC_2_2.py, duration: 1.5 minutes }
+    - { name: TC_DEM_2_10.py, duration: 40 seconds }
+    - { name: TC_DeviceBasicComposition.py, duration: 25 seconds }
+    - { name: TC_DRLK_2_12.py, duration: 30 seconds }
+    - { name: TC_DRLK_2_3.py, duration: 30 seconds }
+    - { name: TC_EEVSE_2_6.py, duration: 30 seconds }
+    - { name: TC_FAN_3_1.py, duration: 15 seconds }
+    - { name: TC_MCORE_FS_1_4.py, duration: 20 seconds }
+    - { name: TC_OPSTATE_2_5.py, duration: 1.25 minutes }
+    - { name: TC_OPSTATE_2_6.py, duration: 35 seconds }
+    - { name: TC_PS_2_3.py, duration: 30 seconds }
+    - { name: TC_RR_1_1.py, duration: 25 seconds }
+    - { name: TC_SWTCH.py, duration: 1 minute }
+    - { name: TC_TIMESYNC_2_10.py, duration: 20 seconds }
+    - { name: TC_TIMESYNC_2_11.py, duration: 30 seconds }
+    - { name: TC_TIMESYNC_2_12.py, duration: 20 seconds }
+    - { name: TC_TIMESYNC_2_7.py, duration: 20 seconds }
+    - { name: TC_TIMESYNC_2_8.py, duration: 1.5 minutes }
