Merge branch 'master' into feature/app-install-flow-public

Author: lazarkov

docs/testing/python.md

@@ -19,6 +19,11 @@ Python tests located in src/python_testing
 
 ## Writing Python tests
 
+-   Defining arguments in the test script
+    -   In order to streamline the configuration and execution of tests, it is
+        essential to define arguments at the top of the test script. This
+        section should include various parameters and their respective values,
+        which will guide the test runner on how to execute the tests.
 -   All test classes inherit from MatterBaseTest in
     [matter_testing_support.py](https://github.com/project-chip/connectedhomeip/blob/master/src/python_testing/matter_testing_support.py)
     -   support for commissioning using the python controller
@@ -36,11 +41,17 @@ Python tests located in src/python_testing
 -   Use Mobly assertions for failing tests
 -   self.step() along with a steps\_ function to mark test plan steps for cert
     tests
--
 
 ### A simple test
 
 ```
+# test-runner-runs: run1
+# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
+# test-runner-run/run1/factoryreset: True
+# test-runner-run/run1/quiet: True
+# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+
 class TC_MYTEST_1_1(MatterBaseTest):
 
     @async_test_body
@@ -74,6 +85,59 @@ The default_matter_test_main() function is used to run the test on the command
 line. These two lines should appear verbatim at the bottom of every python test
 file.
 
+## Defining the test arguments
+
+Below is the format:
+
+```
+# test-runner-runs: <run_identifier>
+# test-runner-run/<run_identifier>/app: ${TYPE_OF_APP}
+# test-runner-run/<run_identifier>/factoryreset: <True|False>
+# test-runner-run/<run_identifier>/quiet: <True|False>
+# test-runner-run/<run_identifier>/app-args: <app_arguments>
+# test-runner-run/<run_identifier>/script-args: <script_arguments>
+```
+
+### Description of Parameters
+
+-   test-runner-runs: Specifies the identifier for the run. This can be any
+    unique identifier.
+
+    -   Example: run1
+
+-   test-runner-run/<run_identifier>/app: Indicates the application to be used
+    in the test. Different app types as needed could be referenced from section
+    [name: Generate an argument environment file ] of the file
+    [.github/workflows/tests.yaml](https://github.com/project-chip/connectedhomeip/blob/master/.github/workflows/tests.yaml)
+
+        -   Example: \${TYPE_OF_APP}
+
+-   test-runner-run/<run_identifier>/factoryreset: Determines whether a factory
+    reset should be performed before the test.
+
+    -   Example: True
+
+-   test-runner-run/<run_identifier>/quiet: Sets the verbosity level of the test
+    run. When set to True, the test run will be quieter.
+
+    -   Example: True
+
+-   test-runner-run/<run_identifier>/app-args: Specifies the arguments to be
+    passed to the application during the test.
+
+    -   Example: --discriminator 1234 --KVS kvs1 --trace-to
+        json:\${TRACE_APP}.json
+
+-   test-runner-run/<run_identifier>/script-args: Specifies the arguments to be
+    passed to the test script.
+    -   Example: --storage-path admin_storage.json --commissioning-method
+        on-network --discriminator 1234 --passcode 20202021 --trace-to
+        json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+
+This structured format ensures that all necessary configurations are clearly
+defined and easily understood, allowing for consistent and reliable test
+execution.
+
 ## Cluster Codegen
 
 -   [Objects.py](https://github.com/project-chip/connectedhomeip/blob/master/src/controller/python/chip/clusters/Objects.py)

examples/platform/linux/Options.cpp

@@ -352,27 +352,28 @@ bool HandleOption(const char * aProgram, OptionSet * aOptions, int aIdentifier,
         break;
 
     case kDeviceOption_Version:
-        LinuxDeviceOptions::GetInstance().payload.version = static_cast<uint8_t>(atoi(aValue));
+        LinuxDeviceOptions::GetInstance().payload.version = static_cast<uint8_t>(strtoul(aValue, nullptr, 0));
         break;
 
     case kDeviceOption_VendorID:
-        LinuxDeviceOptions::GetInstance().payload.vendorID = static_cast<uint16_t>(atoi(aValue));
+        LinuxDeviceOptions::GetInstance().payload.vendorID = static_cast<uint16_t>(strtoul(aValue, nullptr, 0));
         break;
 
     case kDeviceOption_ProductID:
-        LinuxDeviceOptions::GetInstance().payload.productID = static_cast<uint16_t>(atoi(aValue));
+        LinuxDeviceOptions::GetInstance().payload.productID = static_cast<uint16_t>(strtoul(aValue, nullptr, 0));
         break;
 
     case kDeviceOption_CustomFlow:
-        LinuxDeviceOptions::GetInstance().payload.commissioningFlow = static_cast<CommissioningFlow>(atoi(aValue));
+        LinuxDeviceOptions::GetInstance().payload.commissioningFlow = static_cast<CommissioningFlow>(strtoul(aValue, nullptr, 0));
         break;
 
     case kDeviceOption_Capabilities:
-        LinuxDeviceOptions::GetInstance().payload.rendezvousInformation.Emplace().SetRaw(static_cast<uint8_t>(atoi(aValue)));
+        LinuxDeviceOptions::GetInstance().payload.rendezvousInformation.Emplace().SetRaw(
+            static_cast<uint8_t>(strtoul(aValue, nullptr, 0)));
         break;
 
     case kDeviceOption_Discriminator: {
-        uint16_t value = static_cast<uint16_t>(atoi(aValue));
+        uint16_t value = static_cast<uint16_t>(strtoul(aValue, nullptr, 0));
         if (value >= 4096)
         {
             PrintArgError("%s: invalid value specified for discriminator: %s\n", aProgram, aValue);
@@ -386,7 +387,7 @@ bool HandleOption(const char * aProgram, OptionSet * aOptions, int aIdentifier,
     }
 
     case kDeviceOption_Passcode:
-        LinuxDeviceOptions::GetInstance().payload.setUpPINCode = static_cast<uint32_t>(atoi(aValue));
+        LinuxDeviceOptions::GetInstance().payload.setUpPINCode = static_cast<uint32_t>(strtoul(aValue, nullptr, 0));
         break;
 
     case kDeviceOption_Spake2pSaltBase64: {
@@ -476,11 +477,9 @@ bool HandleOption(const char * aProgram, OptionSet * aOptions, int aIdentifier,
     case kDeviceOption_SecuredCommissionerPort:
         LinuxDeviceOptions::GetInstance().securedCommissionerPort = static_cast<uint16_t>(atoi(aValue));
         break;
-    case kCommissionerOption_FabricID: {
-        char * eptr;
-        LinuxDeviceOptions::GetInstance().commissionerFabricId = (chip::FabricId) strtoull(aValue, &eptr, 0);
+    case kCommissionerOption_FabricID:
+        LinuxDeviceOptions::GetInstance().commissionerFabricId = static_cast<chip::FabricId>(strtoull(aValue, nullptr, 0));
         break;
-    }
 #endif
 
     case kDeviceOption_Command:

scripts/py_matter_yamltests/matter_yamltests/hooks.py

@@ -104,7 +104,7 @@ def stop(self, duration: int):
         """
         pass
 
-    def test_start(self, filename: str, name: str, count: int):
+    def test_start(self, filename: str, name: str, count: int, steps: list[str] = []):
         """
         This method is called when the runner starts running a single test.
 
@@ -118,6 +118,9 @@ def test_start(self, filename: str, name: str, count: int):
 
         count: int
             The number of steps from the test that will be run.
+
+        steps: list[str]
+            The computed test step names
         """
         pass
 

scripts/tests/chiptest/__init__.py

@@ -226,50 +226,17 @@ def _GetChipReplUnsupportedTests() -> Set[str]:
     return {
         "Test_AddNewFabricFromExistingFabric.yaml",     # chip-repl does not support GetCommissionerRootCertificate and IssueNocChain command
         "Test_TC_OPCREDS_3_7.yaml",         # chip-repl does not support GetCommissionerRootCertificate and IssueNocChain command
-        "TestEqualities.yaml",              # chip-repl does not support pseudo-cluster commands that return a value
         "TestExampleCluster.yaml",          # chip-repl does not load custom pseudo clusters
         "TestAttributesById.yaml",           # chip-repl does not support AnyCommands (06/06/2023)
         "TestCommandsById.yaml",             # chip-repl does not support AnyCommands (06/06/2023)
         "TestEventsById.yaml",               # chip-repl does not support AnyCommands (06/06/2023)
         "TestReadNoneSubscribeNone.yaml",    # chip-repl does not support AnyCommands (07/27/2023)
-        "Test_TC_DRLK_2_8.yaml",   # Test fails only in chip-repl: Refer--> https://github.com/project-chip/connectedhomeip/pull/27011#issuecomment-1593339855
-        "Test_TC_ACE_1_6.yaml",    # Test fails only in chip-repl: Refer--> https://github.com/project-chip/connectedhomeip/pull/27910#issuecomment-1632485584
         "Test_TC_IDM_1_2.yaml",              # chip-repl does not support AnyCommands (19/07/2023)
-        "TestGroupKeyManagementCluster.yaml",  # chip-repl does not support EqualityCommands (2023-08-04)
         "TestIcdManagementCluster.yaml",   # TODO(#30430): add ICD registration support in chip-repl
         "Test_TC_ICDM_3_4.yaml",           # chip-repl does not support ICD registration
-        "Test_TC_S_2_2.yaml",              # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_MOD_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_MOD_3_2.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_MOD_3_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_MOD_3_4.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_BRBINFO_2_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_DGGEN_2_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_DGGEN_2_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_LWM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_LWM_3_2.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_LWM_3_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_OTCCM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_OTCCM_3_2.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_OTCCM_3_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_G_2_4.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_RVCRUNM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_RVCCLEANM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_TCCM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_TCCM_3_2.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_TCCM_3_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_TCTL_2_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
         # chip-repl and chip-tool disagree on what the YAML here should look like: https://github.com/project-chip/connectedhomeip/issues/29110
         "TestClusterMultiFabric.yaml",
-        "Test_TC_ACL_2_5.yaml",            # chip-repl does not support LastReceivedEventNumber : https://github.com/project-chip/connectedhomeip/issues/28884
-        "Test_TC_ACL_2_6.yaml",            # chip-repl does not support LastReceivedEventNumber : https://github.com/project-chip/connectedhomeip/issues/28884
-        "Test_TC_RVCCLEANM_3_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_BINFO_2_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
         "TestDiagnosticLogs.yaml",          # chip-repl does not implement a BDXTransferServerDelegate
-        "Test_TC_EEVSEM_2_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_EEVSEM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_EEVSEM_3_2.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
-        "Test_TC_EEVSEM_3_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
         "TestDiagnosticLogsDownloadCommand.yaml",  # chip-repl does not implement the bdx download command
     }
 

scripts/tests/chipyaml/tests_logger.py

@@ -161,7 +161,7 @@ def start(self, count: int):
     def stop(self, duration: int):
         print(self.__strings.stop.format(runned=self.__runned, skipped=self.__skipped, duration=duration))
 
-    def test_start(self, filename: str, name: str, count: int):
+    def test_start(self, filename: str, name: str, count: int, steps: list[str] = []):
         print(self.__strings.test_start.format(name=click.style(name, bold=True), count=click.style(count, bold=True)))
 
         if self.__use_test_harness_log_format:

scripts/tests/run_python_test.py

@@ -110,6 +110,10 @@ def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: st
             )
         ]
 
+    if not runs:
+        raise Exception(
+            "No valid runs were found. Make sure you add runs to your file, see https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md document for reference/example.")
+
     for run in runs:
         print(f"Executing {run.py_script_path.split('/')[-1]} {run.run}")
         main_impl(run.app, run.factoryreset, run.factoryreset_app_only, run.app_args,

src/controller/python/chip/yaml/runner.py

@@ -31,14 +31,11 @@
 from chip.exceptions import ChipStackError
 from chip.yaml.data_model_lookup import DataModelLookup
 from chip.yaml.errors import ActionCreationError, UnexpectedActionCreationError
-from matter_yamltests.pseudo_clusters.clusters.delay_commands import DelayCommands
-from matter_yamltests.pseudo_clusters.clusters.log_commands import LogCommands
-from matter_yamltests.pseudo_clusters.clusters.system_commands import SystemCommands
-from matter_yamltests.pseudo_clusters.pseudo_clusters import PseudoClusters
+from matter_yamltests.pseudo_clusters.pseudo_clusters import get_default_pseudo_clusters
 
 from .data_model_lookup import PreDefinedDataModelLookup
 
-_PSEUDO_CLUSTERS = PseudoClusters([DelayCommands(), LogCommands(), SystemCommands()])
+_PSEUDO_CLUSTERS = get_default_pseudo_clusters()
 logger = logging.getLogger('YamlParser')
 
 
@@ -129,8 +126,8 @@ def __init__(self, test_step):
             raise ActionCreationError(f'Default cluster {test_step.cluster} {test_step.command}, not supported')
 
     async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
-        _ = await _PSEUDO_CLUSTERS.execute(self._test_step)
-        return _ActionResult(status=_ActionStatus.SUCCESS, response=None)
+        response = await _PSEUDO_CLUSTERS.execute(self._test_step)
+        return _ActionResult(status=_ActionStatus.SUCCESS, response=response[0])
 
 
 class InvokeAction(BaseAction):
@@ -884,8 +881,12 @@ def decode(self, result: _ActionResult):
         response = result.response
 
         decoded_response = {}
+        if isinstance(response, dict):
+            return response
+
         if isinstance(response, chip.interaction_model.InteractionModelError):
             decoded_response['error'] = stringcase.snakecase(response.status.name).upper()
+            decoded_response['clusterError'] = response.clusterStatus
             return decoded_response
 
         if isinstance(response, chip.interaction_model.Status):
@@ -939,12 +940,13 @@ def decode(self, result: _ActionResult):
                 cluster_id = event.Header.ClusterId
                 cluster_name = self._test_spec_definition.get_cluster_name(cluster_id)
                 event_id = event.Header.EventId
+                event_number = event.Header.EventNumber
                 event_name = self._test_spec_definition.get_event_name(cluster_id, event_id)
                 event_definition = self._test_spec_definition.get_event_by_name(cluster_name, event_name)
                 is_fabric_scoped = bool(event_definition.is_fabric_sensitive)
                 decoded_event = Converter.from_data_model_to_test_definition(
                     self._test_spec_definition, cluster_name, event_definition.fields, event.Data, is_fabric_scoped)
-                decoded_response.append({'value': decoded_event})
+                decoded_response.append({'value': decoded_event, 'eventNumber': event_number})
             return decoded_response
 
         if isinstance(response, ChipStackError):

src/python_testing/hello_external_runner.py

@@ -59,7 +59,7 @@ def start(self, count: int):
     def stop(self, duration: int):
         self.stop_called = True
 
-    def test_start(self, filename: str, name: str, count: int):
+    def test_start(self, filename: str, name: str, count: int, steps: list[str] = []):
         self.test_start_called = True
 
     def test_stop(self, exception: Exception, duration: int):

src/python_testing/hello_test.py

@@ -15,6 +15,13 @@
 #    limitations under the License.
 #
 
+# test-runner-runs: run1
+# test-runner-run/run1/app: ${TYPE_OF_APP}
+# test-runner-run/run1/factoryreset: True
+# test-runner-run/run1/quiet: True
+# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+
 import logging
 
 import chip.clusters as Clusters

src/python_testing/matter_testing_support.py

@@ -274,7 +274,7 @@ def start(self, count: int):
     def stop(self, duration: int):
         logging.info(f'Finished test set, ran for {duration}ms')
 
-    def test_start(self, filename: str, name: str, count: int):
+    def test_start(self, filename: str, name: str, count: int, steps: list[str] = []):
         logging.info(f'Starting test from {filename}: {name} - {count} steps')
 
     def test_stop(self, exception: Exception, duration: int):
@@ -740,7 +740,8 @@ def setup_test(self):
             num_steps = 1 if steps is None else len(steps)
             filename = inspect.getfile(self.__class__)
             desc = self.get_test_desc(test_name)
-            self.runner_hook.test_start(filename=filename, name=desc, count=num_steps)
+            steps_descriptions = [] if steps is None else [step.description for step in steps]
+            self.runner_hook.test_start(filename=filename, name=desc, count=num_steps, steps=steps_descriptions)
             # If we don't have defined steps, we're going to start the one and only step now
             # if there are steps defined by the test, rely on the test calling the step() function
             # to indicates how it is proceeding