Skip to content

Commit 1c604a6

Browse files
Add batch invoke integration tests to CI (project-chip#31642)
* Add batch invoke tests to CI * Restyled by autopep8 * Fix CI and address PR comment * Restyled by autopep8 --------- Co-authored-by: Restyled.io <commits@restyled.io>
1 parent 8c39bd3 commit 1c604a6

File tree

2 files changed

+111
-1
lines changed

2 files changed

+111
-1
lines changed

.github/workflows/tests.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,7 @@ jobs:
467467
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_CGEN_2_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
468468
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_DA_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
469469
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TestGroupTableReports.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
470+
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TestBatchInvoke.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
470471
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TC_TIMESYNC_2_1.py" --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
471472
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TC_TIMESYNC_2_2.py" --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
472473
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TC_TIMESYNC_2_4.py" --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
@@ -484,6 +485,7 @@ jobs:
484485
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-lit-icd-ipv6only-no-ble-no-wifi-tsan-clang-test/lit-icd-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_ICDM_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
485486
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_DA_1_5.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
486487
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_IDM_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
488+
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_IDM_1_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
487489
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_IDM_4_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
488490
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_FAN_3_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
489491
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_FAN_3_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
@@ -591,4 +593,3 @@ jobs:
591593
path: objdir-clone/
592594
# objdirs are big; don't hold on to them too long.
593595
retention-days: 5
594-

src/python_testing/TestBatchInvoke.py

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#
2+
# Copyright (c) 2023 Project CHIP Authors
3+
# All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
import logging
18+
19+
import chip.clusters as Clusters
20+
from chip.interaction_model import InteractionModelError
21+
from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main, type_matches
22+
from mobly import asserts
23+
24+
''' Integration test of batch commands using UnitTesting Cluster
25+
26+
This test is meant to test cases not covered in IDM_1_4 as a result of not being able to control
27+
how DUT processes commands.
28+
'''
29+
30+
31+
class TestBatchInvoke(MatterBaseTest):
32+
33+
@async_test_body
34+
async def test_batch_invoke(self):
35+
dev_ctrl = self.default_controller
36+
dut_node_id = self.dut_node_id
37+
38+
self.print_step(0, "Commissioning - already done")
39+
40+
self.print_step(1, "Get remote node's MaxPathsPerInvoke")
41+
session_parameters = dev_ctrl.GetRemoteSessionParameters(dut_node_id)
42+
max_paths_per_invoke = session_parameters.maxPathsPerInvoke
43+
44+
asserts.assert_greater_equal(max_paths_per_invoke, 2, "Test expects to be able to send at least 2 commands")
45+
asserts.assert_less_equal(max_paths_per_invoke, 65535, "Max Paths per Invoke greater than spec max")
46+
47+
self.print_step(1, "Send simple batch commands to UnitTesting Cluster")
48+
response_size = 7
49+
endpoint = 1
50+
request_1_fill_character = b"a"
51+
request_2_fill_character = b"b"
52+
command = Clusters.UnitTesting.Commands.TestBatchHelperRequest(
53+
sleepBeforeResponseTimeMs=0, sizeOfResponseBuffer=response_size, fillCharacter=ord(request_1_fill_character))
54+
invoke_request_1 = Clusters.Command.InvokeRequestInfo(endpoint, command)
55+
command = Clusters.UnitTesting.Commands.TestSecondBatchHelperRequest(
56+
sleepBeforeResponseTimeMs=0, sizeOfResponseBuffer=response_size, fillCharacter=ord(request_2_fill_character))
57+
invoke_request_2 = Clusters.Command.InvokeRequestInfo(endpoint, command)
58+
try:
59+
result = await dev_ctrl.SendBatchCommands(dut_node_id, [invoke_request_1, invoke_request_2])
60+
except InteractionModelError:
61+
asserts.fail("DUT failed to successfully responded to a InvokeRequest action with two valid commands")
62+
63+
asserts.assert_true(type_matches(result, list), "Unexpected return from SendBatchCommands")
64+
asserts.assert_equal(len(result), 2, "Unexpected number of InvokeResponses sent back from DUT")
65+
asserts.assert_true(type_matches(
66+
result[0], Clusters.UnitTesting.Commands.TestBatchHelperResponse), "Unexpected return type for first InvokeRequest")
67+
asserts.assert_true(type_matches(
68+
result[1], Clusters.UnitTesting.Commands.TestBatchHelperResponse), "Unexpected return type for second InvokeRequest")
69+
asserts.assert_equal(result[0].buffer, request_1_fill_character * response_size,
70+
"Unexpected response to first InvokeRequest")
71+
asserts.assert_equal(result[1].buffer, request_2_fill_character * response_size,
72+
"Unexpected response to second InvokeRequest")
73+
logging.info("DUT successfully responded to a InvokeRequest action with two valid commands")
74+
75+
# TODO(#31434): After TestEventTrigger adds ability to force one response per InvokeResponseMessage
76+
# we should be using that instead of relying on size of response_size. Right now we are relying on,
77+
# the assumption that chip::app::kMaxSecureSduLengthBytes < 1300 bytes.
78+
self.print_step(2, "Send batch commands with large expected response over multiple `InvokeResponseMessage`s")
79+
response_size = 700
80+
endpoint = 1
81+
request_1_fill_character = b"a"
82+
request_2_fill_character = b"b"
83+
# Note that first request is actually delayed trying to get DUT to respond with values out of order
84+
command = Clusters.UnitTesting.Commands.TestBatchHelperRequest(
85+
sleepBeforeResponseTimeMs=50, sizeOfResponseBuffer=response_size, fillCharacter=ord(request_1_fill_character))
86+
invoke_request_1 = Clusters.Command.InvokeRequestInfo(endpoint, command)
87+
command = Clusters.UnitTesting.Commands.TestSecondBatchHelperRequest(
88+
sleepBeforeResponseTimeMs=0, sizeOfResponseBuffer=response_size, fillCharacter=ord(request_2_fill_character))
89+
invoke_request_2 = Clusters.Command.InvokeRequestInfo(endpoint, command)
90+
try:
91+
result = await dev_ctrl.SendBatchCommands(dut_node_id, [invoke_request_1, invoke_request_2])
92+
except InteractionModelError:
93+
asserts.fail("DUT failed to successfully responded to a InvokeRequest action with two valid commands")
94+
95+
asserts.assert_true(type_matches(result, list), "Unexpected return from SendBatchCommands")
96+
asserts.assert_equal(len(result), 2, "Unexpected number of InvokeResponses sent back from DUT")
97+
asserts.assert_true(type_matches(
98+
result[0], Clusters.UnitTesting.Commands.TestBatchHelperResponse), "Unexpected return type for first InvokeRequest")
99+
asserts.assert_true(type_matches(
100+
result[1], Clusters.UnitTesting.Commands.TestBatchHelperResponse), "Unexpected return type for second InvokeRequest")
101+
asserts.assert_equal(result[0].buffer, request_1_fill_character * response_size,
102+
"Unexpected response to first InvokeRequest")
103+
asserts.assert_equal(result[1].buffer, request_2_fill_character * response_size,
104+
"Unexpected response to second InvokeRequest")
105+
logging.info("DUT successfully responded to a InvokeRequest spread accross multiple InvokeResponseMessages")
106+
107+
108+
if __name__ == "__main__":
109+
default_matter_test_main()

0 commit comments

Comments
 (0)