Skip to content

Commit 0d740d0

Browse files
tcarmelveilleuxcecillerestyled-commits
authored
Add a mandatory fence to CI tests config and improve Python testing docs (project-chip#34319)
* Update docs and introduce fences for CI test metadata * Add fences to all tests * Update docs/testing/python.md Co-authored-by: C Freeman <cecille@google.com> * Restyled by prettier-markdown * Restyled by autopep8 * Restyled by isort --------- Co-authored-by: C Freeman <cecille@google.com> Co-authored-by: Restyled.io <commits@restyled.io>
1 parent 049f5b2 commit 0d740d0

File tree

85 files changed

+651
-185
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+651
-185
lines changed

‎docs/testing/python.md

+180-159
Large diffs are not rendered by default.

‎scripts/tests/py/metadata.py

+55-24
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,18 @@
1414
# limitations under the License.
1515

1616
import re
17+
import sys
1718
from dataclasses import dataclass
1819
from typing import Any, Dict, List
1920

2021
import yaml
2122

2223

24+
def bool_from_str(value: str) -> bool:
25+
"""Convert True/true/False/false strings to bool."""
26+
return value.strip().lower() == "true"
27+
28+
2329
@dataclass
2430
class Metadata:
2531
py_script_path: str
@@ -58,16 +64,59 @@ def copy_from_dict(self, attr_dict: Dict[str, Any]) -> None:
5864
self.py_script_path = attr_dict["py_script_path"]
5965

6066
if "factoryreset" in attr_dict:
61-
self.factoryreset = bool(attr_dict["factoryreset"])
67+
self.factoryreset = bool_from_str(attr_dict["factoryreset"])
6268

6369
if "factoryreset_app_only" in attr_dict:
64-
self.factoryreset_app_only = bool(attr_dict["factoryreset_app_only"])
70+
self.factoryreset_app_only = bool_from_str(attr_dict["factoryreset_app_only"])
6571

6672
if "script_gdb" in attr_dict:
67-
self.script_gdb = bool(attr_dict["script_gdb"])
73+
self.script_gdb = bool_from_str(attr_dict["script_gdb"])
6874

6975
if "quiet" in attr_dict:
70-
self.quiet = bool(attr_dict["quiet"])
76+
self.quiet = bool_from_str(attr_dict["quiet"])
77+
78+
79+
def extract_runs_arg_lines(py_script_path: str) -> Dict[str, Dict[str, str]]:
80+
"""Extract the run arguments from the CI test arguments blocks."""
81+
82+
found_ci_args_section = False
83+
done_ci_args_section = False
84+
85+
runs_def_ptrn = re.compile(r'^\s*#\s*test-runner-runs:\s*(?P<run_id>.*)$')
86+
arg_def_ptrn = re.compile(
87+
r'^\s*#\s*test-runner-run/(?P<run_id>[a-zA-Z0-9_]+)/(?P<arg_name>[a-zA-Z0-9_\-]+):\s*(?P<arg_val>.*)$')
88+
89+
runs_arg_lines: Dict[str, Dict[str, str]] = {}
90+
91+
with open(py_script_path, 'r', encoding='utf8') as py_script:
92+
for line_idx, line in enumerate(py_script.readlines()):
93+
line = line.strip()
94+
line_num = line_idx + 1
95+
96+
# Detect the single CI args section, to skip the lines otherwise.
97+
if not done_ci_args_section and line.startswith("# === BEGIN CI TEST ARGUMENTS ==="):
98+
found_ci_args_section = True
99+
elif found_ci_args_section and line.startswith("# === END CI TEST ARGUMENTS ==="):
100+
done_ci_args_section = True
101+
found_ci_args_section = False
102+
103+
runs_match = runs_def_ptrn.match(line)
104+
args_match = arg_def_ptrn.match(line)
105+
106+
if not found_ci_args_section and (runs_match or args_match):
107+
print(f"WARNING: {py_script_path}:{line_num}: Found CI args outside of CI TEST ARGUMENTS block!", file=sys.stderr)
108+
continue
109+
110+
if runs_match:
111+
for run in runs_match.group("run_id").strip().split():
112+
runs_arg_lines[run] = {}
113+
runs_arg_lines[run]['run'] = run
114+
runs_arg_lines[run]['py_script_path'] = py_script_path
115+
116+
elif args_match:
117+
runs_arg_lines[args_match.group("run_id")][args_match.group("arg_name")] = args_match.group("arg_val")
118+
119+
return runs_arg_lines
71120

72121

73122
class MetadataReader:
@@ -126,33 +175,15 @@ def parse_script(self, py_script_path: str) -> List[Metadata]:
126175
the run arguments associated with a particular run defined in
127176
the script file.
128177
"""
129-
130-
runs_def_ptrn = re.compile(r'^\s*#\s*test-runner-runs:\s*(.*)$')
131-
arg_def_ptrn = re.compile(r'^\s*#\s*test-runner-run/([a-zA-Z0-9_]+)/([a-zA-Z0-9_\-]+):\s*(.*)$')
132-
133-
runs_arg_lines: Dict[str, Dict[str, str]] = {}
134178
runs_metadata: List[Metadata] = []
135-
136-
with open(py_script_path, 'r', encoding='utf8') as py_script:
137-
for line in py_script.readlines():
138-
runs_match = runs_def_ptrn.match(line.strip())
139-
args_match = arg_def_ptrn.match(line.strip())
140-
141-
if runs_match:
142-
for run in runs_match.group(1).strip().split():
143-
runs_arg_lines[run] = {}
144-
runs_arg_lines[run]['run'] = run
145-
runs_arg_lines[run]['py_script_path'] = py_script_path
146-
147-
elif args_match:
148-
runs_arg_lines[args_match.group(1)][args_match.group(2)] = args_match.group(3)
179+
runs_arg_lines = extract_runs_arg_lines(py_script_path)
149180

150181
for run, attr in runs_arg_lines.items():
151182
self.__resolve_env_vals__(attr)
152183

153184
metadata = Metadata(
154185
py_script_path=attr.get("py_script_path", ""),
155-
run=attr.get("run", ""),
186+
run=run,
156187
app=attr.get("app", ""),
157188
app_args=attr.get("app_args", ""),
158189
script_args=attr.get("script_args", ""),

‎scripts/tests/py/test_metadata.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,17 @@
2121

2222
class TestMetadataReader(unittest.TestCase):
2323

24-
test_file_content = '''
25-
# test-runner-runs: run1
24+
test_file_content = '''
25+
# === BEGIN CI TEST ARGUMENTS ===
26+
# test-runner-runs: run1
2627
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2728
# test-runner-run/run1/app-args: --discriminator 1234 --trace-to json:${TRACE_APP}.json
2829
# test-runner-run/run1/script-args: --commissioning-method on-network --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
2930
# test-runner-run/run1/factoryreset: True
3031
# test-runner-run/run1/quiet: True
32+
# === END CI TEST ARGUMENTS ===
33+
34+
# test-runner-run/run1/quiet: False
3135
'''
3236

3337
env_file_content = '''

‎src/controller/python/test/test_scripts/mobile-device-test.py

+5
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,17 @@
1919

2020
# Commissioning test.
2121

22+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
23+
# for details about the block below.
24+
#
25+
# === BEGIN CI TEST ARGUMENTS ===
2226
# test-runner-runs: run1
2327
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2428
# test-runner-run/run1/factoryreset: True
2529
# test-runner-run/run1/quiet: True
2630
# test-runner-run/run1/app-args: --trace-to json:${TRACE_APP}.json
2731
# test-runner-run/run1/script-args: --log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
32+
# === END CI TEST ARGUMENTS ===
2833

2934
import asyncio
3035
import os

‎src/python_testing/TC_ACE_1_2.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import logging
2631
import queue

‎src/python_testing/TC_ACE_1_3.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import logging
2631

‎src/python_testing/TC_ACE_1_4.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import sys
2631

‎src/python_testing/TC_ACE_1_5.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import logging
2631

‎src/python_testing/TC_AccessChecker.py

+5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
1+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
2+
# for details about the block below.
3+
#
4+
# === BEGIN CI TEST ARGUMENTS ===
15
# test-runner-runs: run1
26
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
37
# test-runner-run/run1/factoryreset: True
48
# test-runner-run/run1/quiet: True
59
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
610
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
11+
# === END CI TEST ARGUMENTS ===
712

813
import logging
914
from copy import deepcopy

‎src/python_testing/TC_CGEN_2_4.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import logging
2631
import random

‎src/python_testing/TC_DA_1_2.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import os
2631
import random

‎src/python_testing/TC_DA_1_5.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import random
2631

‎src/python_testing/TC_DA_1_7.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --bool-arg allow_sdk_dac:true --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import logging
2631
from glob import glob

‎src/python_testing/TC_DGGEN_2_4.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
import asyncio
2631
import logging

‎src/python_testing/TC_DRLK_2_12.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${CHIP_LOCK_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
from drlk_2_x_common import DRLK_COMMON
2631
from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main

‎src/python_testing/TC_DRLK_2_2.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${CHIP_LOCK_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
from drlk_2_x_common import DRLK_COMMON
2631
from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main

‎src/python_testing/TC_DRLK_2_3.py

+5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
# limitations under the License.
1616
#
1717

18+
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
19+
# for details about the block below.
20+
#
21+
# === BEGIN CI TEST ARGUMENTS ===
1822
# test-runner-runs: run1
1923
# test-runner-run/run1/app: ${CHIP_LOCK_APP}
2024
# test-runner-run/run1/factoryreset: True
2125
# test-runner-run/run1/quiet: True
2226
# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2327
# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
28+
# === END CI TEST ARGUMENTS ===
2429

2530
from drlk_2_x_common import DRLK_COMMON
2631
from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main

0 commit comments

Comments
 (0)