Skip to content

Commit 9306418

Browse files
IDM-10.4: check that CI PICS is not on for cert (project-chip#33817)
* IDM-10.1: check that CI PICS is not on for cert Also adding a test of the test to prove that this works. Added attribute tests as well. * Restyled by autopep8 * Restyled by isort * linter * missed one * whoopsie-doodles, changed the wrong one * add to tests.yaml --------- Co-authored-by: Restyled.io <commits@restyled.io>
1 parent 94d5100 commit 9306418

File tree

6 files changed

+158
-18
lines changed

6 files changed

+158
-18
lines changed

.github/workflows/tests.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,7 @@ jobs:
579579
scripts/run_in_python_env.sh out/venv './scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py'
580580
scripts/run_in_python_env.sh out/venv './src/python_testing/test_testing/test_TC_ICDM_2_1.py'
581581
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestIdChecks.py'
582+
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_IDM_10_4.py'
582583
583584
- name: Uploading core files
584585
uses: actions/upload-artifact@v4

src/python_testing/TC_pics_checker.py

+13-6
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from basic_composition_support import BasicCompositionTests
2121
from global_attribute_ids import GlobalAttributeIds
2222
from matter_testing_support import (AttributePathLocation, ClusterPathLocation, CommandPathLocation, FeaturePathLocation,
23-
MatterBaseTest, TestStep, async_test_body, default_matter_test_main)
23+
MatterBaseTest, ProblemLocation, TestStep, async_test_body, default_matter_test_main)
2424
from mobly import asserts
2525
from pics_support import accepted_cmd_pics_str, attribute_pics_str, feature_pics_str, generated_cmd_pics_str
2626
from spec_parsing_support import build_xml_clusters
@@ -88,11 +88,12 @@ def _add_pics_for_lists(self, cluster_id: int, attribute_id_of_element_list: Glo
8888

8989
def steps_TC_IDM_10_4(self):
9090
return [TestStep(1, "TH performs a wildcard read of all attributes on the endpoint under test"),
91-
TestStep(2, "For every standard cluster: If the cluster is present on the endpoint, ensure the server-side PICS code for the cluster is present in the PICS file (e.g. OO.S for On/Off cluster).If the cluster is not present on the endpoint, ensure the cluster server PICS code is not present in the PICS file."),
92-
TestStep(3, "For every standard cluster, for every attribute in the cluster:If the cluster is present on the endpoint and the attribute ID is present in the AttributeList global attribute within the cluster, ensure the server-side PICS code for the attribute is present in the PICS file (e.g. OO.S.A000 for On/Off cluster’s OnOff attribute).Otherwise, ensure the attribute PICS code is NOT present in the PICS file."),
93-
TestStep(4, "For every cluster present in the spec, for every client → server command in the cluster: If the cluster is present on the endpoint and the command id is present in the accepted commands list, ensure the PICS code for the accepted command is present in the PICS file. Otherwise, ensure the accepted command PICS code is not present in the PICS file."),
94-
TestStep(5, "For every cluster present in the spec, for every server → client command in the cluster: If the cluster is present on the endpoint and the command id is present in the generated commands list, ensure the PICS code for the generated command is present in the PICS file. Otherwise, ensure the generated command PICS code is not present in the PICS file."),
95-
TestStep(6, "For every cluster present in the spec, for every feature in the cluster: If the cluster is present on the endpoint and the feature is marked in the feature map, ensure the PICS code for the feature is present in the PICS file. Otherwise, ensure the feature PICS code is not present in the PICS file.")]
91+
TestStep(2, "For every standard cluster: If the cluster is present on the endpoint, ensure the server-side PICS code for the cluster is present in the PICS file (e.g. OO.S for On/Off cluster).If the cluster is not present on the endpoint, ensure the cluster server PICS code is not present in the PICS file.", "PICS exactly match for server clusters."),
92+
TestStep(3, "For every standard cluster, for every attribute in the cluster:If the cluster is present on the endpoint and the attribute ID is present in the AttributeList global attribute within the cluster, ensure the server-side PICS code for the attribute is present in the PICS file (e.g. OO.S.A000 for On/Off cluster’s OnOff attribute).Otherwise, ensure the attribute PICS code is NOT present in the PICS file.", "PICS exactly match for all attributes in all clusters."),
93+
TestStep(4, "For every cluster present in the spec, for every client → server command in the cluster: If the cluster is present on the endpoint and the command id is present in the accepted commands list, ensure the PICS code for the accepted command is present in the PICS file. Otherwise, ensure the accepted command PICS code is not present in the PICS file.", "PICS exactly match for all accepted commands in all clusters."),
94+
TestStep(5, "For every cluster present in the spec, for every server → client command in the cluster: If the cluster is present on the endpoint and the command id is present in the generated commands list, ensure the PICS code for the generated command is present in the PICS file. Otherwise, ensure the generated command PICS code is not present in the PICS file.", "PICS exactly match for all generated commands in all clusters."),
95+
TestStep(6, "For every cluster present in the spec, for every feature in the cluster: If the cluster is present on the endpoint and the feature is marked in the feature map, ensure the PICS code for the feature is present in the PICS file. Otherwise, ensure the feature PICS code is not present in the PICS file.", "PICS exactly match for all features in all clusters."),
96+
TestStep(7, "Ensure that the PICS_SDK_CI_ONLY PICS does not appear in the PICS file", "CI PICS is not present")]
9697

9798
def test_TC_IDM_10_4(self):
9899
# wildcard read is done in setup_class
@@ -175,6 +176,12 @@ def test_TC_IDM_10_4(self):
175176
location = ClusterPathLocation(endpoint_id=self.endpoint_id, cluster_id=cluster_id)
176177
self._check_and_record_errors(location, required, pics)
177178

179+
self.step(7)
180+
if self.check_pics('PICS_SDK_CI_ONLY'):
181+
self.record_error("PICS check", location=ProblemLocation(),
182+
problem="PICS PICS_SDK_CI_ONLY found in PICS list. This PICS is disallowed for certification.")
183+
self.success = False
184+
178185
if not self.success:
179186
self.fail_current_test("At least one PICS error was found for this endpoint")
180187

src/python_testing/TestMatterTestingSupport.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -625,7 +625,7 @@ def test_xml_pics(self):
625625
self.pics_assert('BINFO.S.A0010', True)
626626
self.pics_assert('BINFO.S.A0011', False)
627627
self.pics_assert('BINFO.S.A0012', True)
628-
self.pics_assert('BINFO.S.A0013', True)
628+
self.pics_assert('BINFO.S.A0013', False)
629629
self.pics_assert('BINFO.S.A0014', False)
630630
self.pics_assert('PICSDOESNOTEXIST', False)
631631

src/python_testing/matter_testing_support.py

+16-10
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
from dataclasses import dataclass, field
3535
from datetime import datetime, timedelta, timezone
3636
from enum import Enum
37-
from typing import List, Optional, Tuple, Union
37+
from typing import List, Optional, Tuple
3838

3939
from chip.tlv import float32, uint
4040

@@ -417,7 +417,13 @@ class CustomCommissioningParameters:
417417

418418

419419
@dataclass
420-
class AttributePathLocation:
420+
class ProblemLocation:
421+
def __str__(self):
422+
return "UNKNOWN"
423+
424+
425+
@dataclass
426+
class AttributePathLocation(ProblemLocation):
421427
endpoint_id: int
422428
cluster_id: Optional[int] = None
423429
attribute_id: Optional[int] = None
@@ -442,7 +448,7 @@ def __str__(self):
442448

443449

444450
@dataclass
445-
class EventPathLocation:
451+
class EventPathLocation(ProblemLocation):
446452
endpoint_id: int
447453
cluster_id: int
448454
event_id: int
@@ -454,7 +460,7 @@ def __str__(self):
454460

455461

456462
@dataclass
457-
class CommandPathLocation:
463+
class CommandPathLocation(ProblemLocation):
458464
endpoint_id: int
459465
cluster_id: int
460466
command_id: int
@@ -466,7 +472,7 @@ def __str__(self):
466472

467473

468474
@dataclass
469-
class ClusterPathLocation:
475+
class ClusterPathLocation(ProblemLocation):
470476
endpoint_id: int
471477
cluster_id: int
472478

@@ -476,7 +482,7 @@ def __str__(self):
476482

477483

478484
@dataclass
479-
class FeaturePathLocation:
485+
class FeaturePathLocation(ProblemLocation):
480486
endpoint_id: int
481487
cluster_id: int
482488
feature_code: str
@@ -500,7 +506,7 @@ class ProblemSeverity(str, Enum):
500506
@dataclass
501507
class ProblemNotice:
502508
test_name: str
503-
location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation]
509+
location: ProblemLocation
504510
severity: ProblemSeverity
505511
problem: str
506512
spec_location: str = ""
@@ -896,13 +902,13 @@ async def check_test_event_triggers_enabled(self):
896902
def print_step(self, stepnum: typing.Union[int, str], title: str) -> None:
897903
logging.info(f'***** Test Step {stepnum} : {title}')
898904

899-
def record_error(self, test_name: str, location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation], problem: str, spec_location: str = ""):
905+
def record_error(self, test_name: str, location: ProblemLocation, problem: str, spec_location: str = ""):
900906
self.problems.append(ProblemNotice(test_name, location, ProblemSeverity.ERROR, problem, spec_location))
901907

902-
def record_warning(self, test_name: str, location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation], problem: str, spec_location: str = ""):
908+
def record_warning(self, test_name: str, location: ProblemLocation, problem: str, spec_location: str = ""):
903909
self.problems.append(ProblemNotice(test_name, location, ProblemSeverity.WARNING, problem, spec_location))
904910

905-
def record_note(self, test_name: str, location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation], problem: str, spec_location: str = ""):
911+
def record_note(self, test_name: str, location: ProblemLocation, problem: str, spec_location: str = ""):
906912
self.problems.append(ProblemNotice(test_name, location, ProblemSeverity.NOTE, problem, spec_location))
907913

908914
def on_fail(self, record):

src/python_testing/test_testing/example_pics_xml_basic_info.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ Draft
182182
<feature>Does the DUT(server) support the CapabilityMinima attribute?</feature>
183183
<reference>9.2.1. Attributes - index.html[pdf]</reference>
184184
<status cond="BINFO.S">M</status>
185-
<support>true</support>
185+
<support>false</support>
186186
</picsItem>
187187
<picsItem>
188188
<itemNumber>BINFO.S.A0014</itemNumber>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#!/usr/bin/env -S python3 -B
2+
#
3+
# Copyright (c) 2024 Project CHIP Authors
4+
# All rights reserved.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
import os
20+
import sys
21+
22+
import chip.clusters as Clusters
23+
from chip.clusters import Attribute
24+
25+
try:
26+
from pics_support import parse_pics_xml
27+
except ImportError:
28+
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
29+
from pics_support import parse_pics_xml
30+
31+
from MockTestRunner import MockTestRunner
32+
33+
# Reachable attribute is off in the pics file
34+
# MaxPathsPerInvoke is not include in the pics file
35+
# Vendor ID is included on ON in the PICS file
36+
37+
38+
def create_read(include_reachable: bool = False, include_max_paths: bool = False, include_vendor_id: bool = True) -> Attribute.AsyncReadTransaction.ReadResponse:
39+
# Attribute read here is set to match the example_pics_xml_basic_info.xml in this directory
40+
bi = Clusters.BasicInformation.Attributes
41+
attrs = {bi.DataModelRevision: 1,
42+
bi.VendorName: 'testVendor',
43+
bi.ProductName: 'testProduct',
44+
bi.ProductID: 0x8000,
45+
bi.NodeLabel: 'label',
46+
bi.Location: 'XX',
47+
bi.HardwareVersion: 1,
48+
bi.HardwareVersionString: 'one',
49+
bi.SoftwareVersion: 2,
50+
bi.SoftwareVersionString: 'two',
51+
bi.ManufacturingDate: 'today',
52+
bi.PartNumber: 'three',
53+
bi.ProductURL: 'example.com',
54+
bi.ProductLabel: 'myProduct',
55+
bi.SerialNumber: 'ABCD1234',
56+
bi.LocalConfigDisabled: False,
57+
bi.UniqueID: 'Hashy-McHashface'}
58+
if include_reachable:
59+
attrs[bi.Reachable] = True
60+
if include_max_paths:
61+
attrs[bi.MaxPathsPerInvoke] = 2
62+
if include_vendor_id:
63+
attrs[bi.VendorID] = 0xFFF1
64+
65+
attrs[bi.AttributeList] = [a.attribute_id for a in attrs.keys()]
66+
attrs[bi.AcceptedCommandList] = []
67+
attrs[bi.GeneratedCommandList] = []
68+
attrs[bi.FeatureMap] = 0
69+
70+
resp = Attribute.AsyncReadTransaction.ReadResponse({}, [], {})
71+
resp.attributes = {0: {Clusters.BasicInformation: attrs}}
72+
73+
tlv_attrs = {a.attribute_id: value for a, value in attrs.items()}
74+
resp.tlvAttributes = {0: {Clusters.BasicInformation.id: tlv_attrs}}
75+
76+
return resp
77+
78+
79+
def main():
80+
# TODO: add the same test for commands and features
81+
script_dir = os.path.dirname(os.path.realpath(__file__))
82+
with open(f'{script_dir}/example_pics_xml_basic_info.xml') as f:
83+
pics = parse_pics_xml(f.read())
84+
test_runner = MockTestRunner('TC_pics_checker.py', 'TC_PICS_Checker', 'test_TC_IDM_10_4', 0, pics)
85+
failures = []
86+
87+
# Success, include vendor ID, which IS in the pics file, and neither of the incorrect ones
88+
resp = create_read()
89+
print(resp)
90+
if not test_runner.run_test_with_mock_read(resp):
91+
failures.append("Test case failure: Vendor ID included - expected pass")
92+
93+
# Failure because Vendor ID is not included in the read, but included in the PICS
94+
resp = create_read(include_vendor_id=False)
95+
if test_runner.run_test_with_mock_read(resp):
96+
failures.append("Test case failure: Vendor ID not included - expected failure")
97+
98+
# Failure because Reachable is included in the read, but not in the PICS
99+
resp = create_read(include_reachable=True)
100+
if test_runner.run_test_with_mock_read(resp):
101+
failures.append("Test case failure: Reachable included but not in PICS- expected failure")
102+
103+
# Failure because MaxPathsPerInvoke is included in the read, but not in the PICS
104+
resp = create_read(include_max_paths=True)
105+
if test_runner.run_test_with_mock_read(resp):
106+
failures.append("Test case failure: MaxPathsPerInvoke included but not in PICS- expected failure")
107+
108+
pics['PICS_SDK_CI_ONLY'] = True
109+
test_runner.config.pics = pics
110+
# This is a success case for the attributes (as seen above), but the test should fail because the CI PICS is added
111+
resp = create_read()
112+
if test_runner.run_test_with_mock_read(resp):
113+
failures.append("Test case failure: SDK CI PICS is included - expected failure")
114+
115+
test_runner.Shutdown()
116+
117+
print(
118+
f"Test of tests: PICS - test response incorrect: {len(failures)}")
119+
for f in failures:
120+
print(f)
121+
122+
return 1 if failures else 0
123+
124+
125+
if __name__ == "__main__":
126+
sys.exit(main())

0 commit comments

Comments
 (0)