From 033cad41090e1be5e2d5d01e71bf39d3db25a09c Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 19 Feb 2025 03:53:01 -0800 Subject: [PATCH 1/8] Creating TestSpecParsingNamespace test module: - Creating with data provided by Cecille in PR here: https://github.com/project-chip/connectedhomeip/pull/37527 - Updated test.yaml to include TestSpecParsingNamespace.py to run in CI - Updated matter_testing support module to include new NamespacePathLocation function for Namespace file locations - Updated spec_parsing to include new functions build_xml_namespaces and parse_namespace, as well as XmlNamespaces and XmlTags dataclasses --- .github/workflows/tests.yaml | 1 + .../TestSpecParsingNamespace.py | 290 ++++++++++++++++++ .../chip/testing/matter_testing.py | 18 +- .../chip/testing/spec_parsing.py | 173 ++++++++++- 4 files changed, 479 insertions(+), 3 deletions(-) create mode 100644 src/python_testing/TestSpecParsingNamespace.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d742598edb1614..69ed355d4b9992 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -539,6 +539,7 @@ jobs: scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestIdChecks.py' scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestMatterTestingSupport.py' scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingDeviceType.py' + scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingNamespace.py' scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingSelection.py' scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingSupport.py' diff --git a/src/python_testing/TestSpecParsingNamespace.py b/src/python_testing/TestSpecParsingNamespace.py new file mode 100644 index 00000000000000..deca631da0a174 --- /dev/null +++ b/src/python_testing/TestSpecParsingNamespace.py @@ -0,0 +1,290 @@ +# +# Copyright (c) 2025 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import xml.etree.ElementTree as ElementTree +from jinja2 import Template +import os + +from chip.testing.matter_testing import (MatterBaseTest, default_matter_test_main, + ProblemNotice, ProblemSeverity, NamespacePathLocation) +from chip.testing.spec_parsing import (XmlNamespace, parse_namespace, + build_xml_namespaces) +from mobly import asserts + +class TestSpecParsingNamespace(MatterBaseTest): + def setup_class(self): + # Get the data model paths + self.dm_1_3 = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "1.3") + self.dm_1_4 = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "1.4") + self.dm_1_4_1 = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "1.4.1") + self.dm_master = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "master") + + # Test data setup + self.namespace_id = 0x0001 + self.namespace_name = "Test Namespace" + self.tags = { + 0x0000: "Tag1", + 0x0001: "Tag2", + 0x0002: "Tag3" + } + + # Template for generating test XML + self.template = Template(""" + + + {% for id, name in tags.items() %} + + {% endfor %} + + """) + + def test_namespace_parsing(self): + """Test basic namespace parsing with valid data""" + xml = self.template.render( + namespace_id=f"0x{self.namespace_id:04X}", + namespace_name=self.namespace_name, + tags=self.tags + ) + et = ElementTree.fromstring(xml) + namespace, problems = parse_namespace(et) + + asserts.assert_equal(len(problems), 0, "Unexpected problems parsing namespace") + asserts.assert_equal(namespace.id, self.namespace_id, "Incorrect namespace ID") + asserts.assert_equal(namespace.name, self.namespace_name, "Incorrect namespace name") + asserts.assert_equal(len(namespace.tags), len(self.tags), "Incorrect number of tags") + + for tag_id, tag_name in self.tags.items(): + asserts.assert_true(tag_id in namespace.tags, f"Tag ID 0x{tag_id:04X} not found") + asserts.assert_equal(namespace.tags[tag_id].name, tag_name, f"Incorrect name for tag 0x{tag_id:04X}") + + def test_bad_namespace_id(self): + """Test parsing with invalid namespace ID""" + xml = self.template.render( + namespace_id="", + namespace_name=self.namespace_name, + tags=self.tags + ) + et = ElementTree.fromstring(xml) + namespace, problems = parse_namespace(et) + asserts.assert_equal(len(problems), 1, "Namespace with blank ID did not generate a problem notice") + + def test_missing_namespace_name(self): + """Test parsing with missing namespace name""" + xml = self.template.render( + namespace_id=f"0x{self.namespace_id:04X}", + namespace_name="", + tags=self.tags + ) + et = ElementTree.fromstring(xml) + namespace, problems = parse_namespace(et) + asserts.assert_equal(len(problems), 1, "Namespace with no name did not generate a problem notice") + + def test_no_tags(self): + """Test parsing with no tags""" + xml = self.template.render( + namespace_id=f"0x{self.namespace_id:04X}", + namespace_name=self.namespace_name, + tags={} + ) + et = ElementTree.fromstring(xml) + namespace, problems = parse_namespace(et) + asserts.assert_equal(len(problems), 0, "Unexpected problems parsing empty namespace") + asserts.assert_equal(len(namespace.tags), 0, "Empty namespace should have no tags") + + def test_spec_files(self): + """Test parsing actual spec files from different versions""" + one_three, _ = build_xml_namespaces(self.dm_1_3) + one_four, one_four_problems = build_xml_namespaces(self.dm_1_4) + one_four_one, one_four_one_problems = build_xml_namespaces(self.dm_1_4_1) + tot, tot_problems = build_xml_namespaces(self.dm_master) + + asserts.assert_equal(len(one_four_problems), 0, "Problems found when parsing 1.4 spec") + asserts.assert_equal(len(one_four_one_problems), 0, "Problems found when parsing 1.4.1 spec") + + # Check version relationships + asserts.assert_greater(len(set(tot.keys()) - set(one_three.keys())), + 0, "Master dir does not contain any namespaces not in 1.3") + asserts.assert_greater(len(set(tot.keys()) - set(one_four.keys())), + 0, "Master dir does not contain any namespaces not in 1.4") + asserts.assert_greater(len(set(one_four.keys()) - set(one_three.keys())), + 0, "1.4 dir does not contain any namespaces not in 1.3") + + # Check version consistency + asserts.assert_equal(set(one_four.keys()) - set(one_four_one.keys()), + set(), "There are some 1.4 namespaces that are unexpectedly not included in the 1.4.1 files") + asserts.assert_equal(set(one_four.keys()) - set(tot.keys()), + set(), "There are some 1.4 namespaces that are unexpectedly not included in the TOT files") + + def validate_namespace_xml(self, xml_file: str) -> list[ProblemNotice]: + # Validating XML namespace files + problems = [] + try: + tree = ElementTree.parse(xml_file) + root = tree.getroot() + + # Check for namespace ID and validate format + namespace_id = root.get('id') + if not namespace_id: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Missing namespace ID in {xml_file}" + )) + else: + # Validate 16-bit hex format (0xNNNN) + try: + # Remove '0x' prefix if present and try to parse + id_value = int(namespace_id.replace('0x', ''), 16) + if id_value < 0 or id_value > 0xFFFF: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Namespace ID {namespace_id} is not a valid 16-bit value in {xml_file}" + )) + + # Check format is exactly 0xNNNN where N is a hex digit + if not namespace_id.lower().startswith('0x') or len(namespace_id) != 6: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Namespace ID {namespace_id} does not follow required format '0xNNNN' in {xml_file}" + )) + except ValueError: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Invalid hex format for namespace ID {namespace_id} in {xml_file}" + )) + + # Check for namespace name + namespace_name = root.get('name', '').strip() + if not namespace_name: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Missing or empty namespace name in {xml_file}" + )) + + # Check tags structure + tags_elem = root.find('tags') + if tags_elem is not None: + for tag in tags_elem.findall('tag'): + # Check tag ID and validate format + tag_id = tag.get('id') + if not tag_id: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Missing tag ID in {xml_file}" + )) + else: + # Validate 16-bit hex format for tags (0xNNNN) + try: + # Remove '0x' prefix if present and try to parse + id_value = int(tag_id.replace('0x', ''), 16) + if id_value < 0 or id_value > 0xFFFF: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Tag ID {tag_id} is not a valid 16-bit value in {xml_file}" + )) + # Check format is exactly 0xNNNN where N is hex digit + if not tag_id.lower().startswith('0x') or len(tag_id) != 6: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Tag ID {tag_id} does not follow required format '0xNNNN' in {xml_file}" + )) + except ValueError: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Invalid hex format for tag ID {tag_id} in {xml_file}" + )) + + # Check tag name + tag_name = tag.get('name', '').strip() + if not tag_name: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Missing or empty tag name in {xml_file}" + )) + + except Exception as e: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Failed to parse {xml_file}: {str(e)}" + )) + + return problems + + def test_all_namespace_files(self): + """Test all namespace XML files in the 1.4 and 1.4.1 data model directories""" + data_model_versions = { + "1.4": self.dm_1_4, + "1.4.1": self.dm_1_4_1, + } + + for version, dm_path in data_model_versions.items(): + namespace_path = os.path.join(dm_path, "namespaces") + if not os.path.exists(namespace_path): + self.print_step("Issue encountered", f"\nSkipping {version} - namespace directory not found") + continue + + for filename in os.listdir(namespace_path): + if not filename.endswith('.xml'): + continue + + filepath = os.path.join(namespace_path, filename) + problems = self.validate_namespace_xml(filepath) + + if problems: + for problem in problems: + self.print_step("problem", problem) + + # Run the same validation we did for generated XML + tree = ElementTree.parse(filepath) + namespace, parse_problems = parse_namespace(tree.getroot()) + + # Verify namespace has required attributes + asserts.assert_true(hasattr(namespace, 'id'), f"Namespace in {filename} missing ID") + asserts.assert_true(hasattr(namespace, 'name'), f"Namespace in {filename} missing name") + asserts.assert_true(hasattr(namespace, 'tags'), f"Namespace in {filename} missing tags dictionary") + + # Verify each tag has required attributes + for tag_id, tag in namespace.tags.items(): + asserts.assert_true(hasattr(tag, 'id'), f"Tag in {filename} missing ID") + asserts.assert_true(hasattr(tag, 'name'), f"Tag in {filename} missing name") + +if __name__ == "__main__": + default_matter_test_main() \ No newline at end of file diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py index d999a0e73122d7..3d1e84bebab376 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py @@ -896,13 +896,27 @@ def __str__(self): msg += f'\n ClusterID: {self.cluster_id}' return msg +@dataclass +class NamespacePathLocation: + """Location in a namespace definition""" + def __init__(self, namespace_id: Optional[int] = None, tag_id: Optional[int] = None): + self.namespace_id = namespace_id + self.tag_id = tag_id + + def __str__(self) -> str: + result = "Namespace" + if self.namespace_id is not None: + result += f" 0x{self.namespace_id:04X}" + if self.tag_id is not None: + result += f" Tag 0x{self.tag_id:04X}" + return result class UnknownProblemLocation: def __str__(self): return '\n Unknown Locations - see message for more details' - -ProblemLocation = typing.Union[ClusterPathLocation, DeviceTypePathLocation, UnknownProblemLocation] +ProblemLocation = typing.Union[ClusterPathLocation, DeviceTypePathLocation, + UnknownProblemLocation, NamespacePathLocation] # ProblemSeverity is not using StrEnum, but rather Enum, since StrEnum only # appeared in 3.11. To make it JSON serializable easily, multiple inheritance diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 3de7bbb5c0559a..866eda9d492dba 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -26,6 +26,7 @@ from enum import Enum, auto from importlib.abc import Traversable from typing import Callable, Optional, Union +import os import chip.clusters as Clusters import chip.testing.conformance as conformance_support @@ -33,7 +34,7 @@ ConformanceParseParameters, feature, is_disallowed, mandatory, optional, or_operation, parse_callable_from_xml, parse_device_type_callable_from_xml) from chip.testing.global_attribute_ids import GlobalAttributeIds -from chip.testing.matter_testing import (AttributePathLocation, ClusterPathLocation, CommandPathLocation, DeviceTypePathLocation, +from chip.testing.matter_testing import (NamespacePathLocation, AttributePathLocation, ClusterPathLocation, CommandPathLocation, DeviceTypePathLocation, UnknownProblemLocation, EventPathLocation, FeaturePathLocation, ProblemLocation, ProblemNotice, ProblemSeverity) from chip.tlv import uint @@ -139,6 +140,53 @@ class XmlDeviceTypeClusterRequirements: def __str__(self): return f'{self.name}: {str(self.conformance)}' +""" +Location, XML namespaces, and XML Tags implementation below this line +""" +''' +@dataclass +class Location: + """Represents a location in a source file for error reporting""" + def __init__(self, file: str = "", line: int = 0): + self.file = file + self.line = line + + def __str__(self): + if self.file and self.line: + return f"{self.file}:{self.line}" + elif self.file: + return self.file + return "" +''' + +@dataclass +class XmlNamespace: + """Represents a namespace definition from XML""" + def __init__(self): + self.id: int = 0 + self.name: str = "" + self.tags: dict[int, XmlTag] = {} + + def __str__(self) -> str: + tags_str = '\n '.join(f"{tag_id:04X}: {tag.name}" + for tag_id, tag in sorted(self.tags.items())) + return f"Namespace 0x{self.id:04X} ({self.name})\n {tags_str}" + +@dataclass +class XmlTag: + """Represents a tag within a namespace""" + def __init__(self): + self.id: int = 0 + self.name: str = "" + self.description: Optional[str] = None + + def __str__(self) -> str: + desc = f" - {self.description}" if self.description else "" + return f"{self.name}{desc}" + +""" +XML namespaces and XML Tags implementation above this line +""" @dataclass class XmlDeviceType: @@ -786,6 +834,129 @@ def combine_attributes(base: dict[uint, XmlAttribute], derived: dict[uint, XmlAt xml_clusters[id] = new +""" +Added XML namespace parsing functions below +""" +def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[ProblemNotice]]: + """Parse a single namespace XML definition""" + problems: list[ProblemNotice] = [] + namespace = XmlNamespace() + + # Parse namespace attributes + namespace_id = et.get('id') + if namespace_id is not None: + try: + namespace.id = int(str(namespace_id), 16) + except (ValueError, TypeError): + problems.append(ProblemNotice( + test_name="Parse Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Invalid namespace ID: {namespace_id}" + )) + else: + problems.append(ProblemNotice( + test_name="Parse Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem="Missing namespace ID" + )) + + # Parse and validate namespace name + namespace.name = et.get('name', '').strip() + if not namespace.name: + problems.append(ProblemNotice( + test_name="Parse Namespace XML", + location=NamespacePathLocation(namespace_id=getattr(namespace, 'id', None)), + severity=ProblemSeverity.WARNING, + problem="Missing or empty namespace name" + )) + + # Parse tags + tags_elem = et.find('tags') + if tags_elem is not None: + for tag_elem in tags_elem.findall('tag'): + tag = XmlTag() + tag_id = tag_elem.get('id') + if tag_id is not None: + try: + tag.id = int(str(tag_id), 16) + except (ValueError, TypeError): + problems.append(ProblemNotice( + test_name="Parse Namespace XML", + location=NamespacePathLocation(namespace_id=namespace.id), + severity=ProblemSeverity.WARNING, + problem=f"Invalid tag ID: {tag_id}" + )) + continue + + tag.name = tag_elem.get('name', '').strip() + if not tag.name: + problems.append(ProblemNotice( + test_name="Parse Namespace XML", + location=NamespacePathLocation(namespace_id=namespace.id, tag_id=getattr(tag, 'id', None)), + severity=ProblemSeverity.WARNING, + problem=f"Missing name for tag {tag.id}" + )) + continue + + desc_elem = tag_elem.find('description') + if desc_elem is not None and desc_elem.text: + tag.description = desc_elem.text.strip() + + namespace.tags[tag.id] = tag + + return namespace, problems + +def build_xml_namespaces(data_model_directory: str) -> tuple[dict[int, XmlNamespace], list[ProblemNotice]]: + """Build a dictionary of namespaces from XML files in the given directory""" + namespace_path = os.path.join(data_model_directory, "namespaces") + namespaces: dict[int, XmlNamespace] = {} + problems: list[ProblemNotice] = [] + + if not os.path.exists(namespace_path): + problems.append(ProblemNotice( + test_name="Build XML Namespaces", + location=UnknownProblemLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Namespace directory not found: {namespace_path}" + )) + return namespaces, problems + + for filename in os.listdir(namespace_path): + if not filename.endswith('.xml'): + continue + + filepath = os.path.join(namespace_path, filename) + try: + tree = ElementTree.parse(filepath) + namespace, parse_problems = parse_namespace(tree.getroot()) + problems.extend(parse_problems) + + if namespace.id in namespaces: + problems.append(ProblemNotice( + test_name="Build XML Namespaces", + location=UnknownProblemLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Duplicate namespace ID {namespace.id:04X} in {filename}" + )) + else: + namespaces[namespace.id] = namespace + + except Exception as e: + problems.append(ProblemNotice( + test_name="Build XML Namespaces", + location=UnknownProblemLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Failed to parse {filename}: {str(e)}" + )) + + return namespaces, problems + +""" +Added XML namespace parsing functions above +""" + def parse_single_device_type(root: ElementTree.Element) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: problems: list[ProblemNotice] = [] device_types: dict[int, XmlDeviceType] = {} From 02c6d80c748b48b82b965a8e15312ec10b10b392 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Wed, 19 Feb 2025 04:22:33 -0800 Subject: [PATCH 2/8] Updating test_metadata: - Added TestSpecParsingNamespace.py as it is a unit test and not run against an app. --- .../chip/testing/spec_parsing.py | 18 +----------------- src/python_testing/test_metadata.yaml | 2 ++ 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 866eda9d492dba..d66b68ca6c317d 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -141,24 +141,8 @@ def __str__(self): return f'{self.name}: {str(self.conformance)}' """ -Location, XML namespaces, and XML Tags implementation below this line +XML namespaces and XML Tags dataclass implementation below this line """ -''' -@dataclass -class Location: - """Represents a location in a source file for error reporting""" - def __init__(self, file: str = "", line: int = 0): - self.file = file - self.line = line - - def __str__(self): - if self.file and self.line: - return f"{self.file}:{self.line}" - elif self.file: - return self.file - return "" -''' - @dataclass class XmlNamespace: """Represents a namespace definition from XML""" diff --git a/src/python_testing/test_metadata.yaml b/src/python_testing/test_metadata.yaml index 5bd91ecf542152..06453b35ddf367 100644 --- a/src/python_testing/test_metadata.yaml +++ b/src/python_testing/test_metadata.yaml @@ -53,6 +53,8 @@ not_automated: reason: Unit test - does not run against an app - name: TestSpecParsingDeviceType.py reason: Unit test - does not run against an app + - name: TestSpecParsingNamespace.py + reason: Unit test - does not run against an app - name: TestConformanceTest.py reason: Unit test - does not run against an app - name: TestSpecParsingSelection.py From c50396735102e7de884d624431024f9c6f05e9d5 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 20 Feb 2025 12:34:12 -0800 Subject: [PATCH 3/8] Updating TestSpecParsingNamespace.py and spec_parsing.py: - Updating to change method to using 0 instead of 16 bit enums - Updating to use PrebuiltDatamodelDirectory instead of os to set filepaths - Updated to utilizing the build_xml_namespaces function in spec_parsing. --- .../TestSpecParsingNamespace.py | 196 ++++++++++-------- .../chip/testing/spec_parsing.py | 88 +++++--- 2 files changed, 162 insertions(+), 122 deletions(-) diff --git a/src/python_testing/TestSpecParsingNamespace.py b/src/python_testing/TestSpecParsingNamespace.py index deca631da0a174..40963fce4f51b2 100644 --- a/src/python_testing/TestSpecParsingNamespace.py +++ b/src/python_testing/TestSpecParsingNamespace.py @@ -15,24 +15,20 @@ # limitations under the License. # -import xml.etree.ElementTree as ElementTree +from importlib.abc import Traversable from jinja2 import Template import os +import xml.etree.ElementTree as ElementTree +import zipfile from chip.testing.matter_testing import (MatterBaseTest, default_matter_test_main, ProblemNotice, ProblemSeverity, NamespacePathLocation) -from chip.testing.spec_parsing import (XmlNamespace, parse_namespace, - build_xml_namespaces) +from chip.testing.spec_parsing import (XmlNamespace, parse_namespace, PrebuiltDataModelDirectory, + build_xml_namespaces, get_data_model_directory, DataModelLevel) from mobly import asserts class TestSpecParsingNamespace(MatterBaseTest): def setup_class(self): - # Get the data model paths - self.dm_1_3 = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "1.3") - self.dm_1_4 = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "1.4") - self.dm_1_4_1 = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "1.4.1") - self.dm_master = os.path.join(os.path.dirname(__file__), "..", "..", "data_model", "master") - # Test data setup self.namespace_id = 0x0001 self.namespace_name = "Test Namespace" @@ -110,14 +106,14 @@ def test_no_tags(self): def test_spec_files(self): """Test parsing actual spec files from different versions""" - one_three, _ = build_xml_namespaces(self.dm_1_3) - one_four, one_four_problems = build_xml_namespaces(self.dm_1_4) - one_four_one, one_four_one_problems = build_xml_namespaces(self.dm_1_4_1) - tot, tot_problems = build_xml_namespaces(self.dm_master) + one_three, _ = build_xml_namespaces(PrebuiltDataModelDirectory.k1_3) + one_four, one_four_problems = build_xml_namespaces(PrebuiltDataModelDirectory.k1_4) + one_four_one, one_four_one_problems = build_xml_namespaces(PrebuiltDataModelDirectory.k1_4_1) + tot, tot_problems = build_xml_namespaces(PrebuiltDataModelDirectory.kMaster) asserts.assert_equal(len(one_four_problems), 0, "Problems found when parsing 1.4 spec") asserts.assert_equal(len(one_four_one_problems), 0, "Problems found when parsing 1.4.1 spec") - + # Check version relationships asserts.assert_greater(len(set(tot.keys()) - set(one_three.keys())), 0, "Master dir does not contain any namespaces not in 1.3") @@ -132,50 +128,49 @@ def test_spec_files(self): asserts.assert_equal(set(one_four.keys()) - set(tot.keys()), set(), "There are some 1.4 namespaces that are unexpectedly not included in the TOT files") - def validate_namespace_xml(self, xml_file: str) -> list[ProblemNotice]: - # Validating XML namespace files - problems = [] + def validate_namespace_xml(self, xml_file: Traversable) -> list[ProblemNotice]: + """Validate namespace XML file""" + problems: list[ProblemNotice] = [] try: - tree = ElementTree.parse(xml_file) - root = tree.getroot() - - # Check for namespace ID and validate format - namespace_id = root.get('id') - if not namespace_id: - problems.append(ProblemNotice( - test_name="Validate Namespace XML", - location=NamespacePathLocation(), - severity=ProblemSeverity.WARNING, - problem=f"Missing namespace ID in {xml_file}" - )) - else: - # Validate 16-bit hex format (0xNNNN) - try: - # Remove '0x' prefix if present and try to parse - id_value = int(namespace_id.replace('0x', ''), 16) - if id_value < 0 or id_value > 0xFFFF: - problems.append(ProblemNotice( - test_name="Validate Namespace XML", - location=NamespacePathLocation(), - severity=ProblemSeverity.WARNING, - problem=f"Namespace ID {namespace_id} is not a valid 16-bit value in {xml_file}" - )) - - # Check format is exactly 0xNNNN where N is a hex digit - if not namespace_id.lower().startswith('0x') or len(namespace_id) != 6: - problems.append(ProblemNotice( - test_name="Validate Namespace XML", - location=NamespacePathLocation(), - severity=ProblemSeverity.WARNING, - problem=f"Namespace ID {namespace_id} does not follow required format '0xNNNN' in {xml_file}" - )) - except ValueError: + with xml_file.open('r', encoding="utf8") as f: + root = ElementTree.parse(f).getroot() + + # Check for namespace ID and validate format + namespace_id = root.get('id') + if not namespace_id: problems.append(ProblemNotice( test_name="Validate Namespace XML", location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, - problem=f"Invalid hex format for namespace ID {namespace_id} in {xml_file}" + problem=f"Missing namespace ID in {xml_file.name}" )) + else: + # Validate 16-bit hex format (0xNNNN) + try: + # Remove '0x' prefix if present and try to parse + id_value = int(namespace_id.replace('0x', ''), 16) + if id_value < 0 or id_value > 0xFFFF: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Namespace ID {namespace_id} is not a valid 16-bit value in {xml_file.name}" + )) + # Check format is exactly 0xNNNN where N is hex digit + if not namespace_id.lower().startswith('0x') or len(namespace_id) != 6: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Namespace ID {namespace_id} does not follow required format '0xNNNN' in {xml_file.name}" + )) + except ValueError: + problems.append(ProblemNotice( + test_name="Validate Namespace XML", + location=NamespacePathLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Invalid hex format for namespace ID {namespace_id} in {xml_file.name}" + )) # Check for namespace name namespace_name = root.get('name', '').strip() @@ -184,7 +179,7 @@ def validate_namespace_xml(self, xml_file: str) -> list[ProblemNotice]: test_name="Validate Namespace XML", location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, - problem=f"Missing or empty namespace name in {xml_file}" + problem=f"Missing or empty namespace name in {xml_file.name}" )) # Check tags structure @@ -198,19 +193,18 @@ def validate_namespace_xml(self, xml_file: str) -> list[ProblemNotice]: test_name="Validate Namespace XML", location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, - problem=f"Missing tag ID in {xml_file}" + problem=f"Missing tag ID in {xml_file.name}" )) else: - # Validate 16-bit hex format for tags (0xNNNN) try: # Remove '0x' prefix if present and try to parse - id_value = int(tag_id.replace('0x', ''), 16) + id_value = int(tag_id.replace('0x', ''), 0) if id_value < 0 or id_value > 0xFFFF: problems.append(ProblemNotice( test_name="Validate Namespace XML", location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, - problem=f"Tag ID {tag_id} is not a valid 16-bit value in {xml_file}" + problem=f"Tag ID {tag_id} is not a valid 16-bit value in {xml_file.name}" )) # Check format is exactly 0xNNNN where N is hex digit if not tag_id.lower().startswith('0x') or len(tag_id) != 6: @@ -218,14 +212,14 @@ def validate_namespace_xml(self, xml_file: str) -> list[ProblemNotice]: test_name="Validate Namespace XML", location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, - problem=f"Tag ID {tag_id} does not follow required format '0xNNNN' in {xml_file}" + problem=f"Tag ID {tag_id} does not follow required format '0xNNNN' in {xml_file.name}" )) except ValueError: problems.append(ProblemNotice( test_name="Validate Namespace XML", location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, - problem=f"Invalid hex format for tag ID {tag_id} in {xml_file}" + problem=f"Invalid hex format for tag ID {tag_id} in {xml_file.name}" )) # Check tag name @@ -235,56 +229,76 @@ def validate_namespace_xml(self, xml_file: str) -> list[ProblemNotice]: test_name="Validate Namespace XML", location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, - problem=f"Missing or empty tag name in {xml_file}" + problem=f"Missing or empty tag name in {xml_file.name}" )) except Exception as e: problems.append(ProblemNotice( test_name="Validate Namespace XML", - location=NamespacePathLocation(), + location=UnknownProblemLocation(), severity=ProblemSeverity.WARNING, - problem=f"Failed to parse {xml_file}: {str(e)}" + problem=f"Failed to parse {xml_file.name}: {str(e)}" )) return problems def test_all_namespace_files(self): - """Test all namespace XML files in the 1.4 and 1.4.1 data model directories""" + """Test all namespace XML files in the data model namespaces directories""" data_model_versions = { - "1.4": self.dm_1_4, - "1.4.1": self.dm_1_4_1, + "1.4": PrebuiltDataModelDirectory.k1_4, + "1.4.1": PrebuiltDataModelDirectory.k1_4_1, } for version, dm_path in data_model_versions.items(): - namespace_path = os.path.join(dm_path, "namespaces") - if not os.path.exists(namespace_path): - self.print_step("Issue encountered", f"\nSkipping {version} - namespace directory not found") - continue - - for filename in os.listdir(namespace_path): - if not filename.endswith('.xml'): - continue - - filepath = os.path.join(namespace_path, filename) - problems = self.validate_namespace_xml(filepath) - + try: + # First get the namespaces + namespaces, problems = build_xml_namespaces(dm_path) if problems: for problem in problems: - self.print_step("problem", problem) - - # Run the same validation we did for generated XML - tree = ElementTree.parse(filepath) - namespace, parse_problems = parse_namespace(tree.getroot()) - - # Verify namespace has required attributes - asserts.assert_true(hasattr(namespace, 'id'), f"Namespace in {filename} missing ID") - asserts.assert_true(hasattr(namespace, 'name'), f"Namespace in {filename} missing name") - asserts.assert_true(hasattr(namespace, 'tags'), f"Namespace in {filename} missing tags dictionary") + print(f" - {problem}") + + # Get the directory for validation + top = get_data_model_directory(dm_path, DataModelLevel.kNamespace) - # Verify each tag has required attributes - for tag_id, tag in namespace.tags.items(): - asserts.assert_true(hasattr(tag, 'id'), f"Tag in {filename} missing ID") - asserts.assert_true(hasattr(tag, 'name'), f"Tag in {filename} missing name") + # Handle both zip files and directories + if isinstance(top, zipfile.Path): + files = [f for f in top.iterdir() if str(f).endswith('.xml')] + else: + files = [f for f in top.iterdir() if f.name.endswith('.xml')] + + # Validate each XML file + for file in files: + validation_problems = self.validate_namespace_xml(file) + if validation_problems: + for problem in validation_problems: + asserts.assert_false( + problem.severity == ProblemSeverity.ERROR, + f"Error in {file.name}: {problem}" + ) + + # Parse and verify the namespace + with file.open('r', encoding="utf8") as xml: + root = ElementTree.parse(xml).getroot() + namespace, parse_problems = parse_namespace(root) + + # Basic attribute verification + asserts.assert_true(hasattr(namespace, 'id'), + f"Namespace in {file.name} missing ID") + asserts.assert_true(hasattr(namespace, 'name'), + f"Namespace in {file.name} missing name") + asserts.assert_true(hasattr(namespace, 'tags'), + f"Namespace in {file.name} missing tags dictionary") + + # Verify each tag + for tag_id, tag in namespace.tags.items(): + asserts.assert_true(hasattr(tag, 'id'), + f"Tag in {file.name} missing ID") + asserts.assert_true(hasattr(tag, 'name'), + f"Tag in {file.name} missing name") + + except Exception as e: + print(f"Error processing {version}: {str(e)}") + raise if __name__ == "__main__": default_matter_test_main() \ No newline at end of file diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index d66b68ca6c317d..a4a9a40cf9dbc2 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -591,6 +591,7 @@ def dirname(self): class DataModelLevel(Enum): kCluster = auto() kDeviceType = auto() + kNamespace = auto() @property def dirname(self): @@ -598,6 +599,8 @@ def dirname(self): return "clusters" if self == DataModelLevel.kDeviceType: return "device_types" + if self == DataModelLevel.kNamespace: + return "namespaces" raise KeyError("Invalid enum: %r" % self) @@ -892,48 +895,71 @@ def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[Problem return namespace, problems -def build_xml_namespaces(data_model_directory: str) -> tuple[dict[int, XmlNamespace], list[ProblemNotice]]: +def build_xml_namespaces(data_model_directory: typing.Union[PrebuiltDataModelDirectory, Traversable]) -> tuple[dict[int, XmlNamespace], list[ProblemNotice]]: """Build a dictionary of namespaces from XML files in the given directory""" - namespace_path = os.path.join(data_model_directory, "namespaces") + namespace_dir = get_data_model_directory(data_model_directory, DataModelLevel.kNamespace) namespaces: dict[int, XmlNamespace] = {} problems: list[ProblemNotice] = [] + + found_xmls = 0 - if not os.path.exists(namespace_path): - problems.append(ProblemNotice( - test_name="Build XML Namespaces", - location=UnknownProblemLocation(), - severity=ProblemSeverity.WARNING, - problem=f"Namespace directory not found: {namespace_path}" - )) - return namespaces, problems + try: + # Handle both zip files and directories + if isinstance(namespace_dir, zipfile.Path): + filenames = [f for f in namespace_dir.iterdir() if str(f).endswith('.xml')] + else: + filenames = [f for f in namespace_dir.iterdir() if f.name.endswith('.xml')] - for filename in os.listdir(namespace_path): - if not filename.endswith('.xml'): - continue + for filename in filenames: + logging.info('Parsing file %s', str(filename)) + found_xmls += 1 - filepath = os.path.join(namespace_path, filename) - try: - tree = ElementTree.parse(filepath) - namespace, parse_problems = parse_namespace(tree.getroot()) - problems.extend(parse_problems) - - if namespace.id in namespaces: + try: + with filename.open('r', encoding="utf8") as xml: + root = ElementTree.parse(xml).getroot() + namespace, parse_problems = parse_namespace(root) + problems.extend(parse_problems) + + if namespace.id in namespaces: + problems.append(ProblemNotice( + test_name="Build XML Namespaces", + location=NamespacePathLocation(namespace_id=namespace.id), + severity=ProblemSeverity.WARNING, + problem=f"Duplicate namespace ID 0x{namespace.id:04X} in {filename.name}" + )) + else: + namespaces[namespace.id] = namespace + + except Exception as e: problems.append(ProblemNotice( test_name="Build XML Namespaces", location=UnknownProblemLocation(), severity=ProblemSeverity.WARNING, - problem=f"Duplicate namespace ID {namespace.id:04X} in {filename}" + problem=f"Failed to parse {filename.name}: {str(e)}" )) - else: - namespaces[namespace.id] = namespace - - except Exception as e: - problems.append(ProblemNotice( - test_name="Build XML Namespaces", - location=UnknownProblemLocation(), - severity=ProblemSeverity.WARNING, - problem=f"Failed to parse {filename}: {str(e)}" - )) + + except Exception as e: + problems.append(ProblemNotice( + test_name="Build XML Namespaces", + location=UnknownProblemLocation(), + severity=ProblemSeverity.WARNING, + problem=f"Failed to access namespace directory: {str(e)}" + )) + + if found_xmls < 1: + logging.warning("No XML files found in the specified namespace directory: %r", namespace_dir) + problems.append(ProblemNotice( + test_name="Build XML Namespaces", + location=UnknownProblemLocation(), + severity=ProblemSeverity.WARNING, + problem=f"No XML files found in namespace directory: {str(namespace_dir)}" + )) + + # Print problems for debugging + if problems: + logging.warning("Found %d problems while parsing namespaces:", len(problems)) + for problem in problems: + logging.warning(" - %s", str(problem)) return namespaces, problems From f5aa80bd317898a312b8249daa921d62d0331af4 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 20 Feb 2025 15:16:03 -0800 Subject: [PATCH 4/8] Update src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py Co-authored-by: C Freeman --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index a4a9a40cf9dbc2..9183b5d369950e 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -867,7 +867,7 @@ def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[Problem tag_id = tag_elem.get('id') if tag_id is not None: try: - tag.id = int(str(tag_id), 16) + tag.id = int(str(tag_id), 0) except (ValueError, TypeError): problems.append(ProblemNotice( test_name="Parse Namespace XML", From 158044cf9b0175cd96b044cc418071b0343b2e01 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 20 Mar 2025 18:13:29 +0000 Subject: [PATCH 5/8] Restyled by autopep8 --- .../TestSpecParsingNamespace.py | 50 ++++++++++--------- .../chip/testing/matter_testing.py | 8 ++- .../chip/testing/spec_parsing.py | 33 ++++++++---- 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/src/python_testing/TestSpecParsingNamespace.py b/src/python_testing/TestSpecParsingNamespace.py index 40963fce4f51b2..72ef93b3d173b6 100644 --- a/src/python_testing/TestSpecParsingNamespace.py +++ b/src/python_testing/TestSpecParsingNamespace.py @@ -21,12 +21,13 @@ import xml.etree.ElementTree as ElementTree import zipfile -from chip.testing.matter_testing import (MatterBaseTest, default_matter_test_main, - ProblemNotice, ProblemSeverity, NamespacePathLocation) +from chip.testing.matter_testing import (MatterBaseTest, default_matter_test_main, + ProblemNotice, ProblemSeverity, NamespacePathLocation) from chip.testing.spec_parsing import (XmlNamespace, parse_namespace, PrebuiltDataModelDirectory, - build_xml_namespaces, get_data_model_directory, DataModelLevel) + build_xml_namespaces, get_data_model_directory, DataModelLevel) from mobly import asserts + class TestSpecParsingNamespace(MatterBaseTest): def setup_class(self): # Test data setup @@ -65,7 +66,7 @@ def test_namespace_parsing(self): asserts.assert_equal(namespace.id, self.namespace_id, "Incorrect namespace ID") asserts.assert_equal(namespace.name, self.namespace_name, "Incorrect namespace name") asserts.assert_equal(len(namespace.tags), len(self.tags), "Incorrect number of tags") - + for tag_id, tag_name in self.tags.items(): asserts.assert_true(tag_id in namespace.tags, f"Tag ID 0x{tag_id:04X} not found") asserts.assert_equal(namespace.tags[tag_id].name, tag_name, f"Incorrect name for tag 0x{tag_id:04X}") @@ -116,17 +117,17 @@ def test_spec_files(self): # Check version relationships asserts.assert_greater(len(set(tot.keys()) - set(one_three.keys())), - 0, "Master dir does not contain any namespaces not in 1.3") + 0, "Master dir does not contain any namespaces not in 1.3") asserts.assert_greater(len(set(tot.keys()) - set(one_four.keys())), - 0, "Master dir does not contain any namespaces not in 1.4") + 0, "Master dir does not contain any namespaces not in 1.4") asserts.assert_greater(len(set(one_four.keys()) - set(one_three.keys())), - 0, "1.4 dir does not contain any namespaces not in 1.3") - + 0, "1.4 dir does not contain any namespaces not in 1.3") + # Check version consistency asserts.assert_equal(set(one_four.keys()) - set(one_four_one.keys()), - set(), "There are some 1.4 namespaces that are unexpectedly not included in the 1.4.1 files") + set(), "There are some 1.4 namespaces that are unexpectedly not included in the 1.4.1 files") asserts.assert_equal(set(one_four.keys()) - set(tot.keys()), - set(), "There are some 1.4 namespaces that are unexpectedly not included in the TOT files") + set(), "There are some 1.4 namespaces that are unexpectedly not included in the TOT files") def validate_namespace_xml(self, xml_file: Traversable) -> list[ProblemNotice]: """Validate namespace XML file""" @@ -134,7 +135,7 @@ def validate_namespace_xml(self, xml_file: Traversable) -> list[ProblemNotice]: try: with xml_file.open('r', encoding="utf8") as f: root = ElementTree.parse(f).getroot() - + # Check for namespace ID and validate format namespace_id = root.get('id') if not namespace_id: @@ -259,7 +260,7 @@ def test_all_namespace_files(self): # Get the directory for validation top = get_data_model_directory(dm_path, DataModelLevel.kNamespace) - + # Handle both zip files and directories if isinstance(top, zipfile.Path): files = [f for f in top.iterdir() if str(f).endswith('.xml')] @@ -280,25 +281,26 @@ def test_all_namespace_files(self): with file.open('r', encoding="utf8") as xml: root = ElementTree.parse(xml).getroot() namespace, parse_problems = parse_namespace(root) - + # Basic attribute verification - asserts.assert_true(hasattr(namespace, 'id'), - f"Namespace in {file.name} missing ID") - asserts.assert_true(hasattr(namespace, 'name'), - f"Namespace in {file.name} missing name") - asserts.assert_true(hasattr(namespace, 'tags'), - f"Namespace in {file.name} missing tags dictionary") + asserts.assert_true(hasattr(namespace, 'id'), + f"Namespace in {file.name} missing ID") + asserts.assert_true(hasattr(namespace, 'name'), + f"Namespace in {file.name} missing name") + asserts.assert_true(hasattr(namespace, 'tags'), + f"Namespace in {file.name} missing tags dictionary") # Verify each tag for tag_id, tag in namespace.tags.items(): - asserts.assert_true(hasattr(tag, 'id'), - f"Tag in {file.name} missing ID") - asserts.assert_true(hasattr(tag, 'name'), - f"Tag in {file.name} missing name") + asserts.assert_true(hasattr(tag, 'id'), + f"Tag in {file.name} missing ID") + asserts.assert_true(hasattr(tag, 'name'), + f"Tag in {file.name} missing name") except Exception as e: print(f"Error processing {version}: {str(e)}") raise + if __name__ == "__main__": - default_matter_test_main() \ No newline at end of file + default_matter_test_main() diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py index 3d1e84bebab376..974151397610c9 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py @@ -896,9 +896,11 @@ def __str__(self): msg += f'\n ClusterID: {self.cluster_id}' return msg + @dataclass class NamespacePathLocation: """Location in a namespace definition""" + def __init__(self, namespace_id: Optional[int] = None, tag_id: Optional[int] = None): self.namespace_id = namespace_id self.tag_id = tag_id @@ -911,12 +913,14 @@ def __str__(self) -> str: result += f" Tag 0x{self.tag_id:04X}" return result + class UnknownProblemLocation: def __str__(self): return '\n Unknown Locations - see message for more details' -ProblemLocation = typing.Union[ClusterPathLocation, DeviceTypePathLocation, - UnknownProblemLocation, NamespacePathLocation] + +ProblemLocation = typing.Union[ClusterPathLocation, DeviceTypePathLocation, + UnknownProblemLocation, NamespacePathLocation] # ProblemSeverity is not using StrEnum, but rather Enum, since StrEnum only # appeared in 3.11. To make it JSON serializable easily, multiple inheritance diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 9183b5d369950e..8900906372c901 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -140,25 +140,31 @@ class XmlDeviceTypeClusterRequirements: def __str__(self): return f'{self.name}: {str(self.conformance)}' + """ XML namespaces and XML Tags dataclass implementation below this line """ + + @dataclass class XmlNamespace: """Represents a namespace definition from XML""" + def __init__(self): self.id: int = 0 self.name: str = "" self.tags: dict[int, XmlTag] = {} def __str__(self) -> str: - tags_str = '\n '.join(f"{tag_id:04X}: {tag.name}" - for tag_id, tag in sorted(self.tags.items())) + tags_str = '\n '.join(f"{tag_id:04X}: {tag.name}" + for tag_id, tag in sorted(self.tags.items())) return f"Namespace 0x{self.id:04X} ({self.name})\n {tags_str}" + @dataclass class XmlTag: """Represents a tag within a namespace""" + def __init__(self): self.id: int = 0 self.name: str = "" @@ -168,10 +174,12 @@ def __str__(self) -> str: desc = f" - {self.description}" if self.description else "" return f"{self.name}{desc}" + """ XML namespaces and XML Tags implementation above this line """ + @dataclass class XmlDeviceType: name: str @@ -824,11 +832,13 @@ def combine_attributes(base: dict[uint, XmlAttribute], derived: dict[uint, XmlAt """ Added XML namespace parsing functions below """ + + def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[ProblemNotice]]: """Parse a single namespace XML definition""" problems: list[ProblemNotice] = [] namespace = XmlNamespace() - + # Parse namespace attributes namespace_id = et.get('id') if namespace_id is not None: @@ -848,7 +858,7 @@ def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[Problem severity=ProblemSeverity.WARNING, problem="Missing namespace ID" )) - + # Parse and validate namespace name namespace.name = et.get('name', '').strip() if not namespace.name: @@ -858,7 +868,7 @@ def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[Problem severity=ProblemSeverity.WARNING, problem="Missing or empty namespace name" )) - + # Parse tags tags_elem = et.find('tags') if tags_elem is not None: @@ -876,7 +886,7 @@ def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[Problem problem=f"Invalid tag ID: {tag_id}" )) continue - + tag.name = tag_elem.get('name', '').strip() if not tag.name: problems.append(ProblemNotice( @@ -895,12 +905,13 @@ def parse_namespace(et: ElementTree.Element) -> tuple[XmlNamespace, list[Problem return namespace, problems + def build_xml_namespaces(data_model_directory: typing.Union[PrebuiltDataModelDirectory, Traversable]) -> tuple[dict[int, XmlNamespace], list[ProblemNotice]]: """Build a dictionary of namespaces from XML files in the given directory""" namespace_dir = get_data_model_directory(data_model_directory, DataModelLevel.kNamespace) namespaces: dict[int, XmlNamespace] = {} problems: list[ProblemNotice] = [] - + found_xmls = 0 try: @@ -913,13 +924,13 @@ def build_xml_namespaces(data_model_directory: typing.Union[PrebuiltDataModelDir for filename in filenames: logging.info('Parsing file %s', str(filename)) found_xmls += 1 - + try: with filename.open('r', encoding="utf8") as xml: root = ElementTree.parse(xml).getroot() namespace, parse_problems = parse_namespace(root) problems.extend(parse_problems) - + if namespace.id in namespaces: problems.append(ProblemNotice( test_name="Build XML Namespaces", @@ -929,7 +940,7 @@ def build_xml_namespaces(data_model_directory: typing.Union[PrebuiltDataModelDir )) else: namespaces[namespace.id] = namespace - + except Exception as e: problems.append(ProblemNotice( test_name="Build XML Namespaces", @@ -963,10 +974,12 @@ def build_xml_namespaces(data_model_directory: typing.Union[PrebuiltDataModelDir return namespaces, problems + """ Added XML namespace parsing functions above """ + def parse_single_device_type(root: ElementTree.Element) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: problems: list[ProblemNotice] = [] device_types: dict[int, XmlDeviceType] = {} From 34c2c6345f145e97ee892af10ddaee103e3cf94f Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 20 Mar 2025 18:13:32 +0000 Subject: [PATCH 6/8] Restyled by isort --- src/python_testing/TestSpecParsingNamespace.py | 12 ++++++------ .../chip/testing/spec_parsing.py | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/python_testing/TestSpecParsingNamespace.py b/src/python_testing/TestSpecParsingNamespace.py index 72ef93b3d173b6..5ac386e8a9170b 100644 --- a/src/python_testing/TestSpecParsingNamespace.py +++ b/src/python_testing/TestSpecParsingNamespace.py @@ -15,16 +15,16 @@ # limitations under the License. # -from importlib.abc import Traversable -from jinja2 import Template import os import xml.etree.ElementTree as ElementTree import zipfile +from importlib.abc import Traversable -from chip.testing.matter_testing import (MatterBaseTest, default_matter_test_main, - ProblemNotice, ProblemSeverity, NamespacePathLocation) -from chip.testing.spec_parsing import (XmlNamespace, parse_namespace, PrebuiltDataModelDirectory, - build_xml_namespaces, get_data_model_directory, DataModelLevel) +from chip.testing.matter_testing import (MatterBaseTest, NamespacePathLocation, ProblemNotice, ProblemSeverity, + default_matter_test_main) +from chip.testing.spec_parsing import (DataModelLevel, PrebuiltDataModelDirectory, XmlNamespace, build_xml_namespaces, + get_data_model_directory, parse_namespace) +from jinja2 import Template from mobly import asserts diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 8900906372c901..8daef8f1c24dc8 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -18,6 +18,7 @@ import importlib import importlib.resources as pkg_resources import logging +import os import typing import xml.etree.ElementTree as ElementTree import zipfile @@ -26,7 +27,6 @@ from enum import Enum, auto from importlib.abc import Traversable from typing import Callable, Optional, Union -import os import chip.clusters as Clusters import chip.testing.conformance as conformance_support @@ -34,8 +34,9 @@ ConformanceParseParameters, feature, is_disallowed, mandatory, optional, or_operation, parse_callable_from_xml, parse_device_type_callable_from_xml) from chip.testing.global_attribute_ids import GlobalAttributeIds -from chip.testing.matter_testing import (NamespacePathLocation, AttributePathLocation, ClusterPathLocation, CommandPathLocation, DeviceTypePathLocation, UnknownProblemLocation, - EventPathLocation, FeaturePathLocation, ProblemLocation, ProblemNotice, ProblemSeverity) +from chip.testing.matter_testing import (AttributePathLocation, ClusterPathLocation, CommandPathLocation, DeviceTypePathLocation, + EventPathLocation, FeaturePathLocation, NamespacePathLocation, ProblemLocation, + ProblemNotice, ProblemSeverity, UnknownProblemLocation) from chip.tlv import uint _PRIVILEGE_STR = { From 03bda6c47cbb22eef95526678c2847b0b820ced7 Mon Sep 17 00:00:00 2001 From: Jake Ororke Date: Thu, 20 Mar 2025 12:34:46 -0700 Subject: [PATCH 7/8] Updating TestSpecParsingNamespace and spec_parsing modules: - Resolving linting errors --- src/python_testing/TestSpecParsingNamespace.py | 5 ++--- .../chip/testing/spec_parsing.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/python_testing/TestSpecParsingNamespace.py b/src/python_testing/TestSpecParsingNamespace.py index 5ac386e8a9170b..17bbe0282ed9c4 100644 --- a/src/python_testing/TestSpecParsingNamespace.py +++ b/src/python_testing/TestSpecParsingNamespace.py @@ -15,14 +15,13 @@ # limitations under the License. # -import os import xml.etree.ElementTree as ElementTree import zipfile from importlib.abc import Traversable from chip.testing.matter_testing import (MatterBaseTest, NamespacePathLocation, ProblemNotice, ProblemSeverity, default_matter_test_main) -from chip.testing.spec_parsing import (DataModelLevel, PrebuiltDataModelDirectory, XmlNamespace, build_xml_namespaces, +from chip.testing.spec_parsing import (DataModelLevel, PrebuiltDataModelDirectory, build_xml_namespaces, get_data_model_directory, parse_namespace) from jinja2 import Template from mobly import asserts @@ -236,7 +235,7 @@ def validate_namespace_xml(self, xml_file: Traversable) -> list[ProblemNotice]: except Exception as e: problems.append(ProblemNotice( test_name="Validate Namespace XML", - location=UnknownProblemLocation(), + location=NamespacePathLocation(), severity=ProblemSeverity.WARNING, problem=f"Failed to parse {xml_file.name}: {str(e)}" )) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 8daef8f1c24dc8..0c2b296b8b1dbd 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -18,7 +18,6 @@ import importlib import importlib.resources as pkg_resources import logging -import os import typing import xml.etree.ElementTree as ElementTree import zipfile From 94bbed8b68ee2de54b9fdc5ec6685de75afc330e Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 24 Mar 2025 14:30:03 +0000 Subject: [PATCH 8/8] Restyled by isort --- src/python_testing/TestSpecParsingNamespace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python_testing/TestSpecParsingNamespace.py b/src/python_testing/TestSpecParsingNamespace.py index 17bbe0282ed9c4..3aad44861892c6 100644 --- a/src/python_testing/TestSpecParsingNamespace.py +++ b/src/python_testing/TestSpecParsingNamespace.py @@ -21,8 +21,8 @@ from chip.testing.matter_testing import (MatterBaseTest, NamespacePathLocation, ProblemNotice, ProblemSeverity, default_matter_test_main) -from chip.testing.spec_parsing import (DataModelLevel, PrebuiltDataModelDirectory, build_xml_namespaces, - get_data_model_directory, parse_namespace) +from chip.testing.spec_parsing import (DataModelLevel, PrebuiltDataModelDirectory, build_xml_namespaces, get_data_model_directory, + parse_namespace) from jinja2 import Template from mobly import asserts