DM XMLs: Add parsing for data types [WIP]

Note that anything with a within-data-type conformance will
have a conformance of optional currently. This is OK since we're
not actually using these for anything expect understanding P
markings currently.

TODO: Still not parsing multi-bit bitfields correctly.
TODO: Needs unit tests
TODO: Probably not passing CI on the test support tests
      because of the new errors with the bitfields.

Author: cecille

src/python_testing/matter_testing_infrastructure/chip/testing/conformance.py

@@ -113,6 +113,10 @@ def is_disallowed(conformance: Callable):
     return conformance(0, [], []).decision == ConformanceDecision.DISALLOWED
 
 
+def is_provisional(conformance: Callable):
+    return conformance(0, [], []).decision == ConformanceDecision.PROVISIONAL
+
+
 @dataclass
 class Conformance(Callable):
     def __call__(self, feature_map: uint, attribute_list: list[uint], all_command_list: list[uint]) -> ConformanceDecisionWithChoice:
@@ -182,7 +186,7 @@ def __str__(self):
 
 class literal(Conformance):
     def __init__(self, value: str):
-        self.value = int(value)
+        self.value = int(value, 0)
 
     def __call__(self):
         # This should never be called

src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py

@@ -23,7 +23,7 @@
 import zipfile
 from copy import deepcopy
 from dataclasses import dataclass
-from enum import Enum, auto
+from enum import Enum, StrEnum, auto
 from importlib.abc import Traversable
 from typing import Callable, Optional, Union
 
@@ -58,6 +58,29 @@ class SpecParsingException(Exception):
 ConformanceCallable = Callable[[uint, list[uint], list[uint]], ConformanceDecision]
 
 
+class DataTypeEnum(StrEnum):
+    kStruct = 'struct'
+    kEnum = 'enum'
+    kBitmap = 'bitmap'
+
+
+@dataclass
+class XmlDataTypeComponent:
+    value: uint
+    name: str
+    # TODO: other fields are available as well - type, constraints etc.
+    conformance: ConformanceCallable
+
+
+@dataclass
+class XmlDataType:
+    data_type: DataTypeEnum
+    name: str
+    components: dict[uint, XmlDataTypeComponent]
+    # if this is None, this is a global struct
+    cluster_ids: Optional[list[uint]]
+
+
 @dataclass
 class XmlFeature:
     code: str
@@ -120,6 +143,9 @@ class XmlCluster:
     generated_commands: dict[uint, XmlCommand]
     unknown_commands: list[XmlCommand]
     events: dict[uint, XmlEvent]
+    structs: dict[str, XmlDataType]
+    enums: dict[str, XmlDataType]
+    bitmaps: dict[str, XmlDataType]
     pics: str
     is_provisional: bool
 
@@ -379,6 +405,56 @@ def str_to_access_type(privilege_str: str) -> Clusters.AccessControl.Enums.Acces
             invoke_access = Clusters.AccessControl.Enums.AccessControlEntryPrivilegeEnum.kUnknownEnumValue
         return (read_access, write_access, invoke_access)
 
+    def _parse_components(self, struct: ElementTree.Element, component_type: DataTypeEnum) -> dict[uint, XmlDataTypeComponent]:
+        @dataclass
+        class ComponentTag:
+            tag: str
+            id_attrib: str
+        component_tags = {DataTypeEnum.kStruct: ComponentTag('field', 'id'), DataTypeEnum.kEnum: ComponentTag(
+            'item', 'value'), DataTypeEnum.kBitmap: ComponentTag('bitfield', 'bit')}
+        components = {}
+        struct_name = struct.attrib['name']
+        location = ClusterPathLocation(0, self._cluster_id)
+        for xml_field in list(struct):
+            if xml_field.tag != component_tags[component_type].tag:
+                continue
+            try:
+                name = xml_field.attrib['name']
+                id = xml_field.attrib[component_tags[component_type].id_attrib]
+            except KeyError:
+                p = ProblemNotice("Spec XML Parsing", location=location,
+                                  severity=ProblemSeverity.WARNING, problem=f"Struct field in {struct_name} with no id or name")
+                self._problems.append(p)
+                continue
+            xml_conformance, problems = get_conformance(xml_field, self._cluster_id)
+            # There are a LOT of struct fields with either arithmetic or desc conformances. We'll just call these as optional if we can't parse
+            # These are currently unused, so this is fine for now.
+            conformance = None
+            if not problems:
+                conformance = self.parse_conformance(xml_conformance)
+            if not conformance:
+                conformance = optional()
+            components[id] = XmlDataTypeComponent(id, name, conformance)
+        return components
+
+    def _parse_data_type(self, data_type: DataTypeEnum) -> dict[str, XmlDataType]:
+        ''' Returns XmlStructs, key is the name.'''
+        data_types = {}
+        container_tags = self._cluster.iter('dataTypes')
+        for container in container_tags:
+            xmls = container.iter(str(data_type))
+            for element in xmls:
+                try:
+                    name = element.attrib['name']
+                except KeyError:
+                    location = ClusterPathLocation(0, self._cluster_id)
+                    self._problems.append(ProblemNotice("Spec XML Parsing", location=location,
+                                          severity=ProblemSeverity.WARNING, problem=f"Struct {element} with no id or name"))
+                    continue
+                data_types[name] = XmlDataType(data_type=data_type, name=name, components=self._parse_components(
+                    element, data_type), cluster_ids=[self._cluster_id])
+        return data_types
+
     def parse_features(self) -> dict[uint, XmlFeature]:
         features = {}
         for element, conformance_xml, _ in self.feature_elements:
@@ -483,7 +559,9 @@ def create_cluster(self) -> XmlCluster:
                           accepted_commands=self.parse_commands(CommandType.ACCEPTED),
                           generated_commands=self.parse_commands(CommandType.GENERATED),
                           unknown_commands=self.parse_unknown_commands(),
-                          events=self.parse_events(), pics=self._pics, is_provisional=self._is_provisional)
+                          events=self.parse_events(),
+                          structs=self._parse_data_type(DataTypeEnum.kStruct), enums=self._parse_data_type(DataTypeEnum.kEnum),
+                          bitmaps=self._parse_data_type(DataTypeEnum.kBitmap), pics=self._pics, is_provisional=self._is_provisional)
 
     def get_problems(self) -> list[ProblemNotice]:
         return self._problems
@@ -768,6 +846,12 @@ def combine_attributes(base: dict[uint, XmlAttribute], derived: dict[uint, XmlAt
             generated_commands.update(c.generated_commands)
             events = deepcopy(base.events)
             events.update(c.events)
+            structs = deepcopy(base.structs)
+            structs.update(c.structs)
+            enums = deepcopy(base.enums)
+            enums.update(c.enums)
+            bitmaps = deepcopy(base.bitmaps)
+            enums.update(c.bitmaps)
             unknown_commands = deepcopy(base.unknown_commands)
             for cmd in c.unknown_commands:
                 if cmd.id in accepted_commands.keys() and cmd.name == accepted_commands[uint(cmd.id)].name:
@@ -781,8 +865,8 @@ def combine_attributes(base: dict[uint, XmlAttribute], derived: dict[uint, XmlAt
             new = XmlCluster(revision=c.revision, derived=c.derived, name=c.name,
                              feature_map=feature_map, attribute_map=attribute_map, command_map=command_map,
                              features=features, attributes=attributes, accepted_commands=accepted_commands,
-                             generated_commands=generated_commands, unknown_commands=unknown_commands, events=events, pics=c.pics,
-                             is_provisional=provisional)
+                             generated_commands=generated_commands, unknown_commands=unknown_commands, events=events, structs=structs,
+                             enums=enums, bitmaps=bitmaps, pics=c.pics, is_provisional=provisional)
             xml_clusters[id] = new