Skip to content

Commit 58c6789

Browse files
authored
Python test whl: Use zip files for data_model (project-chip#37163)
* Python test whl: Use zip files for data_model We are close to the input parameter limit for the builds with the number of files in the data_model directory as-is. With the addition of the 1.4.1 files, we go over. Instead, we can zip the files and have the parser read directly from the zip archive. I think this also makes the whl slightly smaller, which is nice. Spec parsing is tested directly with TestSpecParsingSupport.py and with TestSpecParsingDeviceTypes.py. The TC_DeviceConformance tests also use the parsed spec. * Fix generator script to make the build file in the new format * Add CI checker that build file matches generated * Add an intentional failure so I can see the CI trigger * Add the warning * Add back the CI failure trigger * whoops, missed the part where I check out the code * CI is hard and I am not good at it * Think I got it? Works on act * Revert "Add back the CI failure trigger" This reverts commit 38ad956. * Update src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py
1 parent 4f956ec commit 58c6789

File tree

5 files changed

+113
-44
lines changed

5 files changed

+113
-44
lines changed

.github/workflows/check-data-model-directory-updates.yaml

+22-1
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,25 @@ jobs:
3636
python3 scripts/dm_xml_ci_change_enforcement.py data_model/1.3
3737
- name: Check for changes to 1.4 data_model directory without a SHA update
3838
run: |
39-
python3 scripts/dm_xml_ci_change_enforcement.py data_model/1.4
39+
python3 scripts/dm_xml_ci_change_enforcement.py data_model/1.4
40+
41+
check-data_model-build-file:
42+
name: Check that all data_model files are listed in the data_model_xmls.gni build file
43+
runs-on: ubuntu-latest
44+
container:
45+
image: ghcr.io/project-chip/chip-build
46+
steps:
47+
- name: Checkout
48+
uses: actions/checkout@v4
49+
- name: Setup pip modules we use
50+
run: |
51+
python3 -m venv out/venv
52+
out/venv/bin/pip3 install \
53+
jinja2
54+
- name: Generate build file (data_model_xmls.gni)
55+
run: out/venv/bin/python3 src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py
56+
- name: Ensure git works in current working directory
57+
run: git config --global --add safe.directory `pwd`
58+
- name: Check for uncommited changes
59+
run: |
60+
git diff --exit-code HEAD -- src/python_testing/matter_testing_infrastructure/data_model_xmls.gni

src/python_testing/matter_testing_infrastructure/BUILD.gn

+39-5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import("//build_overrides/pigweed.gni")
1818
import("//src/python_testing/matter_testing_infrastructure/data_model_xmls.gni")
1919
import("$dir_pw_build/python.gni")
2020
import("$dir_pw_build/python_dist.gni")
21+
import("$dir_pw_build/zip.gni")
2122

2223
pw_python_package("chip-testing-module") {
2324
generate_setup = {
@@ -55,6 +56,33 @@ pw_python_package("chip-testing-module") {
5556
]
5657
}
5758

59+
pw_zip("data_model_zip_1_3") {
60+
inputs = []
61+
foreach(file, data_model_XMLS_1_3) {
62+
zip_path = rebase_path(file, "${chip_root}/data_model/1.3/", "/")
63+
inputs += [ "${file} > /${zip_path}" ]
64+
}
65+
output = "${root_out_dir}/data_model/zip_1_3.zip"
66+
}
67+
68+
pw_zip("data_model_zip_1_4") {
69+
inputs = []
70+
foreach(file, data_model_XMLS_1_4) {
71+
zip_path = rebase_path(file, "${chip_root}/data_model/1.4/", "/")
72+
inputs += [ "${file} > /${zip_path}" ]
73+
}
74+
output = "${root_out_dir}/data_model/zip_1_4.zip"
75+
}
76+
77+
pw_zip("data_model_zip_master") {
78+
inputs = []
79+
foreach(file, data_model_XMLS_master) {
80+
zip_path = rebase_path(file, "${chip_root}/data_model/master/", "/")
81+
inputs += [ "${file} > /${zip_path}" ]
82+
}
83+
output = "${root_out_dir}/data_model/zip_master.zip"
84+
}
85+
5886
pw_python_distribution("chip-testing") {
5987
packages = [ ":chip-testing-module" ]
6088

@@ -65,9 +93,15 @@ pw_python_distribution("chip-testing") {
6593
include_extra_files_in_package_data = true
6694
}
6795

68-
# Use the imported data_model_XMLS directly
69-
extra_files = []
70-
foreach(file, data_model_XMLS) {
71-
extra_files += [ "${file} > chip/testing/${file}" ]
72-
}
96+
public_deps = [
97+
":data_model_zip_1_3",
98+
":data_model_zip_1_4",
99+
":data_model_zip_master",
100+
]
101+
102+
extra_files = [
103+
"${root_out_dir}/data_model/zip_1_3.zip > chip/testing/data_model/1.3/allfiles.zip",
104+
"${root_out_dir}/data_model/zip_1_4.zip > chip/testing/data_model/1.4/allfiles.zip",
105+
"${root_out_dir}/data_model/zip_master.zip > chip/testing/data_model/master/allfiles.zip",
106+
]
73107
}

src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import logging
2121
import typing
2222
import xml.etree.ElementTree as ElementTree
23+
import zipfile
2324
from copy import deepcopy
2425
from dataclasses import dataclass
2526
from enum import Enum, auto
@@ -571,8 +572,11 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect
571572
return data_model_directory
572573

573574
# If it's a prebuilt directory, build the path based on the version and data model level
574-
return pkg_resources.files(importlib.import_module('chip.testing')).joinpath(
575-
'data_model').joinpath(data_model_directory.dirname).joinpath(data_model_level.dirname)
575+
zip_path = pkg_resources.files(importlib.import_module('chip.testing')).joinpath(
576+
'data_model').joinpath(data_model_directory.dirname).joinpath('allfiles.zip')
577+
path = zipfile.Path(zip_path)
578+
579+
return path.joinpath(data_model_level.dirname)
576580

577581

578582
def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[uint, XmlCluster], list[ProblemNotice]]:

src/python_testing/matter_testing_infrastructure/data_model_xmls.gni

+12-4
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,15 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14+
#
15+
# This file is generated by
16+
# src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py
17+
#
18+
# Manually re-generate this file on any changes to the data_model directory.
1419

1520
import("//build_overrides/chip.gni")
1621

17-
data_model_XMLS = [
22+
data_model_XMLS_1_3 = [
1823
"${chip_root}/data_model/1.3/clusters/ACL-Cluster.xml",
1924
"${chip_root}/data_model/1.3/clusters/AccountLogin.xml",
2025
"${chip_root}/data_model/1.3/clusters/AdminCommissioningCluster.xml",
@@ -116,7 +121,6 @@ data_model_XMLS = [
116121
"${chip_root}/data_model/1.3/clusters/WindowCovering.xml",
117122
"${chip_root}/data_model/1.3/clusters/bridge-clusters-ActionsCluster.xml",
118123
"${chip_root}/data_model/1.3/clusters/bridge-clusters-BridgedDeviceBasicInformationCluster.xml",
119-
"${chip_root}/data_model/1.3/clusters/cluster_ids.json",
120124
"${chip_root}/data_model/1.3/device_types/Aggregator.xml",
121125
"${chip_root}/data_model/1.3/device_types/AirPurifier.xml",
122126
"${chip_root}/data_model/1.3/device_types/AirQualitySensor.xml",
@@ -180,6 +184,9 @@ data_model_XMLS = [
180184
"${chip_root}/data_model/1.3/device_types/WaterValve.xml",
181185
"${chip_root}/data_model/1.3/device_types/WindowCovering.xml",
182186
"${chip_root}/data_model/1.3/device_types/WindowCoveringController.xml",
187+
]
188+
189+
data_model_XMLS_1_4 = [
183190
"${chip_root}/data_model/1.4/clusters/ACL-Cluster.xml",
184191
"${chip_root}/data_model/1.4/clusters/AccountLogin.xml",
185192
"${chip_root}/data_model/1.4/clusters/AdminCommissioningCluster.xml",
@@ -291,7 +298,6 @@ data_model_XMLS = [
291298
"${chip_root}/data_model/1.4/clusters/WindowCovering.xml",
292299
"${chip_root}/data_model/1.4/clusters/bridge-clusters-ActionsCluster.xml",
293300
"${chip_root}/data_model/1.4/clusters/bridge-clusters-BridgedDeviceBasicInformationCluster.xml",
294-
"${chip_root}/data_model/1.4/clusters/cluster_ids.json",
295301
"${chip_root}/data_model/1.4/device_types/Aggregator.xml",
296302
"${chip_root}/data_model/1.4/device_types/AirPurifier.xml",
297303
"${chip_root}/data_model/1.4/device_types/AirQualitySensor.xml",
@@ -365,6 +371,9 @@ data_model_XMLS = [
365371
"${chip_root}/data_model/1.4/device_types/WaterValve.xml",
366372
"${chip_root}/data_model/1.4/device_types/WindowCovering.xml",
367373
"${chip_root}/data_model/1.4/device_types/WindowCoveringController.xml",
374+
]
375+
376+
data_model_XMLS_master = [
368377
"${chip_root}/data_model/master/clusters/ACL-Cluster.xml",
369378
"${chip_root}/data_model/master/clusters/AccountLogin.xml",
370379
"${chip_root}/data_model/master/clusters/AdminCommissioningCluster.xml",
@@ -482,7 +491,6 @@ data_model_XMLS = [
482491
"${chip_root}/data_model/master/clusters/bridge-clusters-ActionsCluster.xml",
483492
"${chip_root}/data_model/master/clusters/bridge-clusters-BridgedDeviceBasicInformationCluster.xml",
484493
"${chip_root}/data_model/master/clusters/bridge-clusters-EcosystemInformationCluster.xml",
485-
"${chip_root}/data_model/master/clusters/cluster_ids.json",
486494
"${chip_root}/data_model/master/device_types/Aggregator.xml",
487495
"${chip_root}/data_model/master/device_types/AirPurifier.xml",
488496
"${chip_root}/data_model/master/device_types/AirQualitySensor.xml",

src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py

+34-32
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@
1313
# limitations under the License.
1414

1515
"""
16-
Generates a gni file containing all data_model files (generally XML and JSON files)
17-
that are typically used by matter_testing_infrastructure.
16+
Generates a gni file containing all data_model files (XML)
17+
that are used by matter_testing_infrastructure.
1818
1919
These files are to be bundled with whl packages of the matter_testing_infrastructure
2020
so that methods requiring data model files work just by installing the python
@@ -26,19 +26,10 @@
2626

2727
# Set chip_root to be dynamically based on the script's location
2828
chip_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
29-
30-
# Directories to search for .xml and .json files relative to chip_root
31-
directories = [
32-
os.path.join(chip_root, "data_model/1.3/clusters/"),
33-
os.path.join(chip_root, "data_model/1.3/device_types/"),
34-
os.path.join(chip_root, "data_model/1.4/clusters/"),
35-
os.path.join(chip_root, "data_model/1.4/device_types/"),
36-
os.path.join(chip_root, "data_model/master/clusters/"),
37-
os.path.join(chip_root, "data_model/master/device_types/"),
38-
]
29+
data_model_dir = os.path.join(chip_root, "data_model")
3930

4031
# Template for generating the GNI file content with proper indentation
41-
GNI_TEMPLATE = """\
32+
GNI_HEADER = """\
4233
# Copyright (c) 2024 Project CHIP Authors
4334
#
4435
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -52,43 +43,54 @@
5243
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5344
# See the License for the specific language governing permissions and
5445
# limitations under the License.
46+
#
47+
# This file is generated by
48+
# src/python_testing/matter_testing_infrastructure/generate_data_model_xmls_gni.py
49+
#
50+
# Manually re-generate this file on any changes to the data_model directory.
5551
5652
import("//build_overrides/chip.gni")
53+
"""
54+
55+
GNI_TEMPLATE = """\
5756
58-
data_model_XMLS = [
57+
data_model_XMLS_{{ dir }} = [
5958
{% for name in file_list %}
60-
"{{ name }}",
59+
"{{ name }}",
6160
{% endfor %}
6261
]
63-
"""
6462
65-
# Function to find and collect all .xml and .json files
63+
"""
6664

6765

68-
def get_data_model_file_names():
66+
def get_data_model_file_names(directory: str):
67+
''' Function to find and collect all .xml files'''
6968
file_list = []
70-
for directory in directories:
71-
for root, _, files in os.walk(directory):
72-
for file in files:
73-
if file.endswith(".xml") or file.endswith(".json"):
74-
# Replace absolute path with `${chip_root}` for GNI compatibility
75-
relative_path = os.path.join("${chip_root}", os.path.relpath(root, chip_root), file)
76-
file_list.append(relative_path)
69+
for root, _, files in os.walk(directory):
70+
for file in files:
71+
if file.endswith(".xml"):
72+
# Replace absolute path with `${chip_root}` for GNI compatibility
73+
relative_path = os.path.join("${chip_root}", os.path.relpath(root, chip_root), file)
74+
file_list.append(relative_path)
7775
# Sort files alphabetically
7876
file_list.sort()
7977
return file_list
8078

81-
# Main function to generate the data_model_xmls.gni file
82-
8379

8480
def generate_gni_file():
85-
# Step 1: Find all files and create the sorted file list
86-
file_list = get_data_model_file_names()
87-
88-
# Step 2: Render the template with the file list
81+
'''Main function to generate the data_model_xmls.gni file'''
8982
environment = jinja2.Environment(trim_blocks=True, lstrip_blocks=True)
9083
template = environment.from_string(GNI_TEMPLATE)
91-
output_content = template.render(file_list=file_list)
84+
output_content_per_dir = []
85+
86+
# Step 1: Find all files and create the sorted file list
87+
dirs = sorted([d for d in os.listdir(data_model_dir) if os.path.isdir(os.path.join(data_model_dir, d))])
88+
for directory in dirs:
89+
file_list = get_data_model_file_names(os.path.join(data_model_dir, directory))
90+
# Step 2: Render the template with the file list
91+
output_content_per_dir.append(template.render(dir=directory.replace('.', '_'), file_list=file_list))
92+
93+
output_content = GNI_HEADER + "".join(output_content_per_dir)
9294

9395
# Step 3: Dynamically generate the output file path
9496
# Get the script's directory (where this script is located)

0 commit comments

Comments
 (0)