Install Python matter IDL module in matter namespace (#37298)

* Install Python matter IDL module in matter namespace

* Keep Python formatters/checkers configuration in one place

* Map "matter" Python imports as first-party

* Fix spelling warning

* Restyled by prettier-markdown

* Fix import in IDL linter script

* More path updates

* Fix importing matter.idl directly from scripts dir

* Revert ZAP modification

* Fix regression

---------

Co-authored-by: Restyled.io <commits@restyled.io>
Co-authored-by: Andrei Litvin <andy314@gmail.com>

Author: 3 people

.github/.wordlist.txt

@@ -1171,6 +1171,7 @@ PyFunction
 pylint
 PyObject
 pypi
+pyproject
 PyRun
 pytest
 PYTHONPATH

.github/workflows/lint.yml

@@ -151,13 +151,13 @@ jobs:
                       if [ "$idl_file" = './examples/lighting-app-data-mode-no-unique-id/lighting-common/lighting-app.matter' ]; then continue; fi
 
                       # Test files are intentionally small and not spec-compilant, just parse-compliant
-                      if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/cluster_struct_attribute.matter" ]; then continue; fi
-                      if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/global_struct_attribute.matter" ]; then continue; fi
-                      if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/optional_argument.matter" ]; then continue; fi
-                      if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/several_clusters.matter" ]; then continue; fi
-                      if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/simple_attribute.matter" ]; then continue; fi
-                      if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/large_lighting_app.matter" ]; then continue; fi
-                      if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/large_all_clusters_app.matter" ]; then continue; fi
+                      if [ "$idl_file" = "./scripts/py_matter_idl/matter/idl/tests/inputs/cluster_struct_attribute.matter" ]; then continue; fi
+                      if [ "$idl_file" = "./scripts/py_matter_idl/matter/idl/tests/inputs/global_struct_attribute.matter" ]; then continue; fi
+                      if [ "$idl_file" = "./scripts/py_matter_idl/matter/idl/tests/inputs/optional_argument.matter" ]; then continue; fi
+                      if [ "$idl_file" = "./scripts/py_matter_idl/matter/idl/tests/inputs/several_clusters.matter" ]; then continue; fi
+                      if [ "$idl_file" = "./scripts/py_matter_idl/matter/idl/tests/inputs/simple_attribute.matter" ]; then continue; fi
+                      if [ "$idl_file" = "./scripts/py_matter_idl/matter/idl/tests/inputs/large_lighting_app.matter" ]; then continue; fi
+                      if [ "$idl_file" = "./scripts/py_matter_idl/matter/idl/tests/inputs/large_all_clusters_app.matter" ]; then continue; fi
 
                       ./scripts/run_in_build_env.sh "./scripts/idl_lint.py --log-level warn $idl_file" >/dev/null || exit 1
                   done

.github/workflows/tests.yaml

@@ -88,7 +88,7 @@ jobs:
               #
               run: |
                   ./scripts/run_in_build_env.sh \
-                    "./scripts/py_matter_idl/matter_idl/zapxml_parser.py \
+                    "./scripts/py_matter_idl/matter/idl/zapxml_parser.py \
                       --no-print \
                       --log-level info \
                       src/app/zap-templates/zcl/data-model/chip/global-attributes.xml \

.isort.cfg

@@ -68,7 +68,7 @@ exclude:
     - "third_party/android_deps/gradlew" # gradle wrapper generated file
     - "src/controller/python/chip/clusters/Objects.py" # generated file, no point to restyle
     - "src/controller/python/chip/clusters/CHIPClusters.py" # generated file, no point to restyle
-    - "scripts/py_matter_idl/matter_idl/tests/outputs/**/*" # Matches generated output 1:1
+    - "scripts/py_matter_idl/matter/idl/tests/outputs/**/*" # Matches generated output 1:1
     - "scripts/tools/zap/tests/outputs/**/*" # Matches generated output 1:1
     - "examples/chef/sample_app_util/test_files/*.yaml"
     - "examples/chef/zzz_generated/**/*"
@@ -228,8 +228,6 @@ restylers:
       command:
           - autopep8
           - "--in-place"
-          - "--max-line-length"
-          - "132"
       arguments: []
       include:
           - "**/*.py"

.restyled.yaml

@@ -7,7 +7,7 @@ Since it is designed to be easy for both machine and humans to read, it is the
 basis of some tools to make validating zap-based cluster definitions easier.
 
 More details on the format in
-[matter_idl/README.md](../../scripts/py_matter_idl/matter_idl/README.md).
+[matter/idl/README.md](../../scripts/py_matter_idl/matter/idl/README.md).
 
 ## Parsing CSA XML Data definitions
 
@@ -20,12 +20,12 @@ information available in [data_model/README.md](../../data_model/README.md).
 NOTE: scraper is a work in progress, XML data may be incomplete or have errors
 still.
 
-The script `./scripts/py_matter_idl/matter_idl/data_model_xml_parser.py` has the
+The script `./scripts/py_matter_idl/matter/idl/data_model_xml_parser.py` has the
 ability to parse one or more CSA data model XML files and output their content
 in `.matter` format. For example:
 
 ```sh
-./scripts/py_matter_idl/matter_idl/data_model_xml_parser.py data_model/clusters/BooleanState.xml
+./scripts/py_matter_idl/matter/idl/data_model_xml_parser.py data_model/clusters/BooleanState.xml
 ```
 
 The tool supports several options that are useful for development:
@@ -54,7 +54,7 @@ specification:
 As such one can run compares such as:
 
 ```sh
-./scripts/py_matter_idl/matter_idl/data_model_xml_parser.py         \
+./scripts/py_matter_idl/matter/idl/data_model_xml_parser.py         \
      -o out/spec.matter                                             \
      --compare-output out/sdk.matter                                \
      --compare src/controller/data_model/controller-clusters.matter \

docs/guides/matter_idl_tooling.md

@@ -60,22 +60,20 @@ be removed.
 
 We use the following auto-formatters on code:
 
-| Language    | Formatter          | Style File                                                                                 |
-| ----------- | ------------------ | ------------------------------------------------------------------------------------------ |
-| C++         | clang-format       | [.clang-format](https://github.com/project-chip/connectedhomeip/blob/master/.clang-format) |
-| Objective-C | clang-format       | [.clang-format](https://github.com/project-chip/connectedhomeip/blob/master/.clang-format) |
-| java        | google-java-format | N/A                                                                                        |
-| Python      | pep8, isort, ruff  | [.restyled.yaml][restyle_link] (command line), [isort][isort_link], [ruff][ruff_link]      |
-| YAML        | prettier           | None                                                                                       |
-| JSON        | prettier           | None                                                                                       |
-| markdown    | prettier           | None                                                                                       |
-
-[restyle_link]:
-    https://github.com/project-chip/connectedhomeip/blob/master/.restyled.yaml
-[isort_link]:
-    https://github.com/project-chip/connectedhomeip/blob/master/.isort.cfg
-[ruff_link]:
-    https://github.com/project-chip/connectedhomeip/blob/master/ruff.toml
+| Language    | Formatter          | Style File                         |
+| ----------- | ------------------ | ---------------------------------- |
+| C++         | clang-format       | [.clang-format][clang_format_link] |
+| Objective-C | clang-format       | [.clang-format][clang_format_link] |
+| java        | google-java-format | N/A                                |
+| Python      | pep8, isort, ruff  | [pyproject.toml][pyproject_link]   |
+| YAML        | prettier           | None                               |
+| JSON        | prettier           | None                               |
+| markdown    | prettier           | None                               |
+
+[clang_format_link]:
+    https://github.com/project-chip/connectedhomeip/blob/master/.clang-format
+[pyproject_link]:
+    https://github.com/project-chip/connectedhomeip/blob/master/pyproject.toml
 
 All pull requests run formatting checks using these tools before merge is
 allowed. Generated code is not run through restyle.

docs/style/CODING_STYLE_GUIDE.md

@@ -91,7 +91,7 @@ specific codegen.
 `*.matter` files are both human and machine readable. Code that can process
 these files is available at `scripts/py_matter_idl` and `scripts/codegen.py`.
 You can read the
-[scripts/py_matter_idl/matter_idl/README.md](../scripts/py_matter_idl/matter_idl/README.md)
+[scripts/py_matter_idl/matter/idl/README.md](../scripts/py_matter_idl/matter/idl/README.md)
 for details of how things work.
 
 `scripts/codegen.py` can generate various outputs based on an input `*.matter`
@@ -222,7 +222,7 @@ Code pre-generation can be used:
     generation at build time or to save the code generation time at the expense
     of running code generation for every possible zap/generation type
 -   To check changes in generated code across versions, beyond the comparisons
-    of golden image tests in `scripts/py_matter_idl/matter_idl/tests`
+    of golden image tests in `scripts/py_matter_idl/matter/idl/tests`
 
 The script to trigger code pre-generation is `scripts/codepregen.py` and
 requires the pre-generation output directory as an argument

docs/zap_and_codegen/code_generation.md

@@ -1,3 +1,13 @@
+[tool.autopep8]
+max_line_length = 132
+
+[tool.isort]
+line_length = 132
+known_first_party = "matter"
+
+[tool.ruff]
+line-length = 132
+target-version = "py310"
 exclude = [
     ".environment",
     ".git",
@@ -10,12 +20,9 @@ exclude = [
     # TODO(#37698)
     "docs/development_controllers/chip-repl/*.ipynb",
 ]
-target-version = "py310"
-
-line-length = 132
 
-[lint]
+[tool.ruff.lint]
 select = ["E4", "E7", "E9", "F"]
 ignore = [
-    "E721" # We use it for good reasons
+    "E721", # We use it for good reasons
 ]

ruff.toml pyproject.toml

@@ -26,13 +26,13 @@
     _has_coloredlogs = False
 
 try:
-    from matter_idl.matter_idl_parser import CreateParser
+    from matter.idl.matter_idl_parser import CreateParser
 except ImportError:
     import os
     sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'py_matter_idl')))
-    from matter_idl.matter_idl_parser import CreateParser
+    from matter.idl.matter_idl_parser import CreateParser
 
-from matter_idl.backwards_compatibility import is_backwards_compatible
+from matter.idl.backwards_compatibility import is_backwards_compatible
 
 # Supported log levels, mapping string values required for argument
 # parsing into logging constants

scripts/backwards_compatibility_checker.py

@@ -25,15 +25,15 @@
     _has_coloredlogs = False
 
 try:
-    from matter_idl.matter_idl_parser import CreateParser
+    from matter.idl.matter_idl_parser import CreateParser
 except ImportError:
     import os
     sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'py_matter_idl')))
-    from matter_idl.matter_idl_parser import CreateParser
+    from matter.idl.matter_idl_parser import CreateParser
 
 # isort: off
-from matter_idl.generators import FileSystemGeneratorStorage, GeneratorStorage
-from matter_idl.generators.registry import CodeGenerator, GENERATORS
+from matter.idl.generators import FileSystemGeneratorStorage, GeneratorStorage
+from matter.idl.generators.registry import CodeGenerator, GENERATORS
 
 
 class ListGeneratedFilesStorage(GeneratorStorage):

scripts/codegen.py

@@ -21,13 +21,13 @@
 import coloredlogs
 
 try:
-    from matter_idl import matter_idl_parser
+    from matter.idl import matter_idl_parser
 except ImportError:
     sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'py_matter_idl')))
-    from matter_idl import matter_idl_parser
+    from matter.idl import matter_idl_parser
 
 # isort: off
-import matter_idl.lint
+import matter.idl.lint
 
 
 # Supported log levels, mapping string values required for argument
@@ -64,7 +64,7 @@ def main(log_level, rules, idl_path):
 
     lint_rules = []
     logging.info("Loading rules from %s" % rules)
-    lint_rules.extend(matter_idl.lint.CreateParser(rules).parse())
+    lint_rules.extend(matter.idl.lint.CreateParser(rules).parse())
 
     logging.info("Parsing idl from %s" % idl_path)
     idl_tree = matter_idl_parser.CreateParser().parse(open(idl_path, "rt").read(), file_name=idl_path)

scripts/idl_lint.py

@@ -29,37 +29,37 @@ pw_python_package("matter_idl") {
   inputs = matter_idl_generator_templates
   inputs += [
     # Dependency grammar
-    "matter_idl/matter_grammar.lark",
+    "matter/idl/matter_grammar.lark",
 
-    #marker file to indicate to mypy that matter_idl is type-annotated
-    "matter_idl/py.typed",
+    # Marker file to indicate to mypy that matter_idl is type-annotated
+    "matter/idl/py.typed",
 
     # Unit test data
-    "matter_idl/tests/available_tests.yaml",
-    "matter_idl/tests/inputs/cluster_struct_attribute.matter",
-    "matter_idl/tests/inputs/global_struct_attribute.matter",
-    "matter_idl/tests/inputs/optional_argument.matter",
-    "matter_idl/tests/inputs/several_clusters.matter",
-    "matter_idl/tests/inputs/simple_attribute.matter",
-    "matter_idl/tests/outputs/several_clusters/java/ChipClusters.java",
-    "matter_idl/tests/outputs/several_clusters/java/ChipEventStructs.java",
-    "matter_idl/tests/outputs/several_clusters/java/ChipStructs.java",
-    "matter_idl/tests/outputs/several_clusters/java/ClusterInfoMapping.java",
-    "matter_idl/tests/outputs/several_clusters/java/ClusterIDMapping.java",
-    "matter_idl/tests/outputs/several_clusters/java/ClusterWriteMapping.java",
+    "matter/idl/tests/available_tests.yaml",
+    "matter/idl/tests/inputs/cluster_struct_attribute.matter",
+    "matter/idl/tests/inputs/global_struct_attribute.matter",
+    "matter/idl/tests/inputs/optional_argument.matter",
+    "matter/idl/tests/inputs/several_clusters.matter",
+    "matter/idl/tests/inputs/simple_attribute.matter",
+    "matter/idl/tests/outputs/several_clusters/java/ChipClusters.java",
+    "matter/idl/tests/outputs/several_clusters/java/ChipEventStructs.java",
+    "matter/idl/tests/outputs/several_clusters/java/ChipStructs.java",
+    "matter/idl/tests/outputs/several_clusters/java/ClusterInfoMapping.java",
+    "matter/idl/tests/outputs/several_clusters/java/ClusterIDMapping.java",
+    "matter/idl/tests/outputs/several_clusters/java/ClusterWriteMapping.java",
   ]
 
   sources = matter_idl_generator_sources
 
   tests = [
-    "matter_idl/test_backwards_compatibility.py",
-    "matter_idl/test_case_conversion.py",
-    "matter_idl/test_data_model_xml.py",
-    "matter_idl/test_matter_idl_parser.py",
-    "matter_idl/test_generators.py",
-    "matter_idl/test_idl_generator.py",
-    "matter_idl/test_supported_types.py",
-    "matter_idl/test_zapxml.py",
+    "matter/idl/test_backwards_compatibility.py",
+    "matter/idl/test_case_conversion.py",
+    "matter/idl/test_data_model_xml.py",
+    "matter/idl/test_matter_idl_parser.py",
+    "matter/idl/test_generators.py",
+    "matter/idl/test_idl_generator.py",
+    "matter/idl/test_supported_types.py",
+    "matter/idl/test_zapxml.py",
   ]
 
   # TODO: at a future time consider enabling all (* or missing) here to get

scripts/py_matter_idl/BUILD.gn

@@ -16,7 +16,7 @@ scripts/py_matter_idl/examples/matter_idl_plugin:
    `--generator custom:<plugin_path>:<plugin_module_name>` argument and package
    name like `--option package:com.example.matter.proto`
 
-```
+```shell
 # From top-of-tree in this example
 ./scripts/codegen.py \
   --generator custom:./scripts/py_matter_idl/examples:matter_idl_plugin \

scripts/py_matter_idl/examples/README.md

@@ -14,8 +14,8 @@
 
 import os
 
-from matter_idl.generators import CodeGenerator, GeneratorStorage
-from matter_idl.matter_idl_types import Cluster, Command, Field, Idl
+from matter.idl.generators import CodeGenerator, GeneratorStorage
+from matter.idl.matter_idl_types import Cluster, Command, Field, Idl
 
 
 def toUpperSnakeCase(s):