From 80e16cb6f9c9148063c3d7de836d820cbe3966be Mon Sep 17 00:00:00 2001 From: cecille Date: Wed, 1 May 2024 17:25:45 -0400 Subject: [PATCH 01/13] YAML: add checker to run in CI --- .github/workflows/cert_test_checks.yaml | 31 ++++++++++++++ scripts/tests/chiptest/__init__.py | 4 +- scripts/tests/matter_yaml_linter.py | 56 +++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cert_test_checks.yaml create mode 100644 scripts/tests/matter_yaml_linter.py diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml new file mode 100644 index 00000000000000..6e9a7175bcaad3 --- /dev/null +++ b/.github/workflows/cert_test_checks.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2024 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Check for common problems in certification tests + +on: + pull_request: + paths: + - "src/app/tests/suites/certification/**" + +jobs: + check-certification-tests: + name: Certification test checks + runs-on: ubuntu-latest + if: "python scripts/tests/matter_yaml_linter.py" + steps: + - name: Error Message + run: echo Problems found in certification tests + - name: Fail Job + run: exit 1 \ No newline at end of file diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 623017e2e2dd97..26f7e943931b89 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -420,8 +420,8 @@ def AllReplYamlTests(): yield test -def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, treat_chip_tool_unsupported_as_in_development=True, use_short_run_name=True): +def AllChipToolYamlTests(use_short_run_name: bool = False): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, treat_chip_tool_unsupported_as_in_development=True, use_short_run_name=use_short_run_name): yield test diff --git a/scripts/tests/matter_yaml_linter.py b/scripts/tests/matter_yaml_linter.py new file mode 100644 index 00000000000000..679eac3adde769 --- /dev/null +++ b/scripts/tests/matter_yaml_linter.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2024 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re +import sys +from chiptest import AllChipToolYamlTests +from pathlib import Path + +DEFAULT_CHIP_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) + +# TODO: These tests need to be re-written. Please see https://github.com/project-chip/connectedhomeip/issues/32620 +KNOWN_BAD_UNIT_TESTING = set(('Test_TC_S_2_2.yaml', 'Test_TC_S_2_3.yaml')) + + +def _is_cert_test(path): + return "certification" in os.path.dirname(path) + + +def main(): + ret = 0 + bad_tests = set() + for test in AllChipToolYamlTests(use_short_run_name=False): + with open(test.run_name, "r") as f: + # Unit testing cluster is disallowed in cert tests, but permissible in general integration tests + unit_test_lines = {} + if _is_cert_test(test.run_name): + unit_test_lines = {lineno: line.strip() for lineno, line in enumerate( + f) if re.search('cluster: "Unit Testing"', line)} + if unit_test_lines: + print( + f'Found certification test using Unit Testing cluster: {test.name}') + for line, val in unit_test_lines.items(): + print(f'\t{line+1}: {val}') + bad_tests.add(Path(test.run_name).name) + + if bad_tests - KNOWN_BAD_UNIT_TESTING: + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From e3b66ec358922e7acbc352dc1cdf916a5c737b36 Mon Sep 17 00:00:00 2001 From: cecille Date: Wed, 1 May 2024 17:26:47 -0400 Subject: [PATCH 02/13] DNS: make a small change to a yaml test to see the CI run --- src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml index 604dd0f2c159fd..321ef8396242b2 100644 --- a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml +++ b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml @@ -13,7 +13,7 @@ # limitations under the License. # Auto-generated scripts for harness use only, please review before automation. The endpoints and cluster names are currently set to default -name: 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] +name: 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] testytesty PICS: - MCORE.IDM.C.InvokeRequest From ab66859ae16ed5673e5622a452afc4e000b94374 Mon Sep 17 00:00:00 2001 From: cecille Date: Wed, 1 May 2024 17:36:43 -0400 Subject: [PATCH 03/13] Use python3, fix name --- .github/workflows/cert_test_checks.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml index 6e9a7175bcaad3..7ec7f2d5f0315c 100644 --- a/.github/workflows/cert_test_checks.yaml +++ b/.github/workflows/cert_test_checks.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Check for common problems in certification tests +name: Certification test checks on: pull_request: @@ -21,9 +21,9 @@ on: jobs: check-certification-tests: - name: Certification test checks + name: Check for common problems in certification tests runs-on: ubuntu-latest - if: "python scripts/tests/matter_yaml_linter.py" + if: "python3 scripts/tests/matter_yaml_linter.py" steps: - name: Error Message run: echo Problems found in certification tests From bc2b354ead882f4b32162b7c89ad1b044c73f425 Mon Sep 17 00:00:00 2001 From: cecille Date: Wed, 1 May 2024 17:41:54 -0400 Subject: [PATCH 04/13] I have no idea what I'm doing --- .github/workflows/cert_test_checks.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml index 7ec7f2d5f0315c..c6271491e28317 100644 --- a/.github/workflows/cert_test_checks.yaml +++ b/.github/workflows/cert_test_checks.yaml @@ -23,9 +23,7 @@ jobs: check-certification-tests: name: Check for common problems in certification tests runs-on: ubuntu-latest - if: "python3 scripts/tests/matter_yaml_linter.py" steps: - - name: Error Message - run: echo Problems found in certification tests - - name: Fail Job - run: exit 1 \ No newline at end of file + - name: Run checks + run: | + ./scripts/run_in_build_env.sh "python3 scripts/tests/matter_yaml_linter.py \ No newline at end of file From 1b8edfbb29cf43a26be3a42096dfb09776f4976c Mon Sep 17 00:00:00 2001 From: cecille Date: Wed, 1 May 2024 17:43:11 -0400 Subject: [PATCH 05/13] seriously, cecille... --- .github/workflows/cert_test_checks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml index c6271491e28317..06b4e23073d3c9 100644 --- a/.github/workflows/cert_test_checks.yaml +++ b/.github/workflows/cert_test_checks.yaml @@ -26,4 +26,4 @@ jobs: steps: - name: Run checks run: | - ./scripts/run_in_build_env.sh "python3 scripts/tests/matter_yaml_linter.py \ No newline at end of file + ./scripts/run_in_build_env.sh "python3 scripts/tests/matter_yaml_linter.py" \ No newline at end of file From c9fe5545e7bf07d420bab53018bea9f2f6701ca8 Mon Sep 17 00:00:00 2001 From: cecille Date: Wed, 1 May 2024 17:49:43 -0400 Subject: [PATCH 06/13] maybe? --- .github/workflows/cert_test_checks.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml index 06b4e23073d3c9..8b516b05c86efe 100644 --- a/.github/workflows/cert_test_checks.yaml +++ b/.github/workflows/cert_test_checks.yaml @@ -24,6 +24,12 @@ jobs: name: Check for common problems in certification tests runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Checkout submodules & Bootstrap + uses: ./.github/actions/checkout-submodules-and-bootstrap + with: + platform: linux - name: Run checks run: | ./scripts/run_in_build_env.sh "python3 scripts/tests/matter_yaml_linter.py" \ No newline at end of file From e0aea2b16e28b31aa7dc6ffa72ebe7268c97a3ae Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Thu, 2 May 2024 10:08:15 -0400 Subject: [PATCH 07/13] trial and error is just like knowing what you're doing --- .github/workflows/cert_test_checks.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml index 8b516b05c86efe..44d545a6636897 100644 --- a/.github/workflows/cert_test_checks.yaml +++ b/.github/workflows/cert_test_checks.yaml @@ -23,13 +23,13 @@ jobs: check-certification-tests: name: Check for common problems in certification tests runs-on: ubuntu-latest + + container: + image: ghcr.io/project-chip/chip-build + steps: - name: Checkout uses: actions/checkout@v4 - - name: Checkout submodules & Bootstrap - uses: ./.github/actions/checkout-submodules-and-bootstrap - with: - platform: linux - name: Run checks run: | - ./scripts/run_in_build_env.sh "python3 scripts/tests/matter_yaml_linter.py" \ No newline at end of file + python3 scripts/tests/matter_yaml_linter.py From fc1825bda33f2cb7cf7b475f1a7f93282336ee6a Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 2 May 2024 14:08:59 +0000 Subject: [PATCH 08/13] Restyled by prettier-yaml --- src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml index 321ef8396242b2..fa33b674368f51 100644 --- a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml +++ b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml @@ -13,7 +13,9 @@ # limitations under the License. # Auto-generated scripts for harness use only, please review before automation. The endpoints and cluster names are currently set to default -name: 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] testytesty +name: + 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] + testytesty PICS: - MCORE.IDM.C.InvokeRequest From 05ed3a2c988412ca843cfbee00e66da3fb94a1ca Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 2 May 2024 14:09:02 +0000 Subject: [PATCH 09/13] Restyled by isort --- scripts/tests/matter_yaml_linter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/tests/matter_yaml_linter.py b/scripts/tests/matter_yaml_linter.py index 679eac3adde769..eafa1e84ef5c2a 100644 --- a/scripts/tests/matter_yaml_linter.py +++ b/scripts/tests/matter_yaml_linter.py @@ -16,9 +16,10 @@ import os import re import sys -from chiptest import AllChipToolYamlTests from pathlib import Path +from chiptest import AllChipToolYamlTests + DEFAULT_CHIP_ROOT = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) From 8b55d7b57be16e7396e5132b8c4c6221e237be56 Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Thu, 2 May 2024 11:21:08 -0400 Subject: [PATCH 10/13] Revert "Restyled by prettier-yaml" This reverts commit fc1825bda33f2cb7cf7b475f1a7f93282336ee6a. --- src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml index fa33b674368f51..321ef8396242b2 100644 --- a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml +++ b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml @@ -13,9 +13,7 @@ # limitations under the License. # Auto-generated scripts for harness use only, please review before automation. The endpoints and cluster names are currently set to default -name: - 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] - testytesty +name: 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] testytesty PICS: - MCORE.IDM.C.InvokeRequest From 147ccd7d8c78a659a65b95a9706bc128aceea8f5 Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Thu, 2 May 2024 11:21:32 -0400 Subject: [PATCH 11/13] Revert "DNS: make a small change to a yaml test to see the CI run" This reverts commit e3b66ec358922e7acbc352dc1cdf916a5c737b36. --- src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml index 321ef8396242b2..604dd0f2c159fd 100644 --- a/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml +++ b/src/app/tests/suites/certification/Test_TC_IDM_1_1.yaml @@ -13,7 +13,7 @@ # limitations under the License. # Auto-generated scripts for harness use only, please review before automation. The endpoints and cluster names are currently set to default -name: 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] testytesty +name: 3.1.1. [TC-IDM-1.1] Invoke Request Action from DUT to TH - [{DUT_Client}] PICS: - MCORE.IDM.C.InvokeRequest From a548a863740bc8267b6e9d60e53ebd0a10524e48 Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Thu, 2 May 2024 16:39:51 -0400 Subject: [PATCH 12/13] lint --- scripts/tests/matter_yaml_linter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/tests/matter_yaml_linter.py b/scripts/tests/matter_yaml_linter.py index eafa1e84ef5c2a..4fdcb765ce6e0c 100644 --- a/scripts/tests/matter_yaml_linter.py +++ b/scripts/tests/matter_yaml_linter.py @@ -32,7 +32,6 @@ def _is_cert_test(path): def main(): - ret = 0 bad_tests = set() for test in AllChipToolYamlTests(use_short_run_name=False): with open(test.run_name, "r") as f: From 9d1d04b8f4128e7fa91afa914ba351fe4d427ab1 Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Fri, 3 May 2024 12:15:21 -0400 Subject: [PATCH 13/13] Wrong default --- scripts/tests/chiptest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 26f7e943931b89..9ae18675cadf78 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -420,7 +420,7 @@ def AllReplYamlTests(): yield test -def AllChipToolYamlTests(use_short_run_name: bool = False): +def AllChipToolYamlTests(use_short_run_name: bool = True): for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, treat_chip_tool_unsupported_as_in_development=True, use_short_run_name=use_short_run_name): yield test