YAML linter: Add check for manual steps, remove known bad unit tests

.github/workflows/cert_test_checks.yaml

@@ -18,6 +18,8 @@ on:
   pull_request:
     paths:
       - "src/app/tests/suites/certification/**"
+permissions:
+  contents: read
 
 jobs:
   check-certification-tests:
@@ -30,6 +32,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Run checks
         run: |
           python3 scripts/tests/matter_yaml_linter.py

scripts/tests/matter_yaml_linter.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 import os
 import re
+import subprocess
 import sys
 from pathlib import Path
 
@@ -23,15 +24,12 @@
 DEFAULT_CHIP_ROOT = os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..', '..'))
 
-# TODO: These tests need to be re-written. Please see https://github.com/project-chip/connectedhomeip/issues/32620
-KNOWN_BAD_UNIT_TESTING = set(('Test_TC_S_2_2.yaml', 'Test_TC_S_2_3.yaml'))
-
 
 def _is_cert_test(path):
     return "certification" in os.path.dirname(path)
 
 
-def main():
+def check_unit_testing():
     bad_tests = set()
     for test in AllChipToolYamlTests(use_short_run_name=False):
         with open(test.run_name, "r") as f:
@@ -47,10 +45,37 @@ def main():
                     print(f'\t{line+1}: {val}')
                 bad_tests.add(Path(test.run_name).name)
 
-    if bad_tests - KNOWN_BAD_UNIT_TESTING:
+    if bad_tests:
         return 1
     return 0
 
 
+def check_manual_steps():
+    # Doing this on a test-by-test basis so the log message is more obvious
+    bad_test = False
+    # We are operating in a VM, and although there is a checkout, it is working in a scratch directory
+    # where the ownership is different than the runner.
+    # Adding an exception for this directory so that git can function properly.
+    subprocess.run("git config --global --add safe.directory '*'", shell=True)
+    for test in AllChipToolYamlTests(use_short_run_name=False):
+
+        cmd = f'git diff HEAD^..HEAD --unified=0 -- {test.run_name}'
+        output = subprocess.check_output(cmd, shell=True).decode().splitlines()
+        user_prompt_added = [line for line in output if re.search(r'^\+.*UserPrompt.*', line)]
+        user_prompt_removed = [line for line in output if re.search(r'^\-.*UserPrompt.*', line)]
+        if len(user_prompt_added) > len(user_prompt_removed):
+            print(f'Found YAML test with additional manual steps: {test.name}')
+            bad_test = True
+    if bad_test:
+        return 1
+    return 0
+
+
+def main():
+    ret = check_unit_testing()
+    ret += check_manual_steps()
+    return ret
+
+
 if __name__ == '__main__':
     sys.exit(main())