YAML linter: Add check for manual steps, remove known bad unit tests

Author: cecille

scripts/tests/matter_yaml_linter.py

@@ -16,22 +16,20 @@
 import os
 import re
 import sys
+import subprocess
 from pathlib import Path
 
 from chiptest import AllChipToolYamlTests
 
 DEFAULT_CHIP_ROOT = os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..', '..'))
 
-# TODO: These tests need to be re-written. Please see https://github.com/project-chip/connectedhomeip/issues/32620
-KNOWN_BAD_UNIT_TESTING = set(('Test_TC_S_2_2.yaml', 'Test_TC_S_2_3.yaml'))
-
 
 def _is_cert_test(path):
     return "certification" in os.path.dirname(path)
 
 
-def main():
+def check_unit_testing():
     bad_tests = set()
     for test in AllChipToolYamlTests(use_short_run_name=False):
         with open(test.run_name, "r") as f:
@@ -47,10 +45,31 @@ def main():
                     print(f'\t{line+1}: {val}')
                 bad_tests.add(Path(test.run_name).name)
 
-    if bad_tests - KNOWN_BAD_UNIT_TESTING:
+    if bad_tests:
         return 1
     return 0
 
 
+def check_manual_steps():
+    # Doing this on a test-by-test basis so the log message is more obvious
+    bad_tests = set()
+    for test in AllChipToolYamlTests(use_short_run_name=False):
+        cmd = ['git', 'diff', 'HEAD^..HEAD', '--unified=0', '--', test.run_name]
+        output = subprocess.check_output(cmd).decode().splitlines()
+        user_prompt_added = [line for line in output if re.search('^\+.*UserPrompt.*', line)]
+        user_prompt_removed = [line for line in output if re.search('^\-.*UserPrompt.*', line)]
+        if len(user_prompt_added) > len(user_prompt_removed):
+            print(f'Found YAML test with additional manual steps: {test.name}')
+    if bad_tests:
+        return 1
+    return 0
+
+
+def main():
+    ret = check_unit_testing()
+    ret += check_manual_steps()
+    return ret
+
+
 if __name__ == '__main__':
     sys.exit(main())