TC-DA-1.2: cover PAI incorrect PID encoding (#32717)

* TC-DA-1.2: add test to tests.yaml

This wasn't here before. I want to see this fail before adding the fix.

* actually set exit value

Also, apparently the formatting rules changed. OK...

* Add check for incorrectly formatted fallback PID on PAI

Author: cecille

.github/workflows/tests.yaml

@@ -559,6 +559,7 @@ jobs:
                   scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-rvc-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-rvc-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_file json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RVCOPSTATE_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS examples/rvc-app/rvc-common/pics/rvc-app-pics-values --endpoint 1 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
                   scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-rvc-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-rvc-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_file json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RVCOPSTATE_2_3.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS examples/rvc-app/rvc-common/pics/rvc-app-pics-values --endpoint 1 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
                   scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-rvc-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-rvc-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_file json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RVCOPSTATE_2_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS examples/rvc-app/rvc-common/pics/rvc-app-pics-values --endpoint 1 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
+                  scripts/run_in_python_env.sh out/venv './src/python_testing/test_testing/test_TC_DA_1_2.py'
             - name: Uploading core files
               uses: actions/upload-artifact@v4
               if: ${{ failure() && !env.ACT }}

src/python_testing/TC_DA_1_2.py

@@ -57,6 +57,7 @@ def parse_single_vidpid_from_common_name(commonName: str, tag_str: str) -> str:
 
     s = sp[1][:4]
     if not s.isupper() or len(s) != 4:
+        asserts.fail(f"Improperly encoded PID or VID when using fallback encoding {tag_str}:{s}")
         return None
 
     return s
@@ -132,6 +133,8 @@ def steps_TC_DA_1_2(self):
                 TestStep("6.7", "Verify CD security_information", "security_information = 0"),
                 TestStep("6.8", "Verify CD version_number", "version_number is an integer in range 0..65535"),
                 TestStep("6.9", "Verify CD certification_type", "certification_type has a value between 1..2"),
+                TestStep("7.0", "Extract the Vendor ID (VID) and Product ID (PID) from the DAC. Extract the VID from the PAI. Extract the PID from the PAI, if present",
+                         "VID and PID are present and properly encoded in the DAC. VID is present and properly encoded in the PAI. If the PID is present in the PAI, it is properly encoded"),
                 TestStep("7.1", "", "If the dac_origin_vendor_id is present in the CD, confirm the dac_origin_product_id is also present. If the dac_origin_vendor_id is not present in the CD, confirm the dac_origin_product_id is also not present."),
                 TestStep("7.2", "If the Certification Declaration has both the dac_origin_vendor_id and the dac_origin_product_id fields, verify dac_origin fields",
                          ("* The Vendor ID (VID) in the DAC subject and PAI subject are the same as the dac_origin_vendor_id field in the Certification Declaration.\n"
@@ -309,9 +312,10 @@ async def test_TC_DA_1_2(self):
         else:
             asserts.assert_in(certification_type, [1, 2], "Certification type is out of range")
 
-        self.step("7.1")
+        self.step("7.0")
         dac_vid, dac_pid, pai_vid, pai_pid = parse_ids_from_certs(parsed_dac, parsed_pai)
 
+        self.step("7.1")
         has_origin_vid = 9 in cd.keys()
         has_origin_pid = 10 in cd.keys()
         if has_origin_pid != has_origin_vid:

src/python_testing/test_testing/test_TC_DA_1_2.py

@@ -19,8 +19,10 @@
 import json
 import os
 import subprocess
+import sys
 
-CHIP_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../..'))
+CHIP_ROOT = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../../..'))
 RUNNER_SCRIPT_DIR = os.path.join(CHIP_ROOT, 'scripts/tests')
 
 
@@ -30,13 +32,15 @@ def run_single_test(dac_provider: str, product_id: int, factory_reset: bool = Fa
     if factory_reset:
         reset = ' --factoryreset'
 
-    app = os.path.join(CHIP_ROOT, 'out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app')
+    app = os.path.join(
+        CHIP_ROOT, 'out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app')
 
     # Certs in the commissioner_dut directory use 0x8000 as the PID
     app_args = '--discriminator 1234 --KVS kvs1 --product-id ' + \
         str(product_id) + ' --vendor-id 65521 --dac_provider ' + dac_provider
 
-    ci_pics_values = os.path.abspath(os.path.join(CHIP_ROOT, 'src/app/tests/suites/certification/ci-pics-values'))
+    ci_pics_values = os.path.abspath(os.path.join(
+        CHIP_ROOT, 'src/app/tests/suites/certification/ci-pics-values'))
     script_args = '--storage-path admin_storage.json --discriminator 1234 --passcode 20202021 --dut-node-id 1 --PICS ' + \
         str(ci_pics_values)
 
@@ -45,28 +49,34 @@ def run_single_test(dac_provider: str, product_id: int, factory_reset: bool = Fa
     if factory_reset:
         script_args = script_args + ' --commissioning-method on-network'
 
-    script = os.path.abspath(os.path.join(CHIP_ROOT, 'src/python_testing/TC_DA_1_2.py'))
+    script = os.path.abspath(os.path.join(
+        CHIP_ROOT, 'src/python_testing/TC_DA_1_2.py'))
 
     # run_python_test uses click so call as a command
-    run_python_test = os.path.abspath(os.path.join(RUNNER_SCRIPT_DIR, 'run_python_test.py'))
+    run_python_test = os.path.abspath(os.path.join(
+        RUNNER_SCRIPT_DIR, 'run_python_test.py'))
     cmd = str(run_python_test) + reset + ' --app ' + str(app) + ' --app-args "' + \
-        app_args + '" --script ' + str(script) + ' --script-args "' + script_args + '"'
+        app_args + '" --script ' + \
+        str(script) + ' --script-args "' + script_args + '"'
 
     return subprocess.call(cmd, shell=True)
 
 
 def main():
-    cert_path = os.path.abspath(os.path.join(CHIP_ROOT, 'credentials/development/commissioner_dut'))
+    cert_path = os.path.abspath(os.path.join(
+        CHIP_ROOT, 'credentials/development/commissioner_dut'))
 
     # Commission first using a known good set, then run the rest of the tests without recommissioning
-    path = str(os.path.join(cert_path, "struct_cd_authorized_paa_list_count1_valid/test_case_vector.json"))
+    path = str(os.path.join(
+        cert_path, "struct_cd_authorized_paa_list_count1_valid/test_case_vector.json"))
     run_single_test(path, 32768, factory_reset=True)
 
     test_cases = {'struct_cd': 32768, 'fallback_encoding': 177}
 
     # struct_cd_version_number_wrong - excluded because this is a DCL test not covered by cert
     # struct_cd_cert_id_mismatch - excluded because this is a DCL test not covered by cert
-    exclude_cases = ['struct_cd_version_number_wrong', 'struct_cd_cert_id_mismatch']
+    exclude_cases = ['struct_cd_version_number_wrong',
+                     'struct_cd_cert_id_mismatch']
 
     passes = []
     for p in os.listdir(cert_path):
@@ -92,7 +102,7 @@ def main():
             print('INCORRECT: ' + p[0])
             retval = 1
 
-    return retval
+    sys.exit(retval)
 
 
 if __name__ == '__main__':