From 78b52b7f9a36ba8539ed028dd95b3e312f59ca60 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani <152916324+vatsalghelani-csa@users.noreply.github.com> Date: Wed, 18 Sep 2024 15:00:24 -0400 Subject: [PATCH 1/5] Script updated with comments on why the scripts have been included from the glob run --- src/python_testing/execute_python_tests.py | 101 +++++++++------------ 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/src/python_testing/execute_python_tests.py b/src/python_testing/execute_python_tests.py index de8908ca13be25..1cf4a68fb0f4f6 100644 --- a/src/python_testing/execute_python_tests.py +++ b/src/python_testing/execute_python_tests.py @@ -55,65 +55,52 @@ def main(search_directory, env_file): # Define the files and patterns to exclude excluded_patterns = { - "MinimalRepresentation.py", - "TC_CNET_4_4.py", - "TC_CCTRL_2_1.py", - "TC_CCTRL_2_2.py", - "TC_CCTRL_2_3.py", - "TC_DGGEN_3_2.py", - "TC_EEVSE_Utils.py", - "TC_ECOINFO_2_1.py", - "TC_ECOINFO_2_2.py", - "TC_EWATERHTRBase.py", - "TC_EWATERHTR_2_1.py", - "TC_EWATERHTR_2_2.py", - "TC_EWATERHTR_2_3.py", - "TC_EnergyReporting_Utils.py", - "TC_OpstateCommon.py", - "TC_pics_checker.py", - "TC_TMP_2_1.py", - "TC_MCORE_FS_1_1.py", - "TC_MCORE_FS_1_2.py", - "TC_MCORE_FS_1_3.py", - "TC_MCORE_FS_1_4.py", - "TC_MCORE_FS_1_5.py", - "TC_OCC_3_1.py", - "TC_OCC_3_2.py", - "TC_BRBINFO_4_1.py", - "TestCommissioningTimeSync.py", - "TestConformanceSupport.py", - "TestChoiceConformanceSupport.py", - "TC_DEMTestBase.py", - "choice_conformance_support.py", - "TestConformanceTest.py", # Unit test of the conformance test (TC_DeviceConformance) - does not run against an app. - "TestIdChecks.py", - "TestSpecParsingDeviceType.py", - "TestConformanceTest.py", - "TestMatterTestingSupport.py", - "TestSpecParsingSupport.py", - "TestTimeSyncTrustedTimeSource.py", - "basic_composition_support.py", - "conformance_support.py", - "drlk_2_x_common.py", - "execute_python_tests.py", - "global_attribute_ids.py", - "hello_external_runner.py", - "hello_test.py", - "matter_testing_support.py", - "pics_support.py", - "spec_parsing_support.py", - "taglist_and_topology_test_support.py", - "test_plan_support.py", - "test_plan_table_generator.py" + "MinimalRepresentation.py", # Code/Test not being used or not shared code for any other tests + "TC_CNET_4_4.py", # It has no CI execution block, is not executed in CI + "TC_CCTRL_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_CCTRL_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_CCTRL_2_3.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_DGGEN_3_2.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_DGGEN_3_2.py + "TC_EEVSE_Utils.py", # Shared code for TC_EEVSE, not a standalone test + "TC_ECOINFO_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_ECOINFO_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_EWATERHTRBase.py", # Shared code for TC_EWATERHTR, not a standalone test + "TC_EnergyReporting_Utils.py", # Shared code for TC_EEM and TC_EPM, not a standalone test + "TC_OpstateCommon.py", # Shared code for TC_OPSTATE, not a standalone test + "TC_pics_checker.py", # Unit test/shared code of src/python_testing/test_testing/test_IDM_10_4.py + "TC_TMP_2_1.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_TMP_2_1.py + "TC_MCORE_FS_1_1.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_2.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_3.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_4.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_5.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_OCC_3_1.py", # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change + "TC_OCC_3_2.py", # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change + "TC_BRBINFO_4_1.py", # This test requires a TH_ICD_SERVER application, hence not ready run with CI + "TestCommissioningTimeSync.py", # Code/Test not being used or not shared code for any other tests + "TestConformanceSupport.py", # Unit test - does not run against an app + "TestChoiceConformanceSupport.py", # Unit test - does not run against an app + "TC_DEMTestBase.py", # Shared code for TC_DEM, not a standalone test + "TestConformanceTest.py", # Unit test of the conformance test (TC_DeviceConformance) - does not run against an app + "TestIdChecks.py", # Unit test - does not run against an app + "TestSpecParsingDeviceType.py", # Unit test - does not run against an app + "TestConformanceTest.py", # Unit test - does not run against an app + "TestMatterTestingSupport.py", # Unit test - does not run against an app + "TestSpecParsingSupport.py", # Unit test - does not run against an app + "TestTimeSyncTrustedTimeSource.py", # Unit test and shared code for scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py + "drlk_2_x_common.py", # Shared code for TC_DRLK, not a standalone test + "execute_python_tests.py", # Python code that runs all the python tests from src/python_testing (This code itself run via tests.yaml) + "hello_external_runner.py", # Code/Test not being used or not shared code for any other tests + "hello_test.py", # Is a template for tests + "test_plan_support.py", # Shared code for TC_*, not a standalone test + "test_plan_table_generator.py" # Code/Test not being used or not shared code for any other tests } - """ - Explanation for excluded files: - The above list of files are excluded from the tests as they are either not app-specific tests - or are run through a different block of tests mentioned in tests.yaml. - This is to ensure that only relevant test scripts are executed, avoiding redundancy and ensuring - the correctness of the test suite. - """ + # Explanation for excluded files: + # The above list of files are excluded from the tests as they are either not app-specific tests + # or are run through a different block of tests mentioned in tests.yaml. + # This is to ensure that only relevant test scripts are executed, avoiding redundancy and ensuring + # the correctness of the test suite. # Get all .py files in the directory all_python_files = glob.glob(os.path.join(search_directory, "*.py")) From 07eae0e8c45ba24c07a9ae89eeaa8363812d7271 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 18 Sep 2024 19:07:33 +0000 Subject: [PATCH 2/5] Restyled by autopep8 --- src/python_testing/execute_python_tests.py | 77 +++++++++++----------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/python_testing/execute_python_tests.py b/src/python_testing/execute_python_tests.py index 1cf4a68fb0f4f6..58268e37e6201a 100644 --- a/src/python_testing/execute_python_tests.py +++ b/src/python_testing/execute_python_tests.py @@ -55,45 +55,46 @@ def main(search_directory, env_file): # Define the files and patterns to exclude excluded_patterns = { - "MinimalRepresentation.py", # Code/Test not being used or not shared code for any other tests - "TC_CNET_4_4.py", # It has no CI execution block, is not executed in CI - "TC_CCTRL_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now - "TC_CCTRL_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now - "TC_CCTRL_2_3.py", # They rely on example applications that inter-communicate and there is no example app that works right now - "TC_DGGEN_3_2.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_DGGEN_3_2.py - "TC_EEVSE_Utils.py", # Shared code for TC_EEVSE, not a standalone test - "TC_ECOINFO_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now - "TC_ECOINFO_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now - "TC_EWATERHTRBase.py", # Shared code for TC_EWATERHTR, not a standalone test - "TC_EnergyReporting_Utils.py", # Shared code for TC_EEM and TC_EPM, not a standalone test - "TC_OpstateCommon.py", # Shared code for TC_OPSTATE, not a standalone test - "TC_pics_checker.py", # Unit test/shared code of src/python_testing/test_testing/test_IDM_10_4.py - "TC_TMP_2_1.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_TMP_2_1.py - "TC_MCORE_FS_1_1.py", # This test requires a TH_SERVER application, hence not ready run with CI - "TC_MCORE_FS_1_2.py", # This test requires a TH_SERVER application, hence not ready run with CI - "TC_MCORE_FS_1_3.py", # This test requires a TH_SERVER application, hence not ready run with CI - "TC_MCORE_FS_1_4.py", # This test requires a TH_SERVER application, hence not ready run with CI - "TC_MCORE_FS_1_5.py", # This test requires a TH_SERVER application, hence not ready run with CI - "TC_OCC_3_1.py", # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change - "TC_OCC_3_2.py", # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change - "TC_BRBINFO_4_1.py", # This test requires a TH_ICD_SERVER application, hence not ready run with CI - "TestCommissioningTimeSync.py", # Code/Test not being used or not shared code for any other tests - "TestConformanceSupport.py", # Unit test - does not run against an app - "TestChoiceConformanceSupport.py", # Unit test - does not run against an app - "TC_DEMTestBase.py", # Shared code for TC_DEM, not a standalone test + "MinimalRepresentation.py", # Code/Test not being used or not shared code for any other tests + "TC_CNET_4_4.py", # It has no CI execution block, is not executed in CI + "TC_CCTRL_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_CCTRL_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_CCTRL_2_3.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_DGGEN_3_2.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_DGGEN_3_2.py + "TC_EEVSE_Utils.py", # Shared code for TC_EEVSE, not a standalone test + "TC_ECOINFO_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_ECOINFO_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now + "TC_EWATERHTRBase.py", # Shared code for TC_EWATERHTR, not a standalone test + "TC_EnergyReporting_Utils.py", # Shared code for TC_EEM and TC_EPM, not a standalone test + "TC_OpstateCommon.py", # Shared code for TC_OPSTATE, not a standalone test + "TC_pics_checker.py", # Unit test/shared code of src/python_testing/test_testing/test_IDM_10_4.py + "TC_TMP_2_1.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_TMP_2_1.py + "TC_MCORE_FS_1_1.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_2.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_3.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_4.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_MCORE_FS_1_5.py", # This test requires a TH_SERVER application, hence not ready run with CI + "TC_OCC_3_1.py", # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change + "TC_OCC_3_2.py", # There are CI issues for the test cases that implements manually controlling sensor device for the occupancy state ON/OFF change + "TC_BRBINFO_4_1.py", # This test requires a TH_ICD_SERVER application, hence not ready run with CI + "TestCommissioningTimeSync.py", # Code/Test not being used or not shared code for any other tests + "TestConformanceSupport.py", # Unit test - does not run against an app + "TestChoiceConformanceSupport.py", # Unit test - does not run against an app + "TC_DEMTestBase.py", # Shared code for TC_DEM, not a standalone test "TestConformanceTest.py", # Unit test of the conformance test (TC_DeviceConformance) - does not run against an app - "TestIdChecks.py", # Unit test - does not run against an app - "TestSpecParsingDeviceType.py", # Unit test - does not run against an app - "TestConformanceTest.py", # Unit test - does not run against an app - "TestMatterTestingSupport.py", # Unit test - does not run against an app - "TestSpecParsingSupport.py", # Unit test - does not run against an app - "TestTimeSyncTrustedTimeSource.py", # Unit test and shared code for scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py - "drlk_2_x_common.py", # Shared code for TC_DRLK, not a standalone test - "execute_python_tests.py", # Python code that runs all the python tests from src/python_testing (This code itself run via tests.yaml) - "hello_external_runner.py", # Code/Test not being used or not shared code for any other tests - "hello_test.py", # Is a template for tests - "test_plan_support.py", # Shared code for TC_*, not a standalone test - "test_plan_table_generator.py" # Code/Test not being used or not shared code for any other tests + "TestIdChecks.py", # Unit test - does not run against an app + "TestSpecParsingDeviceType.py", # Unit test - does not run against an app + "TestConformanceTest.py", # Unit test - does not run against an app + "TestMatterTestingSupport.py", # Unit test - does not run against an app + "TestSpecParsingSupport.py", # Unit test - does not run against an app + "TestTimeSyncTrustedTimeSource.py", # Unit test and shared code for scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py + "drlk_2_x_common.py", # Shared code for TC_DRLK, not a standalone test + # Python code that runs all the python tests from src/python_testing (This code itself run via tests.yaml) + "execute_python_tests.py", + "hello_external_runner.py", # Code/Test not being used or not shared code for any other tests + "hello_test.py", # Is a template for tests + "test_plan_support.py", # Shared code for TC_*, not a standalone test + "test_plan_table_generator.py" # Code/Test not being used or not shared code for any other tests } # Explanation for excluded files: From d33abd5c55d9aaa8669c80b49bb90e095c199913 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 18 Sep 2024 15:13:26 -0400 Subject: [PATCH 3/5] Removed exclusion test script explanation --- src/python_testing/execute_python_tests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/python_testing/execute_python_tests.py b/src/python_testing/execute_python_tests.py index 58268e37e6201a..0d2c02ffc0d98c 100644 --- a/src/python_testing/execute_python_tests.py +++ b/src/python_testing/execute_python_tests.py @@ -97,12 +97,6 @@ def main(search_directory, env_file): "test_plan_table_generator.py" # Code/Test not being used or not shared code for any other tests } - # Explanation for excluded files: - # The above list of files are excluded from the tests as they are either not app-specific tests - # or are run through a different block of tests mentioned in tests.yaml. - # This is to ensure that only relevant test scripts are executed, avoiding redundancy and ensuring - # the correctness of the test suite. - # Get all .py files in the directory all_python_files = glob.glob(os.path.join(search_directory, "*.py")) From df535c2cedfb8a4b8e4799480e644209ca785c2d Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 18 Sep 2024 15:40:49 -0400 Subject: [PATCH 4/5] Fixed comments on the test scripts --- src/python_testing/execute_python_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python_testing/execute_python_tests.py b/src/python_testing/execute_python_tests.py index 0d2c02ffc0d98c..29ccddafcf57fb 100644 --- a/src/python_testing/execute_python_tests.py +++ b/src/python_testing/execute_python_tests.py @@ -60,15 +60,15 @@ def main(search_directory, env_file): "TC_CCTRL_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now "TC_CCTRL_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now "TC_CCTRL_2_3.py", # They rely on example applications that inter-communicate and there is no example app that works right now - "TC_DGGEN_3_2.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_DGGEN_3_2.py + "TC_DGGEN_3_2.py", # src/python_testing/test_testing/test_TC_DGGEN_3_2.py is the Unit test of this test "TC_EEVSE_Utils.py", # Shared code for TC_EEVSE, not a standalone test "TC_ECOINFO_2_1.py", # They rely on example applications that inter-communicate and there is no example app that works right now "TC_ECOINFO_2_2.py", # They rely on example applications that inter-communicate and there is no example app that works right now "TC_EWATERHTRBase.py", # Shared code for TC_EWATERHTR, not a standalone test "TC_EnergyReporting_Utils.py", # Shared code for TC_EEM and TC_EPM, not a standalone test "TC_OpstateCommon.py", # Shared code for TC_OPSTATE, not a standalone test - "TC_pics_checker.py", # Unit test/shared code of src/python_testing/test_testing/test_IDM_10_4.py - "TC_TMP_2_1.py", # Unit test/shared code of src/python_testing/test_testing/test_TC_TMP_2_1.py + "TC_pics_checker.py", # Currently isn't enabled because we don't have any examples with conformant PICS + "TC_TMP_2_1.py", # src/python_testing/test_testing/test_TC_TMP_2_1.py is the Unit test of this test "TC_MCORE_FS_1_1.py", # This test requires a TH_SERVER application, hence not ready run with CI "TC_MCORE_FS_1_2.py", # This test requires a TH_SERVER application, hence not ready run with CI "TC_MCORE_FS_1_3.py", # This test requires a TH_SERVER application, hence not ready run with CI From 20dc8ee9a85d236083ce1ead9c47ad5c95706203 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 19 Sep 2024 09:58:53 -0400 Subject: [PATCH 5/5] Fix excluded scripts for adding support scripts still in the directory --- src/python_testing/execute_python_tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/python_testing/execute_python_tests.py b/src/python_testing/execute_python_tests.py index 29ccddafcf57fb..1f8621ca2a9b05 100644 --- a/src/python_testing/execute_python_tests.py +++ b/src/python_testing/execute_python_tests.py @@ -94,7 +94,15 @@ def main(search_directory, env_file): "hello_external_runner.py", # Code/Test not being used or not shared code for any other tests "hello_test.py", # Is a template for tests "test_plan_support.py", # Shared code for TC_*, not a standalone test - "test_plan_table_generator.py" # Code/Test not being used or not shared code for any other tests + "test_plan_table_generator.py", # Code/Test not being used or not shared code for any other tests + "basic_composition_support.py", # Test support/shared code script, not a standalone test + "choice_conformance_support.py", # Test support/shared code script, not a standalone test + "conformance_support.py", # Test support/shared code script, not a standalone test + "global_attribute_ids.py", # Test support/shared code script, not a standalone test + "matter_testing_support.py", # Test support/shared code script, not a standalone test + "pics_support.py", # Test support/shared code script, not a standalone test + "spec_parsing_support.py", # Test support/shared code script, not a standalone test + "taglist_and_topology_test_support.py" # Test support/shared code script, not a standalone test } # Get all .py files in the directory