From 81c15fe5ea87af8cb8a35bc0e26b639548365680 Mon Sep 17 00:00:00 2001 From: cecille Date: Tue, 26 Mar 2024 11:02:29 -0400 Subject: [PATCH 01/11] Python scriptiong: Add a test plan generator script --- src/python_testing/__init__.py | 0 .../test_plan_table_generator.py | 59 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 src/python_testing/__init__.py create mode 100755 src/python_testing/test_plan_table_generator.py diff --git a/src/python_testing/__init__.py b/src/python_testing/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py new file mode 100755 index 00000000000000..329b7b6f64db26 --- /dev/null +++ b/src/python_testing/test_plan_table_generator.py @@ -0,0 +1,59 @@ +#!/usr/bin/env -S python3 -B +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import click +import importlib +import os +import string + +from pathlib import Path +from matter_testing_support import generate_mobly_test_config, MatterTestConfig + + +def indent_multiline(multiline: str, num_spaces: int) -> str: + ''' Indents subsequent lines of a multiline string by num_spaces spaces''' + s = multiline.split('\n') + s = [(num_spaces * ' ' + line.lstrip()).rstrip() for line in s] + return '\n'.join(s).lstrip() + + +@click.command() +@click.argument('filename', type=click.Path(exists=True)) +@click.argument('classname', type=str) +@click.argument('test', type=str) +def main(filename, classname, test): + module = importlib.import_module(Path(os.path.basename(filename)).stem) + test_class = getattr(module, classname) + config = generate_mobly_test_config(MatterTestConfig()) + test_instance = test_class(config) + steps = test_instance.get_test_steps(test) + indent = 6 + s = ('[cols="5%,45%,45%"]\n' + '|===\n' + '|**#** |*Test Step*|*Expected Outcome*\n') + for step in steps: + # add 2 to indent for a| and | at start + s += f'|{step.test_plan_number:<{indent}}a|{indent_multiline(step.description, indent+3)}\n' + padding = (indent + 1) * ' ' + s += f'{padding}a|{indent_multiline(step.expectation, indent+3)}\n\n' + s += '|===\n' + + print(s) + + +if __name__ == "__main__": + main() From b0434e10f10c109fa11de495d11d166993f7d304 Mon Sep 17 00:00:00 2001 From: cecille Date: Tue, 2 Apr 2024 17:35:35 -0400 Subject: [PATCH 02/11] Fix linter --- src/python_testing/test_plan_table_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index 329b7b6f64db26..f02ea52d730f5d 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -18,7 +18,6 @@ import click import importlib import os -import string from pathlib import Path from matter_testing_support import generate_mobly_test_config, MatterTestConfig From c429a16b50754222d7ece5e216fd08063afd793a Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 2 Apr 2024 21:36:18 +0000 Subject: [PATCH 03/11] Restyled by isort --- src/python_testing/test_plan_table_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index f02ea52d730f5d..9d6242270218ed 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -15,12 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import click import importlib import os - from pathlib import Path -from matter_testing_support import generate_mobly_test_config, MatterTestConfig + +import click +from matter_testing_support import MatterTestConfig, generate_mobly_test_config def indent_multiline(multiline: str, num_spaces: int) -> str: From 91e7b6d533350355004bf4024021b0b76bc367a8 Mon Sep 17 00:00:00 2001 From: cecille Date: Wed, 3 Apr 2024 10:26:35 -0400 Subject: [PATCH 04/11] fix spacing --- src/python_testing/test_plan_table_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index 9d6242270218ed..cbdccce6bce0a0 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -43,7 +43,7 @@ def main(filename, classname, test): indent = 6 s = ('[cols="5%,45%,45%"]\n' '|===\n' - '|**#** |*Test Step*|*Expected Outcome*\n') + '|**#** |*Test Step*|*Expected Outcome*\n') for step in steps: # add 2 to indent for a| and | at start s += f'|{step.test_plan_number:<{indent}}a|{indent_multiline(step.description, indent+3)}\n' From 415ee0a491ea7506a1cf7035309ab037d606ccec Mon Sep 17 00:00:00 2001 From: cecille Date: Fri, 5 Apr 2024 17:52:17 -0400 Subject: [PATCH 05/11] update spacing on expected outcome --- src/python_testing/test_plan_table_generator.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index cbdccce6bce0a0..c3e672ffca62f9 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -41,14 +41,18 @@ def main(filename, classname, test): test_instance = test_class(config) steps = test_instance.get_test_steps(test) indent = 6 + header_num = f'{"**#**":<{indent}}' + header_num_step = f'|{header_num} |*TestStep* ' s = ('[cols="5%,45%,45%"]\n' '|===\n' - '|**#** |*Test Step*|*Expected Outcome*\n') + f'{header_num_step}|*Expected Outcome*\n') for step in steps: - # add 2 to indent for a| and | at start - s += f'|{step.test_plan_number:<{indent}}a|{indent_multiline(step.description, indent+3)}\n' - padding = (indent + 1) * ' ' - s += f'{padding}a|{indent_multiline(step.expectation, indent+3)}\n\n' + step_num = f'|{step.test_plan_number:<{indent}}a|' + s += f'{step_num}{indent_multiline(step.description, len(step_num))}\n' + + padding = (len(header_num_step) - 1) * ' ' + # add 2 to indent for a| at start + s += f'{padding}a|{indent_multiline(step.expectation, len(padding)+2)}\n\n' s += '|===\n' print(s) From 3b86616e49c8b48e548ef28312b1c90a7c40e559 Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Tue, 23 Apr 2024 13:56:31 -0400 Subject: [PATCH 06/11] remove init file - it snuck in, but isn't needed --- src/python_testing/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/python_testing/__init__.py diff --git a/src/python_testing/__init__.py b/src/python_testing/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 From 18aae7bf7c3938dac298a98f480a63e73baddf03 Mon Sep 17 00:00:00 2001 From: cecille Date: Thu, 25 Apr 2024 11:02:46 -0400 Subject: [PATCH 07/11] Add documnetation and better error reporting. --- src/python_testing/matter_testing_support.py | 4 +- .../test_plan_table_generator.py | 41 +++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/python_testing/matter_testing_support.py b/src/python_testing/matter_testing_support.py index 9ec12dca5fb081..3296b04e1ee0e1 100644 --- a/src/python_testing/matter_testing_support.py +++ b/src/python_testing/matter_testing_support.py @@ -684,10 +684,10 @@ def get_test_steps(self, test: str) -> list[TestStep]: in order using self.step(number), where number is the test_plan_number from each TestStep. ''' - steps = self._get_defined_test_steps(test) + steps = self.get_defined_test_steps(test) return [TestStep(1, "Run entire test")] if steps is None else steps - def _get_defined_test_steps(self, test: str) -> list[TestStep]: + def get_defined_test_steps(self, test: str) -> list[TestStep]: steps_name = 'steps_' + test[5:] try: fn = getattr(self, steps_name) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index c3e672ffca62f9..40b5438f943a75 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -15,8 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import logging import importlib import os +import sys from pathlib import Path import click @@ -35,11 +37,42 @@ def indent_multiline(multiline: str, num_spaces: int) -> str: @click.argument('classname', type=str) @click.argument('test', type=str) def main(filename, classname, test): - module = importlib.import_module(Path(os.path.basename(filename)).stem) - test_class = getattr(module, classname) + ''' + This script generates the Test Procedure table for the test plans document + from the python script steps. In order to use this generator, the test + automation script conform to the following requirements: + - automated in python + - test implements the steps_ function to provide steps information to the TH + - TestStep list returned from the steps_ function includes both description + and expectation fields + - test does not gate any steps on PICS (top level PICS ok) + + + Usage: test_plan_table_generator.py filename classname test + + filename - name of the file where the test is automated + classname - name of the MatterBaseTest class + test - name of the test to generate the table for (include the test_ portion) + ''' + try: + module = importlib.import_module(Path(os.path.basename(filename)).stem) + except ModuleNotFoundError as e: + logging.error(f'Unable to load python module from {filename}. Please ensure this is a valid python file path') + return -1 + + try: + test_class = getattr(module, classname) + except AttributeError as e: + logging.error(f'Unable to load the test class {classname}. Please ensure this class is implemented in {filename}') + return -1 + config = generate_mobly_test_config(MatterTestConfig()) test_instance = test_class(config) - steps = test_instance.get_test_steps(test) + steps = test_instance.get_defined_test_steps(test) + if not steps: + logging.error(f'Unable to find steps for test {test}. Please ensure the steps_ function is implemented') + return -1 + indent = 6 header_num = f'{"**#**":<{indent}}' header_num_step = f'|{header_num} |*TestStep* ' @@ -59,4 +92,4 @@ def main(filename, classname, test): if __name__ == "__main__": - main() + sys.exit(main()) From d5c26db898c336aa8ba2faac93769627829f0caf Mon Sep 17 00:00:00 2001 From: cecille Date: Thu, 25 Apr 2024 11:10:22 -0400 Subject: [PATCH 08/11] make print a little nicer for help --- src/python_testing/test_plan_table_generator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index 40b5438f943a75..ba9c5f2d3e991c 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -41,18 +41,18 @@ def main(filename, classname, test): This script generates the Test Procedure table for the test plans document from the python script steps. In order to use this generator, the test automation script conform to the following requirements: - - automated in python - - test implements the steps_ function to provide steps information to the TH - - TestStep list returned from the steps_ function includes both description - and expectation fields - - test does not gate any steps on PICS (top level PICS ok) + + - automated in python\n + - test implements the steps_ function to provide steps information to the TH\n + - TestStep list returned from the steps_ function includes both description and expectation fields\n + - test does not gate any steps on PICS (top level PICS ok)\n Usage: test_plan_table_generator.py filename classname test - filename - name of the file where the test is automated - classname - name of the MatterBaseTest class - test - name of the test to generate the table for (include the test_ portion) + filename - name of the file where the test is automated\n + classname - name of the MatterBaseTest class\n + test - name of the test to generate the table for (include the test_ portion)\n ''' try: module = importlib.import_module(Path(os.path.basename(filename)).stem) From 399b11675032c65b568116ec779abd92bb99d475 Mon Sep 17 00:00:00 2001 From: cecille Date: Thu, 25 Apr 2024 13:25:47 -0400 Subject: [PATCH 09/11] linter... --- src/python_testing/test_plan_table_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index ba9c5f2d3e991c..c0b3df9daa45cd 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -56,13 +56,13 @@ def main(filename, classname, test): ''' try: module = importlib.import_module(Path(os.path.basename(filename)).stem) - except ModuleNotFoundError as e: + except ModuleNotFoundError: logging.error(f'Unable to load python module from {filename}. Please ensure this is a valid python file path') return -1 try: test_class = getattr(module, classname) - except AttributeError as e: + except AttributeError: logging.error(f'Unable to load the test class {classname}. Please ensure this class is implemented in {filename}') return -1 From 6164fc64ac42472f04c23aa178889e6888bb3916 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 25 Apr 2024 17:30:38 +0000 Subject: [PATCH 10/11] Restyled by isort --- src/python_testing/test_plan_table_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py index c0b3df9daa45cd..bed4fa3eaa747e 100755 --- a/src/python_testing/test_plan_table_generator.py +++ b/src/python_testing/test_plan_table_generator.py @@ -15,8 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import logging import importlib +import logging import os import sys from pathlib import Path From 7401a03a75de5836ef9e9783a2401727942e7934 Mon Sep 17 00:00:00 2001 From: cecille Date: Fri, 26 Apr 2024 09:46:00 -0400 Subject: [PATCH 11/11] make defined steps function public --- src/python_testing/matter_testing_support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python_testing/matter_testing_support.py b/src/python_testing/matter_testing_support.py index 3296b04e1ee0e1..4861cae7ea8c61 100644 --- a/src/python_testing/matter_testing_support.py +++ b/src/python_testing/matter_testing_support.py @@ -781,7 +781,7 @@ def setup_test(self): self.step_skipped = False if self.runner_hook and not self.is_commissioning: test_name = self.current_test_info.name - steps = self._get_defined_test_steps(test_name) + steps = self.get_defined_test_steps(test_name) num_steps = 1 if steps is None else len(steps) filename = inspect.getfile(self.__class__) desc = self.get_test_desc(test_name) @@ -977,7 +977,7 @@ def on_pass(self, record): self.runner_hook.step_success(logger=None, logs=None, duration=step_duration, request=None) # TODO: this check could easily be annoying when doing dev. flag it somehow? Ditto with the in-order check - steps = self._get_defined_test_steps(record.test_name) + steps = self.get_defined_test_steps(record.test_name) if steps is None: # if we don't have a list of steps, assume they were all run all_steps_run = True