Allow to specify run delay between app and test script (project-chip#35216)

* Allow to specify run delay between app and test script

* Restyled by prettier-markdown

* Explain the use case of the delay

---------

Co-authored-by: Restyled.io <commits@restyled.io>

Author: arkq

docs/testing/python.md

@@ -635,9 +635,19 @@ markers must be present.
 
 -   `test-runner-run/<run_identifier>/script-args`: Specifies the arguments to
     be passed to the test script.
+
     -   Example:
         `--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto`
 
+-   `test-runner-run/<run_identifier>/script-start-delay`: Specifies the number
+    of seconds to wait before starting the test script. This parameter can be
+    used to allow the application to initialize itself properly before the test
+    script will try to commission it (e.g. in case if the application needs to
+    be commissioned to some other controller first). By default, the delay is 0
+    seconds.
+
+    -   Example: `10`
+
 This structured format ensures that all necessary configurations are clearly
 defined and easily understood, allowing for consistent and reliable test
 execution.

scripts/tests/run_python_test.py

@@ -87,11 +87,14 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st
                                                                              'mobile-device-test.py'), help='Test script to use.')
 @click.option("--script-args", type=str, default='',
               help='Script arguments, can use placeholders like {SCRIPT_BASE_NAME}.')
+@click.option("--script-start-delay", type=int, default=0,
+              help='Delay in seconds before starting the script.')
 @click.option("--script-gdb", is_flag=True,
               help='Run script through gdb')
 @click.option("--quiet", is_flag=True, help="Do not print output from passing tests. Use this flag in CI to keep github log sizes manageable.")
 @click.option("--load-from-env", default=None, help="YAML file that contains values for environment variables.")
-def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str, script: str, script_args: str, script_gdb: bool, quiet: bool, load_from_env):
+def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str,
+         script: str, script_args: str, script_start_delay: int, script_gdb: bool, quiet: bool, load_from_env):
     if load_from_env:
         reader = MetadataReader(load_from_env)
         runs = reader.parse_script(script)
@@ -103,6 +106,7 @@ def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: st
                 app=app,
                 app_args=app_args,
                 script_args=script_args,
+                script_start_delay=script_start_delay,
                 factoryreset=factoryreset,
                 factoryreset_app_only=factoryreset_app_only,
                 script_gdb=script_gdb,
@@ -117,10 +121,11 @@ def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: st
     for run in runs:
         print(f"Executing {run.py_script_path.split('/')[-1]} {run.run}")
         main_impl(run.app, run.factoryreset, run.factoryreset_app_only, run.app_args,
-                  run.py_script_path, run.script_args, run.script_gdb, run.quiet)
+                  run.py_script_path, run.script_args, run.script_start_delay, run.script_gdb, run.quiet)
 
 
-def main_impl(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str, script: str, script_args: str, script_gdb: bool, quiet: bool):
+def main_impl(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str,
+              script: str, script_args: str, script_start_delay: int, script_gdb: bool, quiet: bool):
 
     app_args = app_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0])
     script_args = script_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0])
@@ -175,6 +180,8 @@ def main_impl(app: str, factoryreset: bool, factoryreset_app_only: bool, app_arg
         DumpProgramOutputToQueue(
             log_cooking_threads, Fore.GREEN + "APP " + Style.RESET_ALL, app_process, stream_output, log_queue)
 
+    time.sleep(script_start_delay)
+
     script_command = [script, "--paa-trust-store-path", os.path.join(DEFAULT_CHIP_ROOT, MATTER_DEVELOPMENT_PAA_ROOT_CERTS),
                       '--log-format', '%(message)s', "--app-pid", str(app_pid)] + shlex.split(script_args)
 

src/python_testing/matter_testing_infrastructure/metadata_parser/metadata.py

@@ -33,6 +33,7 @@ class Metadata:
     app: str
     app_args: str
     script_args: str
+    script_start_delay: int = 0
     factoryreset: bool = False
     factoryreset_app_only: bool = False
     script_gdb: bool = False
@@ -60,6 +61,9 @@ def copy_from_dict(self, attr_dict: Dict[str, Any]) -> None:
         if "script-args" in attr_dict:
             self.script_args = attr_dict["script-args"]
 
+        if "script-start-delay" in attr_dict:
+            self.script_start_delay = int(attr_dict["script-start-delay"])
+
         if "py_script_path" in attr_dict:
             self.py_script_path = attr_dict["py_script_path"]
 
@@ -187,6 +191,7 @@ def parse_script(self, py_script_path: str) -> List[Metadata]:
                 app=attr.get("app", ""),
                 app_args=attr.get("app_args", ""),
                 script_args=attr.get("script_args", ""),
+                script_start_delay=int(attr.get("script_start_delay", 0)),
                 factoryreset=bool(attr.get("factoryreset", False)),
                 factoryreset_app_only=bool(attr.get("factoryreset_app_only", False)),
                 script_gdb=bool(attr.get("script_gdb", False)),