fix: collect artifacts on test setup/teardown (#117)

Author: storenth

pytest.ini

@@ -0,0 +1,5 @@
+; [pytest]
+; log_cli=true
+; log_cli_level=DEBUG
+; log_cli_format=%(asctime)s.%(msecs)03d %(levelname)s %(name)s:%(filename)s:%(lineno)s: %(message)s
+; log_cli_date_format=%Y-%m-%d %H:%M:%S

pytest_playwright/pytest_playwright.py

@@ -18,7 +18,7 @@
 import sys
 import warnings
 from typing import Any, Callable, Dict, Generator, List, Optional
-
+import logging
 import pytest
 from playwright.sync_api import (
     Browser,
@@ -33,6 +33,8 @@
 import tempfile
 
 
+log = logging.getLogger(__name__)
+
 artifacts_folder = tempfile.TemporaryDirectory(prefix="playwright-pytest-")
 
 
@@ -74,6 +76,34 @@ def pytest_configure(config: Any) -> None:
         "markers",
         "browser_context_args(**kwargs): provide additional arguments to browser.new_context()",
     )
+    log.debug("pytest_configure")
+    class Teardown:
+        failed = False
+    setattr(config, "teardown", Teardown)
+
+
+def pytest_runtest_teardown(item):
+    # import faulthandler
+    # faulthandler.dump_traceback(all_threads=True)
+    log.debug("pytest_runtest_teardown")
+    item.config.teardown.failed = True
+
+
+def pytest_sessionfinish(session, exitstatus):
+    log.debug("pytest_runtest_teardown")
+    log.debug(f"{exitstatus=}")
+
+
+def pytest_exception_interact(node, call, report):
+    log.debug("1pytest_exception_interact")
+    log.debug(f"{node.config.teardown.failed=}")
+    log.debug(f"{node.session.testsfailed=}")
+    excinfo = call.exc_info if hasattr(call, "exc_info") else None
+    log.debug(f"{excinfo=}")
+
+    if call.when == "teardown":
+        log.debug("2pytest_exception_interact")
+        node.config.teardown.failed = True
 
 
 # Making test result information available in fixtures
@@ -256,10 +286,52 @@ def context(
         )
 
     yield context
+    # import traceback
+    # log.debug(f"{dir(traceback)=}")
+    # log.debug(f"{traceback.print_last()=}")
+    # stack = traceback.extract_stack()
+    # log.debug(f"{stack=}")
+    # for frame_summary in stack:
+    #     log.debug(f"{frame_summary.filename=}")
+    #     log.debug(f"{frame_summary.name=}")
+    #     log.debug(f"{frame_summary.colno=}")
+
+
+    # log.debug(f"{traceback.print_stack()=}")
 
+    # log.debug(f"{traceback.print_exc()=}")
+
+    log.debug(f"{request.session.testscollected=}")
+    log.debug(f"{request.session.exitstatus=}")
+    log.debug(f"{request.session.testsfailed=}")
+
+    if request.session.testsfailed:
+        log.debug("Only print if failed")
+
+    log.debug("context")
     # If request.node is missing rep_call, then some error happened during execution
     # that prevented teardown, but should still be counted as a failure
-    failed = request.node.rep_call.failed if hasattr(request.node, "rep_call") else True
+    failed_setup = request.node.rep_setup.failed if hasattr(request.node, "rep_setup") else False
+    failed_call = request.node.rep_call.failed if hasattr(request.node, "rep_call") else False
+    failed_teardown = request.node.rep_teardown.failed if hasattr(request.node, "rep_teardown") else False
+
+    failed_xteardown = request.config.teardown.failed if hasattr(request.config, "teardown") else False
+
+
+    failed = failed_setup or failed_call or failed_xteardown
+    log.debug(f"{failed=}")
+    log.debug(f"{failed_setup=}")
+    log.debug(f"{failed_call=}")
+    log.debug(f"{failed_teardown=}")
+    log.debug(f"{failed_xteardown=}")
+
+
+    log.debug(f"{hasattr(request.node, 'rep_setup')=}")
+    log.debug(f"{hasattr(request.node, 'rep_call')=}")
+    log.debug(f"{hasattr(request.node, 'rep_teardown')=}")
+
+    log.debug(f"{hasattr(request.config, 'teardown')=}")
+
 
     if capture_trace:
         retain_trace = tracing_option == "on" or (
@@ -272,9 +344,11 @@ def context(
             context.tracing.stop()
 
     screenshot_option = pytestconfig.getoption("--screenshot")
+    log.debug(f"{screenshot_option=}")
     capture_screenshot = screenshot_option == "on" or (
         failed and screenshot_option == "only-on-failure"
     )
+    log.debug(f"{capture_screenshot=}")
     if capture_screenshot:
         for index, page in enumerate(pages):
             human_readable_status = "failed" if failed else "finished"

tests/test_playwright.py

@@ -703,6 +703,73 @@ def test_failing(page):
     _assert_folder_tree(test_results_dir, expected)
 
 
+def test_artifacts_retain_on_setup_failure(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        @pytest.fixture
+        def failed_setup_call(page):
+            assert 1 == page.evaluate("1 + 1")
+            yield page
+
+        def test_failing(page, failed_setup_call):
+            assert 2 == page.evaluate("1 + 1")
+        """
+    )
+    result = testdir.runpytest(
+        "--screenshot",
+        "only-on-failure"
+    )
+    result.assert_outcomes(errors=1)
+    test_results_dir = os.path.join(testdir.tmpdir, "test-results")
+
+    expected = [
+        {
+            "name": "test-artifacts-retain-on-setup-failure-py-test-failing-chromium",
+            "children": [
+                {
+                    "name": "test-failed-1.png",
+                }
+            ],
+        }
+    ]
+    _assert_folder_tree(test_results_dir, expected)
+
+# request.config.teardown.failed = False
+def test_artifacts_on_teardown(testdir: pytest.Testdir) -> None:
+    testdir.makepyfile(
+        """
+        import pytest
+        @pytest.fixture
+        def failed_teardown_call(page, request):
+            yield page
+            assert 1 == page.evaluate("1 + 1")
+
+        def test_passing(page, failed_teardown_call):
+            assert 2 == page.evaluate("1 + 1")
+        """
+    )
+    result = testdir.runpytest(
+        "--screenshot",
+        "only-on-failure"
+    )
+    result.assert_outcomes(passed=1, errors=1)  # сделать чтобы скрин не сохранялся при успешном тесте (и все успешные тирдауны и сетапы)
+    test_results_dir = os.path.join(testdir.tmpdir, "test-results")
+    print(f"{os.listdir(test_results_dir)=}")
+    print(os.listdir(f"{test_results_dir}/test-artifacts-on-teardown-py-test-passing-chromium"))
+    expected = [
+        {
+            "name": "test-artifacts-on-teardown-py-test-passing-chromium",
+            "children": [
+                {
+                    "name": "test-failed-1.png",
+                }
+            ],
+        }
+    ]
+    _assert_folder_tree(test_results_dir, expected)
+
+
 def test_should_work_with_test_names_which_exceeds_256_characters(
     testdir: pytest.Testdir,
 ) -> None:
@@ -734,8 +801,10 @@ def _assert_folder_tree(root: str, expected_tree: List[Any]) -> None:
     for file in expected_tree:
         if isinstance(file["name"], str):
             if "children" in file:
+                print(f"{file=}")
                 assert os.path.isdir(os.path.join(root, file["name"]))
             else:
+                print(f"{file=}")
                 assert os.path.isfile(os.path.join(root, file["name"]))
         if isinstance(file["name"], re.Pattern):
             assert any([file["name"].match(item) for item in os.listdir(root)])