Only store failed tests mpl-pytest (#113)

* Only include failed tests when uploading

* test: try a failure live

* Update ultraplot/tests/test_1dplots.py

---------

Co-authored-by: Matthew R. Becker <beckermr@users.noreply.github.com>

Author: cvanelteren

.github/workflows/build-ultraplot.yml

@@ -81,7 +81,7 @@ jobs:
           micromamba activate ultraplot-dev
           mkdir -p results
           python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')"
-          pytest --mpl --mpl-baseline-path=baseline --mpl-generate-summary=html --mpl-results-path=./results/ --mpl-default-style="./ultraplot.yml"
+          pytest --mpl --mpl-baseline-path=baseline --mpl-generate-summary=html --mpl-results-path=./results/ --mpl-default-style="./ultraplot.yml" --store-failed-only ultraplot/tests
 
       # Return the html output of the comparison even if failed
       - name: Upload comparison failures

ultraplot/tests/conftest.py

@@ -0,0 +1,72 @@
+import os, shutil, pytest, re
+from pathlib import Path
+import warnings
+
+warnings.simplefilter("ignore")
+warnings.filterwarnings(
+    "ignore", message="Bad key .* in file .*ultraplot.yml", module="matplotlib"
+)
+
+
+# Define command line option
+def pytest_addoption(parser):
+    parser.addoption(
+        "--store-failed-only",
+        action="store_true",
+        help="Store only failed matplotlib comparison images",
+    )
+
+
+class StoreFailedMplPlugin:
+    def __init__(self, config):
+        self.config = config
+
+        # Get base directories as Path objects
+        self.result_dir = Path(config.getoption("--mpl-results-path", "./results"))
+        self.baseline_dir = Path(config.getoption("--mpl-baseline-path", "./baseline"))
+
+        print(f"Store Failed MPL Plugin initialized")
+        print(f"Result dir: {self.result_dir}")
+
+    def _has_mpl_marker(self, report: pytest.TestReport):
+        """Check if the test has the mpl_image_compare marker."""
+        return report.keywords.get("mpl_image_compare", False)
+
+    def _remove_success(self, report: pytest.TestReport):
+        """Remove successful test images."""
+
+        pattern = r"(?P<sep>::|/)|\[|\]|\.py"
+        name = re.sub(
+            pattern,
+            lambda m: "." if m.group("sep") else "_" if m.group(0) == "[" else "",
+            report.nodeid,
+        )
+        target = (self.result_dir / name).absolute()
+        if target.is_dir():
+            print(f"Removing {report.nodeid}")
+            shutil.rmtree(target)
+        else:
+            print(f"Did not find {report.nodeid}")
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_logreport(self, report):
+        """Hook that processes each test report."""
+        if report.when == "call":
+            # Delete successfull tests
+            if report.failed == False:
+                if self._has_mpl_marker(report):
+                    self._remove_success(report)
+            else:
+                print(f"{report.failed=}")
+                print(f"Skipping {report.nodeid}")
+
+
+# Register the plugin if the option is used
+def pytest_configure(config):
+    print("Configuring StoreFailedMplPlugin")
+    try:
+        if config.getoption("--store-failed-only", False):
+            print("Registering StoreFailedMplPlugin")
+            config.pluginmanager.register(StoreFailedMplPlugin(config))
+    except Exception as e:
+        print(f"Error during plugin configuration: {e}")