Merge pull request #153 from Kitware/main

Cut PiPy 0.6.0 release

README.md

@@ -11,8 +11,8 @@ is built using [trame][1] by the [kitware][2] team.
 
 - Explore image datasets in COCO format.
 - Apply parametrized image degradation (such as blur) to the images.
-- Benchmark dataset resilience with a differential PCA|UMAP analysis of the
-  embeddings of the images and its transformation.
+- Benchmark dataset resilience with a differential PCA|UMAP analysis on the
+  embeddings of the images and their transformations.
 - Evaluate object detection DL models in both the source images and its
   transformations.
 - When possible it will attempt to utilize the user GPU as much as possible to
@@ -28,25 +28,41 @@ pip install nrtk-explorer
 
 ## Usage
 
+Explore Hugging Face hosted [dataset](https://huggingface.co/datasets/rafaelpadilla/coco2017):
+
 ```bash
-# get some sample data
-git clone https://github.com/vicentebolea/nrtk_explorer_datasets.git
+nrtk-explorer --dataset rafaelpadilla/coco2017
+```
 
-# Run the application on given dataset
-nrtk-explorer --dataset ./nrtk_explorer_datasets/OIRDS_v1_0/oirds.json
+Compare inference results for Hugging Face hosted models:
+
+```bash
+nrtk-explorer --dataset cppe-5 --models qubvel-hf/detr-resnet-50-finetuned-10k-cppe5 ashaduzzaman/detr_finetuned_cppe5
 ```
 
-![nrtk explorer usage](https://github.com/user-attachments/assets/86a61485-471c-4b94-872e-943cb9da52a1)
+2 COCO format datasets are available at: https://github.com/vicentebolea/nrtk_explorer_datasets/
 
-Some COCO image datasets available at: https://github.com/vicentebolea/nrtk_explorer_datasets/
+```bash
+git clone https://github.com/vicentebolea/nrtk_explorer_datasets.git
+nrtk-explorer --dataset ./nrtk_explorer_datasets/coco-od-2017/mini_val2017.json ./nrtk_explorer_datasets/OIRDS_v1_0/oirds.json
+```
 
 ## CLI flags and options
 
-- `-h|--help` show the help for the command line options, it inherit trame
+- `--dataset` specify the path to a [COCO dataset](https://roboflow.com/formats/coco-json) JSON file,
+  a [Hugging Face dataset](https://huggingface.co/datasets?task_categories=task_categories:object-detection) repository name,
+  or a directory loadable by the [Dataset](https://huggingface.co/docs/datasets/index) library.
+  You can specify multiple datasets using a space as the
+  separator. Example: `nrtk-explorer --dataset ../foo-dir/coco.json cppe-5`
+- `--download` Cache Hugging Face Hub datasets locally instead of streaming them.
+  When datasets are streamed, nrtk-explorer limits the number of loaded images.
+- `--models` specify the Hugging Face Hub [object detection](https://huggingface.co/models?pipeline_tag=object-detection&library=transformers&sort=trending)
+  repository name or a directory loadable by the [Transformers](https://huggingface.co/docs/transformers/index) library. Load multiple models using space as the separator.  
+  Example: `nrtk-explorer --models hustvl/yolos-tiny facebook/detr-resnet-50`
+- `-h|--help` show the help for the command line options. nrtk-explorer inherits the trame
   command line options and flags.
-- `--dataset` specify the path to a json file describing a COCO
-  image dataset. You can specify multiple COCO datasets using a comma `,` as a
-  separator. Example usage: `nrtk_explorer --dataset /foo-dir/coco.json, ../bar-dir/baz.json`
+
+![nrtk explorer usage](https://github.com/user-attachments/assets/86a61485-471c-4b94-872e-943cb9da52a1)
 
 ## Contribute to NRTK_EXPLORER
 
@@ -62,12 +78,12 @@ pytest .
 
 For more details on setting up a development environment see [DEVELOPMENT docs](docs/source/manual/DEVELOPMENT.rst).
 
-[1]: https://trame.readthedocs.io/en/latest/
-[2]: https://www.kitware.com/
-[3]: https://cocodataset.org/
-
 ### Create release
 
 1. Merge `main` to `release` with a _merge commit_.
 2. Run "Create Release" workflow with workflow from `release` branch.
 3. Merge `release` to `main` with a _merge commit_.
+
+[1]: https://trame.readthedocs.io/en/latest/
+[2]: https://www.kitware.com/
+[3]: https://cocodataset.org/

pyproject.toml

@@ -30,8 +30,8 @@ dependencies = [
     "accelerate",
     "numpy",
     "Pillow",
-    "pybsm>=0.6",
-    "scikit-learn==1.5.2",
+    "pybsm>=0.6,<=0.9.0",
+    "scikit-learn>=1.6.0",
     "smqtk_image_io",
     "tabulate",
     "timm>=1.0.3",
@@ -40,8 +40,10 @@ dependencies = [
     "trame>=3.7",
     "trame-quasar",
     "transformers",
+    "datasets[vision]",
     "umap-learn",
     "nrtk[headless]>=0.12.0",
+    "trame-annotations>=0.4.0",
 ]
 
 [project.optional-dependencies]

src/nrtk_explorer/app/core.py

@@ -4,7 +4,7 @@
 from trame.widgets import html
 from trame_server.utils.namespace import Translator
 from nrtk_explorer.library.filtering import FilterProtocol
-from nrtk_explorer.library.dataset import get_dataset, get_image_fpath
+from nrtk_explorer.library.dataset import get_dataset, expand_hugging_face_datasets
 from nrtk_explorer.library.debounce import debounce
 
 from nrtk_explorer.app.images.images import Images
@@ -47,14 +47,23 @@ def __init__(self, server=None):
             "--dataset",
             nargs="+",
             default=DEFAULT_DATASETS,
-            help="Path of the json file describing the image dataset",
+            help="Path to the JSON file describing the image dataset",
+        )
+
+        self.server.cli.add_argument(
+            "--download",
+            action="store_true",
+            default=False,
+            help="Download Hugging Face Hub datasets instead of streaming them",
         )
 
         known_args, _ = self.server.cli.parse_known_args()
-        self.input_paths = known_args.dataset
+        dataset_identifiers = expand_hugging_face_datasets(
+            known_args.dataset, not known_args.download
+        )
+        self.input_paths = dataset_identifiers
         self.state.current_dataset = self.input_paths[0]
 
-        self.ctrl.get_image_fpath = lambda i: get_image_fpath(i, self.state.current_dataset)
         images = Images(server=self.server)
 
         self._transforms_app = TransformsApp(
@@ -107,7 +116,7 @@ def on_server_ready(self, *args, **kwargs):
 
     def on_dataset_change(self, **kwargs):
         self.state.dataset_ids = []  # sampled images
-        self.context.dataset = get_dataset(self.state.current_dataset, force_reload=True)
+        self.context.dataset = get_dataset(self.state.current_dataset)
         self.state.num_images_max = len(self.context.dataset.imgs)
         self.state.num_images = min(self.state.num_images_max, NUM_IMAGES_DEFAULT)
         self.state.dirty("num_images")  # Trigger resample_images()

src/nrtk_explorer/app/embeddings.py

@@ -78,7 +78,7 @@ def on_feature_extraction_model_change(self, **kwargs):
     def on_current_dataset_change(self, **kwargs):
         self.state.num_elements_disabled = True
         if self.context.dataset is None:
-            self.context.dataset = get_dataset(self.state.current_dataset, force_reload=True)
+            self.context.dataset = get_dataset(self.state.current_dataset)
 
         self.state.num_elements_max = len(list(self.context.dataset.imgs))
         self.state.num_elements_disabled = False

src/nrtk_explorer/app/images/annotations.py

@@ -3,7 +3,7 @@
 from PIL import Image
 from nrtk_explorer.app.images.cache import LruCache
 from nrtk_explorer.library.object_detector import ObjectDetector
-from nrtk_explorer.library.coco_utils import partition
+from nrtk_explorer.library.scoring import partition
 
 
 ANNOTATION_CACHE_SIZE = 1000
@@ -81,8 +81,7 @@ def get_annotations(self, detector: ObjectDetector, id_to_image: Dict[str, Image
             )
 
         predictions.update(**cached_predictions)
-        # match input order because of scoring code assumptions
-        return {id: predictions[id] for id in id_to_image.keys()}
+        return predictions
 
     def cache_clear(self):
         self.cache.clear()

src/nrtk_explorer/app/images/images.py

@@ -1,5 +1,5 @@
 import base64
-import io
+from io import BytesIO
 from PIL import Image
 from trame.decorators import TrameApp, change, controller
 from nrtk_explorer.app.images.image_ids import (
@@ -13,7 +13,7 @@
 
 def convert_to_base64(img: Image.Image) -> str:
     """Convert image to base64 string"""
-    buf = io.BytesIO()
+    buf = BytesIO()
     img.save(buf, format="png")
     return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
 
@@ -33,8 +33,10 @@ def __init__(self, server):
         )
 
     def _load_image(self, dataset_id: str):
-        image_path = self.server.controller.get_image_fpath(int(dataset_id))
-        return Image.open(image_path)
+        img = self.server.context.dataset.get_image(int(dataset_id))
+        img.load()  # Avoid OSError(24, 'Too many open files')
+        # transforms and base64 encoding require RGB mode
+        return img.convert("RGB") if img.mode != "RGB" else img
 
     def get_image(self, dataset_id: str, **kwargs):
         """For cache side effects pass on_add_item and on_clear_item callbacks as kwargs"""

src/nrtk_explorer/app/images/stateful_annotations.py

@@ -2,6 +2,7 @@
 from typing import Any, Union, Callable
 from .annotations import GroundTruthAnnotations, DetectionAnnotations
 from trame.decorators import TrameApp, change
+from nrtk_explorer.library.annotations import to_annotation
 from nrtk_explorer.app.images.image_ids import (
     image_id_to_result_id,
 )
@@ -16,33 +17,10 @@ def delete_annotation_from_state(state: Any, image_id: str):
     delete_state(state, image_id_to_result_id(image_id))
 
 
-def prediction_to_annotations(state, predictions):
-    annotations = []
-    for prediction in predictions:
-        # if no matching category in dataset JSON, category_id will be None
-        category_id = None
-        for cat_id, cat in state.annotation_categories.items():
-            if cat["name"] == prediction["label"]:
-                category_id = cat_id
-
-        bbox = prediction["box"]
-        annotations.append(
-            {
-                "category_id": category_id,
-                "label": prediction["label"],
-                "bbox": [
-                    bbox["xmin"],
-                    bbox["ymin"],
-                    bbox["xmax"] - bbox["xmin"],
-                    bbox["ymax"] - bbox["ymin"],
-                ],
-            }
-        )
-    return annotations
-
-
-def add_prediction_to_state(state: Any, image_id: str, prediction: Any):
-    state[image_id_to_result_id(image_id)] = prediction_to_annotations(state, prediction)
+def add_predictions_to_state(context: Any, state: Any, image_id: str, predictions: Any):
+    state[image_id_to_result_id(image_id)] = [
+        to_annotation(context.dataset, prediction) for prediction in predictions
+    ]
 
 
 AnnotationsFactoryConstructorType = Union[
@@ -67,7 +45,7 @@ def __init__(
             add_to_cache_callback, delete_from_cache_callback
         )
 
-    @change("current_dataset", "object_detection_model")
+    @change("current_dataset", "inference_model")
     def _cache_clear(self, **kwargs):
         self.annotations_factory.cache_clear()
 
@@ -83,5 +61,5 @@ def make_stateful_predictor(server):
     return StatefulAnnotations(
         DetectionAnnotations,
         server,
-        add_to_cache_callback=partial(add_prediction_to_state, server.state),
+        add_to_cache_callback=partial(add_predictions_to_state, server.context, server.state),
     )