From b31e677b06d0cfb2502f6ce327a3833077263802 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:11:59 -0400 Subject: [PATCH 1/6] add .nwb to known zarr suffixes --- dandi/consts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/consts.py b/dandi/consts.py index 34391185a..59b422165 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -167,7 +167,7 @@ def urls(self) -> Iterator[str]: VIDEO_FILE_EXTENSIONS = [".mp4", ".avi", ".wmv", ".mov", ".flv", ".mkv"] VIDEO_FILE_MODULES = ["processing", "acquisition"] -ZARR_EXTENSIONS = [".ngff", ".zarr"] +ZARR_EXTENSIONS = [".ngff", ".zarr", ".nwb"] #: Maximum allowed depth of a Zarr directory tree MAX_ZARR_DEPTH = 7 From 50499999b89e4faff61f897bcf0339aaa8649da3 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:32:30 -0400 Subject: [PATCH 2/6] add zarr backend to pynwb_utils --- dandi/pynwb_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dandi/pynwb_utils.py b/dandi/pynwb_utils.py index b7cfaa6d4..076d4d54f 100644 --- a/dandi/pynwb_utils.py +++ b/dandi/pynwb_utils.py @@ -13,6 +13,7 @@ from fscacher import PersistentCache import h5py import hdmf +import hdmf_zarr from packaging.version import Version import pynwb from pynwb import NWBHDF5IO @@ -482,14 +483,15 @@ def get_object_id(path: str | Path | Readable) -> Any: def make_nwb_file( - filename: StrPath, *args: Any, cache_spec: bool = False, **kwargs: Any + filename: StrPath, *args: Any, cache_spec: bool = False, backend: Literal["hdf5", "zarr"] = "hdf5", **kwargs: Any ) -> StrPath: """A little helper to produce an .nwb file in the path using NWBFile Note: it doesn't cache_spec by default """ + backends = dict(hdf5=pynwb.NWBHDF5IO, zarr=hdmf_zarr.NWBZarrIO) nwbfile = pynwb.NWBFile(*args, **kwargs) - with pynwb.NWBHDF5IO(filename, "w") as io: + with backends[backend](filename, "w") as io: io.write(nwbfile, cache_spec=cache_spec) return filename From 9bfb9ca0fca9b0d54db5771a3e6f7f59a44b841b Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:36:49 -0400 Subject: [PATCH 3/6] add extra zarr file to existing directory fixture --- dandi/tests/fixtures.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dandi/tests/fixtures.py b/dandi/tests/fixtures.py index 2ad4505ab..5977d53d3 100644 --- a/dandi/tests/fixtures.py +++ b/dandi/tests/fixtures.py @@ -229,6 +229,17 @@ def organized_nwb_dir2( ), **simple1_nwb_metadata, ) + make_nwb_file( + tmp_path / "simple1_zarr.nwb", + backend="zarr", + subject=pynwb.file.Subject( + subject_id="lizard001", + date_of_birth=datetime(2016, 12, 1, tzinfo=tzutc()), + sex="F", + species="Gekko gecko", + ), + **simple1_nwb_metadata, + ) (tmp_path / dandiset_metadata_file).write_text("{}\n") r = CliRunner().invoke(organize, ["-f", "move", "--dandiset-path", str(tmp_path)]) assert r.exit_code == 0, r.stdout From 5d48f40a59fcba470582f02c4566444839306c18 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:37:56 -0400 Subject: [PATCH 4/6] add hdmf_zarr to requirements --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index b9326e731..9a6e12b44 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,7 @@ install_requires = fasteners fscacher >= 0.3.0 hdmf != 3.5.0 + hdmf_zarr humanize interleave ~= 0.1 joblib From 71e8c3d743dfa2c4fceea62c434463b825a1a8e8 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:43:13 -0400 Subject: [PATCH 5/6] black and import --- dandi/pynwb_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dandi/pynwb_utils.py b/dandi/pynwb_utils.py index 076d4d54f..b13d3ae53 100644 --- a/dandi/pynwb_utils.py +++ b/dandi/pynwb_utils.py @@ -6,7 +6,7 @@ import os.path as op from pathlib import Path import re -from typing import IO, Any, Dict, List, Optional, Tuple, TypeVar, Union, cast +from typing import IO, Any, Dict, List, Literal, Optional, Tuple, TypeVar, Union, cast import warnings import dandischema @@ -483,7 +483,11 @@ def get_object_id(path: str | Path | Readable) -> Any: def make_nwb_file( - filename: StrPath, *args: Any, cache_spec: bool = False, backend: Literal["hdf5", "zarr"] = "hdf5", **kwargs: Any + filename: StrPath, + *args: Any, + cache_spec: bool = False, + backend: Literal["hdf5", "zarr"] = "hdf5", + **kwargs: Any, ) -> StrPath: """A little helper to produce an .nwb file in the path using NWBFile From 14c9c1e4a2f401223f0ee136962408c84376054a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 4 Jan 2024 11:33:16 -0500 Subject: [PATCH 6/6] Update dandi/consts.py --- dandi/consts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/consts.py b/dandi/consts.py index 59b422165..34391185a 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -167,7 +167,7 @@ def urls(self) -> Iterator[str]: VIDEO_FILE_EXTENSIONS = [".mp4", ".avi", ".wmv", ".mov", ".flv", ".mkv"] VIDEO_FILE_MODULES = ["processing", "acquisition"] -ZARR_EXTENSIONS = [".ngff", ".zarr", ".nwb"] +ZARR_EXTENSIONS = [".ngff", ".zarr"] #: Maximum allowed depth of a Zarr directory tree MAX_ZARR_DEPTH = 7