diff --git a/e2e/features/test_generic.py b/e2e/features/test_generic.py new file mode 100644 index 00000000..b9b11e2f --- /dev/null +++ b/e2e/features/test_generic.py @@ -0,0 +1,124 @@ +import pytest +from typing import Dict +import json +from pathlib import Path +import shutil +import siibra +from nibabel.nifti1 import Nifti1Image +from pandas import DataFrame + +CUSTOM_CONF_FOLDER = "./custom-configurations/" + + +def create_json(conf: Dict): + if conf["@type"] == "siibra/feature/tabular/v0.1": + folderpath = Path(CUSTOM_CONF_FOLDER + "features/tabular/") + elif conf["@type"] == "siibra/feature/image/v0.1": + folderpath = Path(CUSTOM_CONF_FOLDER + "features/images/") + else: + raise NotImplementedError(f"There is no generic feature type '{conf['@type']}'") + + folderpath.mkdir(parents=True, exist_ok=True) + filepath = folderpath.joinpath(conf.get("name") + ".json") + with open(filepath, "wt") as fp: + json.dump(conf, fp=fp) + + return filepath.as_posix() + + +siibra.spaces + +generic_feature_configs = [ + { + "config": { + "@type": "siibra/feature/tabular/v0.1", + "name": "bla at ACBL", + "region": "ACBL", + "modality": "any modality", + "description": "this describes the feature", + "species": "Homo sapiens", + "file": "https://object.cscs.ch/v1/AUTH_227176556f3c4bb38df9feea4b91200c/hbp-d000045_receptors-human-7A_pub/v1.0/7A_pr_examples/5-HT1A/7A_pr_5-HT1A.tsv", + }, + "queries": [ + (siibra.get_region("julich 3.1", "acbl"), siibra.features.generic.Tabular), + (siibra.get_region("julich 3.1", "basal ganglia"), siibra.features.generic.Tabular), + (siibra.get_region("julich 3.1", "acbl right"), siibra.features.generic.Tabular), + (siibra.get_region("julich 3.0", "ventral striatum"), siibra.features.generic.Tabular), + ], + }, + { + "config": { + "@type": "siibra/feature/tabular/v0.1", + "name": "Cochlear nucleus, ventral part data", + "region": "Cochlear nucleus, ventral part", + "modality": "any modality", + "description": "this describes the feature", + "species": "Rattus norvegicus", + "file": "https://object.cscs.ch/v1/AUTH_227176556f3c4bb38df9feea4b91200c/hbp-d000045_receptors-human-7A_pub/v1.0/7A_pr_examples/5-HT1A/7A_pr_5-HT1A.tsv", + }, + "queries": [ + (siibra.get_region("Waxholm 4", "Cochlear nucleus, ventral part"), siibra.features.generic.Tabular), + (siibra.get_region("Waxholm 4", "Rhombencephalon"), siibra.features.generic.Tabular), + (siibra.get_region("Waxholm 4", "Ventral cochlear nucleus, anterior part"), siibra.features.generic.Tabular), + ], + }, + { + "config": { + "@type": "siibra/feature/image/v0.1", + "name": "some name for custom image feature", + "modality": "foo", + "ebrains": {"openminds/DatasetVersion": "73c1fa55-d099-4854-8cda-c9a403c6080a"}, + "space": {"@id": "minds/core/referencespace/v1.0.0/d5717c4a-0fa1-46e6-918c-b8003069ade8"}, + "providers": { + "neuroglancer/precomputed": "https://1um.brainatlas.eu/registered_sections/bigbrain/B20_0102/precomputed" + }, + }, + "queries": [ + ('waxholm', siibra.features.generic.Image), + ('waxholm', siibra.features.generic.Image), + ('waxholm', siibra.features.generic.Image), + ], + }, +] + +conf_jsons = [] +for conf in generic_feature_configs: + conf_jsons.append(create_json(conf["config"])) + + +@pytest.mark.parametrize("conf_path", conf_jsons) +def test_digestion(conf_path: str): + with open(conf_path, 'rt') as fp: + conf = json.load(fp) + f = siibra.from_json(conf_path) + assert f.category == 'generic' + if conf["@type"] == "siibra/feature/image/v0.1": + assert isinstance(f, siibra.features.generic.Image) + assert isinstance(f.fetch(), (Nifti1Image, dict)) + elif conf["@type"] == "siibra/feature/tabular/v0.1": + assert isinstance(f, siibra.features.generic.Tabular) + assert isinstance(f.data, DataFrame) + else: + raise ValueError(f'type {conf["@type"]} does not match any predefined generic types for testing.') + + +siibra.extend_configuration(CUSTOM_CONF_FOLDER) +queries = [q for qs in generic_feature_configs for q in qs["queries"]] + + +@pytest.mark.parametrize("query_concept, query_type", queries) +def test_generic_feature_query(query_concept, query_type: siibra.features.Feature): + if isinstance(query_concept, str): + query_concept = siibra.spaces.get(query_concept) # TODO: check why match method fails + fts = [ + f + for f in siibra.features.get(query_concept, query_type) + if isinstance(f, query_type) + ] + assert len(fts) > 0 + + +@pytest.fixture(scope="module", autouse=True) +def cleanup(): + yield + shutil.rmtree(CUSTOM_CONF_FOLDER) diff --git a/siibra/configuration/factory.py b/siibra/configuration/factory.py index 381d1b73..2b603613 100644 --- a/siibra/configuration/factory.py +++ b/siibra/configuration/factory.py @@ -25,13 +25,14 @@ from ..commons import logger, Species from ..features import anchor, connectivity from ..features.tabular import ( + tabular, receptor_density_profile, receptor_density_fingerprint, cell_density_profile, layerwise_cell_density, regional_timeseries_activity, ) -from ..features.image import sections, volume_of_interest +from ..features.image import image, sections, volume_of_interest from ..core import atlas, parcellation, space, region from ..locations import point, pointcloud, boundingbox from ..retrieval import datasets, repositories @@ -391,6 +392,19 @@ def build_receptor_density_fingerprint(cls, spec): prerelease=spec.get("prerelease", False), ) + @classmethod + @build_type("siibra/feature/tabular/v0.1") + def build_generic_tabular(cls, spec): + return tabular.Tabular( + file=spec["file"], + description=spec.get("description"), + modality=spec.get("modality"), + anchor=cls.extract_anchor(spec), + datasets=cls.extract_datasets(spec), + id=spec.get("@id", None), + prerelease=spec.get("prerelease", False), + ) + @classmethod @build_type("siibra/feature/fingerprint/celldensity/v0.1") def build_cell_density_fingerprint(cls, spec): @@ -492,6 +506,22 @@ def build_volume_of_interest(cls, spec): f"No method for building image section feature type {modality}." ) + @classmethod + @build_type("siibra/feature/image/v0.1") + def build_generic_image_feature(cls, spec): + kwargs = { + "name": spec.get("name"), + "modality": spec.get("modality"), + "region": spec.get("region", None), + "space_spec": spec.get("space"), + "providers": cls.build_volumeproviders(spec.get("providers")), + "datasets": cls.extract_datasets(spec), + "bbox": cls.build_boundingbox(spec), + "id": spec.get("@id", None), + "prerelease": spec.get("prerelease", False), + } + return image.Image(**kwargs) + @classmethod @build_type("siibra/feature/connectivitymatrix/v0.3") def build_connectivity_matrix(cls, spec): diff --git a/siibra/features/feature.py b/siibra/features/feature.py index 9fccf68f..f6fef437 100644 --- a/siibra/features/feature.py +++ b/siibra/features/feature.py @@ -232,7 +232,11 @@ def _get_instances(cls, **kwargs) -> List['Feature']: from ..configuration.configuration import Configuration conf = Configuration() Configuration.register_cleanup(cls._clean_instances) - assert cls._configuration_folder in conf.folders + try: + assert cls._configuration_folder in conf.folders + except AssertionError: + logger.info(f"'{cls._configuration_folder}' has no configuration jsons.") + return [] cls._preconfigured_instances = [ o for o in conf.build_objects(cls._configuration_folder) if isinstance(o, cls) @@ -464,14 +468,12 @@ def _decode_concept(cls, concepts: List[str]) -> concept.AtlasConcept: def _parse_featuretype(cls, feature_type: str) -> List[Type['Feature']]: ftypes = sorted({ feattype - for FeatCls, feattypes in cls._SUBCLASSES.items() - if all(w.lower() in FeatCls.__name__.lower() for w in feature_type.split()) + for FeatDataCls, feattypes in cls._SUBCLASSES.items() + if all(w.lower() in FeatDataCls.__name__.lower() for w in feature_type.split()) for feattype in feattypes + if getattr(feattype, 'category') }, key=lambda t: t.__name__) - if len(ftypes) > 1: - return [ft for ft in ftypes if getattr(ft, 'category')] - else: - return list(ftypes) + return sorted(ftypes, key=lambda t: t.__name__) @classmethod def _livequery(cls, concept: Union[region.Region, parcellation.Parcellation, space.Space], **kwargs) -> List['Feature']: @@ -502,7 +504,7 @@ def _livequery(cls, concept: Union[region.Region, parcellation.Parcellation, spa def _match( cls, concept: Union[structure.BrainStructure, space.Space], - feature_type: Union[str, Type['Feature'], list], + feature_type: Union[str, Type['Feature'], List['Feature']], **kwargs ) -> List['Feature']: """ @@ -524,13 +526,13 @@ def _match( """ if isinstance(feature_type, list): # a list of feature types is given, collect match results on those - assert all( - (isinstance(t, str) or issubclass(t, cls)) - for t in feature_type - ) + ftypes = list(dict.fromkeys( + cls._parse_featuretype(ft) if isinstance(ft, str) else ft + for ft in feature_type + )) return list(dict.fromkeys( sum(( - cls._match(concept, t, **kwargs) for t in feature_type + cls._match(concept, t, **kwargs) for t in ftypes ), []) )) diff --git a/siibra/features/image/image.py b/siibra/features/image/image.py index 6350078c..68ff25e9 100644 --- a/siibra/features/image/image.py +++ b/siibra/features/image/image.py @@ -53,7 +53,12 @@ def __str__(self): return f"Bounding box of image in {self.space.name}" -class Image(feature.Feature, _volume.Volume): +class Image( + feature.Feature, + _volume.Volume, + configuration_folder="features/images", + category="generic" +): def __init__( self, diff --git a/siibra/features/tabular/receptor_density_fingerprint.py b/siibra/features/tabular/receptor_density_fingerprint.py index 466e0e6e..518e6d76 100644 --- a/siibra/features/tabular/receptor_density_fingerprint.py +++ b/siibra/features/tabular/receptor_density_fingerprint.py @@ -55,12 +55,12 @@ def __init__( description=self.DESCRIPTION, modality="Neurotransmitter receptor density", anchor=anchor, + file=tsvfile, data=None, # lazy loading below datasets=datasets, id=id, prerelease=prerelease, ) - self._loader = requests.HttpRequest(tsvfile) @property def unit(self) -> str: diff --git a/siibra/features/tabular/tabular.py b/siibra/features/tabular/tabular.py index a50cfb87..def43d4f 100644 --- a/siibra/features/tabular/tabular.py +++ b/siibra/features/tabular/tabular.py @@ -22,9 +22,10 @@ from .. import feature from .. import anchor as _anchor from ...commons import logger +from ...retrieval import requests -class Tabular(feature.Feature): +class Tabular(feature.Feature, category="generic", configuration_folder="features/tabular"): """ Represents a table of different measures anchored to a brain location. @@ -42,7 +43,8 @@ def __init__( description: str, modality: str, anchor: _anchor.AnatomicalAnchor, - data: pd.DataFrame, # sample x feature dimension + file: str = None, + data: pd.DataFrame = None, # sample x feature dimension datasets: list = [], id: str = None, prerelease: bool = False, @@ -56,10 +58,15 @@ def __init__( id=id, prerelease=prerelease ) + self._loader = None if file is None else requests.HttpRequest(file) + if file is not None: + assert data is None self._data_cached = data @property def data(self): + if self._loader is not None: + self._data_cached = self._loader.get() return self._data_cached.copy() def _to_zip(self, fh: ZipFile): diff --git a/siibra/livequeries/allen.py b/siibra/livequeries/allen.py index aefb1809..fe433f6a 100644 --- a/siibra/livequeries/allen.py +++ b/siibra/livequeries/allen.py @@ -47,6 +47,20 @@ def is_allen_api_microarray_service_available(): return response["success"] +def parse_gene(spec): + if isinstance(spec, str): + return [GENE_NAMES.get(spec)] + elif isinstance(spec, dict): + assert all(k in spec for k in ['symbol', 'description']) + assert spec['symbol'] in GENE_NAMES + return [spec] + elif isinstance(spec, list): + return [g for s in spec for g in parse_gene(s)] + else: + logger.error("Invalid specification of gene:", spec) + return [] + + class InvalidAllenAPIResponseException(Exception): pass @@ -119,21 +133,10 @@ def __init__(self, **kwargs): """ _query.LiveQuery.__init__(self, **kwargs) gene = kwargs.get('gene') - - def parse_gene(spec): - if isinstance(spec, str): - return [GENE_NAMES.get(spec)] - elif isinstance(spec, dict): - assert all(k in spec for k in ['symbol', 'description']) - assert spec['symbol'] in GENE_NAMES - return [spec] - elif isinstance(spec, list): - return [g for s in spec for g in parse_gene(s)] - else: - logger.error("Invalid specification of gene:", spec) - return [] - - self.genes = parse_gene(gene) + if gene is None: + self.genes = [] + else: + self.genes = parse_gene(gene) def query(self, concept: structure.BrainStructure) -> List[GeneExpressions]: if not is_allen_api_microarray_service_available(): @@ -143,6 +146,9 @@ def query(self, concept: structure.BrainStructure) -> List[GeneExpressions]: 'gene expression features. This is a known issue which we are investigating: ' 'https://github.com/FZJ-INM1-BDA/siibra-python/issues/636.' ) + if len(self.genes) == 0: + logger.error("Cannot query for gene expresssions without a specified gene.") + return [] # Match the microarray probes to the query mask. # Record matched instances and their locations.