tested down to task#4 (rename_on_xnat)

Author: mahdieh-dst

conftest.py

@@ -52,9 +52,9 @@ def mr_session(xnat_project, work_dir, request):
                    user=xnat4tests.config.XNAT_USER,
                    password=xnat4tests.config.XNAT_PASSWORD,
                    cache_dir=work_dir / 'cache')
-    dataset = store.load_dataset(xnat_project.id)
+    dataset = store.new_dataset(xnat_project.id)
     # Access single row of dataset
-    return dataset.row(id=session_label)
+    return dataset.row(id=session_label, row_frequency='session')
 
 
 def make_project_name(dataset_name: str, run_prefix: str=None):

dds4xnat/__init__.py

@@ -1,2 +1,2 @@
-from .workflow_dds import dds4xnat_workflow
-from .tasks_dds import *
+from .dds4xnat_workflow import dds4xnat_workflow
+from .dds4xnat_tasks import *

dds4xnat/tasks_dds.py dds4xnat/dds4xnat_tasks.py

@@ -6,7 +6,7 @@
 import pydicom
 import yaml
 from tensorflow.keras.models import load_model
-# import Tools.data_IO as data_IO
+from .tools import data_IO
 import tensorflow as tf
 from .preprocessing import DICOM_preparation_functions as DPF
 from .preprocessing import NIFTI_preparation_functions as NPF
@@ -23,11 +23,11 @@ def download_from_xnat(row: DataRow) -> Path:
     store = row.dataset.store
 
     with store:
-        xproject = store.login[row.dataset.id]
+        xproject = store.login.projects[row.dataset.id]
         xsession = xproject.experiments[row.id]
         xsession.download_dir(str(download_dir)) 
 
-    return download_dir
+    return str(download_dir)
 
 
 @mark.task #---task #2----------------------------------------------
@@ -101,13 +101,13 @@ def is_odd(number):
     NPF.rescale_image_intensity(nifti_slices_folder)
 
     print('Creating label file....')
-    NPF.create_label_file(nifti_slices_folder, images_4D_file)
+    label_file = NPF.create_label_file(nifti_slices_folder, images_4D_file)
 
     elapsed_time = time.time() - start_time
 
     print(elapsed_time)
 
-    return label_file, structured_dcm_dir, 
+    return label_file, structured_dcm_dir
 
 
 @mark.task #---task #3----------------------------------------------
@@ -186,13 +186,18 @@ def load_labels(label_file):
 
 
 @mark.task #---task #4----------------------------------------------
+@mark.annotate({
+    "return": {
+        "info_file": Path
+    }
+})
 
-def rename_on_xnat(out_file: str, download_dir: str, row: DataRow) -> Path:
+def rename_on_xnat(out_file: str, output_folder: Path, download_dir: str, row: DataRow) -> Path:
 
     store = row.dataset.store
 
     with store:
-        xproject = store.login[row.dataset.id]
+        xproject = store.login.projects[row.dataset.id]
         xsession = xproject.experiments[row.id]
 
         # Rename "scan types" of sessions
@@ -204,7 +209,7 @@ def rename_on_xnat(out_file: str, download_dir: str, row: DataRow) -> Path:
         prediction_file = cfg['post_processing']['prediction_file']
         root_dicom_folder = cfg['preprocessing']['root_dicom_folder']
         """
-        
+        info_file = os.path.join(output_folder, 'info.txt')
         base_dir = os.path.dirname(os.path.normpath(download_dir))
         structured_dicom_folder = os.path.join(base_dir,'DICOM_STRUCTURED')
         #root_out_folder = os.path.join(base_dir, 'DICOM_SORTED')
@@ -248,8 +253,15 @@ def rename_on_xnat(out_file: str, download_dir: str, row: DataRow) -> Path:
                 print(predicted_scan_type)
 
                 scan = xsession.scans[scan_id]
+                #renaming scan type on xnat
                 scan.type = prediction_names[i_prediction]
 
+                with open(info_file, 'w') as the_file:
+                    out_elements = [scan_id, unpredicted_scan_type, predicted_scan_type]
+                    the_file.write('\t'.join(out_elements) + '\n')
+    
+    return info_file
+
 
 @mark.task #----------------------------------------------------------
 def cleanup(download_dir: str):

dds4xnat/dds_main/workflow_dds.py dds4xnat/dds4xnat_workflow.py

@@ -1,13 +1,14 @@
 from pydra import Workflow
-from .tasks_dds import *
+from .dds4xnat_tasks import *
 
-def dds4xnat_workflow(name):
+WORKFLOW_INPUTS = [
+    'mr_session', 'dcm2niix_bin', 'fslval_bin',
+    'model_file', 'output_folder']
+
+
+def dds4xnat_workflow(name, **kwargs):
 
-    wf = Workflow(name=name, input_spec=['mr_session',
-                                         'dcm2niix_bin',
-                                         'fslval_bin',
-                                         'model_file',
-                                         'output_folder'])
+    wf = Workflow(name=name, input_spec=WORKFLOW_INPUTS, **kwargs)
 
     # Build workflow here
     wf.add(
@@ -39,8 +40,11 @@ def dds4xnat_workflow(name):
         rename_on_xnat(
             name='rename',
             out_file = wf.predict.lzout.out_file,
-            download_dir=wf.download.lzout.out
+            download_dir=wf.download.lzout.out,
+            output_folder=wf.lzin.output_folder
         )
     )
 
+    wf.set_output([("rename_log", wf.rename.lzout.info_file)])
+
     return wf

dds4xnat/dds_main/tasks_dds.py dds4xnat/dds_main/dds4xnat_tasks.py

@@ -6,10 +6,10 @@
 import pydicom
 import yaml
 from tensorflow.keras.models import load_model
-import Tools.data_IO as data_IO
+from .tools import data_IO
 import tensorflow as tf
-from Preprocessing import DICOM_preparation_functions as DPF
-from Preprocessing import NIFTI_preparation_functions as NPF
+from .preprocessing import DICOM_preparation_functions as DPF
+from .preprocessing import NIFTI_preparation_functions as NPF
 import time
 from arcana.core.data.row import DataRow
 
@@ -23,11 +23,11 @@ def download_from_xnat(row: DataRow) -> Path:
     store = row.dataset.store
 
     with store:
-        xproject = store.login[row.dataset.id]
+        xproject = store.login.projects[row.dataset.id]
         xsession = xproject.experiments[row.id]
         xsession.download_dir(str(download_dir)) 
 
-    return download_dir
+    return str(download_dir)
 
 
 @mark.task #---task #2----------------------------------------------
@@ -101,13 +101,13 @@ def is_odd(number):
     NPF.rescale_image_intensity(nifti_slices_folder)
 
     print('Creating label file....')
-    NPF.create_label_file(nifti_slices_folder, images_4D_file)
+    label_file = NPF.create_label_file(nifti_slices_folder, images_4D_file)
 
     elapsed_time = time.time() - start_time
 
     print(elapsed_time)
 
-    return label_file, structured_dcm_dir, 
+    return label_file, structured_dcm_dir
 
 
 @mark.task #---task #3----------------------------------------------
@@ -186,13 +186,18 @@ def load_labels(label_file):
 
 
 @mark.task #---task #4----------------------------------------------
+@mark.annotate({
+    "return": {
+        "info_file": Path
+    }
+})
 
-def rename_on_xnat(out_file: str, download_dir: str, row: DataRow) -> Path:
+def rename_on_xnat(out_file: str, output_folder: Path, download_dir: str, row: DataRow) -> Path:
 
     store = row.dataset.store
 
     with store:
-        xproject = store.login[row.dataset.id]
+        xproject = store.login.projects[row.dataset.id]
         xsession = xproject.experiments[row.id]
 
         # Rename "scan types" of sessions
@@ -204,7 +209,7 @@ def rename_on_xnat(out_file: str, download_dir: str, row: DataRow) -> Path:
         prediction_file = cfg['post_processing']['prediction_file']
         root_dicom_folder = cfg['preprocessing']['root_dicom_folder']
         """
-        
+        info_file = os.path.join(output_folder, 'info.txt')
         base_dir = os.path.dirname(os.path.normpath(download_dir))
         structured_dicom_folder = os.path.join(base_dir,'DICOM_STRUCTURED')
         #root_out_folder = os.path.join(base_dir, 'DICOM_SORTED')
@@ -248,8 +253,15 @@ def rename_on_xnat(out_file: str, download_dir: str, row: DataRow) -> Path:
                 print(predicted_scan_type)
 
                 scan = xsession.scans[scan_id]
+                #renaming scan type on xnat
                 scan.type = prediction_names[i_prediction]
 
+                with open(info_file, 'w') as the_file:
+                    out_elements = [scan_id, unpredicted_scan_type, predicted_scan_type]
+                    the_file.write('\t'.join(out_elements) + '\n')
+    
+    return info_file
+
 
 @mark.task #----------------------------------------------------------
 def cleanup(download_dir: str):

dds4xnat/workflow_dds.py dds4xnat/dds_main/dds4xnat_workflow.py

@@ -1,13 +1,14 @@
 from pydra import Workflow
-from .tasks_dds import *
+from .dds4xnat_tasks import *
 
-def dds4xnat_workflow(name):
+WORKFLOW_INPUTS = [
+    'mr_session', 'dcm2niix_bin', 'fslval_bin',
+    'model_file', 'output_folder']
+
+
+def dds4xnat_workflow(name, **kwargs):
 
-    wf = Workflow(name=name, input_spec=['mr_session',
-                                         'dcm2niix_bin',
-                                         'fslval_bin',
-                                         'model_file',
-                                         'output_folder'])
+    wf = Workflow(name=name, input_spec=WORKFLOW_INPUTS, **kwargs)
 
     # Build workflow here
     wf.add(
@@ -39,8 +40,11 @@ def dds4xnat_workflow(name):
         rename_on_xnat(
             name='rename',
             out_file = wf.predict.lzout.out_file,
-            download_dir=wf.download.lzout.out
+            download_dir=wf.download.lzout.out,
+            output_folder=wf.lzin.output_folder
         )
     )
 
+    wf.set_output([("rename_log", wf.rename.lzout.info_file)])
+
     return wf

dds4xnat/tools/__init__.py

@@ -0,0 +1,99 @@
+import threading
+import numpy as np
+import SimpleITK as sitk
+
+
+class NiftiGenerator2D_ExtraInput(object):
+
+    def __init__(self, batch_size, image_locations,
+                 labels, image_size, extra_inputs, random_shuffle=True):
+        self.n = len(image_locations)
+        self.batch_size = batch_size
+        self.batch_index = 0
+        self.total_batches_seen = 0
+        self.lock = threading.Lock()
+        self.index_array = None
+        self.index_generator = self._flow_index()
+        self.image_locations = image_locations
+        self.labels = labels
+        self.image_size = image_size
+        self.random_shuffle = random_shuffle
+        self.extra_inputs = extra_inputs
+
+    def _set_index_array(self):
+        if self.random_shuffle:
+            self.index_array = np.random.permutation(self.n)
+        else:
+            self.index_array = np.arange(self.n)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self, *args, **kwargs):
+        return self.next(*args, **kwargs)
+
+    def reset(self):
+        self.batch_index = 0
+        self.total_batches_seen = 0
+        self._set_index_array()
+
+    def _flow_index(self):
+        self.reset()
+        while 1:
+            if self.batch_index == 0:
+                self._set_index_array()
+
+            current_index = (self.batch_index * self.batch_size) % self.n
+            if self.n > current_index + self.batch_size:
+                self.batch_index += 1
+            else:
+                self.batch_index = 0
+            self.total_batches_seen += 1
+
+            if current_index + self.batch_size > self.n:
+                N_missing_samples = (current_index + self.batch_size) - self.n
+                batch_indices_leftover = self.index_array[current_index:]
+                self.reset()
+                batch_indices_filler = self.index_array[0:N_missing_samples]
+                batch_indices = np.concatenate((batch_indices_leftover, batch_indices_filler))
+            else:
+                batch_indices = self.index_array[current_index:
+                                            current_index + self.batch_size]
+            yield batch_indices
+
+    def on_epoch_end(self):
+        self.reset()
+
+    def _get_batch_of_samples(self, index_array):
+        image_tensor = np.zeros((self.batch_size,
+                                 self.image_size[0],
+                                 self.image_size[1],
+                                 1))
+
+        out_labels = self.labels[index_array, :]
+        out_extra_inputs = self.extra_inputs[index_array, :]
+
+        image_locations = self.image_locations[index_array]
+
+        for i_sample, i_image_location in enumerate(image_locations):
+            i_image = sitk.ReadImage(i_image_location, sitk.sitkFloat32)
+            i_image_array = sitk.GetArrayFromImage(i_image)
+            image_tensor[i_sample, :, :, 0] = i_image_array[:, :]
+
+        return [image_tensor, out_extra_inputs], out_labels
+
+    def next(self):
+        with self.lock:
+            index_array = next(self.index_generator)
+
+        return self._get_batch_of_samples(index_array)
+
+    def get_single_image(self, image_path):
+        full_sample_tensor = np.zeros((1,
+                                       self.image_size[0],
+                                       self.image_size[1],
+                                       1))
+        i_image = sitk.ReadImage(image_path, sitk.sitkFloat32)
+        i_image_array = sitk.GetArrayFromImage(i_image)
+        full_sample_tensor[0, :, :, 0] = i_image_array
+        return full_sample_tensor

dds4xnat/tools/data_IO.py

@@ -1,2 +1,3 @@
 arcana
-xnat4tests
+xnat4tests
+tensorflow>=2.9.1

test_requirements.txt

@@ -13,10 +13,10 @@ def test_dicom_sort(mr_session: DataRow, trained_model_file: Path, work_dir: Pat
         name='test_workflow',
         mr_session=mr_session,
         dcm2niix_bin='/usr/local/bin/dcm2niix',
-        fslval_bin='/usr/bin/fsl5.0-fslval',
+        fslval_bin='/usr/local/fsl/bin/fslval',
         model_file=trained_model_file,
         output_folder=output_dir)
 
     # run workflow
-    wf()
+    wf(plugin='serial')