Add HDF5 format movie reader (#331)

* Add HDF5 format movie reader

* Correctly set var_name_hdf5 when loading hdf5 and this param is set

Author: ethanbb

mesmerize_core/caiman_extensions/common.py

@@ -707,10 +707,18 @@ def get_input_movie(
         if reader is not None:
             if not callable(reader):
                 raise TypeError(f"reader must be a callable type, such as a function")
+        else:
+            reader = default_reader
+
+        # add hdf5 variable name if provided
+        main_params = self._series.params['main']
+        if 'var_name_hdf5' in main_params:
+            kwargs['var_name_hdf5'] = main_params['var_name_hdf5']
+        elif 'data' in main_params and 'var_name_hdf5' in main_params['data']:
+            kwargs['var_name_hdf5'] = main_params['data']['var_name_hdf5']
 
-            return reader(path_str, **kwargs)
+        return reader(path_str, **kwargs)
 
-        return default_reader(path_str, **kwargs)
 
     @validate()
     def get_corr_image(self) -> np.ndarray:

mesmerize_core/movie_readers.py

@@ -2,6 +2,8 @@
 from pathlib import Path
 import numpy as np
 from caiman import load_memmap
+import h5py
+import zarr
 
 from .utils import warning_experimental
 from .arrays import LazyTiff
@@ -14,7 +16,7 @@
     HAS_PIMS = False
 
 
-def default_reader(path: str, **kwargs):
+def default_reader(path: str, var_name_hdf5='mov', **kwargs):
     ext = Path(path).suffixes[-1]
     if ext in [".tiff", ".tif", ".btf"]:
         try:
@@ -27,8 +29,11 @@ def default_reader(path: str, **kwargs):
     if ext in [".mmap", ".memmap"]:
         return caiman_memmap_reader(path, **kwargs)
 
-    else:
-        raise ValueError(f"No default movie reader for given file extension: '{ext}'")
+    
+    if ext in ['.hdf5', '.h5', '.nwb', '.mat', '.n5', '.zarr']:
+        return hdf5_reader(path, var_name_hdf5=var_name_hdf5, **kwargs)
+
+    raise ValueError(f"No default movie reader for given file extension: '{ext}'")
 
 
 def tiff_memmap_reader(path: str, **kwargs) -> np.memmap:
@@ -51,3 +56,29 @@ def pims_reader(path: str, **kwargs):
     if not HAS_PIMS:
         raise ModuleNotFoundError("you must install `pims` to use the pims reader")
     return pims.open(path, **kwargs)
+
+
+def hdf5_reader(path: str, var_name_hdf5='mov'):
+    # based on caiman.base.movies.load_iter
+    extension = Path(path).suffix
+    if extension in ['.n5', '.zarr']:  # Thankfully, the zarr library lines up closely with h5py past the initial open
+        f = zarr.open(path, "r")
+        if isinstance(f, zarr.Array):
+            raise RuntimeError('Expected a zarr Group, not an Array')
+    else:
+        try:
+            f = h5py.File(path, "r")
+        except:
+            if extension == '.mat':
+                raise Exception(f"Problem loading {path}: Unknown format. This may be in the original version 1 (non-hdf5) mat format; please convert it first")
+            else:
+                raise Exception(f"Problem loading {path}: Unknown format.")
+    
+    ignore_keys = ['__DATA_TYPES__'] # Known metadata that tools provide, add to this as needed.
+    fkeys = list(filter(lambda x: x not in ignore_keys, f.keys()))
+    if len(fkeys) == 1: # If the hdf5 file we're parsing has only one dataset inside it,
+                        # ignore the arg and pick that dataset
+        var_name_hdf5 = fkeys[0]
+    Y = f.get('acquisition/' + var_name_hdf5 + '/data'
+            if extension == '.nwb' else var_name_hdf5)
+    return Y