Merge branch 'release/2.39.0'

Author: oliche

brainbox/behavior/wheel.py

@@ -1,10 +1,6 @@
 """
 Set of functions to handle wheel data.
 """
-import logging
-import warnings
-import traceback
-
 import numpy as np
 from numpy import pi
 from iblutil.numerical import between_sorted
@@ -68,42 +64,6 @@ def interpolate_position(re_ts, re_pos, freq=1000, kind='linear', fill_gaps=None
     return yinterp, t
 
 
-def velocity(re_ts, re_pos):
-    """
-    (DEPRECATED) Compute wheel velocity from non-uniformly sampled wheel data. Returns the velocity
-    at the same samples locations as the position through interpolation.
-
-    Parameters
-    ----------
-    re_ts : array_like
-        Array of timestamps
-    re_pos: array_like
-        Array of unwrapped wheel positions
-
-    Returns
-    -------
-    np.ndarray
-        numpy array of velocities
-    """
-    for line in traceback.format_stack():
-        print(line.strip())
-
-    msg = 'brainbox.behavior.wheel.velocity will soon be removed. Use velocity_filtered instead.'
-    warnings.warn(msg, FutureWarning)
-    logging.getLogger(__name__).warning(msg)
-
-    dp = np.diff(re_pos)
-    dt = np.diff(re_ts)
-    # Compute raw velocity
-    vel = dp / dt
-    # Compute velocity time scale
-    tv = re_ts[:-1] + dt / 2
-    # interpolate over original time scale
-    if tv.size > 1:
-        ifcn = interpolate.interp1d(tv, vel, fill_value="extrapolate")
-        return ifcn(re_ts)
-
-
 def velocity_filtered(pos, fs, corner_frequency=20, order=8):
     """
     Compute wheel velocity from uniformly sampled wheel data.
@@ -130,83 +90,6 @@ def velocity_filtered(pos, fs, corner_frequency=20, order=8):
     return vel, acc
 
 
-def velocity_smoothed(pos, freq, smooth_size=0.03):
-    """
-    (DEPRECATED) Compute wheel velocity from uniformly sampled wheel data.
-
-    Parameters
-    ----------
-    pos : array_like
-        Array of wheel positions
-    smooth_size : float
-        Size of Gaussian smoothing window in seconds
-    freq : float
-        Sampling frequency of the data
-
-    Returns
-    -------
-    vel : np.ndarray
-        Array of velocity values
-    acc : np.ndarray
-        Array of acceleration values
-    """
-    for line in traceback.format_stack():
-        print(line.strip())
-
-    msg = 'brainbox.behavior.wheel.velocity_smoothed will be removed. Use velocity_filtered instead.'
-    warnings.warn(msg, FutureWarning)
-    logging.getLogger(__name__).warning(msg)
-
-    # Define our smoothing window with an area of 1 so the units won't be changed
-    std_samps = np.round(smooth_size * freq)  # Standard deviation relative to sampling frequency
-    N = std_samps * 6  # Number of points in the Gaussian covering +/-3 standard deviations
-    gauss_std = (N - 1) / 6
-    win = scipy.signal.windows.gaussian(N, gauss_std)
-    win = win / win.sum()  # Normalize amplitude
-
-    # Convolve and multiply by sampling frequency to restore original units
-    vel = np.insert(scipy.signal.convolve(np.diff(pos), win, mode='same'), 0, 0) * freq
-    acc = np.insert(scipy.signal.convolve(np.diff(vel), win, mode='same'), 0, 0) * freq
-
-    return vel, acc
-
-
-def last_movement_onset(t, vel, event_time):
-    """
-    (DEPRECATED) Find the time at which movement started, given an event timestamp that occurred during the
-    movement.
-
-    Movement start is defined as the first sample after the velocity has been zero for at least 50ms.
-    Wheel inputs should be evenly sampled.
-
-    :param t: numpy array of wheel timestamps in seconds
-    :param vel: numpy array of wheel velocities
-    :param event_time: timestamp anywhere during movement of interest, e.g. peak velocity
-    :return: timestamp of movement onset
-    """
-    for line in traceback.format_stack():
-        print(line.strip())
-
-    msg = 'brainbox.behavior.wheel.last_movement_onset has been deprecated. Use get_movement_onset instead.'
-    warnings.warn(msg, FutureWarning)
-    logging.getLogger(__name__).warning(msg)
-
-    # Look back from timestamp
-    threshold = 50e-3
-    mask = t < event_time
-    times = t[mask]
-    vel = vel[mask]
-    t = None  # Initialize
-    for i, t in enumerate(times[::-1]):
-        i = times.size - i
-        idx = np.min(np.where((t - times) < threshold))
-        if np.max(np.abs(vel[idx:i])) < 0.5:
-            break
-
-    # Return timestamp
-    return t
-
-
 def get_movement_onset(intervals, event_times):
     """
     Find the time at which movement started, given an event timestamp that occurred during the

brainbox/io/one.py

@@ -20,6 +20,7 @@
 import spikeglx
 
 import ibldsp.voltage
+from ibldsp.waveform_extraction import WaveformsLoader
 from iblutil.util import Bunch
 from iblatlas.atlas import AllenAtlas, BrainRegions
 from iblatlas import atlas
@@ -975,6 +976,21 @@ def raw_electrophysiology(self, stream=True, band='ap', **kwargs):
             if cbin_file is not None:
                 return spikeglx.Reader(cbin_file)
 
+    def download_raw_waveforms(self, **kwargs):
+        """
+        Downloads raw waveforms extracted from sorting to local disk.
+        """
+        _logger.debug(f"loading waveforms from {self.collection}")
+        return self.one.load_object(
+            self.eid, "waveforms",
+            attribute=["traces", "templates", "table", "channels"],
+            collection=self._get_spike_sorting_collection("pykilosort"), download_only=True, **kwargs
+        )
+
+    def raw_waveforms(self, **kwargs):
+        wf_paths = self.download_raw_waveforms(**kwargs)
+        return WaveformsLoader(wf_paths[0].parent, wfs_dtype=np.float16)
+
     def load_channels(self, **kwargs):
         """
         Loads channels
@@ -1318,6 +1334,7 @@ class SessionLoader:
     one: One = None
     session_path: Path = ''
     eid: str = ''
+    revision: str = ''
     data_info: pd.DataFrame = field(default_factory=pd.DataFrame, repr=False)
     trials: pd.DataFrame = field(default_factory=pd.DataFrame, repr=False)
     wheel: pd.DataFrame = field(default_factory=pd.DataFrame, repr=False)
@@ -1445,7 +1462,7 @@ def load_trials(self, collection=None):
         # itiDuration frequently has a mismatched dimension, and we don't need it, exclude using regex
         self.one.wildcards = False
         self.trials = self.one.load_object(
-            self.eid, 'trials', collection=collection, attribute=r'(?!itiDuration).*').to_df()
+            self.eid, 'trials', collection=collection, attribute=r'(?!itiDuration).*', revision=self.revision or None).to_df()
         self.one.wildcards = True
         self.data_info.loc[self.data_info['name'] == 'trials', 'is_loaded'] = True
 
@@ -1468,7 +1485,7 @@ def load_wheel(self, fs=1000, corner_frequency=20, order=8, collection=None):
         """
         if not collection:
             collection = self._find_behaviour_collection('wheel')
-        wheel_raw = self.one.load_object(self.eid, 'wheel', collection=collection)
+        wheel_raw = self.one.load_object(self.eid, 'wheel', collection=collection, revision=self.revision or None)
         if wheel_raw['position'].shape[0] != wheel_raw['timestamps'].shape[0]:
             raise ValueError("Length mismatch between 'wheel.position' and 'wheel.timestamps")
         # resample the wheel position and compute velocity, acceleration
@@ -1498,7 +1515,7 @@ def load_pose(self, likelihood_thr=0.9, views=['left', 'right', 'body']):
         # empty the dictionary so that if one loads only one view, after having loaded several, the others don't linger
         self.pose = {}
         for view in views:
-            pose_raw = self.one.load_object(self.eid, f'{view}Camera', attribute=['dlc', 'times'])
+            pose_raw = self.one.load_object(self.eid, f'{view}Camera', attribute=['dlc', 'times'], revision=self.revision or None)
             # Double check if video timestamps are correct length or can be fixed
             times_fixed, dlc = self._check_video_timestamps(view, pose_raw['times'], pose_raw['dlc'])
             self.pose[f'{view}Camera'] = likelihood_threshold(dlc, likelihood_thr)
@@ -1525,7 +1542,8 @@ def load_motion_energy(self, views=['left', 'right', 'body']):
         # empty the dictionary so that if one loads only one view, after having loaded several, the others don't linger
         self.motion_energy = {}
         for view in views:
-            me_raw = self.one.load_object(self.eid, f'{view}Camera', attribute=['ROIMotionEnergy', 'times'])
+            me_raw = self.one.load_object(
+                self.eid, f'{view}Camera', attribute=['ROIMotionEnergy', 'times'], revision=self.revision or None)
             # Double check if video timestamps are correct length or can be fixed
             times_fixed, motion_energy = self._check_video_timestamps(
                 view, me_raw['times'], me_raw['ROIMotionEnergy'])
@@ -1550,7 +1568,7 @@ def load_pupil(self, snr_thresh=5.):
             will be considered unusable and will be discarded.
         """
         # Try to load from features
-        feat_raw = self.one.load_object(self.eid, 'leftCamera', attribute=['times', 'features'])
+        feat_raw = self.one.load_object(self.eid, 'leftCamera', attribute=['times', 'features'], revision=self.revision or None)
         if 'features' in feat_raw.keys():
             times_fixed, feats = self._check_video_timestamps('left', feat_raw['times'], feat_raw['features'])
             self.pupil = feats.copy()

brainbox/tests/test_behavior.py

@@ -124,12 +124,6 @@ def test_get_movement_onset(self):
         with self.assertRaises(ValueError):
             wheel.get_movement_onset(intervals, np.random.permutation(self.trials['feedback_times']))
 
-    def test_velocity_deprecation(self):
-        """Ensure brainbox.behavior.wheel.velocity is removed."""
-        from datetime import datetime
-        self.assertTrue(datetime.today() < datetime(2024, 8, 1),
-                        'remove brainbox.behavior.wheel.velocity, velocity_smoothed and last_movement_onset')
-
 
 class TestTraining(unittest.TestCase):
     def setUp(self):

examples/data_release/data_release_behavior.ipynb

@@ -11,18 +11,16 @@
    "source": [
     "# Data Release - Behavior\n",
     "\n",
-    "The International Brain Laboratory is a team of systems and computational neuroscientists, working collaboratively to understand the computations that support decision-making in the brain. To achieve this aim, we have developed a standardized decision-making task in mice in order that probes decision-making. The task requires mice to perform decisions by combining incoming visual evidence with internal beliefs about the dynamic structure of the environment. Please read our accompanying [paper (The International Brain Laboratory et al. 2020)](https://elifesciences.org/articles/63711) for details on the decision-making task and the experiment.\n"
+    "The International Brain Laboratory is a team of systems and computational neuroscientists, working collaboratively to understand the computations that support decision-making in the brain. To achieve this aim, we have developed a standardized decision-making task in mice in order that probes decision-making. The task requires mice to perform decisions by combining incoming visual evidence with internal beliefs about the dynamic structure of the environment. Please read our accompanying [paper (The International Brain Laboratory et al. 2020)]( https://doi.org/10.7554/eLife.63711) for details on the decision-making task and the experiment.\n"
    ]
   },
   {
    "cell_type": "markdown",
    "id": "dd157e91",
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "source": [
     "## Overview of the Data\n",
-    "We have released behavioral data throughout learning from our standardized training pipeline, implemented across 9 labs in 7 institutions. Users can download behavioral data from mice throughout their training, and analyse the transition from novice to expert behavior unfold. The behavioral data is associated with 198 mice up until 2020-03-23, as used in [The International Brain Laboratory et al. 2020](https://elifesciences.org/articles/63711). This dataset contains notably information on the sensory stimuli presented to the mouse, as well as mouse decisions and response times.\n",
+    "We have released behavioral data throughout learning from our standardized training pipeline, implemented across 9 labs in 7 institutions. Users can download behavioral data from mice throughout their training, and analyse the transition from novice to expert behavior unfold. The behavioral data is associated with 198 mice up until 2020-03-23, as used in [The International Brain Laboratory et al. 2020](https://doi.org/10.7554/eLife.63711). This dataset contains notably information on the sensory stimuli presented to the mouse, as well as mouse decisions and response times.\n",
     "\n",
     "\n",
     "## Data structure and download\n",
@@ -33,21 +31,24 @@
     "* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_structure.html) to download an example dataset for one session, and get familiarised with the data structure\n",
     "* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_download.html) to learn how to use the ONE-api to search and download the released datasets\n",
     "* [These instructions](https://int-brain-lab.github.io/iblenv/loading_examples.html) to get familiarised with specific data loading functions\n",
-    "* [These instructions](https://int-brain-lab.github.io/iblenv/dj_docs/dj_public.html) for instructions on how to access this dataset via Datajoint. (warning: this method may be retired in 2023)\n",
     "\n",
     "\n",
     "Note:\n",
     "\n",
-    "* The tag associated to this release is `2021_Q1_IBL_et_al_Behaviour`"
+    "* The tag associated to this release is `2021_Q1_IBL_et_al_Behaviour`\n",
+    "\n",
+    "\n",
+    "## How to cite this dataset\n",
+    "If you are using this dataset for your research please cite the [Behavior Paper](https://doi.org/10.7554/eLife.63711) and see the **How to cite** section in the associated entry in the [AWS open data registry](https://registry.opendata.aws/ibl-behaviour/)."
    ]
   }
  ],
  "metadata": {
   "celltoolbar": "Edit Metadata",
   "kernelspec": {
-   "display_name": "Python [conda env:iblenv] *",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "conda-env-iblenv-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -59,7 +60,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.9.16"
   }
  },
  "nbformat": 4,

examples/data_release/data_release_brainwidemap.ipynb

@@ -11,18 +11,16 @@
    "source": [
     "# Data Release - Brain Wide Map\n",
     "\n",
-    "IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). Please read the associated article [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)."
+    "IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). Please read the associated article [(IBL et al. 2023)](https://doi.org/10.1101/2023.07.04.547681). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)."
    ]
   },
   {
    "cell_type": "markdown",
    "id": "e9be5894",
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "source": [
     "## Overview of the Data\n",
-    "We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. In total, 138 brain regions were recorded in sufficient numbers for inclusion in IBL’s analyses [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2).\n",
+    "We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. In total, 138 brain regions were recorded in sufficient numbers for inclusion in IBL’s analyses [(IBL et al. 2023)](https://doi.org/10.1101/2023.07.04.547681).\n",
     "\n",
     "## Data structure and download\n",
     "The organisation of the data follows the standard IBL data structure.\n",
@@ -38,6 +36,10 @@
     "* The tag associated to this release is `Brainwidemap`\n",
     "\n",
     "\n",
+    "## How to cite this dataset\n",
+    "If you are using this dataset for your research please cite the [Brain Wide Map Paper](https://doi.org/10.1101/2023.07.04.547681) and see the **How to cite** section in the associated entry in the [AWS open data registry](https://registry.opendata.aws/ibl-brain-wide-map/).\n",
+    "\n",
+    "\n",
     "## Updates on the data\n",
     "Note: The section [Overview of the Data](#overview-of-the-data) contains the latest numbers released.\n",
     "\n",
@@ -54,9 +56,9 @@
  "metadata": {
   "celltoolbar": "Edit Metadata",
   "kernelspec": {
-   "display_name": "Python [conda env:iblenv] *",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "conda-env-iblenv-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -68,7 +70,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.9.16"
   }
  },
  "nbformat": 4,