Merge branch 'release/2.21.0'

Author: mayofaulkner

brainbox/behavior/training.py

@@ -1,4 +1,5 @@
 from one.api import ONE
+from one.alf.exceptions import ALFObjectNotFound
 import datetime
 import re
 import numpy as np
@@ -131,7 +132,10 @@ def get_sessions(subj, date=None, one=None):
     sess_dates = []
     if len(sessions) < 3:
         for n, _ in enumerate(sessions):
-            trials_ = one.load_object(sessions[n]['url'].split('/')[-1], 'trials')
+            try:
+                trials_ = one.load_object(sessions[n]['url'].split('/')[-1], 'trials')
+            except ALFObjectNotFound:
+                trials_ = None
 
             if trials_:
                 task_protocol.append(re.search('tasks_(.*)Choice',
@@ -142,7 +146,11 @@ def get_sessions(subj, date=None, one=None):
     else:
         n = 0
         while len(trials) < 3:
-            trials_ = one.load_object(sessions[n]['url'].split('/')[-1], 'trials')
+            print(sessions[n]['url'].split('/')[-1])
+            try:
+                trials_ = one.load_object(sessions[n]['url'].split('/')[-1], 'trials')
+            except ALFObjectNotFound:
+                trials_ = None
 
             if trials_:
                 task_protocol.append(re.search('tasks_(.*)Choice',

brainbox/io/one.py

@@ -706,6 +706,28 @@ def load_wheel_reaction_times(eid, one=None):
     return firstMove_times - trials['goCue_times']
 
 
+def load_iti(trials):
+    """
+    The inter-trial interval (ITI) time for each trial, defined as the period of open-loop grey
+    screen commencing at stimulus off and lasting until the quiescent period at the start of the
+    following trial.  Note that the ITI for the first trial is the time between the first trial
+    and the next, therefore the last value is NaN.
+
+    Parameters
+    ----------
+    trials : one.alf.io.AlfBunch
+        An ALF trials object containing the keys {'intervals', 'stimOff_times'}.
+
+    Returns
+    -------
+    np.array
+        An array of inter-trial intervals, the last value being NaN.
+    """
+    if not {'intervals', 'stimOff_times'} <= trials.keys():
+        raise ValueError('trials must contain keys {"intervals", "stimOff_times"}')
+    return np.r_[(np.roll(trials['intervals'][:, 0], -1) - trials['stimOff_times'])[:-1], np.nan]
+
+
 def load_channels_from_insertion(ins, depths=None, one=None, ba=None):
 
     PROV_2_VAL = {

brainbox/io/spikeglx.py

@@ -132,7 +132,7 @@ def __init__(self, pid, one, typ='ap', cache_folder=None, remove_cached=False):
             self.chunks['chunk_bounds'] = np.array(self.chunks['chunk_bounds'])
         super(Streamer, self).__init__(meta_file, ignore_warnings=True)
 
-    def read(self, nsel=slice(0, 10000), csel=slice(None), sync=True):
+    def read(self, nsel=slice(0, 10000), csel=slice(None), sync=True, volts=True):
         """
         overload the read function by downloading the necessary chunks
         """
@@ -142,7 +142,11 @@ def read(self, nsel=slice(0, 10000), csel=slice(None), sync=True):
         _logger.debug(f'Streamer: caching sample {n0}, (t={n0 / self.fs})')
         self.cache_folder.mkdir(exist_ok=True, parents=True)
         sr = self._download_raw_partial(first_chunk=first_chunk, last_chunk=last_chunk)
-        data = sr[nsel.start - n0: nsel.stop - n0, csel]
+        if not volts:
+            data = np.copy(sr._raw[nsel.start - n0:nsel.stop - n0, csel])
+        else:
+            data = sr[nsel.start - n0: nsel.stop - n0, csel]
+
         sr.close()
         if self.remove_cached:
             shutil.rmtree(self.target_dir)

brainbox/plot.py

@@ -4,15 +4,15 @@
 
 Run the following to set-up the workspace to run the docstring examples:
 >>> from brainbox import processing
->>> import alf.io as aio
+>>> import one.alf.io as alfio
 >>> import numpy as np
 >>> import matplotlib.pyplot as plt
 >>> import ibllib.ephys.spikes as e_spks
 # (*Note, if there is no 'alf' directory, make 'alf' directory from 'ks2' output directory):
 >>> e_spks.ks2_to_alf(path_to_ks_out, path_to_alf_out)
 # Load the alf spikes bunch and clusters bunch, and get a units bunch.
->>> spks_b = aio.load_object(path_to_alf_out, 'spikes')
->>> clstrs_b = aio.load_object(path_to_alf_out, 'clusters')
+>>> spks_b = alfio.load_object(path_to_alf_out, 'spikes')
+>>> clstrs_b = alfio.load_object(path_to_alf_out, 'clusters')
 >>> units_b = processing.get_units_bunch(spks_b)  # may take a few mins to compute
 """
 
@@ -28,7 +28,7 @@
 from brainbox.metrics import single_units
 from brainbox.processing import bincount2D
 from brainbox.io.spikeglx import extract_waveforms
-from ibllib.io import spikeglx
+import spikeglx
 
 
 def feat_vars(units_b, units=None, feat_name='amps', dist='norm', test='ks', cmap_name='coolwarm',

brainbox/tests/test_io.py

@@ -5,6 +5,7 @@
 import shutil
 
 import numpy as np
+import numpy.testing
 
 from brainbox.io import one as bbone
 from one.api import ONE
@@ -66,5 +67,24 @@ def tearDown(self) -> None:
         shutil.rmtree(self.tmpdir)
 
 
-if __name__ == "__main__":
+class TestIO_ONE(unittest.TestCase):
+    """Tests for brainbox.io.one functions that don't require fixtures on disk."""
+    def test_load_iti(self):
+        """Test for brainbox.io.one.load_iti function."""
+        trials = bbone.alfio.AlfBunch({})
+        trials.intervals = np.array([
+            [114.52487625, 117.88103707],
+            [118.5169474, 122.89742147],
+            [123.49302927, 126.12216664],
+            [126.68107337, 129.53872083],
+            [130.11952807, 133.90539162]
+        ])
+        trials.stimOff_times = [117.38098379, 122.39736201, 125.62210278, 129.03865947, 133.4053633]
+        expected = np.array([1.13596361, 1.09566726, 1.05897059, 1.0808686, np.nan])
+        np.testing.assert_array_almost_equal(bbone.load_iti(trials), expected)
+        _ = trials.pop('stimOff_times')
+        self.assertRaises(ValueError, bbone.load_iti, trials)
+
+
+if __name__ == '__main__':
     unittest.main(exit=False, verbosity=2)

examples/data_release/data_release_behavior.ipynb

@@ -29,6 +29,8 @@
     "* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_structure.html) to download an example dataset for one session, and get familiarised with the data structure\n",
     "* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_download.html) to learn how to use the ONE-api to search and download the released datasets\n",
     "* [These instructions](https://int-brain-lab.github.io/iblenv/loading_examples.html) to get familiarised with specific data loading functions\n",
+    "* [These instructions](https://int-brain-lab.github.io/iblenv/dj_docs/dj_public.html) for instructions on how to access this dataset via Datajoint. (warning: this method may be retired in 2023)\n",
+    "\n",
     "\n",
     "Note:\n",
     "\n",

examples/data_release/data_release_spikesorting_benchmarks.ipynb

@@ -0,0 +1,97 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7ceb904d",
+   "metadata": {},
+   "source": [
+    "# Data Release - Spike sorting benchmark datasets\n",
+    "\n",
+    "Spike sorting is the process by which one extracts the spikes information (times, waveforms) from the raw electrophysiology signals. In the case of our Brainwide map dataset, such electrophysiology signals are heterogeneous, changing from one brain region to another.\n",
+    "By looking at recordings in widespread anatomical regions we discovered that the current spike sorting method has a limited range of applicability. Specifically, we uncovered two major issues. Firstly, there is no one-size-fit-all: an algorithm that is well tailored to extract signals for a given region or electrode type may perform poorly for another. Secondly, and most importantly, we had very limited means to conclude on the performance of such algorithms beyond eyes on the data and qualitative judgements.\n",
+    "\n",
+    "In order to facilitate the development of spike sorting algorithms, we aim to provide benchmarks datasets (for a full explanation, see our proposal [Spike Net](https://docs.google.com/document/d/1OA69Ptg58AQnGdmGi6UvZFrngwZDMixil1V7hJX6bNI/edit)). Here, we explain how to download these datasets, taken to represent various anatomical regions from our Brainwide map."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ea567308",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "## Overview of the Data\n",
+    "We have selected 13 recordings for our benchmarks.\n",
+    "\n",
+    "The insertion IDs are :\n",
+    "```python\n",
+    "pids = [\n",
+    "   '1a276285-8b0e-4cc9-9f0a-a3a002978724',\n",
+    "   '1e104bf4-7a24-4624-a5b2-c2c8289c0de7',\n",
+    "   '5d570bf6-a4c6-4bf1-a14b-2c878c84ef0e',\n",
+    "   '5f7766ce-8e2e-410c-9195-6bf089fea4fd',\n",
+    "   '6638cfb3-3831-4fc2-9327-194b76cf22e1',\n",
+    "   '749cb2b7-e57e-4453-a794-f6230e4d0226',\n",
+    "   'd7ec0892-0a6c-4f4f-9d8f-72083692af5c',\n",
+    "   'da8dfec1-d265-44e8-84ce-6ae9c109b8bd',\n",
+    "   'dab512bd-a02d-4c1f-8dbc-9155a163efc0',\n",
+    "   'dc7e9403-19f7-409f-9240-05ee57cb7aea',\n",
+    "   'e8f9fba4-d151-4b00-bee7-447f0f3e752c',\n",
+    "   'eebcaf65-7fa4-4118-869d-a084e84530e2',\n",
+    "   'fe380793-8035-414e-b000-09bfe5ece92a',\n",
+    "]\n",
+    "```\n",
+    "\n",
+    "(Coming soon) In 2023, we will provide 20-30min chunks of raw electrophysiology data processed in a standardised fashion, with manual annotations of spikes and quality metrics for spike detection recall. These will serve as benchmarks for spike sorting algorithm development. In the meantime, you can familiarise yourself with the data heterogeneity by looking at the whole recordings.\n",
+    "\n",
+    "## View the data\n",
+    "You can view the whole electrophysiology data:\n",
+    "\n",
+    "* [Through a specific AWS website](http://benchmarks.internationalbrainlab.org.s3-website-us-east-1.amazonaws.com)\n",
+    "* [Through our main visualisation website](https://viz.internationalbrainlab.org/app) , inputting a PID in the search bar at the top\n",
+    "* By downloading the electrophysiology data (see below), and using the GUI [viewephys](https://github.com/oliche/viewephys) to navigate through it\n",
+    "\n",
+    "## Data structure and download\n",
+    "The organisation of the data follows the standard IBL data structure.\n",
+    "\n",
+    "Please see\n",
+    "\n",
+    "* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_structure.html) to download an example dataset for one session, and get familiarised with the data structure\n",
+    "    * Note that probe insertion ids `pid` are provided here, not session ids `eid`.\n",
+    "    * Note that you will be most interested in the folders [raw_ephys_data](https://int-brain-lab.github.io/iblenv/notebooks_external/data_structure.html#Datasets-in-raw_ephys_data) / [probeXX](https://int-brain-lab.github.io/iblenv/notebooks_external/data_structure.html#Datasets-in-raw_ephys_data/probeXX) for the raw ephys data, and [alf/probeXX/pykilosort](https://int-brain-lab.github.io/iblenv/notebooks_external/data_structure.html#Datasets-in-alf/probeXX/pykilosort) if using the pyKilosort spike sorting output.\n",
+    "* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_download.html) to learn how to use the ONE-api to search and download the released datasets\n",
+    "* [These instructions](https://int-brain-lab.github.io/iblenv/loading_examples.html) to get familiarised with specific data loading functions\n",
+    "    * You will want to load in particular the [raw ephys data](https://int-brain-lab.github.io/iblenv/notebooks_external/loading_raw_ephys_data.html) and the [spike sorting data](https://int-brain-lab.github.io/iblenv/notebooks_external/loading_spikesorting_data.html) for a given `pid`\n",
+    "\n",
+    "\n",
+    "Note:\n",
+    "\n",
+    "* The tag associated to this release is (coming up soon)"
+   ]
+  }
+ ],
+ "metadata": {
+  "celltoolbar": "Edit Metadata",
+  "kernelspec": {
+   "display_name": "Python [conda env:iblenv] *",
+   "language": "python",
+   "name": "conda-env-iblenv-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/exploring_data/data_download.ipynb

@@ -196,8 +196,18 @@
    "execution_count": null,
    "outputs": [],
    "source": [
+    "# Note that tags are associated with datasets originally\n",
+    "# Find datasets that are tagged for the repeated site paper\n",
+    "datasets_rep_site = one.alyx.rest('datasets', 'list', tag='2022_Q2_IBL_et_al_RepeatedSite')\n",
+    "\n",
     "# Find sessions that have data and are tagged for the repeated site paper\n",
-    "sessions_rep_site = one.alyx.rest('sessions', 'list', dataset_types='spikes.times', tag='2022_Q2_IBL_et_al_RepeatedSite')"
+    "# (you have access to the tag endpoint from the session list)\n",
+    "sessions_rep_site = one.alyx.rest('sessions', 'list', dataset_types='spikes.times', tag='2022_Q2_IBL_et_al_RepeatedSite')\n",
+    "\n",
+    "# Find insertions that are tagged\n",
+    "# (you do not have access to the tag endpoint from the insertion list, so you need to create a django query)\n",
+    "ins_str_query = 'datasets__tags__name,2022_Q2_IBL_et_al_RepeatedSite'\n",
+    "insertions_rep_site = one.alyx.rest('insertions', 'list', django=ins_str_query)"
    ],
    "metadata": {
     "collapsed": false
@@ -206,7 +216,7 @@
   {
    "cell_type": "markdown",
    "source": [
-    "However, if you are only interested in data with a specific tag, the cleanest approach is to follow [these instructions](ttps://int-brain-lab.github.io/ONE/FAQ.html#how-do-i-download-the-datasets-cache-for-a-specific-ibl-paper-release) to work with a tag-specific cache table."
+    "However, if you are only interested in data with a specific tag, the cleanest approach is to follow [these instructions](https://int-brain-lab.github.io/ONE/FAQ.html#how-do-i-download-the-datasets-cache-for-a-specific-ibl-paper-release) to work with a tag-specific cache table."
    ],
    "metadata": {
     "collapsed": false
@@ -365,4 +375,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 0
-}
+}

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.20.0'
+__version__ = '2.21.0'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/atlas/allen_structure_tree.csv

@@ -367,12 +367,12 @@ id,atlas_id,name,acronym,st_level,ontology_id,hemisphere_id,weight,parent_struct
 289,1026,Temporal association areas layer 5,TEa5,,1,3,8690,541,7,1,364,/997/8/567/688/695/315/541/289/,15B0B3,,,f,365,475299804,734881840,Temporal association areas layer 5
 729,1081,Temporal association areas layer 6a,TEa6a,,1,3,8690,541,7,1,365,/997/8/567/688/695/315/541/729/,15B0B3,,,f,366,1289955072,734881840,Temporal association areas layer 6a
 786,1088,Temporal association areas layer 6b,TEa6b,,1,3,8690,541,7,1,366,/997/8/567/688/695/315/541/786/,15B0B3,,,f,367,3588912826,734881840,Temporal association areas layer 6b
-922,256,Perirhinal area,PERI,,1,3,8690,315,6,1,367,/997/8/567/688/695/315/922/,0,,,f,368,173995527,734881840,Perirhinal area
-540,1057,Perirhinal area layer 1,PERI1,,1,3,8690,922,7,1,368,/997/8/567/688/695/315/922/540/,0,,,f,369,161492388,734881840,Perirhinal area layer 1
-888,959,Perirhinal area layer 2/3,PERI2/3,,1,3,8690,922,7,1,369,/997/8/567/688/695/315/922/888/,0,,,f,370,1642584549,734881840,Perirhinal area layer 2/3
-692,1076,Perirhinal area layer 5,PERI5,,1,3,8690,922,7,1,370,/997/8/567/688/695/315/922/692/,0,,,f,371,248375741,734881840,Perirhinal area layer 5
-335,890,Perirhinal area layer 6a,PERI6a,,1,3,8690,922,7,1,371,/997/8/567/688/695/315/922/335/,0,,,f,372,1984229208,734881840,Perirhinal area layer 6a
-368,894,Perirhinal area layer 6b,PERI6b,,1,3,8690,922,7,1,372,/997/8/567/688/695/315/922/368/,0,,,f,373,4014849762,734881840,Perirhinal area layer 6b
+922,256,Perirhinal area,PERI,,1,3,8690,315,6,1,367,/997/8/567/688/695/315/922/,0E9684,,,f,368,173995527,734881840,Perirhinal area
+540,1057,Perirhinal area layer 1,PERI1,,1,3,8690,922,7,1,368,/997/8/567/688/695/315/922/540/,0E9684,,,f,369,161492388,734881840,Perirhinal area layer 1
+888,959,Perirhinal area layer 2/3,PERI2/3,,1,3,8690,922,7,1,369,/997/8/567/688/695/315/922/888/,0E9684,,,f,370,1642584549,734881840,Perirhinal area layer 2/3
+692,1076,Perirhinal area layer 5,PERI5,,1,3,8690,922,7,1,370,/997/8/567/688/695/315/922/692/,0E9684,,,f,371,248375741,734881840,Perirhinal area layer 5
+335,890,Perirhinal area layer 6a,PERI6a,,1,3,8690,922,7,1,371,/997/8/567/688/695/315/922/335/,0E9684,,,f,372,1984229208,734881840,Perirhinal area layer 6a
+368,894,Perirhinal area layer 6b,PERI6b,,1,3,8690,922,7,1,372,/997/8/567/688/695/315/922/368/,0E9684,,,f,373,4014849762,734881840,Perirhinal area layer 6b
 895,111,Ectorhinal area,ECT,,1,3,8690,315,6,1,373,/997/8/567/688/695/315/895/,0D9F91,,,f,374,3399607880,734881840,Ectorhinal area
 836,1094,Ectorhinal area/Layer 1,ECT1,,1,3,8690,895,7,1,374,/997/8/567/688/695/315/895/836/,0D9F91,,,f,375,2240743692,734881840,Ectorhinal area/Layer 1
 427,1043,Ectorhinal area/Layer 2/3,ECT2/3,,1,3,8690,895,7,1,375,/997/8/567/688/695/315/895/427/,0D9F91,,,f,376,993691642,734881840,Ectorhinal area/Layer 2/3