Merge branch 'main' of https://github.com/datajoint/element-array-ephys

Author: kushalbakshi

.pre-commit-config.yaml

@@ -0,0 +1,56 @@
+default_stages: [commit, push]
+exclude: (^.github/|^docs/|^images/)
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files # prevent giant files from being committed
+      - id: requirements-txt-fixer
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+        description: Forces to replace line ending by the UNIX 'lf' character.
+
+  # black
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+      - id: black-jupyter
+        args:
+          - --line-length=88
+
+  # isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+        description: Sorts imports in an alphabetical order
+
+  # flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: # arguments to configure flake8
+          # making isort line length compatible with black
+          - "--max-line-length=88"
+          - "--max-complexity=18"
+          - "--select=B,C,E,F,W,T4,B9"
+
+          # these are errors that will be ignored by flake8
+          # https://www.flake8rules.com/rules/{code}.html
+          - "--ignore=E203,E501,W503,W605"
+          # E203 - Colons should not have any space before them.
+          #        Needed for list indexing
+          # E501 - Line lengths are recommended to be no greater than 79 characters.
+          #        Needed as we conform to 88
+          # W503 - Line breaks should occur after the binary operator.
+          #        Needed because not compatible with black
+          # W605 - a backslash-character pair that is not a valid escape sequence now
+          #        generates a DeprecationWarning. This will eventually become a SyntaxError.
+          #        Needed because we use \d as an escape sequence

CHANGELOG.md

@@ -48,7 +48,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Update - Rename module for acute probe insertions from `ephys.py` to `ephys_acute.py`.
 + Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of `ephys_acute.py`.
-+ Add - Module for chronic probe insertions (`ephys_chronic.py`).  
++ Add - Module for chronic probe insertions (`ephys_chronic.py`).
 + Bugfix - Missing `fileTimeSecs` key in SpikeGLX meta file.
 + Update - Move common functions to `element-interface` package.
 + Add - NWB export function

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 # Contribution Guidelines
 
-This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.io/python/community/02-Contribute.html). Please reference the link for more full details.
+This project follows the [DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/). Please reference the link for more full details.

README.md

@@ -8,5 +8,5 @@ fully functional pipeline.
 
 ![diagram](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/diagram_flowchart.svg)
 
-Installation and usage instructions can be found at the 
+Installation and usage instructions can be found at the
 [Element documentation](https://datajoint.com/docs/elements/element-array-ephys).

docs/src/api/make_pages.py

@@ -41,4 +41,3 @@
 
 with mkdocs_gen_files.open("api/navigation.md", "w") as nav_file:
     nav_file.writelines(nav.build_literate_nav())
-    

docs/src/concepts.md

@@ -104,6 +104,14 @@ Tables related to information about physiological recordings and automatic inges
 | CuratedClusting.Unit | A part table containing single unit information after spike sorting and optional curation. |
 | WaveformSet | A table containing spike waveforms for single units. |
 
+### `ephys_report` schema ([API docs](../api/element_array_ephys/ephys_report))
+Tables for storing probe or unit-level visualization results.
+
+| Table | Description |
+| --- | --- |
+| ProbeLevelReport | A table to store drift map figures generated from each recording probe. |
+| UnitLevelReport | A table to store figures (waveforms, autocorrelogram, peak waveform + neighbors) generated for each unit. |
+
 ## Element Development
 
 Through our interviews and direct collaboration on the precursor projects, we identified the common motifs to create the [Array Electrophysiology Element](https://github.com/datajoint/element-array-ephys).

element_array_ephys/__init__.py

@@ -1,7 +1,11 @@
-import datajoint as dj
 import logging
 import os
 
+import datajoint as dj
+
+from . import ephys_acute as ephys
+
+__all__ = ["ephys", "get_logger"]
 
 dj.config["enable_python_native_blobs"] = True
 
@@ -10,6 +14,3 @@ def get_logger(name):
     log = logging.getLogger(name)
     log.setLevel(os.getenv("LOGLEVEL", "INFO"))
     return log
-
-
-from . import ephys_acute as ephys

element_array_ephys/ephys_acute.py

@@ -1,17 +1,17 @@
-import datajoint as dj
+import gc
+import importlib
+import inspect
 import pathlib
 import re
-import numpy as np
-import inspect
-import importlib
-import gc
 from decimal import Decimal
-import pandas as pd
 
-from element_interface.utils import find_root_directory, find_full_path, dict_to_uuid
+import datajoint as dj
+import numpy as np
+import pandas as pd
+from element_interface.utils import dict_to_uuid, find_full_path, find_root_directory
 
-from .readers import spikeglx, kilosort, openephys
-from . import probe, get_logger, ephys_report
+from . import ephys_report, get_logger, probe
+from .readers import kilosort, openephys, spikeglx
 
 log = get_logger(__name__)
 
@@ -127,7 +127,7 @@ class AcquisitionSoftware(dj.Lookup):
     """
 
     definition = """  # Software used for recording of neuropixels probes
-    acq_software: varchar(24)    
+    acq_software: varchar(24)
     """
     contents = zip(["SpikeGLX", "Open Ephys"])
 
@@ -265,7 +265,7 @@ class EphysRecording(dj.Imported):
 
     definition = """
     # Ephys recording from a probe insertion for a given session.
-    -> ProbeInsertion      
+    -> ProbeInsertion
     ---
     -> probe.ElectrodeConfig
     -> AcquisitionSoftware
@@ -473,9 +473,9 @@ class Electrode(dj.Part):
 
         definition = """
         -> master
-        -> probe.ElectrodeConfig.Electrode  
+        -> probe.ElectrodeConfig.Electrode
         ---
-        lfp: longblob               # (uV) recorded lfp at this electrode 
+        lfp: longblob               # (uV) recorded lfp at this electrode
         """
 
     # Only store LFP for every 9th channel, due to high channel density,
@@ -629,7 +629,7 @@ class ClusteringParamSet(dj.Lookup):
     # Parameter set to be used in a clustering procedure
     paramset_idx:  smallint
     ---
-    -> ClusteringMethod    
+    -> ClusteringMethod
     paramset_desc: varchar(128)
     param_set_hash: uuid
     unique index (param_set_hash)
@@ -694,7 +694,7 @@ class ClusterQualityLabel(dj.Lookup):
 
     Attributes:
         cluster_quality_label (foreign key, varchar(100) ): Cluster quality type.
-        cluster_quality_description (varchar(4000) ): Description of the cluster quality type.
+        cluster_quality_description ( varchar(4000) ): Description of the cluster quality type.
     """
 
     definition = """
@@ -718,7 +718,7 @@ class ClusteringTask(dj.Manual):
     Attributes:
         EphysRecording (foreign key): EphysRecording primary key.
         ClusteringParamSet (foreign key): ClusteringParamSet primary key.
-        clustering_outdir_dir (varchar (255) ): Relative path to output clustering results.
+        clustering_output_dir ( varchar (255) ): Relative path to output clustering results.
         task_mode (enum): `Trigger` computes clustering or and `load` imports existing data.
     """
 
@@ -808,14 +808,14 @@ class Clustering(dj.Imported):
     Attributes:
         ClusteringTask (foreign key): ClusteringTask primary key.
         clustering_time (datetime): Time when clustering results are generated.
-        package_version (varchar(16) ): Package version used for a clustering analysis.
+        package_version ( varchar(16) ): Package version used for a clustering analysis.
     """
 
     definition = """
     # Clustering Procedure
     -> ClusteringTask
     ---
-    clustering_time: datetime  # time of generation of this set of clustering results 
+    clustering_time: datetime  # time of generation of this set of clustering results
     package_version='': varchar(16)
     """
 
@@ -857,6 +857,10 @@ def make(self, key):
                         spikeglx_meta_filepath.parent
                     )
                     spikeglx_recording.validate_file("ap")
+                    run_CatGT = (
+                        params.pop("run_CatGT", True)
+                        and "_tcat." not in spikeglx_meta_filepath.stem
+                    )
 
                     if clustering_method.startswith("pykilosort"):
                         kilosort_triggering.run_pykilosort(
@@ -877,7 +881,7 @@ def make(self, key):
                             ks_output_dir=kilosort_dir,
                             params=params,
                             KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}',
-                            run_CatGT=True,
+                            run_CatGT=run_CatGT,
                         )
                         run_kilosort.run_modules()
                 elif acq_software == "Open Ephys":
@@ -930,22 +934,22 @@ class Curation(dj.Manual):
         Clustering (foreign key): Clustering primary key.
         curation_id (foreign key, int): Unique curation ID.
         curation_time (datetime): Time when curation results are generated.
-        curation_output_dir (varchar(255) ): Output directory of the curated results.
+        curation_output_dir ( varchar(255) ): Output directory of the curated results.
         quality_control (bool): If True, this clustering result has undergone quality control.
         manual_curation (bool): If True, manual curation has been performed on this clustering result.
-        curation_note (varchar(2000) ): Notes about the curation task.
+        curation_note ( varchar(2000) ): Notes about the curation task.
     """
 
     definition = """
     # Manual curation procedure
     -> Clustering
     curation_id: int
     ---
-    curation_time: datetime             # time of generation of this set of curated clustering results 
+    curation_time: datetime             # time of generation of this set of curated clustering results
     curation_output_dir: varchar(255)   # output directory of the curated results, relative to root data directory
     quality_control: bool               # has this clustering result undergone quality control?
     manual_curation: bool               # has manual curation been performed on this clustering result?
-    curation_note='': varchar(2000)  
+    curation_note='': varchar(2000)
     """
 
     def create1_from_clustering_task(self, key, curation_note=""):
@@ -994,7 +998,7 @@ class CuratedClustering(dj.Imported):
 
     definition = """
     # Clustering results of a curation.
-    -> Curation    
+    -> Curation
     """
 
     class Unit(dj.Part):
@@ -1021,7 +1025,7 @@ class Unit(dj.Part):
         spike_count: int         # how many spikes in this recording for this unit
         spike_times: longblob    # (s) spike times of this unit, relative to the start of the EphysRecording
         spike_sites : longblob   # array of electrode associated with each spike
-        spike_depths=null : longblob  # (um) array of depths associated with each spike, relative to the (0, 0) of the probe    
+        spike_depths=null : longblob  # (um) array of depths associated with each spike, relative to the (0, 0) of the probe
         """
 
     def make(self, key):
@@ -1149,8 +1153,8 @@ class Waveform(dj.Part):
         # Spike waveforms and their mean across spikes for the given unit
         -> master
         -> CuratedClustering.Unit
-        -> probe.ElectrodeConfig.Electrode  
-        --- 
+        -> probe.ElectrodeConfig.Electrode
+        ---
         waveform_mean: longblob   # (uV) mean waveform across spikes of the given unit
         waveforms=null: longblob  # (uV) (spike x sample) waveforms of a sampling of spikes at the given electrode for the given unit
         """
@@ -1281,7 +1285,7 @@ class QualityMetrics(dj.Imported):
 
     definition = """
     # Clusters and waveforms metrics
-    -> CuratedClustering    
+    -> CuratedClustering
     """
 
     class Cluster(dj.Part):
@@ -1306,26 +1310,26 @@ class Cluster(dj.Part):
             contamination_rate (float): Frequency of spikes in the refractory period.
         """
 
-        definition = """   
+        definition = """
         # Cluster metrics for a particular unit
         -> master
         -> CuratedClustering.Unit
         ---
-        firing_rate=null: float # (Hz) firing rate for a unit 
+        firing_rate=null: float # (Hz) firing rate for a unit
         snr=null: float  # signal-to-noise ratio for a unit
         presence_ratio=null: float  # fraction of time in which spikes are present
         isi_violation=null: float   # rate of ISI violation as a fraction of overall rate
         number_violation=null: int  # total number of ISI violations
         amplitude_cutoff=null: float  # estimate of miss rate based on amplitude histogram
         isolation_distance=null: float  # distance to nearest cluster in Mahalanobis space
-        l_ratio=null: float  # 
+        l_ratio=null: float  #
         d_prime=null: float  # Classification accuracy based on LDA
         nn_hit_rate=null: float  # Fraction of neighbors for target cluster that are also in target cluster
         nn_miss_rate=null: float # Fraction of neighbors outside target cluster that are in target cluster
         silhouette_score=null: float  # Standard metric for cluster overlap
         max_drift=null: float  # Maximum change in spike depth throughout recording
-        cumulative_drift=null: float  # Cumulative change in spike depth throughout recording 
-        contamination_rate=null: float # 
+        cumulative_drift=null: float  # Cumulative change in spike depth throughout recording
+        contamination_rate=null: float #
         """
 
     class Waveform(dj.Part):
@@ -1345,7 +1349,7 @@ class Waveform(dj.Part):
             velocity_below (float) inverse velocity of waveform propagation from soma toward the bottom of the probe.
         """
 
-        definition = """   
+        definition = """
         # Waveform metrics for a particular unit
         -> master
         -> CuratedClustering.Unit