diff --git a/.gitignore b/.gitignore
index 722f2f0b..0db702a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -107,6 +107,7 @@ venv/
ENV/
env.bak/
venv.bak/
+deepcell-env/
# Spyder project settings
.spyderproject
diff --git a/backend/deepcell_label/client.py b/backend/deepcell_label/client.py
new file mode 100644
index 00000000..b2430423
--- /dev/null
+++ b/backend/deepcell_label/client.py
@@ -0,0 +1,28 @@
+import asyncio
+import os
+import pickle
+import zlib
+
+import websockets
+from flask import current_app
+
+
+# connects to cell SAM server
+async def perform_send(to_send):
+ uri = os.environ['CELLSAM_SERVER'] # to be replaced with cell SAM server uri
+ async with websockets.connect(uri, ping_interval=None) as websocket:
+ data = {'img': to_send}
+ print(uri)
+ pkt = zlib.compress(pickle.dumps(data))
+ await websocket.send(pkt)
+ print('sent')
+ pkt_received = await websocket.recv()
+ print('received')
+ mask = pickle.loads(zlib.decompress(pkt_received))
+ return mask
+
+
+def send_to_server(to_send):
+ current_app.logger.info('Sent to server to generate mask for cellSAM')
+ mask = asyncio.run(perform_send(to_send))
+ return mask
diff --git a/backend/deepcell_label/label.py b/backend/deepcell_label/label.py
index 5cd7c5a5..b75fa778 100644
--- a/backend/deepcell_label/label.py
+++ b/backend/deepcell_label/label.py
@@ -13,6 +13,8 @@
from skimage.morphology import dilation, disk, erosion, flood, square
from skimage.segmentation import morphological_chan_vese, watershed
+from deepcell_label.client import send_to_server
+
class Edit(object):
"""
@@ -76,6 +78,9 @@ def load(self, labels_zip):
self.action = edit['action']
self.height = edit['height']
self.width = edit['width']
+ self.d1, self.d2, self.d3, self.d4 = [
+ edit[k] for k in ['d1', 'd2', 'd3', 'd4']
+ ]
self.args = edit.get('args', None)
# TODO: specify write mode per cell?
self.write_mode = edit.get('writeMode', 'overlap')
@@ -102,7 +107,8 @@ def load(self, labels_zip):
if 'raw.dat' in zf.namelist():
with zf.open('raw.dat') as f:
raw = np.frombuffer(f.read(), np.uint8)
- self.raw = np.reshape(raw, (self.width, self.height))
+ self.rawOriginal = np.reshape(raw, (self.d1, self.d2, self.d3, self.d4))
+ self.raw = self.rawOriginal[0][0]
elif self.action in self.raw_required:
raise ValueError(
f'Include raw array in raw.json to use action {self.action}.'
@@ -418,3 +424,36 @@ def action_dilate(self, cell):
mask = self.get_mask(cell)
dilated = dilation(mask, square(3))
self.add_mask(dilated, cell)
+
+ def action_select_channels(self, channels):
+ self.nuclear_channel = int(channels[0])
+ self.wholecell_channel = int(channels[1])
+
+ def action_segment_all(self, channels):
+ nuclear_channel = channels[0]
+ wholecell_channel = channels[1]
+ to_send = []
+ if self.d1 == 1:
+ to_send = self.raw.reshape(self.d3, self.d4, self.d1)
+ elif self.d1 > 1:
+ nuclear = self.rawOriginal[nuclear_channel][0]
+ wholecell = self.rawOriginal[wholecell_channel][0]
+ to_send = np.stack([nuclear, wholecell], axis=-1)
+ mask = send_to_server(to_send)
+ self.labels = mask.astype(np.int32)
+ if len(self.labels.shape) == 2:
+ self.labels = np.expand_dims(np.expand_dims(self.labels, 0), 3)
+ cells = []
+ for t in range(self.labels.shape[0]):
+ for c in range(self.labels.shape[-1]):
+ for value in np.unique(self.labels[t, :, :, c]):
+ if value != 0:
+ cells.append(
+ {
+ 'cell': int(value),
+ 'value': int(value),
+ 't': int(t),
+ 'c': int(c),
+ }
+ )
+ self.cells = cells
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 4a671ec7..9d4e9345 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -10,9 +10,10 @@ mysqlclient~=2.1.0
numpy
python-decouple~=3.1
python-dotenv~=0.19.2
-python-magic~=0.4.25
+# python-magic~=0.4.25
requests~=2.29.0
scikit-image~=0.19.0
sqlalchemy~=1.3.24
tifffile
imagecodecs
+websockets
diff --git a/frontend/src/Project/EditControls/EditControls.js b/frontend/src/Project/EditControls/EditControls.js
index 5c582a78..1a22c439 100644
--- a/frontend/src/Project/EditControls/EditControls.js
+++ b/frontend/src/Project/EditControls/EditControls.js
@@ -6,6 +6,7 @@ import CellControls from './CellControls';
import CellTypeControls from './CellTypeControls';
import TrackingControls from './DivisionsControls';
import SegmentControls from './SegmentControls';
+import SegmentSamControls from './SegmentSamControls';
function TabPanel(props) {
const { children, value, index, ...other } = props;
@@ -28,14 +29,16 @@ function EditControls() {
const value = useSelector(labelMode, (state) => {
return state.matches('editSegment')
? 0
- : state.matches('editCells')
+ : state.matches('editSegmentSam')
? 1
- : state.matches('editDivisions')
+ : state.matches('editCells')
? 2
- : state.matches('editCellTypes')
+ : state.matches('editDivisions')
? 3
- : state.matches('editSpots')
+ : state.matches('editCellTypes')
? 4
+ : state.matches('editSpots')
+ ? 5
: false;
});
@@ -51,15 +54,18 @@ function EditControls() {
-
+
-
+
-
+
+
+
+
diff --git a/frontend/src/Project/EditControls/EditTabs.js b/frontend/src/Project/EditControls/EditTabs.js
index fdabcae2..4a1a7603 100644
--- a/frontend/src/Project/EditControls/EditTabs.js
+++ b/frontend/src/Project/EditControls/EditTabs.js
@@ -13,14 +13,16 @@ function EditTabs() {
const value = useSelector(labelMode, (state) => {
return state.matches('editSegment')
? 0
- : state.matches('editCells')
+ : state.matches('editSegmentSam')
? 1
- : state.matches('editDivisions')
+ : state.matches('editCells')
? 2
- : state.matches('editCellTypes')
+ : state.matches('editDivisions')
? 3
- : state.matches('editSpots')
+ : state.matches('editCellTypes')
? 4
+ : state.matches('editSpots')
+ ? 5
: false;
});
const handleChange = (event, newValue) => {
@@ -29,15 +31,18 @@ function EditTabs() {
labelMode.send('EDIT_SEGMENT');
break;
case 1:
- labelMode.send('EDIT_CELLS');
+ labelMode.send('EDIT_SEGMENT_SAM');
break;
case 2:
- labelMode.send('EDIT_DIVISIONS');
+ labelMode.send('EDIT_CELLS');
break;
case 3:
- labelMode.send('EDIT_CELLTYPES');
+ labelMode.send('EDIT_DIVISIONS');
break;
case 4:
+ labelMode.send('EDIT_CELLTYPES');
+ break;
+ case 5:
labelMode.send('EDIT_SPOTS');
break;
default:
@@ -72,6 +77,7 @@ function EditTabs() {
variant='scrollable'
>
+
diff --git a/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons.js b/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons.js
new file mode 100644
index 00000000..b94b4e8e
--- /dev/null
+++ b/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons.js
@@ -0,0 +1,22 @@
+import { useSelector } from '@xstate/react';
+import { FormLabel } from '@mui/material';
+import Box from '@mui/material/Box';
+import ButtonGroup from '@mui/material/ButtonGroup';
+import SegmentAllButton from './ActionButtons/SegmentAllButton';
+import { useRaw } from '../../ProjectContext';
+
+function ActionButtons() {
+ const raw = useRaw();
+ const layers = useSelector(raw, (state) => state.context.layers);
+ const layer = layers[0];
+ return (
+
+ Actions
+
+
+
+
+ );
+}
+
+export default ActionButtons;
diff --git a/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons/ActionButton.js b/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons/ActionButton.js
new file mode 100644
index 00000000..2c1e09c5
--- /dev/null
+++ b/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons/ActionButton.js
@@ -0,0 +1,36 @@
+import Button from '@mui/material/Button';
+import Tooltip from '@mui/material/Tooltip';
+import { bind } from 'mousetrap';
+import React, { useEffect } from 'react';
+
+// for adding tooltip to disabled buttons
+// from https://stackoverflow.com/questions/61115913
+
+const ActionButton = ({ tooltipText, disabled, onClick, hotkey, ...other }) => {
+ const adjustedButtonProps = {
+ disabled: disabled,
+ component: disabled ? 'div' : undefined,
+ onClick: disabled ? undefined : onClick,
+ };
+
+ useEffect(() => {
+ bind(hotkey, onClick);
+ }, [hotkey, onClick]);
+
+ return (
+
+
+
+ );
+};
+
+export default ActionButton;
diff --git a/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons/SegmentAllButton.js b/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons/SegmentAllButton.js
new file mode 100644
index 00000000..06df810e
--- /dev/null
+++ b/frontend/src/Project/EditControls/SegmentSamControls/ActionButtons/SegmentAllButton.js
@@ -0,0 +1,102 @@
+import { useSelector } from '@xstate/react';
+import React, { useCallback, useState } from 'react';
+import { useEditSegment, useSelect, useRaw, useSegmentApi } from '../../../ProjectContext';
+import ActionButton from './ActionButton';
+import { MenuItem, TextField, LinearProgress, Box } from '@mui/material';
+import Grid from '@mui/material/Grid';
+
+function LayerSelector({ layer, channelType }) {
+ const segment = useEditSegment();
+ const nuclearChannel = useSelector(segment, (state) => state.context.nuclearChannel);
+ const wholeCellChannel = useSelector(segment, (state) => state.context.wholeCellChannel);
+
+ const raw = useRaw();
+ const names = useSelector(raw, (state) => state.context.channelNames);
+
+ const onChangeNuclear = (e) => {
+ segment.send({ type: 'SET_NUCLEAR_CHANNEL', nuclearChannel: Number(e.target.value) });
+ };
+
+ const onChangeWholeCell = (e) => {
+ segment.send({ type: 'SET_WHOLE_CELL_CHANNEL', wholeCellChannel: Number(e.target.value) });
+ };
+
+ return channelType == 'nuclear' ? (
+
+ {names.map((opt, index) => (
+
+ ))}
+
+ ) : (
+
+ {names.map((opt, index) => (
+
+ ))}
+
+ );
+}
+
+function SegmentAllButton({ props, layer }) {
+ const segment = useEditSegment();
+ const segmentAPI = useSegmentApi();
+ const segmentFinished = useSelector(segmentAPI, (state) => state.matches('idle'));
+ const grayscale = useSelector(segment, (state) => state.matches('display.grayscale'));
+
+ const segmentAction = useCallback(() => {
+ setSegmentButtonClicked(true);
+ segment.send('SEGMENTALL');
+ }, [segment]);
+
+ const [segmentButtonClicked, setSegmentButtonClicked] = useState(false);
+
+ const tooltipText = (
+
+ Generate all the segmentation masks M
+
+ );
+
+ return (
+
+ Select Nuclear Channel
+
+
+
+ Select Whole Cell Channel
+
+
+
+
+
+ Segment All
+
+
+
+ {segmentButtonClicked && !segmentFinished && }
+
+
+ );
+}
+
+export default SegmentAllButton;
diff --git a/frontend/src/Project/EditControls/SegmentSamControls/SegmentSamControls.js b/frontend/src/Project/EditControls/SegmentSamControls/SegmentSamControls.js
new file mode 100644
index 00000000..4633992b
--- /dev/null
+++ b/frontend/src/Project/EditControls/SegmentSamControls/SegmentSamControls.js
@@ -0,0 +1,12 @@
+import Box from '@mui/material/Box';
+import ActionButtons from './ActionButtons';
+
+function SegmentSamControls() {
+ return (
+
+
+
+ );
+}
+
+export default SegmentSamControls;
diff --git a/frontend/src/Project/EditControls/SegmentSamControls/index.js b/frontend/src/Project/EditControls/SegmentSamControls/index.js
new file mode 100644
index 00000000..302d0218
--- /dev/null
+++ b/frontend/src/Project/EditControls/SegmentSamControls/index.js
@@ -0,0 +1,3 @@
+import SegmentSamControls from './SegmentSamControls';
+
+export default SegmentSamControls;
diff --git a/frontend/src/Project/service/edit/editMachine.js b/frontend/src/Project/service/edit/editMachine.js
index af9acbe4..043dfa62 100644
--- a/frontend/src/Project/service/edit/editMachine.js
+++ b/frontend/src/Project/service/edit/editMachine.js
@@ -45,6 +45,7 @@ const createEditMachine = ({ eventBuses, undoRef }) =>
checkTool: {
always: [
{ cond: ({ tool }) => tool === 'editSegment', target: 'editSegment' },
+ { cond: ({ tool }) => tool === 'editSegmentSam', target: 'editSegmentSam' },
{ cond: ({ tool }) => tool === 'editCells', target: 'editCells' },
{ cond: ({ tool }) => tool === 'editDivisions', target: 'editDivisions' },
{ cond: ({ tool }) => tool === 'editCellTypes', target: 'editCellTypes' },
@@ -59,6 +60,13 @@ const createEditMachine = ({ eventBuses, undoRef }) =>
mousedown: { actions: forwardTo('editSegment') },
},
},
+ editSegmentSam: {
+ entry: [assign({ tool: 'editSegmentSam' }), send({ type: 'ENTER_TAB' })],
+ on: {
+ mouseup: { actions: forwardTo('editSegmentSam') },
+ mousedown: { actions: forwardTo('editSegmentSam') },
+ },
+ },
editDivisions: {
entry: [
assign({ tool: 'editDivisions' }),
@@ -103,6 +111,7 @@ const createEditMachine = ({ eventBuses, undoRef }) =>
SAVE: { actions: 'save' },
RESTORE: { target: '.checkTool', actions: ['restore', respond('RESTORED')] },
EDIT_SEGMENT: 'editSegment',
+ EDIT_SEGMENT_SAM: 'editSegmentSam',
EDIT_DIVISIONS: 'editDivisions',
EDIT_CELLS: 'editCells',
EDIT_CELLTYPES: 'editCellTypes',
@@ -118,6 +127,7 @@ const createEditMachine = ({ eventBuses, undoRef }) =>
restore: assign((_, { tool }) => ({ tool })),
spawnTools: assign((context) => ({
editSegmentRef: spawn(createEditSegmentMachine(context), 'editSegment'),
+ editSegmentSamRef: spawn(createEditSegmentMachine(context), 'editSegmentSam'),
editDivisionsRef: spawn(createEditDivisionsMachine(context), 'editDivisions'),
editCellsRef: spawn(createEditCellsMachine(context), 'editCells'),
editCellTypesRef: spawn(createEditCellTypesMachine(context), 'editCellTypes'),
diff --git a/frontend/src/Project/service/edit/segment/editSegmentMachine.js b/frontend/src/Project/service/edit/segment/editSegmentMachine.js
index 465a91c8..cea480c9 100644
--- a/frontend/src/Project/service/edit/segment/editSegmentMachine.js
+++ b/frontend/src/Project/service/edit/segment/editSegmentMachine.js
@@ -15,7 +15,15 @@ import createWatershedMachine from './watershedMachine';
const { pure, respond } = actions;
const colorTools = ['brush', 'select', 'trim', 'flood'];
-const grayscaleTools = ['brush', 'select', 'trim', 'flood', 'threshold', 'watershed'];
+const grayscaleTools = [
+ 'brush',
+ 'select',
+ 'trim',
+ 'flood',
+ 'threshold',
+ 'watershed',
+ 'segment_all',
+];
const panTools = ['select', 'trim', 'flood', 'watershed'];
const noPanTools = ['brush', 'threshold'];
@@ -41,6 +49,8 @@ const createEditSegmentMachine = (context) =>
tool: 'select',
tools: null,
eventBuses: context.eventBuses,
+ nuclearChannel: 0,
+ wholeCellChannel: 1,
},
type: 'parallel',
states: {
@@ -102,6 +112,10 @@ const createEditSegmentMachine = (context) =>
SAVE: { actions: 'save' },
RESTORE: { actions: ['restore', respond('RESTORED')] },
+
+ SEGMENTALL: { actions: 'segment_all' },
+ SET_WHOLE_CELL_CHANNEL: { actions: 'set_whole_cell_channel' },
+ SET_NUCLEAR_CHANNEL: { actions: 'set_nuclear_channel' },
},
},
{
@@ -134,6 +148,10 @@ const createEditSegmentMachine = (context) =>
}),
}),
forwardToTool: forwardTo((ctx) => ctx.tools[ctx.tool]),
+ set_whole_cell_channel: assign({
+ wholeCellChannel: (_, { wholeCellChannel }) => wholeCellChannel,
+ }),
+ set_nuclear_channel: assign({ nuclearChannel: (_, { nuclearChannel }) => nuclearChannel }),
erode: send(
(ctx) => ({
type: 'EDIT',
@@ -158,6 +176,14 @@ const createEditSegmentMachine = (context) =>
}),
{ to: 'arrays' }
),
+ segment_all: send(
+ (ctx) => ({
+ type: 'EDIT',
+ action: 'segment_all',
+ args: { channels: [ctx.nuclearChannel, ctx.wholeCellChannel] },
+ }),
+ { to: 'arrays' }
+ ),
},
}
);
diff --git a/frontend/src/Project/service/labels/arraysMachine.js b/frontend/src/Project/service/labels/arraysMachine.js
index 7a763111..b8850cbe 100644
--- a/frontend/src/Project/service/labels/arraysMachine.js
+++ b/frontend/src/Project/service/labels/arraysMachine.js
@@ -49,7 +49,10 @@ const createArraysMachine = (context) =>
states: {
loading: {
on: {
- LOADED: { target: 'done', actions: ['setRaw', 'setLabeled', 'setRawOriginal'] },
+ LOADED: {
+ target: 'done',
+ actions: ['setRaw', 'setLabeled', 'setRawOriginal'],
+ },
},
},
done: { type: 'final' },
@@ -69,14 +72,16 @@ const createArraysMachine = (context) =>
},
onDone: {
target: 'idle',
- actions: ['sendLabeledFrame', 'sendRawFrame'],
+ actions: ['sendLabeledFrame', 'sendRawFrame', 'sendArrays'],
},
},
idle: {
// TODO: factor out raw and labeled states (and/or machines)
on: {
EDIT: { target: 'editing', actions: forwardTo('api') },
- SET_T: { actions: ['setT', 'sendLabeledFrame', 'sendRawFrame'] },
+ SET_T: {
+ actions: ['setT', 'sendLabeledFrame', 'sendRawFrame', 'sendArrays', 'setGray'],
+ },
SET_FEATURE: { actions: ['setFeature', 'sendLabeledFrame'] },
SET_CHANNEL: { actions: ['setChannel', 'sendRawFrame'] },
GET_ARRAYS: { actions: 'sendArrays' },
@@ -158,7 +163,7 @@ const createArraysMachine = (context) =>
sendRawFrame: send(
(ctx, evt) => ({
type: 'RAW',
- raw: ctx.raw[ctx.channel][ctx.t],
+ raw: ctx.raw,
}),
{ to: 'eventBus' }
),
diff --git a/frontend/src/Project/service/labels/cellsMachine.js b/frontend/src/Project/service/labels/cellsMachine.js
index cf8b7b54..68ed019f 100644
--- a/frontend/src/Project/service/labels/cellsMachine.js
+++ b/frontend/src/Project/service/labels/cellsMachine.js
@@ -177,6 +177,7 @@ const createCellsMachine = ({ eventBuses, undoRef }) =>
send({ type: 'CELLS', cells }, { to: 'eventBus' }),
];
}),
+ // uploadSegmentSam: assign({cells: }),
sendCells: send((ctx) => ({ type: 'CELLS', cells: ctx.cells }), { to: 'eventBus' }),
// needs to be pure because assign events have priority
// NOTE: this is changing in xstate v5, can revert to assign when that's released
diff --git a/frontend/src/Project/service/labels/segmentApiMachine.js b/frontend/src/Project/service/labels/segmentApiMachine.js
index f57100e6..fdf9ae1e 100644
--- a/frontend/src/Project/service/labels/segmentApiMachine.js
+++ b/frontend/src/Project/service/labels/segmentApiMachine.js
@@ -5,6 +5,7 @@
import * as zip from '@zip.js/zip.js';
import { assign, Machine, sendParent } from 'xstate';
import { fromEventBus } from '../eventBus';
+import { flattenDeep } from 'lodash';
/** Splits a 1D Int32Array buffer into a 2D list of Int32Array with height and width. */
function splitRows(buffer, width, height) {
@@ -20,7 +21,17 @@ function splitRows(buffer, width, height) {
async function makeEditZip(context, event) {
const { labeled, raw, cells, writeMode, t, c } = context;
const { action, args } = event;
- const edit = { width: labeled[0].length, height: labeled.length, action, args, writeMode };
+ const edit = {
+ width: labeled[0].length,
+ height: labeled.length,
+ action,
+ args,
+ writeMode,
+ d1: raw.length,
+ d2: raw[0].length,
+ d3: raw[0][0].length,
+ d4: raw[0][0][0].length,
+ };
const zipWriter = new zip.ZipWriter(new zip.BlobWriter('application/zip'));
// Required files
@@ -31,9 +42,15 @@ async function makeEditZip(context, event) {
);
await zipWriter.add('labeled.dat', new zip.BlobReader(new Blob(labeled)));
// Optional files
- const usesRaw = action === 'active_contour' || action === 'threshold' || action === 'watershed';
+
+ const usesRaw =
+ action === 'active_contour' ||
+ action === 'threshold' ||
+ action === 'watershed' ||
+ action === 'segment_all' ||
+ action === 'select_channels';
if (usesRaw) {
- await zipWriter.add('raw.dat', new zip.BlobReader(new Blob(raw)));
+ await zipWriter.add('raw.dat', new zip.BlobReader(new Blob(flattenDeep(raw))));
}
const zipBlob = await zipWriter.close();
@@ -89,7 +106,7 @@ const createSegmentApiMachine = ({ eventBuses }) =>
invoke: [
{
id: 'arrays',
- src: fromEventBus('editSegment', () => eventBuses.arrays, ['LABELED', 'RAW']),
+ src: fromEventBus('editSegment', () => eventBuses.arrays, ['LABELED', 'RAW', 'ARRAYS']),
},
{ id: 'cells', src: fromEventBus('editSegment', () => eventBuses.cells, 'CELLS') },
{ src: fromEventBus('editSegment', () => eventBuses.image, 'SET_T') },
@@ -109,6 +126,7 @@ const createSegmentApiMachine = ({ eventBuses }) =>
on: {
LABELED: { actions: 'setLabeled' },
RAW: { actions: 'setRaw' },
+ ARRAYS: { actions: 'setRaw' },
CELLS: { actions: 'setCells' },
SET_T: { actions: 'setT' },
SET_FEATURE: { actions: 'setC' },