From 12eb20ceaac84ded1bf632fb2b3f7bec42b2cb3a Mon Sep 17 00:00:00 2001 From: Mark Rivers Date: Sat, 27 Dec 2025 11:17:21 -0600 Subject: [PATCH 1/3] Changed how NDArrays are generated when multiple thresholds are enabled and the Stream2 interface is used. Previously 3-D NDArrays were created, with the third dimension being the threshold index. This does not work when the driver does not decompress the data, because the arrays are individually compressed but a 3-D array must be compressed as a unit. The driver now only creates 2-D NDArrays, and creates new NDAttributes, ThresholdName and ThresholdEnergy to identify which threshold an NDArray contains. The driver sends all thresholds on asyn address 0, and threshold N on asyn address N (N=1 to number of enabled thresholds). --- eigerApp/src/eigerDetector.cpp | 127 ++++++++++++------------ eigerApp/src/stream2Api.cpp | 170 ++++++++++++++++----------------- eigerApp/src/streamApi.h | 7 +- 3 files changed, 157 insertions(+), 147 deletions(-) diff --git a/eigerApp/src/eigerDetector.cpp b/eigerApp/src/eigerDetector.cpp index ca68e23..d598942 100644 --- a/eigerApp/src/eigerDetector.cpp +++ b/eigerApp/src/eigerDetector.cpp @@ -41,6 +41,9 @@ #define ENERGY_EPSILON 0.05 #define WAVELENGTH_EPSILON 0.0005 +// Maximum number of thresholds Pilatus4 has 4 +#define MAX_THRESHOLDS 4 + // Error message formatters #define ERR(msg) asynPrint(pasynUserSelf, ASYN_TRACE_ERROR, "%s::%s: %s\n", \ driverName, functionName, msg) @@ -173,7 +176,7 @@ eigerDetector::eigerDetector (const char *portName, const char *serverHostname, int maxBuffers, size_t maxMemory, int priority, int stackSize) - : ADDriver(portName, 2, 0, maxBuffers, maxMemory, + : ADDriver(portName, MAX_THRESHOLDS+1, 0, maxBuffers, maxMemory, 0, 0, /* No interfaces beyond ADDriver.cpp */ ASYN_CANBLOCK | /* ASYN_CANBLOCK=1 */ ASYN_MULTIDEVICE, /* ASYN_MULTIDEVICE=1 */ @@ -1374,6 +1377,7 @@ void eigerDetector::streamTask (void) } int err; stream_header_t header = {}; + int numThresholds = 1; for(;;) { unlock(); @@ -1424,7 +1428,7 @@ void eigerDetector::streamTask (void) if (streamVersion == STREAM_VERSION_STREAM) { err = mStreamAPI->waitFrame(&endFrames); } else { - err = mStream2API->waitFrame(&endFrames); + err = mStream2API->waitFrame(&endFrames, &numThresholds); } lock(); if (err == STREAM_SUCCESS) { @@ -1448,66 +1452,69 @@ void eigerDetector::streamTask (void) break; } - NDArray *pArray; - int decompress; - mStreamDecompress->get(decompress); - bool tsIsSet = false; - if (streamVersion == STREAM_VERSION_STREAM) { - err = mStreamAPI->getFrame(&pArray, pNDArrayPool, decompress); - } else { - err = mStream2API->getFrame(&pArray, pNDArrayPool, decompress, streamAsTsSource); - tsIsSet = streamAsTsSource; - } - int imageCounter, numImagesCounter, arrayCallbacks; - getIntegerParam(NDArrayCounter, &imageCounter); - getIntegerParam(ADNumImagesCounter, &numImagesCounter); - getIntegerParam(NDArrayCallbacks, &arrayCallbacks); - - // The data returned from the StreamAPIs is unsigned. - // Bad pixels and gaps are very large positive numbers, which makes autoscaling difficult - // Optionally change the data type to signed. - // This improves autoscaling, but reduces the count range by 2X. - int signedData; - mSignedData->get(signedData); - if (signedData) { - int dataType = pArray->dataType; - switch (pArray->dataType) { - case NDUInt8: - pArray->dataType = NDInt8; - break; - case NDUInt16: - pArray->dataType = NDInt16; - break; - case NDUInt32: - pArray->dataType = NDInt32; - break; - default: - ERR_ARGS("Unknown data type=%d", dataType); + for (int thresh=0; threshget(decompress); + bool tsIsSet = false; + if (streamVersion == STREAM_VERSION_STREAM) { + err = mStreamAPI->getFrame(&pArray, pNDArrayPool, decompress); + } else { + err = mStream2API->getFrame(&pArray, pNDArrayPool, thresh, decompress, streamAsTsSource); + tsIsSet = streamAsTsSource; + } + int imageCounter, numImagesCounter, arrayCallbacks; + getIntegerParam(NDArrayCounter, &imageCounter); + getIntegerParam(ADNumImagesCounter, &numImagesCounter); + getIntegerParam(NDArrayCallbacks, &arrayCallbacks); + + // The data returned from the StreamAPIs is unsigned. + // Bad pixels and gaps are very large positive numbers, which makes autoscaling difficult + // Optionally change the data type to signed. + // This improves autoscaling, but reduces the count range by 2X. + int signedData; + mSignedData->get(signedData); + if (signedData) { + int dataType = pArray->dataType; + switch (pArray->dataType) { + case NDUInt8: + pArray->dataType = NDInt8; + break; + case NDUInt16: + pArray->dataType = NDInt16; + break; + case NDUInt32: + pArray->dataType = NDInt32; + break; + default: + ERR_ARGS("Unknown data type=%d", dataType); + } + } + + // Put the frame number and timestamp into the buffer + pArray->uniqueId = imageCounter; + + // Only call updateTimeStamps if the stream2 has not set the ts itself + if (!tsIsSet) + updateTimeStamps(pArray); + + // Update Omega angle for this frame + ++mFrameNumber; + + // Get any attributes that have been defined for this driver + this->getAttributes(pArray->pAttributeList); + + // Call the NDArray callback + if (arrayCallbacks) { + doCallbacksGenericPointer(pArray, NDArrayData, 0); + doCallbacksGenericPointer(pArray, NDArrayData, thresh+1); } + setIntegerParam(NDArrayCounter, ++imageCounter); + setIntegerParam(ADNumImagesCounter, ++numImagesCounter); + + callParamCallbacks(); + pArray->release(); } - - // Put the frame number and timestamp into the buffer - pArray->uniqueId = imageCounter; - - // Only call updateTimeStamps if the stream2 has not set the ts itself - if (!tsIsSet) - updateTimeStamps(pArray); - - // Update Omega angle for this frame - ++mFrameNumber; - - // Get any attributes that have been defined for this driver - this->getAttributes(pArray->pAttributeList); - - // Call the NDArray callback - if (arrayCallbacks) - doCallbacksGenericPointer(pArray, NDArrayData, 0); - - setIntegerParam(NDArrayCounter, ++imageCounter); - setIntegerParam(ADNumImagesCounter, ++numImagesCounter); - - callParamCallbacks(); - pArray->release(); } end: diff --git a/eigerApp/src/stream2Api.cpp b/eigerApp/src/stream2Api.cpp index 77de57a..48ded22 100644 --- a/eigerApp/src/stream2Api.cpp +++ b/eigerApp/src/stream2Api.cpp @@ -185,10 +185,14 @@ int Stream2API::getHeader (stream_header_t *header, int timeout) mImage_size_x = sm->image_size_x; mImage_size_y = sm->image_size_y; mNumber_of_images = sm->number_of_images; + mThresholdEnergy.clear(); + for (int i=0; i<(int)sm->threshold_energy.len; i++) { + mThresholdEnergy.push_back(sm->threshold_energy.ptr[i]); + } return STREAM_SUCCESS; } -int Stream2API::waitFrame (int *end, int timeout) +int Stream2API::waitFrame (int *end, int *numThresholds, int timeout) { //const char *functionName = "waitFrame"; int err = STREAM_SUCCESS; @@ -206,6 +210,8 @@ int Stream2API::waitFrame (int *end, int timeout) return err; } mImageMsg = (stream2_image_msg *)s2msg; + mNumThresholds = (int)mImageMsg->data.len; + *numThresholds = mNumThresholds; if (mImageMsg->type == STREAM2_MSG_END) { @@ -214,7 +220,7 @@ int Stream2API::waitFrame (int *end, int timeout) return err; } -int Stream2API::getFrame (NDArray **pArrayOut, NDArrayPool *pNDArrayPool, int decompress, bool extractTimeStamp) +int Stream2API::getFrame (NDArray **pArrayOut, NDArrayPool *pNDArrayPool, int thresh, int decompress, bool extractTimeStamp) { const char *functionName = "getFrame"; int err = STREAM_SUCCESS; @@ -224,102 +230,96 @@ int Stream2API::getFrame (NDArray **pArrayOut, NDArrayPool *pNDArrayPool, int de NDArray *pArray; char encoding[32]; NDDataType_t dataType; - int numThresholds = mImageMsg->data.len; switch (mImageMsg->type) { - case STREAM2_MSG_IMAGE: - for (int i = 0; i < numThresholds; i++) { - struct stream2_image_data *pSID = &mImageMsg->data.ptr[i]; - struct stream2_multidim_array mda = pSID->data; - dims[0] = mda.dim[1]; - dims[1] = mda.dim[0]; - numDims = 2; - if (numThresholds > 1) { - dims[2] = numThresholds; - numDims = 3; - } - struct stream2_typed_array *pS2Array = &mda.array; - stream2_typed_array_tag s2DataType = (stream2_typed_array_tag)pS2Array->tag; - struct stream2_bytes *pSB = &pS2Array->data; - compressedSize = pSB->len; - uncompressedSize = pSB->len; - struct stream2_compression *pCompression = &pSB->compression; - if (pCompression->algorithm != NULL) { - uncompressedSize = pCompression->orig_size; - strcpy(encoding, pCompression->algorithm); - } - switch (s2DataType) { - case STREAM2_TYPED_ARRAY_UINT8: - dataType = NDUInt8; - break; - case STREAM2_TYPED_ARRAY_UINT16_LITTLE_ENDIAN: - dataType = NDUInt16; - break; - case STREAM2_TYPED_ARRAY_UINT32_LITTLE_ENDIAN: - dataType = NDUInt32; - break; - default: - ERR_ARGS("unknown dataType %d", s2DataType); - err = STREAM_ERROR; - goto error; + case STREAM2_MSG_IMAGE: { + struct stream2_image_data *pSID = &mImageMsg->data.ptr[thresh]; + struct stream2_multidim_array mda = pSID->data; + dims[0] = mda.dim[1]; + dims[1] = mda.dim[0]; + numDims = 2; + struct stream2_typed_array *pS2Array = &mda.array; + stream2_typed_array_tag s2DataType = (stream2_typed_array_tag)pS2Array->tag; + struct stream2_bytes *pSB = &pS2Array->data; + compressedSize = pSB->len; + uncompressedSize = pSB->len; + struct stream2_compression *pCompression = &pSB->compression; + if (pCompression->algorithm != NULL) { + uncompressedSize = pCompression->orig_size; + strcpy(encoding, pCompression->algorithm); + } + switch (s2DataType) { + case STREAM2_TYPED_ARRAY_UINT8: + dataType = NDUInt8; + break; + case STREAM2_TYPED_ARRAY_UINT16_LITTLE_ENDIAN: + dataType = NDUInt16; + break; + case STREAM2_TYPED_ARRAY_UINT32_LITTLE_ENDIAN: + dataType = NDUInt32; + break; + default: + ERR_ARGS("unknown dataType %d", s2DataType); + err = STREAM_ERROR; + goto error; + } + + if(!(pArray = pNDArrayPool->alloc(numDims, dims, dataType, 0, NULL))) + { + ERR("failed to allocate NDArray for frame"); + err = STREAM_ERROR; + goto error; + } + + // Get frame data + // If data is uncompressed we can copy directly into NDArray + if (pCompression->algorithm == NULL) + { + memcpy((char *)pArray->pData, pSB->ptr, uncompressedSize); + } + else + { + if (decompress) + { + uncompress(pSB->ptr, (char *)pArray->pData, encoding, compressedSize, uncompressedSize, dataType); } - - // On first threshold we create the NDArray - if (i == 0) { - if(!(pArray = pNDArrayPool->alloc(numDims, dims, dataType, 0, NULL))) + else + { + const unsigned char *pInput = pSB->ptr; + if (strcmp(encoding, "lz4") == 0) { - ERR("failed to allocate NDArray for frame"); - err = STREAM_ERROR; - goto error; + pArray->codec.name = "lz4"; } - } - // Get frame data - // If data is uncompressed we can copy directly into NDArray - if (pCompression->algorithm == NULL) - { - memcpy((char *)pArray->pData + (i*uncompressedSize), pSB->ptr, uncompressedSize); - } - else - { - if (decompress) + else if (strcmp(encoding, "bslz4") == 0) { - uncompress(pSB->ptr, (char *)pArray->pData + (i*uncompressedSize), encoding, - compressedSize, uncompressedSize, dataType); + pArray->codec.name = "bslz4"; + pInput += 12; + compressedSize -= 12; } - else - { - const unsigned char *pInput = pSB->ptr; - if (strcmp(encoding, "lz4") == 0) - { - pArray->codec.name = "lz4"; - } - else if (strcmp(encoding, "bslz4") == 0) - { - pArray->codec.name = "bslz4"; - pInput += 12; - compressedSize -= 12; - } - else { - ERR_ARGS("unknown encoding %s", encoding); - } - pArray->compressedSize = compressedSize; - memcpy(pArray->pData, pInput, compressedSize); - - if (extractTimeStamp) { - epicsTimeStamp ts = extractTimeStampFromMessage(mImageMsg); - pArray->epicsTS = ts; - pArray->timeStamp = ts.secPastEpoch + ts.nsec/1.e9; - } + else { + ERR_ARGS("unknown encoding %s", encoding); + } + pArray->compressedSize = compressedSize; + memcpy(pArray->pData, pInput, compressedSize); + + if (extractTimeStamp) { + epicsTimeStamp ts = extractTimeStampFromMessage(mImageMsg); + pArray->epicsTS = ts; + pArray->timeStamp = ts.secPastEpoch + ts.nsec/1.e9; } } - *pArrayOut = pArray; } + pArray->pAttributeList->add("ThresholdName", "Threshold name", NDAttrString, mThresholdEnergy[thresh].channel); + pArray->pAttributeList->add("ThresholdEnergy", "Threshold energy (eV)", NDAttrFloat64, (void *)&(mThresholdEnergy[thresh].energy)); + *pArrayOut = pArray; break; + } default: - ERR_ARGS("unknown unexpected message types %d", mImageMsg->type); - break; + ERR_ARGS("unexpected message type %d", mImageMsg->type); } error: - zmq_msg_close(&mMsg); + if (thresh == mNumThresholds-1) { + zmq_msg_close(&mMsg); + } return err; } diff --git a/eigerApp/src/streamApi.h b/eigerApp/src/streamApi.h index fea20da..a080d4e 100644 --- a/eigerApp/src/streamApi.h +++ b/eigerApp/src/streamApi.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,8 @@ class Stream2API uint64_t mImage_size_x; uint64_t mImage_size_y; uint64_t mNumber_of_images; + int mNumThresholds; + std::vector mThresholdEnergy; struct { std::string tsStr; epicsTimeStamp ts; @@ -62,8 +65,8 @@ class Stream2API Stream2API (const char *hostname); ~Stream2API (void); int getHeader (stream_header_t *header, int timeout = 0); - int waitFrame (int *end, int timeout = 0); - int getFrame (NDArray **pArray, NDArrayPool *pNDArrayPool, int decompress, bool extractTimeStamp); + int waitFrame (int *end, int *numThresholds, int timeout = 0); + int getFrame (NDArray **pArray, NDArrayPool *pNDArrayPool, int thresh, int decompress, bool extractTimeStamp); }; From 36de099a7e2495e81b7bf885a4c3b81fc9570158 Mon Sep 17 00:00:00 2001 From: Mark Rivers Date: Sat, 27 Dec 2025 11:55:39 -0600 Subject: [PATCH 2/3] Updated docs for using Stream2 with multiple thresholds --- docs/ADEiger/eiger.rst | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/ADEiger/eiger.rst b/docs/ADEiger/eiger.rst index 2a5fdac..e7a307f 100644 --- a/docs/ADEiger/eiger.rst +++ b/docs/ADEiger/eiger.rst @@ -234,11 +234,19 @@ NDArrays. Otherwise a third-party client can listen on that socket for data. The format of the packets is specified in the Eiger SIMPLON API documentation. -The NDArrays received by the Stream1 API are 2-dimensional, [NX, NY]. -If only a single threshold is enabled then the NDArrays received by the -Stream2 API are also 2-dimensional. -If more than one threshold is enabled then the NDArrays are 3-dimensional, -[NX, NY, NThreshholds]. +If StreamVersion=Stream2 and if more than one threshold is enabled then the driver +generates one NDArray for each threshold in successive order for the enabled thresholds. +The driver sends the NDArrays for all thresholds on asyn address 0. +It sends only the NDArrays for threshold N on address N (N=1 to number of enabled thresholds). +Plugins can thus use asyn address 0 to receive NDArrays for all thresholds, address 1 +to receive only the first enabled threshold, etc. + +Stream2 adds 2 new NDAttributes for each NDArray. These attributes identify which threshold that NDArray contains. + +- ThresholdName is an NDAttrString attribute containing the name of the threshold as reported by the Stream2 interface. + These are "threshold_1", "threshold_2", etc. +- ThresholdEnergy is an NDAttrFloat64 attribute containing the energy of the threshold as reported by the Stream2 interface + in units of eV. The data sent from the Eiger server is unsigned 32-bit, 16-bit, or 8-bit integers, depending on the exposure time. From 667705fceabdaf7fbb38391dee0415d8cb4c930b Mon Sep 17 00:00:00 2001 From: Mark Rivers Date: Sat, 27 Dec 2025 11:56:09 -0600 Subject: [PATCH 3/3] Updates for R3-6 --- RELEASE.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 6d1b279..9fd75ff 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,15 +9,22 @@ https://github.com/areaDetector/ADEiger/tags Release Notes ============= -R3-6 (June XXX, 2025) +R3-6 (January XXX, 2026) ---- * Added support for the Stream2 interface. Stream2 supports multiple thresholds. - Added new StreamVersion record to select the Stream or Stream2 interface. - If Stream2 is selected and more than one threshold is enabled then the - dimensions of NDArrays created when DataSource is Stream will be [NumX, NumY, NumThresholds]. - - ROI plugins can be used to select individual the thresholds to send to other plugins, - such as statistics, PVA for viewing, etc. -* Added support for Pilatus4 detectors. + driver creates one NDArray for each threshold. + - The driver sends the NDArrays for all thresholds on asyn address 0. It sends + only the NDArray for threshold N on address N (N=1 to number of enabled thresholds). + - Plugins can thus use asyn address 0 to receive NDArrays for all thresholds, address 1 + to receive only the first enabled threshold, etc. + - Stream2 adds 2 new NDAttributes for each NDArray. These attributes identify which threshold that NDArray contains. + - ThresholdName is an NDAttrString attribute containing the name of the threshold as reported by the Stream2 interface. + These are "threshold_1", "threshold_2", etc. + - ThresholdEnergy is an NDAttrFloat64 attribute containing the energy of the threshold as reported by the Stream2 interface + in units of eV. +* Added support for Pilatus4 detectors. Thanks to Tejus Guruswamy for this. * Added new FWHDF5Format record for the FileWriter interface. - This record allows selecting the "Legacy" format, or the "v2024.2" format. v2024.2 supports saving multiple thresholds. @@ -38,6 +45,9 @@ R3-6 (June XXX, 2025) For 16-bit data this would be a problem when there are over 32K counts per pixel. Since the maximum count rate is about 2e6 counts/s there should never be more than 20K counts in 0.01 seconds, and there should thus be no problem. +* Added new StreamAsTSSource record. If this is set to Yes, and if the data is coming from the Stream2 + interface, then the NDArray timeStamp and epicsTS fields are taken from the timestamp information + sent by the detector over the Stream2 interface. Thanks to Bruno Martins for this. * BEFORE RELEASE. Check the function of InternalEnable mode to see if it works now. The documentation said it was flaky in firmware 1.5.0, so it probably works now and the documentation should be fixed in several places.