From 9e476ab20ad051db9b51cd1002b98154e805c715 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Mon, 4 Aug 2025 14:35:18 -0400 Subject: [PATCH 1/2] ENH: include additional metadata for SRX --- pyxrf/model/fileio.py | 8 +++- pyxrf/model/load_data_from_db.py | 75 ++++++++++++++++++++++++++++++++ pyxrf/model/scan_metadata.py | 10 +++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/pyxrf/model/fileio.py b/pyxrf/model/fileio.py index 38875da4..7f50be5f 100644 --- a/pyxrf/model/fileio.py +++ b/pyxrf/model/fileio.py @@ -165,7 +165,6 @@ def _get_pyxrf_version_str(self): """ # Determine the current version of PyXRF - global pyxrf_version pyxrf_version_str = pyxrf_version if pyxrf_version_str[0].lower() != "v": pyxrf_version_str = f"v{pyxrf_version_str}" @@ -1511,7 +1510,12 @@ def read_hdf_APS( # Convert ndarrays to lists (they were lists before they were saved) if isinstance(value, np.ndarray): value = list(value) - mdata[key] = value + if "|" in key: + k1, k2 = key.split("|", 1) + mdata.setdefault(k1, {}) + mdata[k1][k2] = value + else: + mdata[key] = value data = f["xrfmap"] fname = file_name.split(".")[0] diff --git a/pyxrf/model/load_data_from_db.py b/pyxrf/model/load_data_from_db.py index 8c5ed752..fc995b53 100644 --- a/pyxrf/model/load_data_from_db.py +++ b/pyxrf/model/load_data_from_db.py @@ -669,6 +669,27 @@ def _get_metadata_value_from_descriptor_document(hdr, *, data_key, stream_name=" return value +def _get_metadata_value_from_descriptor_document_by_pattern(hdr, *, pattern, stream_name="baseline"): + """ + Returns all variables in the the first occurrence of each variable with the name containing + the patter in specified document stream. Returns empty dictionary if the variable is not found. + """ + value_dict = {} + docs = hdr.documents(stream_name=stream_name) + for name, doc in docs: + if (name != "event") or ("descriptor" not in doc): + continue + try: + values = {k: v for k, v in doc["data"].items if re.search(pattern, k)} + for k, v in values.items(): + if k not in value_dict: + value_dict[k] = v + except Exception: + pass + + return value_dict + + def _get_metadata_all_from_descriptor_document(hdr, *, data_key, stream_name="baseline"): """ Returns the list of the recorded values of variables with the name ``data_key`` in @@ -703,6 +724,21 @@ def _get_metadata_value_from_descriptor_document_tiled(hdr, *, data_key, stream_ return value +def _get_metadata_value_from_descriptor_document_by_pattern_tiled(hdr, *, pattern, stream_name="baseline"): + """ + Returns all variables in the the first occurrence of each variable with the name containing + the patter in specified document stream. Returns empty dictionary if the variable is not found. + """ + value_dict = {} + docs = hdr[stream_name]["data"] + + for k, v in docs.items(): + if re.search(pattern, k): + value_dict[k] = v.compute()[0] + + return value_dict + + def _get_metadata_all_from_descriptor_document_tiled(hdr, *, data_key, stream_name="baseline"): """ Returns the list of the recorded values of variables with the name ``data_key`` in @@ -2280,6 +2316,24 @@ def map_data2D_srx_new_tiled( if v is not None: mdata["instrument_beam_current"] = v + v = _get_metadata_all_from_descriptor_document_tiled( + hdr, data_key="nano_det_sample2detector", stream_name="baseline" + ) + if v is not None: + mdata["instrument_sample_to_detector"] = v + + v = _get_metadata_value_from_descriptor_document_by_pattern_tiled( + hdr, pattern="attenuators", stream_name="baseline" + ) + if v: + mdata["instrument_attenuators_config"] = v + + v = _get_metadata_value_from_descriptor_document_by_pattern_tiled( + hdr, pattern="preamp", stream_name="baseline" + ) + if v: + mdata["instrument_preamps_config"] = v + for ax in ["X", "Y", "Z"]: v = _get_metadata_all_from_descriptor_document_tiled( hdr, data_key=f"nanoKB_interferometer_pos{ax}", stream_name="baseline" @@ -3982,6 +4036,27 @@ def incorrect_type_msg(channel, data_type): if "file_software" not in metadata_prepared: metadata_prepared.update(metadata_software_version) + metadata_prepared2 = metadata_prepared + metadata_prepared = {} + for k, v in metadata_prepared2.items(): + if isinstance(v, dict): + for k2, v2 in v.items(): + metadata_prepared[k + "|" + k2] = v2 + else: + metadata_prepared[k] = v + + metadata_prepared2 = metadata_prepared + metadata_prepared = {} + for k, v in metadata_prepared2.items(): + if isinstance(v, np.int64): + metadata_prepared[k] = int(v) + elif isinstance(v, np.float64): + metadata_prepared[k] = float(v) + elif isinstance(v, np.str_): + metadata_prepared[k] = str(v) + else: + metadata_prepared[k] = v + if metadata_prepared: # We assume, that metadata does not contain repeated keys. Otherwise the # entry with the last occurrence of the key will override the previous ones. diff --git a/pyxrf/model/scan_metadata.py b/pyxrf/model/scan_metadata.py index a7507594..46b996a5 100644 --- a/pyxrf/model/scan_metadata.py +++ b/pyxrf/model/scan_metadata.py @@ -1,3 +1,4 @@ +import pprint import re import textwrap @@ -64,6 +65,9 @@ def values(self): def items(self): return self._values.items() + def setdefault(self, key, default=None): + return self._values.setdefault(key, default) + def update(self, source_dict): self._values.update(source_dict) @@ -131,6 +135,9 @@ def cap(s): # Now remove spaces at the beginning of the line, since the key will be # printed instead of the spaces val = val.lstrip() + elif isinstance(v, dict): + val = "\n" + pprint.pformat(v) + val = val.replace("\n", "\n ") else: val = v @@ -221,6 +228,9 @@ def _gen_default_descriptions(self): "instrument_mono_incident_energy": "incident energy", "instrument_beam_current": "ring current, mA", "instrument_detectors": "detectors", + "instrument_sample_to_detector": "sample-to-detector distance, mm", + "instrument_attenuators_config": "attenuators configuration", + "instrument_preamps_config": "preamp configuration", "sample_name": "sample name", "experiment_plan_name": "plan name", "experiment_plan_type": "plan type", From 3518b4751a4322519dea354e3fa8b7db5fda0e14 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Tue, 5 Aug 2025 10:14:08 -0400 Subject: [PATCH 2/2] TST: fixed unit test --- pyxrf/core/tests/test_map_processing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyxrf/core/tests/test_map_processing.py b/pyxrf/core/tests/test_map_processing.py index aefe4f87..3dbe3971 100644 --- a/pyxrf/core/tests/test_map_processing.py +++ b/pyxrf/core/tests/test_map_processing.py @@ -47,9 +47,10 @@ def test_dask_client_create(tmpdir): # Set the number of workers to some strange number (11 is unusual) # (pass another kwarg in addition to default) - client = dask_client_create(n_workers=11) + n_workers_requested = 3 + client = dask_client_create(n_workers=n_workers_requested) n_workers = len(client.scheduler_info()["workers"]) - assert n_workers == 11, "The number of workers was set incorrectly" + assert n_workers == n_workers_requested, "The number of workers was set incorrectly" client.close() assert not os.path.exists(dask_worker_space_path), "Temporary directory was created in the current directory"