From 74a815a845c18533bbbed9f7da9d7d6df3203c7c Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 11 Apr 2022 21:34:32 +0200 Subject: [PATCH 1/9] Add chunk iterator --- hdf5/src/hl.rs | 2 + hdf5/src/hl/chunks.rs | 89 ++++++++++++++++++++++++++++++++++++++++++ hdf5/src/hl/dataset.rs | 9 +++++ 3 files changed, 100 insertions(+) create mode 100644 hdf5/src/hl/chunks.rs diff --git a/hdf5/src/hl.rs b/hdf5/src/hl.rs index 5c5081777..f8fb42bc6 100644 --- a/hdf5/src/hl.rs +++ b/hdf5/src/hl.rs @@ -1,4 +1,6 @@ pub mod attribute; +#[cfg(feature = "1.13.0")] +pub mod chunks; pub mod container; pub mod dataset; pub mod dataspace; diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs new file mode 100644 index 000000000..ff6db8af7 --- /dev/null +++ b/hdf5/src/hl/chunks.rs @@ -0,0 +1,89 @@ +use crate::internal_prelude::*; +use hdf5_sys::h5d::H5Dchunk_iter; + +/// Borrowed version of [ChunkInfo](crate::hl::Dataset::ChunkInfo) +#[derive(Debug)] +pub struct ChunkInfoBorrowed<'a> { + pub offset: &'a [hsize_t], + pub filter_mask: u32, + pub addr: u64, + pub size: u64, +} + +struct RustCallback { + ndims: usize, + callback: F, +} + +extern "C" fn chunks_callback( + offset: *const hsize_t, filter_mask: u32, addr: haddr_t, nbytes: u32, op_data: *mut c_void, +) -> herr_t +where + F: FnMut(ChunkInfoBorrowed) -> i32, +{ + unsafe { + std::panic::catch_unwind(|| { + let data: *mut RustCallback = op_data.cast::>(); + let ndims = (*data).ndims; + let callback = &mut (*data).callback; + + let offset = std::slice::from_raw_parts(offset, ndims); + + let info = + ChunkInfoBorrowed { offset, filter_mask, addr: addr as u64, size: nbytes as u64 }; + + callback(info) + }) + .unwrap_or(-1) + } +} + +pub(crate) fn visit(ds: &Dataset, callback: F) -> Result<()> +where + F: for<'a> FnMut(ChunkInfoBorrowed<'a>) -> i32, +{ + let mut data = RustCallback:: { ndims: ds.ndim(), callback }; + + h5try!(H5Dchunk_iter( + ds.id(), + H5P_DEFAULT, + Some(chunks_callback::), + std::ptr::addr_of_mut!(data).cast() + )); + + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn chunks_visit() { + with_tmp_file(|f| { + let ds = f.new_dataset::().no_chunk().shape((4, 4)).create("nochunk").unwrap(); + assert_err_re!(visit(&ds, |_| 0), "not a chunked dataset"); + + let ds = f.new_dataset::().shape([3, 2]).chunk([1, 1]).create("chunk").unwrap(); + ds.write(&ndarray::arr2(&[[1, 2], [3, 4], [5, 6]])).unwrap(); + + let mut i = 0; + let f = |c: ChunkInfoBorrowed| { + match i { + 0 => assert_eq!(c.offset, [0, 0]), + 1 => assert_eq!(c.offset, [0, 1]), + 2 => assert_eq!(c.offset, [1, 0]), + 3 => assert_eq!(c.offset, [1, 1]), + 4 => assert_eq!(c.offset, [2, 0]), + 5 => assert_eq!(c.offset, [2, 1]), + _ => unreachable!(), + } + assert_eq!(c.size, std::mem::size_of::() as u64); + i += 1; + 0 + }; + visit(&ds, f).unwrap(); + assert_eq!(i, 6); + }) + } +} diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index d0cc1dc03..a4e82c990 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -171,6 +171,15 @@ impl Dataset { self.dcpl().map_or(None, |pl| pl.chunk()) } + /// Visit all chunks + #[cfg(feature = "1.13.0")] + pub fn chunks_visit(&self, callback: F) -> Result<()> + where + F: for<'a> FnMut(crate::hl::chunks::ChunkInfoBorrowed<'a>) -> i32, + { + crate::hl::chunks::visit(self, callback) + } + /// Returns the absolute byte offset of the dataset in the file if such offset is defined /// (which is not the case for datasets that are chunked, compact or not allocated yet). pub fn offset(&self) -> Option { From fca779ac1bfd1b90ad9cbdf79d9ccd0b3205f265 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 12 Apr 2022 11:47:48 +0200 Subject: [PATCH 2/9] Make ChunkInfoBorrowed visible --- hdf5/src/hl/chunks.rs | 4 ++-- hdf5/src/lib.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index ff6db8af7..e59af5cff 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -1,10 +1,10 @@ use crate::internal_prelude::*; use hdf5_sys::h5d::H5Dchunk_iter; -/// Borrowed version of [ChunkInfo](crate::hl::Dataset::ChunkInfo) +/// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo) #[derive(Debug)] pub struct ChunkInfoBorrowed<'a> { - pub offset: &'a [hsize_t], + pub offset: &'a [u64], pub filter_mask: u32, pub addr: u64, pub size: u64, diff --git a/hdf5/src/lib.rs b/hdf5/src/lib.rs index 587b0539c..7fc022c67 100644 --- a/hdf5/src/lib.rs +++ b/hdf5/src/lib.rs @@ -77,6 +77,8 @@ mod export { } pub mod dataset { + #[cfg(feature = "1.13.0")] + pub use crate::hl::chunks::ChunkInfoBorrowed; #[cfg(feature = "1.10.5")] pub use crate::hl::dataset::ChunkInfo; pub use crate::hl::dataset::{Chunk, Dataset, DatasetBuilder}; From a4e1605300a76b43106ded097c6e6f3f9e96e055 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 12 Apr 2022 11:59:04 +0200 Subject: [PATCH 3/9] Move ChunkInfo to chunks file --- hdf5/src/hl.rs | 1 - hdf5/src/hl/chunks.rs | 218 +++++++++++++++++++++++++++-------------- hdf5/src/hl/dataset.rs | 62 +----------- hdf5/src/lib.rs | 4 +- 4 files changed, 153 insertions(+), 132 deletions(-) diff --git a/hdf5/src/hl.rs b/hdf5/src/hl.rs index f8fb42bc6..ac813036f 100644 --- a/hdf5/src/hl.rs +++ b/hdf5/src/hl.rs @@ -1,5 +1,4 @@ pub mod attribute; -#[cfg(feature = "1.13.0")] pub mod chunks; pub mod container; pub mod dataset; diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index e59af5cff..d2d464f5f 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -1,89 +1,165 @@ use crate::internal_prelude::*; -use hdf5_sys::h5d::H5Dchunk_iter; -/// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo) -#[derive(Debug)] -pub struct ChunkInfoBorrowed<'a> { - pub offset: &'a [u64], +#[cfg(feature = "1.10.5")] +use hdf5_sys::h5d::{H5Dget_chunk_info, H5Dget_num_chunks}; + +#[cfg(feature = "1.10.5")] +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChunkInfo { + /// Array with a size equal to the dataset’s rank whose elements contain 0-based + /// logical positions of the chunk’s first element in each dimension. + pub offset: Vec, + /// Filter mask that indicates which filters were used with the chunk when written. + /// A zero value indicates that all enabled filters are applied on the chunk. + /// A filter is skipped if the bit corresponding to the filter’s position in + /// the pipeline (0 ≤ position < 32) is turned on. pub filter_mask: u32, + /// Chunk address in the file. pub addr: u64, + /// Chunk size in bytes. pub size: u64, } -struct RustCallback { - ndims: usize, - callback: F, -} +#[cfg(feature = "1.10.5")] +impl ChunkInfo { + pub(crate) fn new(ndim: usize) -> Self { + let offset = vec![0; ndim]; + Self { offset, filter_mask: 0, addr: 0, size: 0 } + } -extern "C" fn chunks_callback( - offset: *const hsize_t, filter_mask: u32, addr: haddr_t, nbytes: u32, op_data: *mut c_void, -) -> herr_t -where - F: FnMut(ChunkInfoBorrowed) -> i32, -{ - unsafe { - std::panic::catch_unwind(|| { - let data: *mut RustCallback = op_data.cast::>(); - let ndims = (*data).ndims; - let callback = &mut (*data).callback; - - let offset = std::slice::from_raw_parts(offset, ndims); - - let info = - ChunkInfoBorrowed { offset, filter_mask, addr: addr as u64, size: nbytes as u64 }; - - callback(info) - }) - .unwrap_or(-1) + /// Returns positional indices of disabled filters. + pub fn disabled_filters(&self) -> Vec { + (0..32).filter(|i| self.filter_mask & (1 << i) != 0).collect() } } -pub(crate) fn visit(ds: &Dataset, callback: F) -> Result<()> -where - F: for<'a> FnMut(ChunkInfoBorrowed<'a>) -> i32, -{ - let mut data = RustCallback:: { ndims: ds.ndim(), callback }; - - h5try!(H5Dchunk_iter( - ds.id(), - H5P_DEFAULT, - Some(chunks_callback::), - std::ptr::addr_of_mut!(data).cast() - )); +#[cfg(feature = "1.10.5")] +pub(crate) fn chunk_info(ds: &Dataset, index: usize) -> Option { + if !ds.is_chunked() { + return None; + } + h5lock!(ds.space().map_or(None, |s| { + let mut chunk_info = ChunkInfo::new(ds.ndim()); + h5check(H5Dget_chunk_info( + ds.id(), + s.id(), + index as _, + chunk_info.offset.as_mut_ptr(), + &mut chunk_info.filter_mask, + &mut chunk_info.addr, + &mut chunk_info.size, + )) + .map(|_| chunk_info) + .ok() + })) +} - Ok(()) +#[cfg(feature = "1.10.5")] +pub(crate) fn get_num_chunks(ds: &Dataset) -> Option { + if !ds.is_chunked() { + return None; + } + h5lock!(ds.space().map_or(None, |s| { + let mut n: hsize_t = 0; + h5check(H5Dget_num_chunks(ds.id(), s.id(), &mut n)).map(|_| n as _).ok() + })) } -#[cfg(test)] -mod test { +#[cfg(feature = "1.13.0")] +mod one_thirteen { use super::*; + use hdf5_sys::h5d::H5Dchunk_iter; + /// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo) + #[derive(Debug)] + pub struct ChunkInfoBorrowed<'a> { + pub offset: &'a [u64], + pub filter_mask: u32, + pub addr: u64, + pub size: u64, + } + + struct RustCallback { + ndims: usize, + callback: F, + } + + extern "C" fn chunks_callback( + offset: *const hsize_t, filter_mask: u32, addr: haddr_t, nbytes: u32, op_data: *mut c_void, + ) -> herr_t + where + F: FnMut(ChunkInfoBorrowed) -> i32, + { + unsafe { + std::panic::catch_unwind(|| { + let data: *mut RustCallback = op_data.cast::>(); + let ndims = (*data).ndims; + let callback = &mut (*data).callback; + + let offset = std::slice::from_raw_parts(offset, ndims); + + let info = ChunkInfoBorrowed { + offset, + filter_mask, + addr: addr as u64, + size: nbytes as u64, + }; + + callback(info) + }) + .unwrap_or(-1) + } + } + + pub(crate) fn visit(ds: &Dataset, callback: F) -> Result<()> + where + F: for<'a> FnMut(ChunkInfoBorrowed<'a>) -> i32, + { + let mut data = RustCallback:: { ndims: ds.ndim(), callback }; + + h5try!(H5Dchunk_iter( + ds.id(), + H5P_DEFAULT, + Some(chunks_callback::), + std::ptr::addr_of_mut!(data).cast() + )); + + Ok(()) + } + + #[cfg(test)] + mod test { + use super::*; + + #[test] + fn chunks_visit() { + with_tmp_file(|f| { + let ds = f.new_dataset::().no_chunk().shape((4, 4)).create("nochunk").unwrap(); + assert_err_re!(visit(&ds, |_| 0), "not a chunked dataset"); + + let ds = + f.new_dataset::().shape([3, 2]).chunk([1, 1]).create("chunk").unwrap(); + ds.write(&ndarray::arr2(&[[1, 2], [3, 4], [5, 6]])).unwrap(); - #[test] - fn chunks_visit() { - with_tmp_file(|f| { - let ds = f.new_dataset::().no_chunk().shape((4, 4)).create("nochunk").unwrap(); - assert_err_re!(visit(&ds, |_| 0), "not a chunked dataset"); - - let ds = f.new_dataset::().shape([3, 2]).chunk([1, 1]).create("chunk").unwrap(); - ds.write(&ndarray::arr2(&[[1, 2], [3, 4], [5, 6]])).unwrap(); - - let mut i = 0; - let f = |c: ChunkInfoBorrowed| { - match i { - 0 => assert_eq!(c.offset, [0, 0]), - 1 => assert_eq!(c.offset, [0, 1]), - 2 => assert_eq!(c.offset, [1, 0]), - 3 => assert_eq!(c.offset, [1, 1]), - 4 => assert_eq!(c.offset, [2, 0]), - 5 => assert_eq!(c.offset, [2, 1]), - _ => unreachable!(), - } - assert_eq!(c.size, std::mem::size_of::() as u64); - i += 1; - 0 - }; - visit(&ds, f).unwrap(); - assert_eq!(i, 6); - }) + let mut i = 0; + let f = |c: ChunkInfoBorrowed| { + match i { + 0 => assert_eq!(c.offset, [0, 0]), + 1 => assert_eq!(c.offset, [0, 1]), + 2 => assert_eq!(c.offset, [1, 0]), + 3 => assert_eq!(c.offset, [1, 1]), + 4 => assert_eq!(c.offset, [2, 0]), + 5 => assert_eq!(c.offset, [2, 1]), + _ => unreachable!(), + } + assert_eq!(c.size, std::mem::size_of::() as u64); + i += 1; + 0 + }; + visit(&ds, f).unwrap(); + assert_eq!(i, 6); + }) + } } } +#[cfg(feature = "1.13.0")] +pub use one_thirteen::*; diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index a4e82c990..1a3770fc3 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -8,8 +8,6 @@ use hdf5_sys::h5d::{ H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, H5Dset_extent, }; -#[cfg(feature = "1.10.5")] -use hdf5_sys::h5d::{H5Dget_chunk_info, H5Dget_num_chunks}; use hdf5_sys::h5l::H5Ldelete; use hdf5_sys::h5p::H5P_DEFAULT; use hdf5_sys::h5z::H5Z_filter_t; @@ -66,36 +64,6 @@ impl Deref for Dataset { } } -#[cfg(feature = "1.10.5")] -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ChunkInfo { - /// Array with a size equal to the dataset’s rank whose elements contain 0-based - /// logical positions of the chunk’s first element in each dimension. - pub offset: Vec, - /// Filter mask that indicates which filters were used with the chunk when written. - /// A zero value indicates that all enabled filters are applied on the chunk. - /// A filter is skipped if the bit corresponding to the filter’s position in - /// the pipeline (0 ≤ position < 32) is turned on. - pub filter_mask: u32, - /// Chunk address in the file. - pub addr: u64, - /// Chunk size in bytes. - pub size: u64, -} - -#[cfg(feature = "1.10.5")] -impl ChunkInfo { - pub(crate) fn new(ndim: usize) -> Self { - let offset = vec![0; ndim]; - Self { offset, filter_mask: 0, addr: 0, size: 0 } - } - - /// Returns positional indices of disabled filters. - pub fn disabled_filters(&self) -> Vec { - (0..32).filter(|i| self.filter_mask & (1 << i) != 0).collect() - } -} - impl Dataset { /// Returns a copy of the dataset access property list. pub fn access_plist(&self) -> Result { @@ -135,35 +103,13 @@ impl Dataset { #[cfg(feature = "1.10.5")] /// Returns the number of chunks if the dataset is chunked. pub fn num_chunks(&self) -> Option { - if !self.is_chunked() { - return None; - } - h5lock!(self.space().map_or(None, |s| { - let mut n: hsize_t = 0; - h5check(H5Dget_num_chunks(self.id(), s.id(), &mut n)).map(|_| n as _).ok() - })) + crate::hl::chunks::get_num_chunks(self) } #[cfg(feature = "1.10.5")] /// Retrieves the chunk information for the chunk specified by its index. - pub fn chunk_info(&self, index: usize) -> Option { - if !self.is_chunked() { - return None; - } - h5lock!(self.space().map_or(None, |s| { - let mut chunk_info = ChunkInfo::new(self.ndim()); - h5check(H5Dget_chunk_info( - self.id(), - s.id(), - index as _, - chunk_info.offset.as_mut_ptr(), - &mut chunk_info.filter_mask, - &mut chunk_info.addr, - &mut chunk_info.size, - )) - .map(|_| chunk_info) - .ok() - })) + pub fn chunk_info(&self, index: usize) -> Option { + crate::hl::chunks::chunk_info(self, index) } /// Returns the chunk shape if the dataset is chunked. @@ -175,7 +121,7 @@ impl Dataset { #[cfg(feature = "1.13.0")] pub fn chunks_visit(&self, callback: F) -> Result<()> where - F: for<'a> FnMut(crate::hl::chunks::ChunkInfoBorrowed<'a>) -> i32, + F: for<'a> FnMut(crate::dataset::ChunkInfoBorrowed<'a>) -> i32, { crate::hl::chunks::visit(self, callback) } diff --git a/hdf5/src/lib.rs b/hdf5/src/lib.rs index 7fc022c67..59c7bff91 100644 --- a/hdf5/src/lib.rs +++ b/hdf5/src/lib.rs @@ -77,10 +77,10 @@ mod export { } pub mod dataset { + #[cfg(feature = "1.10.5")] + pub use crate::hl::chunks::ChunkInfo; #[cfg(feature = "1.13.0")] pub use crate::hl::chunks::ChunkInfoBorrowed; - #[cfg(feature = "1.10.5")] - pub use crate::hl::dataset::ChunkInfo; pub use crate::hl::dataset::{Chunk, Dataset, DatasetBuilder}; pub use crate::hl::plist::dataset_access::*; pub use crate::hl::plist::dataset_create::*; From 76172a5d2f25fa0c8551395087c225cce3c73c75 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 12 Apr 2022 12:19:12 +0200 Subject: [PATCH 4/9] Add convenience methods on ChunkInfoBorrowed --- hdf5/src/hl/chunks.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index d2d464f5f..460d417dc 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -69,8 +69,9 @@ pub(crate) fn get_num_chunks(ds: &Dataset) -> Option { mod one_thirteen { use super::*; use hdf5_sys::h5d::H5Dchunk_iter; + /// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo) - #[derive(Debug)] + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ChunkInfoBorrowed<'a> { pub offset: &'a [u64], pub filter_mask: u32, @@ -78,6 +79,24 @@ mod one_thirteen { pub size: u64, } + impl<'a> ChunkInfoBorrowed<'a> { + /// Returns positional indices of disabled filters. + pub fn disabled_filters(&self) -> Vec { + (0..32).filter(|i| self.filter_mask & (1 << i) != 0).collect() + } + } + + impl<'a> From> for ChunkInfo { + fn from(val: ChunkInfoBorrowed<'a>) -> Self { + Self { + offset: val.offset.to_owned(), + filter_mask: val.filter_mask, + addr: val.addr, + size: val.size, + } + } + } + struct RustCallback { ndims: usize, callback: F, From 307b3094b34669a218a4b18917b67e749e4aa5c9 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 17 Jan 2023 12:05:05 +0100 Subject: [PATCH 5/9] Fix 1.14 function diff --- hdf5/src/hl/chunks.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index 460d417dc..4be7940b2 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -65,7 +65,7 @@ pub(crate) fn get_num_chunks(ds: &Dataset) -> Option { })) } -#[cfg(feature = "1.13.0")] +#[cfg(feature = "1.14.0")] mod one_thirteen { use super::*; use hdf5_sys::h5d::H5Dchunk_iter; @@ -103,7 +103,8 @@ mod one_thirteen { } extern "C" fn chunks_callback( - offset: *const hsize_t, filter_mask: u32, addr: haddr_t, nbytes: u32, op_data: *mut c_void, + offset: *const hsize_t, filter_mask: c_uint, addr: haddr_t, size: hsize_t, + op_data: *mut c_void, ) -> herr_t where F: FnMut(ChunkInfoBorrowed) -> i32, @@ -116,12 +117,8 @@ mod one_thirteen { let offset = std::slice::from_raw_parts(offset, ndims); - let info = ChunkInfoBorrowed { - offset, - filter_mask, - addr: addr as u64, - size: nbytes as u64, - }; + let info = + ChunkInfoBorrowed { offset, filter_mask, addr: addr as u64, size: size as u64 }; callback(info) }) From 41a64ed84c2069b2b912fb653571a1477c3f4fb6 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 9 Jun 2023 21:19:03 +0200 Subject: [PATCH 6/9] Apply suggestions from code review Co-authored-by: Ivan Smirnov Signed-off-by: Magnus Ulimoen --- hdf5/src/hl/chunks.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index 4be7940b2..689ae187d 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -10,6 +10,7 @@ pub struct ChunkInfo { /// logical positions of the chunk’s first element in each dimension. pub offset: Vec, /// Filter mask that indicates which filters were used with the chunk when written. + /// /// A zero value indicates that all enabled filters are applied on the chunk. /// A filter is skipped if the bit corresponding to the filter’s position in /// the pipeline (0 ≤ position < 32) is turned on. @@ -72,7 +73,7 @@ mod one_thirteen { /// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo) #[derive(Clone, Debug, PartialEq, Eq)] - pub struct ChunkInfoBorrowed<'a> { + pub struct ChunkInfoRef<'a> { pub offset: &'a [u64], pub filter_mask: u32, pub addr: u64, @@ -97,9 +98,10 @@ mod one_thirteen { } } + #[repr(C)] struct RustCallback { - ndims: usize, - callback: F, + pub ndims: hsize_t, + pub callback: F, } extern "C" fn chunks_callback( From 5bc25e58f5c30bd6e5df6ea7f3d1d3d2b835a2b3 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 9 Jun 2023 21:25:08 +0200 Subject: [PATCH 7/9] Fixup types and ChunkInfoRef --- hdf5/src/hl/chunks.rs | 37 ++++++++++++++++++------------------- hdf5/src/hl/dataset.rs | 2 +- hdf5/src/lib.rs | 4 ++-- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index 689ae187d..7cc0921e3 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -8,7 +8,7 @@ use hdf5_sys::h5d::{H5Dget_chunk_info, H5Dget_num_chunks}; pub struct ChunkInfo { /// Array with a size equal to the dataset’s rank whose elements contain 0-based /// logical positions of the chunk’s first element in each dimension. - pub offset: Vec, + pub offset: Vec, /// Filter mask that indicates which filters were used with the chunk when written. /// /// A zero value indicates that all enabled filters are applied on the chunk. @@ -16,9 +16,9 @@ pub struct ChunkInfo { /// the pipeline (0 ≤ position < 32) is turned on. pub filter_mask: u32, /// Chunk address in the file. - pub addr: u64, + pub addr: haddr_t, /// Chunk size in bytes. - pub size: u64, + pub size: hsize_t, } #[cfg(feature = "1.10.5")] @@ -67,28 +67,28 @@ pub(crate) fn get_num_chunks(ds: &Dataset) -> Option { } #[cfg(feature = "1.14.0")] -mod one_thirteen { +mod v1_14_0 { use super::*; use hdf5_sys::h5d::H5Dchunk_iter; /// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo) #[derive(Clone, Debug, PartialEq, Eq)] pub struct ChunkInfoRef<'a> { - pub offset: &'a [u64], + pub offset: &'a [hsize_t], pub filter_mask: u32, - pub addr: u64, - pub size: u64, + pub addr: haddr_t, + pub size: hsize_t, } - impl<'a> ChunkInfoBorrowed<'a> { + impl<'a> ChunkInfoRef<'a> { /// Returns positional indices of disabled filters. pub fn disabled_filters(&self) -> Vec { (0..32).filter(|i| self.filter_mask & (1 << i) != 0).collect() } } - impl<'a> From> for ChunkInfo { - fn from(val: ChunkInfoBorrowed<'a>) -> Self { + impl<'a> From> for ChunkInfo { + fn from(val: ChunkInfoRef<'a>) -> Self { Self { offset: val.offset.to_owned(), filter_mask: val.filter_mask, @@ -109,7 +109,7 @@ mod one_thirteen { op_data: *mut c_void, ) -> herr_t where - F: FnMut(ChunkInfoBorrowed) -> i32, + F: FnMut(ChunkInfoRef) -> i32, { unsafe { std::panic::catch_unwind(|| { @@ -117,10 +117,9 @@ mod one_thirteen { let ndims = (*data).ndims; let callback = &mut (*data).callback; - let offset = std::slice::from_raw_parts(offset, ndims); + let offset = std::slice::from_raw_parts(offset, ndims as usize); - let info = - ChunkInfoBorrowed { offset, filter_mask, addr: addr as u64, size: size as u64 }; + let info = ChunkInfoRef { offset, filter_mask, addr, size }; callback(info) }) @@ -130,9 +129,9 @@ mod one_thirteen { pub(crate) fn visit(ds: &Dataset, callback: F) -> Result<()> where - F: for<'a> FnMut(ChunkInfoBorrowed<'a>) -> i32, + F: for<'a> FnMut(ChunkInfoRef<'a>) -> i32, { - let mut data = RustCallback:: { ndims: ds.ndim(), callback }; + let mut data = RustCallback:: { ndims: ds.ndim() as _, callback }; h5try!(H5Dchunk_iter( ds.id(), @@ -159,7 +158,7 @@ mod one_thirteen { ds.write(&ndarray::arr2(&[[1, 2], [3, 4], [5, 6]])).unwrap(); let mut i = 0; - let f = |c: ChunkInfoBorrowed| { + let f = |c: ChunkInfoRef| { match i { 0 => assert_eq!(c.offset, [0, 0]), 1 => assert_eq!(c.offset, [0, 1]), @@ -179,5 +178,5 @@ mod one_thirteen { } } } -#[cfg(feature = "1.13.0")] -pub use one_thirteen::*; +#[cfg(feature = "1.14.0")] +pub use v1_14_0::*; diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 1a3770fc3..feb0633c2 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -121,7 +121,7 @@ impl Dataset { #[cfg(feature = "1.13.0")] pub fn chunks_visit(&self, callback: F) -> Result<()> where - F: for<'a> FnMut(crate::dataset::ChunkInfoBorrowed<'a>) -> i32, + F: for<'a> FnMut(crate::dataset::ChunkInfoRef<'a>) -> i32, { crate::hl::chunks::visit(self, callback) } diff --git a/hdf5/src/lib.rs b/hdf5/src/lib.rs index 59c7bff91..5f6c34116 100644 --- a/hdf5/src/lib.rs +++ b/hdf5/src/lib.rs @@ -79,8 +79,8 @@ mod export { pub mod dataset { #[cfg(feature = "1.10.5")] pub use crate::hl::chunks::ChunkInfo; - #[cfg(feature = "1.13.0")] - pub use crate::hl::chunks::ChunkInfoBorrowed; + #[cfg(feature = "1.14.0")] + pub use crate::hl::chunks::ChunkInfoRef; pub use crate::hl::dataset::{Chunk, Dataset, DatasetBuilder}; pub use crate::hl::plist::dataset_access::*; pub use crate::hl::plist::dataset_create::*; From 6b671265761182d64d448696a785d36bcc2c6cf8 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 9 Jun 2023 23:16:15 +0200 Subject: [PATCH 8/9] Document ChunkInfo --- hdf5/src/hl/chunks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hdf5/src/hl/chunks.rs b/hdf5/src/hl/chunks.rs index 7cc0921e3..1de68115b 100644 --- a/hdf5/src/hl/chunks.rs +++ b/hdf5/src/hl/chunks.rs @@ -5,6 +5,7 @@ use hdf5_sys::h5d::{H5Dget_chunk_info, H5Dget_num_chunks}; #[cfg(feature = "1.10.5")] #[derive(Clone, Debug, PartialEq, Eq)] +/// Information on a chunk in a Dataset pub struct ChunkInfo { /// Array with a size equal to the dataset’s rank whose elements contain 0-based /// logical positions of the chunk’s first element in each dimension. From e3ffa48f04ad2739d80e5f443fb40edebba9c738 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 9 Jun 2023 23:50:03 +0200 Subject: [PATCH 9/9] Add to changelog --- CHANGELOG.md | 2 ++ hdf5/src/hl/dataset.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a94195c3..d1d3db2d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,13 @@ - Support field renaming via `#[hdf5(rename = "new_name")]` helper attribute. - Add a `ByteReader` which implements `std::io::{Read, Seek}` for 1D `u8` datasets. Usage via `Dataset::as_byte_reader()`. +- Add `chunk_visit` to visit all chunks in a dataset. ### Changed - The `H5Type` derive macro now uses `proc-macro-error` to emit error messages. - MSRV is now `1.64.0` and Rust edition has now been bumped to 2021. +- Types in ChunkInfo has been changed to match HDF5 ### Fixed diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index feb0633c2..f8e9ee64e 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -118,7 +118,7 @@ impl Dataset { } /// Visit all chunks - #[cfg(feature = "1.13.0")] + #[cfg(feature = "1.14.0")] pub fn chunks_visit(&self, callback: F) -> Result<()> where F: for<'a> FnMut(crate::dataset::ChunkInfoRef<'a>) -> i32,