perf: copy contiguous hyperslab and point reads run by run
A 256 x 256 hyperslab of a contiguous f32 dataset read at an eighth of h5py's speed: partial_read copied the bounding box out of the file, the extractor then walked it element by element (a recursive call and two bounds checks per element) into a second buffer, and read_f32_selection converted that into a third. Selections of contiguous data are now copied straight from the file, one memcpy per run of elements contiguous in the file (gather.rs: a block along the last dimension, touching blocks as one range, whole rows merged), with no zero-filled intermediate and no full copy for large selections. The typed selection readers copy into their Vec<T> directly when the dataset stores T natively (new data_read::read_selection_native and sealed NativeElement trait, which the read_as_* fast paths now share; read_as_u64 gains one) and convert as before otherwise. The general extractor used by the chunked paths runs on the same run walker, keeping its old handling of unvalidated selections. Checked against h5py (contiguous_read_interop.rs) for strided, blocked, adjacent-block and whole-row hyperslabs, points and empty selections of every 1-8-byte type in both byte orders, ranks 1-4. Also keeps the huge-page threshold constant out of no_std builds, where it was unused. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -3,11 +3,13 @@
|
||||
//!
|
||||
//! [`crate::data_read::read_raw_data_selection`] used to decode the *entire*
|
||||
//! dataset and then pick elements out of it, so reading a 64x64 window of a
|
||||
//! large dataset took about as long as reading all of it. Here the selection's
|
||||
//! bounding box is materialised instead — only the rows of a contiguous
|
||||
//! dataset, or only the chunks, that overlap it — and the existing extractor
|
||||
//! runs over that small buffer with the selection translated to the box's
|
||||
//! origin. Extraction semantics are therefore exactly the full-read ones.
|
||||
//! large dataset took about as long as reading all of it. A contiguous
|
||||
//! dataset's selection is now copied straight out of the file, one `memcpy`
|
||||
//! per contiguous run of selected elements (`crate::gather`). For chunked
|
||||
//! data the selection's bounding box is materialised — only the chunks that
|
||||
//! overlap it — and the extractor runs over that small buffer with the
|
||||
//! selection translated to the box's origin. Extraction semantics are
|
||||
//! therefore exactly the full-read ones.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::string as alloc_or_std;
|
||||
@@ -250,10 +252,33 @@ pub fn read_selection(
|
||||
if dims.is_empty() || elem_size == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let total = dataspace.checked_num_elements()?;
|
||||
// Contiguous data is addressable in place: copy the selection's runs
|
||||
// straight out of it, whatever fraction of the dataset it covers, with no
|
||||
// intermediate box (and no full copy for a large selection).
|
||||
if let (
|
||||
DataLayout::Contiguous {
|
||||
address: Some(address),
|
||||
..
|
||||
},
|
||||
Selection::Hyperslab { .. } | Selection::Points(_),
|
||||
) = (layout, selection)
|
||||
{
|
||||
validate(selection, dims)?;
|
||||
let base = usize::try_from(*address)
|
||||
.map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?;
|
||||
let data = file_data
|
||||
.get(base..)
|
||||
.and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?))
|
||||
.ok_or(FormatError::UnexpectedEof {
|
||||
expected: base,
|
||||
available: file_data.len(),
|
||||
})?;
|
||||
return crate::gather::gather::<u8>(data, dims, elem_size, selection).map(Some);
|
||||
}
|
||||
let Some((box_start, box_extent)) = bounding_box(selection, dims) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let total = dataspace.checked_num_elements()?;
|
||||
let box_elements = box_extent
|
||||
.iter()
|
||||
.try_fold(1u64, |acc, &e| acc.checked_mul(e))
|
||||
@@ -265,30 +290,6 @@ pub fn read_selection(
|
||||
let mut boxed = alloc_output(checked_byte_len(box_elements, elem_size)?)?;
|
||||
|
||||
match layout {
|
||||
DataLayout::Contiguous {
|
||||
address: Some(address),
|
||||
..
|
||||
} => {
|
||||
let base = usize::try_from(*address)
|
||||
.map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?;
|
||||
let data = file_data
|
||||
.get(base..)
|
||||
.and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?))
|
||||
.ok_or(FormatError::UnexpectedEof {
|
||||
expected: base,
|
||||
available: file_data.len(),
|
||||
})?;
|
||||
let origin = vec![0u64; dims.len()];
|
||||
copy_overlap(
|
||||
data,
|
||||
&origin,
|
||||
dims,
|
||||
&mut boxed,
|
||||
&box_start,
|
||||
&box_extent,
|
||||
elem_size,
|
||||
);
|
||||
}
|
||||
DataLayout::Chunked {
|
||||
btree_address: Some(_),
|
||||
..
|
||||
|
||||
Reference in New Issue
Block a user