perf(py): read an index list one group of chunks at a time
Each run of consecutive indices was its own uncached hyperslab read, so a list over a compressed chunked dataset decoded the same chunk once per run (d[range(0, 200000, 40)] over 20 gzip chunks: 8 s, h5py 0.014 s). Plan::reads now groups the indices — a group ends only where a whole chunk holds no selected index, or, unchunked, at a gap over 64 KiB — and the selected rows are gathered from each group's block in Rust. Now 3.8 ms (h5py 4.1 ms, release, tank). The new test (1-D, 2-D and contiguous, compared with h5py, 2 s bound) took 5.8 s before. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -135,6 +135,33 @@ pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResul
|
||||
})
|
||||
}
|
||||
|
||||
/// The chunk shape of a chunked dataset (one entry per dataset dimension),
|
||||
/// or `None` for other layouts or a layout message that does not parse.
|
||||
pub(crate) fn chunk_shape(
|
||||
file: &clawhdf5_rs::File,
|
||||
hdr: &ObjectHeader,
|
||||
rank: usize,
|
||||
) -> Option<Vec<u64>> {
|
||||
let sb = file.superblock();
|
||||
let msg = hdr
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::DataLayout)?;
|
||||
match clawhdf5_format::data_layout::DataLayout::parse(&msg.data, sb.offset_size, sb.length_size)
|
||||
.ok()?
|
||||
{
|
||||
clawhdf5_format::data_layout::DataLayout::Chunked {
|
||||
chunk_dimensions, ..
|
||||
} if chunk_dimensions.len() >= rank => Some(
|
||||
chunk_dimensions[..rank]
|
||||
.iter()
|
||||
.map(|&d| u64::from(d))
|
||||
.collect(),
|
||||
),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_null(space: &Dataspace) -> bool {
|
||||
space.space_type == DataspaceType::Null
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user