quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

fix(format): selections of v4 implicit-index chunked data no longer panic

Browse Source 
read_raw_data_selection's chunked fallback (taken when partial_read
declines, e.g. a bounding box over half the dataset) handed the layout's
chunk dimensions, element-size dimension included, to
generate_implicit_chunks, which indexed past the dataset rank. It then
decoded the whole dataset regardless, so the enumeration is gone: the
arm decodes and extracts for every chunk index.

The new test reads small and large hyperslabs of all five v4 indexes
written by h5py and compares with h5py's values; it panicked before.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:49:54 -05:00
co-authored by Claude Opus 5.5
parent c3850a0b66
commit 3bcd443e63
4 changed files with 260 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_read.rs
+7 -75
View File
@@ -356,85 +356,17 @@ pub fn read_raw_data_selection(
}
DataLayout::Chunked {
chunk_dimensions,
btree_address,
version,
chunk_index_type,
..
} => {
// `partial_read` declined (a bounding box covering most of the
// dataset, or a selection it doesn't box), so decode every chunk
// and pick the selection out, whatever the chunk index. This arm
// used to enumerate the chunks first — passing the layout's
// chunk dimensions, element-size dimension included, to the
// implicit-index generator, which then indexed past the rank and
// panicked — only to decode the full dataset anyway.
crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?;
// For chunked data, only read chunks that intersect the selection
let chunk_dims: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let rank = dims.len();
// Collect chunk info from B-tree
let chunks = if *version == 4 {
match chunk_index_type {
Some(2) => {
// Implicit index
crate::chunked_read::generate_implicit_chunks(
btree_address.unwrap_or(0),
dims,
chunk_dimensions,
elem_size as u32,
)
}
_ => {
if let Some(_addr) = btree_address {
// Use extensible array or fixed array
// Fall back to full read for complex v4 index types
let full_data = read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
)?;
return extract_selection_from_buffer(
&full_data, dims, elem_size, selection,
);
} else {
return Ok(Vec::new());
}
}
}
} else {
// v3: B-tree v1
if let Some(addr) = btree_address {
crate::chunked_read::collect_chunk_info_checked(
file_data,
*addr,
chunk_dimensions,
offset_size,
length_size,
)?
} else {
return Ok(Vec::new());
}
};
// Filter chunks to only those that intersect the selection
let intersecting: Vec<_> = chunks
.iter()
.filter(|ci| {
let offsets: Vec<u64> = ci.offsets.iter().take(rank).copied().collect();
selection.intersects_chunk(&offsets, &chunk_dims[..rank])
})
.collect();
if intersecting.is_empty() {
return Ok(Vec::new());
}
// Decompress only the intersecting chunks
let _chunk_total_bytes: usize =
chunk_dims.iter().map(|&d| d as usize).product::<usize>() * elem_size;
let _element_size_u32 = elem_size as u32;
// First, assemble only the intersecting chunks into a partial buffer,
// then extract the selection. For simplicity, we assemble into a full
// dataset buffer and extract (same as contiguous path).
let full_data = read_raw_data_full(
file_data,
layout,