quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p2-python-bindings' into feat/p2-perf-coverage

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	README.md
This commit is contained in:
osobh
2026-09-26 09:10:57 -05:00
parent 956e55c76a 8dcce084ca
commit 73a01f1256
26 changed files with 3541 additions and 568 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_read.rs
+12 -78
View File
@@ -288,9 +288,11 @@ pub fn read_raw_data_indexed(
/// Read raw bytes for only the selected elements of a dataset.
///
/// For chunked layouts, only chunks that intersect the selection are read
/// and decompressed. For compact/contiguous layouts, the full data is read
/// and then the selection is extracted.
/// When the selection's bounding box covers at most half the dataset, only
/// that box is materialised — the overlapping rows of a contiguous dataset,
/// the overlapping chunks of a chunked one, whatever its chunk index (see
/// [`crate::partial_read`]). Otherwise, and for compact and virtual
/// layouts, the whole dataset is decoded and the selection extracted.
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_selection(
file_data: &[u8],
@@ -358,85 +360,17 @@ pub fn read_raw_data_selection(
}
DataLayout::Chunked {
chunk_dimensions,
btree_address,
version,
chunk_index_type,
..
} => {
// `partial_read` declined (a bounding box covering most of the
// dataset, or a selection it doesn't box), so decode every chunk
// and pick the selection out, whatever the chunk index. This arm
// used to enumerate the chunks first — passing the layout's
// chunk dimensions, element-size dimension included, to the
// implicit-index generator, which then indexed past the rank and
// panicked — only to decode the full dataset anyway.
crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?;
// For chunked data, only read chunks that intersect the selection
let chunk_dims: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let rank = dims.len();
// Collect chunk info from B-tree
let chunks = if *version == 4 {
match chunk_index_type {
Some(2) => {
// Implicit index
crate::chunked_read::generate_implicit_chunks(
btree_address.unwrap_or(0),
dims,
chunk_dimensions,
elem_size as u32,
)
}
_ => {
if let Some(_addr) = btree_address {
// Use extensible array or fixed array
// Fall back to full read for complex v4 index types
let full_data = read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
)?;
return extract_selection_from_buffer(
&full_data, dims, elem_size, selection,
);
} else {
return Ok(Vec::new());
}
}
}
} else {
// v3: B-tree v1
if let Some(addr) = btree_address {
crate::chunked_read::collect_chunk_info_checked(
file_data,
*addr,
chunk_dimensions,
offset_size,
length_size,
)?
} else {
return Ok(Vec::new());
}
};
// Filter chunks to only those that intersect the selection
let intersecting: Vec<_> = chunks
.iter()
.filter(|ci| {
let offsets: Vec<u64> = ci.offsets.iter().take(rank).copied().collect();
selection.intersects_chunk(&offsets, &chunk_dims[..rank])
})
.collect();
if intersecting.is_empty() {
return Ok(Vec::new());
}
// Decompress only the intersecting chunks
let _chunk_total_bytes: usize =
chunk_dims.iter().map(|&d| d as usize).product::<usize>() * elem_size;
let _element_size_u32 = elem_size as u32;
// First, assemble only the intersecting chunks into a partial buffer,
// then extract the selection. For simplicity, we assemble into a full
// dataset buffer and extract (same as contiguous path).
let full_data = read_raw_data_full(
file_data,
layout,