quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p2b-remaining-conformance' into feat/p2b-scale

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/chunked_read.rs
This commit is contained in:
osobh
2026-09-26 11:57:35 -05:00
parent 4b02e7d068 55e0e7e9cf
commit 9a73299594
36 changed files with 3297 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_read.rs
+299 -95
View File
@@ -588,26 +588,41 @@ pub fn collect_chunk_info(
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(
let _ = length_size;
let mut chunks = Vec::new();
parse_chunk_node(
file_data,
btree_address,
ndims,
None,
offset_size,
length_size,
0,
)
&mut chunks,
)?;
Ok(chunks)
}
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
/// layout message's list, element size last), checking every key of the
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
/// must be a multiple of its chunk dimension. That includes the keys that
/// only bound a node (internal-node keys and each node's final key), which
/// is where a corrupt chunk dimension shows when the chunks themselves all
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
/// chunks were read at the wrong place, or the dataset read as fill values.
/// The chunks of a v1 B-tree chunk index as libhdf5 reads them, for a
/// layout with these `chunk_dimensions` (the layout message's list, element
/// size last).
///
/// Every key of the B-tree is checked as libhdf5 checks it
/// (`H5D__btree_decode_key`): each coordinate offset must be a multiple of
/// its chunk dimension. That includes the keys that only bound a node
/// (internal-node keys and each node's final key), which is where a
/// corrupt chunk dimension shows when the chunks themselves all start at
/// offset 0 in that dimension (`cve-2018-11205`). A key that fails ("bad
/// coordinate offset") means a corrupt index or chunk dimension.
///
/// libhdf5 does not read a chunk by walking the tree: it looks each chunk
/// up (`H5B_find` with `H5D__btree_cmp3` and `H5D__btree_found`), comparing
/// the element-size coordinate too, which it asks for as 0. So a chunk is
/// returned only where that lookup finds it: a key whose element-size
/// coordinate is not 0 is found in a 1-D dataset (the comparison looks at
/// that coordinate only against the next key) but not in a dataset of rank
/// 2 or more (`cve-2025-44905` `/Shuffle_float_data_le`), which then reads
/// as fill values, and a tree whose keys are out of order loses the chunks
/// libhdf5's binary search misses.
pub fn collect_chunk_info_checked(
file_data: &[u8],
btree_address: u64,
@@ -615,15 +630,141 @@ pub fn collect_chunk_info_checked(
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(
let _ = length_size;
let ndims = chunk_dimensions.len();
if ndims == 0 {
return Err(FormatError::ChunkedReadError(
"chunk layout has no dimensions".into(),
));
}
let mut stored = Vec::new();
let mut root = parse_chunk_node(
file_data,
btree_address,
chunk_dimensions.len(),
ndims,
Some(chunk_dimensions),
offset_size,
length_size,
0,
)
&mut stored,
)?;
// Keys were checked to be multiples of non-zero dimensions.
root.scale_keys(chunk_dimensions);
// Look every stored chunk's position up. A lookup finds a chunk only at
// that chunk's own position, so each is returned at most once.
let mut returned = vec![false; stored.len()];
let mut wanted = vec![0u64; ndims];
for chunk in &stored {
for (w, (&o, &d)) in wanted
.iter_mut()
.zip(chunk.offsets.iter().zip(chunk_dimensions))
{
*w = o / u64::from(d);
}
wanted[ndims - 1] = 0;
if let Some(i) = root.find(&wanted) {
returned[i] = true;
}
}
Ok(stored
.into_iter()
.zip(returned)
.filter_map(|(c, r)| r.then_some(c))
.collect())
}
/// A node of a v1 B-tree chunk index: its `n + 1` keys (`ndims`
/// coordinates each, flattened; byte offsets as stored, or scaled by the
/// chunk dimensions after [`ChunkNode::scale_keys`]) and its `n` children,
/// a leaf's as indices into the list of stored chunks.
struct ChunkNode {
ndims: usize,
keys: Vec<u64>,
children: ChunkChildren,
}
enum ChunkChildren {
Nodes(Vec<ChunkNode>),
Chunks(Vec<usize>),
}
impl ChunkNode {
fn key(&self, i: usize) -> &[u64] {
&self.keys[i * self.ndims..(i + 1) * self.ndims]
}
fn len(&self) -> usize {
match &self.children {
ChunkChildren::Nodes(n) => n.len(),
ChunkChildren::Chunks(c) => c.len(),
}
}
fn scale_keys(&mut self, dims: &[u32]) {
for (k, &d) in self.keys.iter_mut().zip(dims.iter().cycle()) {
*k /= u64::from(d);
}
if let ChunkChildren::Nodes(nodes) = &mut self.children {
for n in nodes {
n.scale_keys(dims);
}
}
}
/// `H5B_find_helper` over scaled keys: binary search for the child
/// whose keys bracket `scaled`, then `H5D__btree_found` at the leaf.
/// Returns the index of the chunk found.
fn find(&self, scaled: &[u64]) -> Option<usize> {
let (mut lt, mut rt) = (0, self.len());
let mut idx = 0;
let mut cmp = core::cmp::Ordering::Greater;
while lt < rt && cmp != core::cmp::Ordering::Equal {
idx = (lt + rt) / 2;
cmp = btree_cmp3(self.key(idx), scaled, self.key(idx + 1));
if cmp == core::cmp::Ordering::Less {
rt = idx;
} else {
lt = idx + 1;
}
}
if cmp != core::cmp::Ordering::Equal {
return None;
}
match &self.children {
ChunkChildren::Nodes(nodes) => nodes[idx].find(scaled),
ChunkChildren::Chunks(chunks) => {
// "Is this *really* the requested chunk?"
let lt_key = self.key(idx);
let found = scaled
.iter()
.zip(lt_key)
.all(|(&s, &k)| s < k.wrapping_add(1));
found.then_some(chunks[idx])
}
}
}
}
/// `H5D__btree_cmp3`: where `scaled` falls against a child's left and
/// right keys. `Less` is left of the child, `Greater` right of it. With a
/// rank-1 dataset (two coordinates, element size last) libhdf5 compares
/// only the first coordinate, and the second against the right key.
fn btree_cmp3(lt: &[u64], scaled: &[u64], rt: &[u64]) -> core::cmp::Ordering {
use core::cmp::Ordering;
if scaled.len() == 2 {
if scaled[0] > rt[0] || (scaled[0] == rt[0] && scaled[1] >= rt[1]) {
Ordering::Greater
} else if scaled[0] < lt[0] {
Ordering::Less
} else {
Ordering::Equal
}
} else if scaled >= rt {
Ordering::Greater
} else if scaled < lt {
Ordering::Less
} else {
Ordering::Equal
}
}
/// Check one v1 B-tree chunk key's offsets (see
@@ -640,24 +781,25 @@ fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), Fo
}
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
/// chunk size and filter mask) and check them when `chunk_dimensions` is
/// given.
/// chunk size and filter mask) into `out`, checking them when
/// `chunk_dimensions` is given.
fn read_key_offsets(
file_data: &[u8],
pos: usize,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
) -> Result<Vec<u64>, FormatError> {
let mut offsets = Vec::with_capacity(ndims);
out: &mut Vec<u64>,
) -> Result<(), FormatError> {
let start = out.len();
let mut kp = pos + 8;
for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
out.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize;
}
if let Some(dims) = chunk_dimensions {
check_key_offsets(&offsets, dims)?;
check_key_offsets(&out[start..], dims)?;
}
Ok(offsets)
Ok(())
}
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
@@ -668,15 +810,17 @@ const CHUNK_KEY_OFFSET_SIZE: u8 = 8;
/// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
const MAX_CHUNK_BTREE_DEPTH: usize = 64;
fn collect_chunk_info_inner(
/// Parse the v1 B-tree chunk index node at `btree_address` and its
/// subtree, appending its chunks to `stored` in tree order.
fn parse_chunk_node(
file_data: &[u8],
btree_address: u64,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
offset_size: u8,
_length_size: u8,
depth: usize,
) -> Result<Vec<ChunkInfo>, FormatError> {
stored: &mut Vec<ChunkInfo>,
) -> Result<ChunkNode, FormatError> {
if depth > MAX_CHUNK_BTREE_DEPTH {
return Err(FormatError::NestingDepthExceeded);
}
@@ -711,74 +855,68 @@ fn collect_chunk_info_inner(
.and_then(|n| n.checked_add(8))
.ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
if node_level == 0 {
// Leaf node: keys and children interleaved
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
let mut chunks = Vec::with_capacity(entries_used);
for _ in 0..entries_used {
// Parse key
let chunk_size = u32::from_le_bytes([
file_data[pos],
file_data[pos + 1],
file_data[pos + 2],
file_data[pos + 3],
]);
let filter_mask = u32::from_le_bytes([
file_data[pos + 4],
file_data[pos + 5],
file_data[pos + 6],
file_data[pos + 7],
]);
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
// Parse child address
let address = read_offset(file_data, pos, offset_size)?;
pos += os;
chunks.push(ChunkInfo {
let mut keys = Vec::with_capacity((entries_used + 1) * ndims);
let mut chunks = Vec::new();
let mut child_addrs = Vec::new();
for _ in 0..entries_used {
let chunk_size = u32::from_le_bytes([
file_data[pos],
file_data[pos + 1],
file_data[pos + 2],
file_data[pos + 3],
]);
let filter_mask = u32::from_le_bytes([
file_data[pos + 4],
file_data[pos + 5],
file_data[pos + 6],
file_data[pos + 7],
]);
let k = keys.len();
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
pos += key_size;
let address = read_offset(file_data, pos, offset_size)?;
pos += os;
if node_level == 0 {
chunks.push(stored.len());
stored.push(ChunkInfo {
chunk_size,
filter_mask,
offsets,
offsets: keys[k..].to_vec(),
address,
});
} else {
child_addrs.push(address);
}
// The final key only bounds the node; libhdf5 still checks it.
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
Ok(chunks)
}
// The final key only bounds the node; libhdf5 still checks it.
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
let children = if node_level == 0 {
ChunkChildren::Chunks(chunks)
} else {
// Internal node: recurse into children
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
let mut child_addrs = Vec::with_capacity(entries_used);
for _ in 0..entries_used {
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
let child_addr = read_offset(file_data, pos, offset_size)?;
child_addrs.push(child_addr);
pos += os;
}
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
let mut all_chunks = Vec::new();
let mut nodes = Vec::with_capacity(child_addrs.len());
for child_addr in child_addrs {
let child_chunks = collect_chunk_info_inner(
nodes.push(parse_chunk_node(
file_data,
child_addr,
ndims,
chunk_dimensions,
offset_size,
_length_size,
depth + 1,
)?;
all_chunks.extend(child_chunks);
stored,
)?);
}
Ok(all_chunks)
}
ChunkChildren::Nodes(nodes)
};
Ok(ChunkNode {
ndims,
keys,
children,
})
}
/// Generate ChunkInfo entries for an implicit index (v4 index type 2).
@@ -1087,6 +1225,47 @@ pub fn list_chunks(
Ok((chunks, chunk_dims))
}
/// [`list_chunks`] for reading the chunks through `pipeline`: a dataset
/// without filters stores every chunk at the chunk's full size, and a chunk
/// the index records at another size is refused, as libhdf5 refuses it
/// ("incorrect chunk size returned from index for unfiltered chunk"). Such
/// a chunk was read at its recorded size, with the rest of the chunk left
/// as zeros or fill values: `cve-2025-44904`'s `Scale_offset_float_data_le`
/// has chunks of 38 and 37 bytes for 48-byte chunks, where HDF5 2.0 reads
/// whatever its buffer held for the missing bytes.
pub fn list_chunks_for_read(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
elem_size: usize,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
) -> Result<(Vec<ChunkInfo>, Vec<usize>), FormatError> {
let (chunks, chunk_dims) = list_chunks(
file_data,
layout,
dataspace,
elem_size,
offset_size,
length_size,
)?;
if pipeline.is_none_or(|p| p.filters.is_empty()) {
let chunk_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?;
if let Some(c) = chunks
.iter()
.find(|c| c.address != u64::MAX && c.chunk_size as usize != chunk_bytes)
{
return Err(FormatError::ChunkedReadError(format!(
"incorrect chunk size returned from index for unfiltered chunk at {:?}: \
{} bytes, expected {chunk_bytes}",
c.offsets, c.chunk_size
)));
}
}
Ok((chunks, chunk_dims))
}
/// The chunk cache a full read may use (`None` without `std`).
#[cfg(feature = "std")]
pub(crate) type CacheRef<'a> = Option<&'a ChunkCache>;
@@ -1114,11 +1293,12 @@ pub(crate) fn read_chunked_full<O>(
check_chunk_element_size(layout, datatype, offset_size)?;
let elem_size = datatype.type_size() as usize;
let list = || {
list_chunks(
list_chunks_for_read(
file_data,
layout,
dataspace,
elem_size,
pipeline,
offset_size,
length_size,
)
@@ -1429,11 +1609,12 @@ pub fn read_chunked_data_sweep(
// lookup is keyed by this dataset's chunk-index address, so another
// dataset's index or chunks are never used for this read.
let chunks = cache.chunks_for(addr, rank, || {
list_chunks(
list_chunks_for_read(
file_data,
layout,
dataspace,
elem_size,
pipeline,
offset_size,
length_size,
)
@@ -1570,11 +1751,12 @@ pub fn read_chunked_data_indexed(
addr,
rank,
|| {
list_chunks(
list_chunks_for_read(
file_data,
layout,
dataspace,
elem_size,
pipeline,
offset_size,
length_size,
)
@@ -1874,6 +2056,25 @@ mod tests {
/// Build a B-tree v1 type 1 leaf node with given chunk infos.
fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec<u8> {
build_chunk_btree_leaf_to(chunks, &vec![0; ndims], offset_size)
}
/// A leaf whose final key is the one libhdf5 writes past the last chunk:
/// each coordinate of the last chunk plus its chunk dimension (the
/// element size last).
fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u32], offset_size: u8) -> Vec<u8> {
let last = &chunks.last().expect("a chunk").offsets;
let end: Vec<u64> = dims
.iter()
.enumerate()
.map(|(d, &c)| last.get(d).copied().unwrap_or(0) + u64::from(c))
.collect();
build_chunk_btree_leaf_to(chunks, &end, offset_size)
}
/// A leaf holding `chunks`, with final key `end`.
fn build_chunk_btree_leaf_to(chunks: &[ChunkInfo], end: &[u64], offset_size: u8) -> Vec<u8> {
let ndims = end.len();
let _os = offset_size as usize;
let entries_used = chunks.len() as u16;
let mut buf = Vec::new();
@@ -1915,8 +2116,8 @@ mod tests {
// checks; 0 always is)
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
for _ in 0..ndims {
write_offset(&mut buf, 0, 8);
for &e in end {
write_offset(&mut buf, e, 8);
}
buf
@@ -1932,8 +2133,11 @@ mod tests {
offsets,
address,
};
let good =
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
let good = build_chunk_btree_leaf_dims(
&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)],
&[10, 8],
8,
);
assert_eq!(
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
.unwrap()
@@ -2164,7 +2368,6 @@ mod tests {
) -> (Vec<u8>, DataLayout, Dataspace) {
let os: u8 = 8;
let elem_size = 8usize;
let ndims = 2; // rank(1) + 1
let total = values.len();
// Place chunk data starting at offset 0x2000
@@ -2195,12 +2398,13 @@ mod tests {
}
// Build B-tree at offset 0x100
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
let dims = [chunk_size_elems as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_size_elems as u32, elem_size as u32],
chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64),
version: 3,
chunk_index_type: None,
@@ -2530,7 +2734,6 @@ mod tests {
let os: u8 = 8;
let elem_size = 8usize;
let ndims = 2;
let chunk_elems = 10usize;
let total = 20usize;
@@ -2569,12 +2772,13 @@ mod tests {
data_offset += compressed.len() + 16; // some padding
}
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
let dims = [chunk_elems as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_elems as u32, elem_size as u32],
chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64),
version: 3,
chunk_index_type: None,
@@ -2612,7 +2816,6 @@ mod tests {
// 4x6 dataset with chunk size 2x3 => 4 chunks
let os: u8 = 8;
let elem_size = 4usize; // f32
let ndims = 3; // rank(2) + 1
let ds_dims = [4usize, 6];
let chunk_dims = [2usize, 3];
@@ -2652,12 +2855,13 @@ mod tests {
}
}
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
let dims = [chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32],
chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64),
version: 3,
chunk_index_type: None,