quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): read a v1 chunk B-tree where libhdf5's lookup finds chunks

Browse Source 
libhdf5 does not walk the chunk B-tree to read a dataset: it looks each
chunk up (H5B_find with H5D__btree_cmp3 and H5D__btree_found), asking for
the element-size coordinate as 0. collect_chunk_info_checked now parses
the tree with its keys and returns each stored chunk only when that
lookup, replayed over the scaled keys, finds it.

A key with a non-zero element-size coordinate is therefore found in a
1-D dataset (cmp3 compares only the first coordinate there, and found
compares with <=) and missed in a dataset of rank 2 or more, which reads
fill values. The previous commit refused every such key, which refused
1-D files libhdf5 reads correctly; before that, the rank-2 case read the
chunk's data where h5py reads fill values (cve-2025-44905
/Shuffle_float_data_le, now identical to h5py, so it leaves the
conformance report's list of libhdf5 bugs).

Test: chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them
compares 1-D and 2-D files against h5py's values. It fails on the
previous commit (the 1-D file is refused) and with the refusal removed
(the 2-D file reads 0..23 where h5py reads fill values).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 11:35:13 -05:00
co-authored by Claude Opus 5.5
parent 378afa1584
commit 742ed4dfb8
4 changed files with 322 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_read.rs
+252 -103
View File
@@ -316,26 +316,41 @@ pub fn collect_chunk_info(
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(
let _ = length_size;
let mut chunks = Vec::new();
parse_chunk_node(
file_data,
btree_address,
ndims,
None,
offset_size,
length_size,
0,
)
&mut chunks,
)?;
Ok(chunks)
}
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
/// layout message's list, element size last), checking every key of the
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
/// must be a multiple of its chunk dimension. That includes the keys that
/// only bound a node (internal-node keys and each node's final key), which
/// is where a corrupt chunk dimension shows when the chunks themselves all
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
/// chunks were read at the wrong place, or the dataset read as fill values.
/// The chunks of a v1 B-tree chunk index as libhdf5 reads them, for a
/// layout with these `chunk_dimensions` (the layout message's list, element
/// size last).
///
/// Every key of the B-tree is checked as libhdf5 checks it
/// (`H5D__btree_decode_key`): each coordinate offset must be a multiple of
/// its chunk dimension. That includes the keys that only bound a node
/// (internal-node keys and each node's final key), which is where a
/// corrupt chunk dimension shows when the chunks themselves all start at
/// offset 0 in that dimension (`cve-2018-11205`). A key that fails ("bad
/// coordinate offset") means a corrupt index or chunk dimension.
///
/// libhdf5 does not read a chunk by walking the tree: it looks each chunk
/// up (`H5B_find` with `H5D__btree_cmp3` and `H5D__btree_found`), comparing
/// the element-size coordinate too, which it asks for as 0. So a chunk is
/// returned only where that lookup finds it: a key whose element-size
/// coordinate is not 0 is found in a 1-D dataset (the comparison looks at
/// that coordinate only against the next key) but not in a dataset of rank
/// 2 or more (`cve-2025-44905` `/Shuffle_float_data_le`), which then reads
/// as fill values, and a tree whose keys are out of order loses the chunks
/// libhdf5's binary search misses.
pub fn collect_chunk_info_checked(
file_data: &[u8],
btree_address: u64,
@@ -343,15 +358,141 @@ pub fn collect_chunk_info_checked(
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(
let _ = length_size;
let ndims = chunk_dimensions.len();
if ndims == 0 {
return Err(FormatError::ChunkedReadError(
"chunk layout has no dimensions".into(),
));
}
let mut stored = Vec::new();
let mut root = parse_chunk_node(
file_data,
btree_address,
chunk_dimensions.len(),
ndims,
Some(chunk_dimensions),
offset_size,
length_size,
0,
)
&mut stored,
)?;
// Keys were checked to be multiples of non-zero dimensions.
root.scale_keys(chunk_dimensions);
// Look every stored chunk's position up. A lookup finds a chunk only at
// that chunk's own position, so each is returned at most once.
let mut returned = vec![false; stored.len()];
let mut wanted = vec![0u64; ndims];
for chunk in &stored {
for (w, (&o, &d)) in wanted
.iter_mut()
.zip(chunk.offsets.iter().zip(chunk_dimensions))
{
*w = o / u64::from(d);
}
wanted[ndims - 1] = 0;
if let Some(i) = root.find(&wanted) {
returned[i] = true;
}
}
Ok(stored
.into_iter()
.zip(returned)
.filter_map(|(c, r)| r.then_some(c))
.collect())
}
/// A node of a v1 B-tree chunk index: its `n + 1` keys (`ndims`
/// coordinates each, flattened; byte offsets as stored, or scaled by the
/// chunk dimensions after [`ChunkNode::scale_keys`]) and its `n` children,
/// a leaf's as indices into the list of stored chunks.
struct ChunkNode {
ndims: usize,
keys: Vec<u64>,
children: ChunkChildren,
}
enum ChunkChildren {
Nodes(Vec<ChunkNode>),
Chunks(Vec<usize>),
}
impl ChunkNode {
fn key(&self, i: usize) -> &[u64] {
&self.keys[i * self.ndims..(i + 1) * self.ndims]
}
fn len(&self) -> usize {
match &self.children {
ChunkChildren::Nodes(n) => n.len(),
ChunkChildren::Chunks(c) => c.len(),
}
}
fn scale_keys(&mut self, dims: &[u32]) {
for (k, &d) in self.keys.iter_mut().zip(dims.iter().cycle()) {
*k /= u64::from(d);
}
if let ChunkChildren::Nodes(nodes) = &mut self.children {
for n in nodes {
n.scale_keys(dims);
}
}
}
/// `H5B_find_helper` over scaled keys: binary search for the child
/// whose keys bracket `scaled`, then `H5D__btree_found` at the leaf.
/// Returns the index of the chunk found.
fn find(&self, scaled: &[u64]) -> Option<usize> {
let (mut lt, mut rt) = (0, self.len());
let mut idx = 0;
let mut cmp = core::cmp::Ordering::Greater;
while lt < rt && cmp != core::cmp::Ordering::Equal {
idx = (lt + rt) / 2;
cmp = btree_cmp3(self.key(idx), scaled, self.key(idx + 1));
if cmp == core::cmp::Ordering::Less {
rt = idx;
} else {
lt = idx + 1;
}
}
if cmp != core::cmp::Ordering::Equal {
return None;
}
match &self.children {
ChunkChildren::Nodes(nodes) => nodes[idx].find(scaled),
ChunkChildren::Chunks(chunks) => {
// "Is this *really* the requested chunk?"
let lt_key = self.key(idx);
let found = scaled
.iter()
.zip(lt_key)
.all(|(&s, &k)| s < k.wrapping_add(1));
found.then_some(chunks[idx])
}
}
}
}
/// `H5D__btree_cmp3`: where `scaled` falls against a child's left and
/// right keys. `Less` is left of the child, `Greater` right of it. With a
/// rank-1 dataset (two coordinates, element size last) libhdf5 compares
/// only the first coordinate, and the second against the right key.
fn btree_cmp3(lt: &[u64], scaled: &[u64], rt: &[u64]) -> core::cmp::Ordering {
use core::cmp::Ordering;
if scaled.len() == 2 {
if scaled[0] > rt[0] || (scaled[0] == rt[0] && scaled[1] >= rt[1]) {
Ordering::Greater
} else if scaled[0] < lt[0] {
Ordering::Less
} else {
Ordering::Equal
}
} else if scaled >= rt {
Ordering::Greater
} else if scaled < lt {
Ordering::Less
} else {
Ordering::Equal
}
}
/// Check one v1 B-tree chunk key's offsets (see
@@ -368,24 +509,25 @@ fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), Fo
}
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
/// chunk size and filter mask) and check them when `chunk_dimensions` is
/// given.
/// chunk size and filter mask) into `out`, checking them when
/// `chunk_dimensions` is given.
fn read_key_offsets(
file_data: &[u8],
pos: usize,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
) -> Result<Vec<u64>, FormatError> {
let mut offsets = Vec::with_capacity(ndims);
out: &mut Vec<u64>,
) -> Result<(), FormatError> {
let start = out.len();
let mut kp = pos + 8;
for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
out.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize;
}
if let Some(dims) = chunk_dimensions {
check_key_offsets(&offsets, dims)?;
check_key_offsets(&out[start..], dims)?;
}
Ok(offsets)
Ok(())
}
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
@@ -396,15 +538,17 @@ const CHUNK_KEY_OFFSET_SIZE: u8 = 8;
/// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
const MAX_CHUNK_BTREE_DEPTH: usize = 64;
fn collect_chunk_info_inner(
/// Parse the v1 B-tree chunk index node at `btree_address` and its
/// subtree, appending its chunks to `stored` in tree order.
fn parse_chunk_node(
file_data: &[u8],
btree_address: u64,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
offset_size: u8,
_length_size: u8,
depth: usize,
) -> Result<Vec<ChunkInfo>, FormatError> {
stored: &mut Vec<ChunkInfo>,
) -> Result<ChunkNode, FormatError> {
if depth > MAX_CHUNK_BTREE_DEPTH {
return Err(FormatError::NestingDepthExceeded);
}
@@ -439,85 +583,68 @@ fn collect_chunk_info_inner(
.and_then(|n| n.checked_add(8))
.ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
if node_level == 0 {
// Leaf node: keys and children interleaved
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
let mut chunks = Vec::with_capacity(entries_used);
for _ in 0..entries_used {
// Parse key
let chunk_size = u32::from_le_bytes([
file_data[pos],
file_data[pos + 1],
file_data[pos + 2],
file_data[pos + 3],
]);
let filter_mask = u32::from_le_bytes([
file_data[pos + 4],
file_data[pos + 5],
file_data[pos + 6],
file_data[pos + 7],
]);
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
// A chunk's key carries 0 in the element-size dimension. libhdf5
// compares that coordinate too when it looks a chunk up
// (`H5D__btree_found`), so whether it finds a chunk keyed
// otherwise depends on where the key falls; in `cve-2025-44905`
// `/Shuffle_float_data_le` (offset 4096) it does not, and h5py
// reads fill values there. Such a key is refused here.
if chunk_dimensions.is_some() && offsets.last().is_some_and(|&o| o != 0) {
return Err(FormatError::ChunkedReadError(format!(
"chunk key {offsets:?} has a non-zero element offset"
)));
}
pos += key_size;
// Parse child address
let address = read_offset(file_data, pos, offset_size)?;
pos += os;
chunks.push(ChunkInfo {
let mut keys = Vec::with_capacity((entries_used + 1) * ndims);
let mut chunks = Vec::new();
let mut child_addrs = Vec::new();
for _ in 0..entries_used {
let chunk_size = u32::from_le_bytes([
file_data[pos],
file_data[pos + 1],
file_data[pos + 2],
file_data[pos + 3],
]);
let filter_mask = u32::from_le_bytes([
file_data[pos + 4],
file_data[pos + 5],
file_data[pos + 6],
file_data[pos + 7],
]);
let k = keys.len();
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
pos += key_size;
let address = read_offset(file_data, pos, offset_size)?;
pos += os;
if node_level == 0 {
chunks.push(stored.len());
stored.push(ChunkInfo {
chunk_size,
filter_mask,
offsets,
offsets: keys[k..].to_vec(),
address,
});
} else {
child_addrs.push(address);
}
// The final key only bounds the node; libhdf5 still checks it.
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
Ok(chunks)
}
// The final key only bounds the node; libhdf5 still checks it.
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
let children = if node_level == 0 {
ChunkChildren::Chunks(chunks)
} else {
// Internal node: recurse into children
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
let mut child_addrs = Vec::with_capacity(entries_used);
for _ in 0..entries_used {
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
let child_addr = read_offset(file_data, pos, offset_size)?;
child_addrs.push(child_addr);
pos += os;
}
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
let mut all_chunks = Vec::new();
let mut nodes = Vec::with_capacity(child_addrs.len());
for child_addr in child_addrs {
let child_chunks = collect_chunk_info_inner(
nodes.push(parse_chunk_node(
file_data,
child_addr,
ndims,
chunk_dimensions,
offset_size,
_length_size,
depth + 1,
)?;
all_chunks.extend(child_chunks);
stored,
)?);
}
Ok(all_chunks)
}
ChunkChildren::Nodes(nodes)
};
Ok(ChunkNode {
ndims,
keys,
children,
})
}
/// Generate ChunkInfo entries for an implicit index (v4 index type 2).
@@ -1754,6 +1881,25 @@ mod tests {
/// Build a B-tree v1 type 1 leaf node with given chunk infos.
fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec<u8> {
build_chunk_btree_leaf_to(chunks, &vec![0; ndims], offset_size)
}
/// A leaf whose final key is the one libhdf5 writes past the last chunk:
/// each coordinate of the last chunk plus its chunk dimension (the
/// element size last).
fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u32], offset_size: u8) -> Vec<u8> {
let last = &chunks.last().expect("a chunk").offsets;
let end: Vec<u64> = dims
.iter()
.enumerate()
.map(|(d, &c)| last.get(d).copied().unwrap_or(0) + u64::from(c))
.collect();
build_chunk_btree_leaf_to(chunks, &end, offset_size)
}
/// A leaf holding `chunks`, with final key `end`.
fn build_chunk_btree_leaf_to(chunks: &[ChunkInfo], end: &[u64], offset_size: u8) -> Vec<u8> {
let ndims = end.len();
let _os = offset_size as usize;
let entries_used = chunks.len() as u16;
let mut buf = Vec::new();
@@ -1795,8 +1941,8 @@ mod tests {
// checks; 0 always is)
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
for _ in 0..ndims {
write_offset(&mut buf, 0, 8);
for &e in end {
write_offset(&mut buf, e, 8);
}
buf
@@ -1812,8 +1958,11 @@ mod tests {
offsets,
address,
};
let good =
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
let good = build_chunk_btree_leaf_dims(
&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)],
&[10, 8],
8,
);
assert_eq!(
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
.unwrap()
@@ -2044,7 +2193,6 @@ mod tests {
) -> (Vec<u8>, DataLayout, Dataspace) {
let os: u8 = 8;
let elem_size = 8usize;
let ndims = 2; // rank(1) + 1
let total = values.len();
// Place chunk data starting at offset 0x2000
@@ -2075,12 +2223,13 @@ mod tests {
}
// Build B-tree at offset 0x100
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
let dims = [chunk_size_elems as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_size_elems as u32, elem_size as u32],
chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64),
version: 3,
chunk_index_type: None,
@@ -2218,7 +2367,6 @@ mod tests {
let os: u8 = 8;
let elem_size = 8usize;
let ndims = 2;
let chunk_elems = 10usize;
let total = 20usize;
@@ -2257,12 +2405,13 @@ mod tests {
data_offset += compressed.len() + 16; // some padding
}
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
let dims = [chunk_elems as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_elems as u32, elem_size as u32],
chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64),
version: 3,
chunk_index_type: None,
@@ -2300,7 +2449,6 @@ mod tests {
// 4x6 dataset with chunk size 2x3 => 4 chunks
let os: u8 = 8;
let elem_size = 4usize; // f32
let ndims = 3; // rank(2) + 1
let ds_dims = [4usize, 6];
let chunk_dims = [2usize, 3];
@@ -2340,12 +2488,13 @@ mod tests {
}
}
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
let dims = [chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32],
chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64),
version: 3,
chunk_index_type: None,