Chunked reads beat an h5py process pool; unlimited writer B-trees; Blosc2; 599/697 conformance #16

Merged
osobh merged 48 commits from feat/p2b-scale into main 2026-09-26 17:42:16 +00:00
4 changed files with 322 additions and 125 deletions
Showing only changes of commit 742ed4dfb8 - Show all commits
+9 -2
View File
@@ -40,8 +40,15 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug.
- shuffle uses its own parameter as the element size, as libhdf5 does - shuffle uses its own parameter as the element size, as libhdf5 does
(`cve-2025-44905`); (`cve-2025-44905`);
- an unfiltered chunk the index records at other than the chunk's size is - an unfiltered chunk the index records at other than the chunk's size is
refused (it read with zeros for the missing bytes; `cve-2025-44904`), refused (it read with zeros for the missing bytes; `cve-2025-44904`);
as is a chunk B-tree key with a non-zero element offset. - a v1 B-tree chunk index is read as libhdf5 reads it: each chunk is
looked up the way `H5B_find` / `H5D__btree_cmp3` / `H5D__btree_found`
look it up, and a chunk that lookup does not find reads as fill values.
A key with a non-zero element-size coordinate is found in a 1-D dataset
and not in one of rank 2 or more (`cve-2025-44905`
`/Shuffle_float_data_le`, which read the chunk's data where h5py reads
fill values); an interim fix refused every such key, including 1-D
files libhdf5 reads correctly.
- **Refused as libhdf5 refuses them:** a v1 group with an empty link name - **Refused as libhdf5 refuses them:** a v1 group with an empty link name
fails its listing (`FormatError::InvalidLinkName`; lookups still work, fails its listing (`FormatError::InvalidLinkName`; lookups still work,
`cve-2021-46244`); dataspaces with more than 32 dimensions, a rank on a `cve-2021-46244`); dataspaces with more than 32 dimensions, a rank on a
+2 -5
View File
@@ -100,8 +100,8 @@ def is_h5py_be_vlen(i):
# Objects the reference (h5py 3.16 / HDF5 2.0) reads only because of an # Objects the reference (h5py 3.16 / HDF5 2.0) reads only because of an
# HDF5 2.0 bug, and that clawhdf5 refuses: each one reads past a buffer or # HDF5 2.0 bug, and that clawhdf5 refuses: each one reads past a buffer or
# returns bytes the file does not hold, and libhdf5's develop branch refuses the # returns bytes the file does not hold, and libhdf5's develop branch refuses all
# first three. (file, object) -> why. Checked 2026-09-26 against HDF5 2.0.0 # three. (file, object) -> why. Checked 2026-09-26 against HDF5 2.0.0
# and HDFGroup/hdf5 develop sources; see docs/known-issues.md. # and HDFGroup/hdf5 develop sources; see docs/known-issues.md.
LIBHDF5_BUGS = { LIBHDF5_BUGS = {
("cve_hdf5/cvefiles/cve-2025-2308.h5", "/Scale_offset_long_long_data_le"): ("cve_hdf5/cvefiles/cve-2025-2308.h5", "/Scale_offset_long_long_data_le"):
@@ -114,9 +114,6 @@ LIBHDF5_BUGS = {
("hdf5/test/testfiles/bad_nbit_parms_walk.h5", "/Nbit_int_data_le"): ("hdf5/test/testfiles/bad_nbit_parms_walk.h5", "/Nbit_int_data_le"):
"an N-Bit parameter list one value short: HDF5 2.0 reads past the list; libhdf5's own " "an N-Bit parameter list one value short: HDF5 2.0 reads past the list; libhdf5's own "
"test (`test_filter_bad_params`, test/dsets.c) now requires the read to fail", "test (`test_filter_bad_params`, test/dsets.c) now requires the read to fail",
("cve_hdf5/cvefiles/cve-2025-44905.h5", "/Shuffle_float_data_le"):
"a chunk B-tree key with element offset 4096: libhdf5's lookup misses the chunk and "
"returns fill values for data the file holds; clawhdf5 refuses the key",
} }
+252 -103
View File
@@ -316,26 +316,41 @@ pub fn collect_chunk_info(
offset_size: u8, offset_size: u8,
length_size: u8, length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> { ) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner( let _ = length_size;
let mut chunks = Vec::new();
parse_chunk_node(
file_data, file_data,
btree_address, btree_address,
ndims, ndims,
None, None,
offset_size, offset_size,
length_size,
0, 0,
) &mut chunks,
)?;
Ok(chunks)
} }
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the /// The chunks of a v1 B-tree chunk index as libhdf5 reads them, for a
/// layout message's list, element size last), checking every key of the /// layout with these `chunk_dimensions` (the layout message's list, element
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset /// size last).
/// must be a multiple of its chunk dimension. That includes the keys that ///
/// only bound a node (internal-node keys and each node's final key), which /// Every key of the B-tree is checked as libhdf5 checks it
/// is where a corrupt chunk dimension shows when the chunks themselves all /// (`H5D__btree_decode_key`): each coordinate offset must be a multiple of
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails /// its chunk dimension. That includes the keys that only bound a node
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the /// (internal-node keys and each node's final key), which is where a
/// chunks were read at the wrong place, or the dataset read as fill values. /// corrupt chunk dimension shows when the chunks themselves all start at
/// offset 0 in that dimension (`cve-2018-11205`). A key that fails ("bad
/// coordinate offset") means a corrupt index or chunk dimension.
///
/// libhdf5 does not read a chunk by walking the tree: it looks each chunk
/// up (`H5B_find` with `H5D__btree_cmp3` and `H5D__btree_found`), comparing
/// the element-size coordinate too, which it asks for as 0. So a chunk is
/// returned only where that lookup finds it: a key whose element-size
/// coordinate is not 0 is found in a 1-D dataset (the comparison looks at
/// that coordinate only against the next key) but not in a dataset of rank
/// 2 or more (`cve-2025-44905` `/Shuffle_float_data_le`), which then reads
/// as fill values, and a tree whose keys are out of order loses the chunks
/// libhdf5's binary search misses.
pub fn collect_chunk_info_checked( pub fn collect_chunk_info_checked(
file_data: &[u8], file_data: &[u8],
btree_address: u64, btree_address: u64,
@@ -343,15 +358,141 @@ pub fn collect_chunk_info_checked(
offset_size: u8, offset_size: u8,
length_size: u8, length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> { ) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner( let _ = length_size;
let ndims = chunk_dimensions.len();
if ndims == 0 {
return Err(FormatError::ChunkedReadError(
"chunk layout has no dimensions".into(),
));
}
let mut stored = Vec::new();
let mut root = parse_chunk_node(
file_data, file_data,
btree_address, btree_address,
chunk_dimensions.len(), ndims,
Some(chunk_dimensions), Some(chunk_dimensions),
offset_size, offset_size,
length_size,
0, 0,
) &mut stored,
)?;
// Keys were checked to be multiples of non-zero dimensions.
root.scale_keys(chunk_dimensions);
// Look every stored chunk's position up. A lookup finds a chunk only at
// that chunk's own position, so each is returned at most once.
let mut returned = vec![false; stored.len()];
let mut wanted = vec![0u64; ndims];
for chunk in &stored {
for (w, (&o, &d)) in wanted
.iter_mut()
.zip(chunk.offsets.iter().zip(chunk_dimensions))
{
*w = o / u64::from(d);
}
wanted[ndims - 1] = 0;
if let Some(i) = root.find(&wanted) {
returned[i] = true;
}
}
Ok(stored
.into_iter()
.zip(returned)
.filter_map(|(c, r)| r.then_some(c))
.collect())
}
/// A node of a v1 B-tree chunk index: its `n + 1` keys (`ndims`
/// coordinates each, flattened; byte offsets as stored, or scaled by the
/// chunk dimensions after [`ChunkNode::scale_keys`]) and its `n` children,
/// a leaf's as indices into the list of stored chunks.
struct ChunkNode {
ndims: usize,
keys: Vec<u64>,
children: ChunkChildren,
}
enum ChunkChildren {
Nodes(Vec<ChunkNode>),
Chunks(Vec<usize>),
}
impl ChunkNode {
fn key(&self, i: usize) -> &[u64] {
&self.keys[i * self.ndims..(i + 1) * self.ndims]
}
fn len(&self) -> usize {
match &self.children {
ChunkChildren::Nodes(n) => n.len(),
ChunkChildren::Chunks(c) => c.len(),
}
}
fn scale_keys(&mut self, dims: &[u32]) {
for (k, &d) in self.keys.iter_mut().zip(dims.iter().cycle()) {
*k /= u64::from(d);
}
if let ChunkChildren::Nodes(nodes) = &mut self.children {
for n in nodes {
n.scale_keys(dims);
}
}
}
/// `H5B_find_helper` over scaled keys: binary search for the child
/// whose keys bracket `scaled`, then `H5D__btree_found` at the leaf.
/// Returns the index of the chunk found.
fn find(&self, scaled: &[u64]) -> Option<usize> {
let (mut lt, mut rt) = (0, self.len());
let mut idx = 0;
let mut cmp = core::cmp::Ordering::Greater;
while lt < rt && cmp != core::cmp::Ordering::Equal {
idx = (lt + rt) / 2;
cmp = btree_cmp3(self.key(idx), scaled, self.key(idx + 1));
if cmp == core::cmp::Ordering::Less {
rt = idx;
} else {
lt = idx + 1;
}
}
if cmp != core::cmp::Ordering::Equal {
return None;
}
match &self.children {
ChunkChildren::Nodes(nodes) => nodes[idx].find(scaled),
ChunkChildren::Chunks(chunks) => {
// "Is this *really* the requested chunk?"
let lt_key = self.key(idx);
let found = scaled
.iter()
.zip(lt_key)
.all(|(&s, &k)| s < k.wrapping_add(1));
found.then_some(chunks[idx])
}
}
}
}
/// `H5D__btree_cmp3`: where `scaled` falls against a child's left and
/// right keys. `Less` is left of the child, `Greater` right of it. With a
/// rank-1 dataset (two coordinates, element size last) libhdf5 compares
/// only the first coordinate, and the second against the right key.
fn btree_cmp3(lt: &[u64], scaled: &[u64], rt: &[u64]) -> core::cmp::Ordering {
use core::cmp::Ordering;
if scaled.len() == 2 {
if scaled[0] > rt[0] || (scaled[0] == rt[0] && scaled[1] >= rt[1]) {
Ordering::Greater
} else if scaled[0] < lt[0] {
Ordering::Less
} else {
Ordering::Equal
}
} else if scaled >= rt {
Ordering::Greater
} else if scaled < lt {
Ordering::Less
} else {
Ordering::Equal
}
} }
/// Check one v1 B-tree chunk key's offsets (see /// Check one v1 B-tree chunk key's offsets (see
@@ -368,24 +509,25 @@ fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), Fo
} }
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its /// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
/// chunk size and filter mask) and check them when `chunk_dimensions` is /// chunk size and filter mask) into `out`, checking them when
/// given. /// `chunk_dimensions` is given.
fn read_key_offsets( fn read_key_offsets(
file_data: &[u8], file_data: &[u8],
pos: usize, pos: usize,
ndims: usize, ndims: usize,
chunk_dimensions: Option<&[u32]>, chunk_dimensions: Option<&[u32]>,
) -> Result<Vec<u64>, FormatError> { out: &mut Vec<u64>,
let mut offsets = Vec::with_capacity(ndims); ) -> Result<(), FormatError> {
let start = out.len();
let mut kp = pos + 8; let mut kp = pos + 8;
for _ in 0..ndims { for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?); out.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize; kp += CHUNK_KEY_OFFSET_SIZE as usize;
} }
if let Some(dims) = chunk_dimensions { if let Some(dims) = chunk_dimensions {
check_key_offsets(&offsets, dims)?; check_key_offsets(&out[start..], dims)?;
} }
Ok(offsets) Ok(())
} }
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the /// Width of each chunk offset in a v1 chunk B-tree key, independent of the
@@ -396,15 +538,17 @@ const CHUNK_KEY_OFFSET_SIZE: u8 = 8;
/// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`. /// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
const MAX_CHUNK_BTREE_DEPTH: usize = 64; const MAX_CHUNK_BTREE_DEPTH: usize = 64;
fn collect_chunk_info_inner( /// Parse the v1 B-tree chunk index node at `btree_address` and its
/// subtree, appending its chunks to `stored` in tree order.
fn parse_chunk_node(
file_data: &[u8], file_data: &[u8],
btree_address: u64, btree_address: u64,
ndims: usize, ndims: usize,
chunk_dimensions: Option<&[u32]>, chunk_dimensions: Option<&[u32]>,
offset_size: u8, offset_size: u8,
_length_size: u8,
depth: usize, depth: usize,
) -> Result<Vec<ChunkInfo>, FormatError> { stored: &mut Vec<ChunkInfo>,
) -> Result<ChunkNode, FormatError> {
if depth > MAX_CHUNK_BTREE_DEPTH { if depth > MAX_CHUNK_BTREE_DEPTH {
return Err(FormatError::NestingDepthExceeded); return Err(FormatError::NestingDepthExceeded);
} }
@@ -439,85 +583,68 @@ fn collect_chunk_info_inner(
.and_then(|n| n.checked_add(8)) .and_then(|n| n.checked_add(8))
.ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?; .ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
if node_level == 0 { // key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
// Leaf node: keys and children interleaved let needed = entries_used * (key_size + os) + key_size;
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N] ensure_len(file_data, pos, needed)?;
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
let mut chunks = Vec::with_capacity(entries_used); let mut keys = Vec::with_capacity((entries_used + 1) * ndims);
for _ in 0..entries_used { let mut chunks = Vec::new();
// Parse key let mut child_addrs = Vec::new();
let chunk_size = u32::from_le_bytes([ for _ in 0..entries_used {
file_data[pos], let chunk_size = u32::from_le_bytes([
file_data[pos + 1], file_data[pos],
file_data[pos + 2], file_data[pos + 1],
file_data[pos + 3], file_data[pos + 2],
]); file_data[pos + 3],
let filter_mask = u32::from_le_bytes([ ]);
file_data[pos + 4], let filter_mask = u32::from_le_bytes([
file_data[pos + 5], file_data[pos + 4],
file_data[pos + 6], file_data[pos + 5],
file_data[pos + 7], file_data[pos + 6],
]); file_data[pos + 7],
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; ]);
// A chunk's key carries 0 in the element-size dimension. libhdf5 let k = keys.len();
// compares that coordinate too when it looks a chunk up read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
// (`H5D__btree_found`), so whether it finds a chunk keyed pos += key_size;
// otherwise depends on where the key falls; in `cve-2025-44905` let address = read_offset(file_data, pos, offset_size)?;
// `/Shuffle_float_data_le` (offset 4096) it does not, and h5py pos += os;
// reads fill values there. Such a key is refused here. if node_level == 0 {
if chunk_dimensions.is_some() && offsets.last().is_some_and(|&o| o != 0) { chunks.push(stored.len());
return Err(FormatError::ChunkedReadError(format!( stored.push(ChunkInfo {
"chunk key {offsets:?} has a non-zero element offset"
)));
}
pos += key_size;
// Parse child address
let address = read_offset(file_data, pos, offset_size)?;
pos += os;
chunks.push(ChunkInfo {
chunk_size, chunk_size,
filter_mask, filter_mask,
offsets, offsets: keys[k..].to_vec(),
address, address,
}); });
} else {
child_addrs.push(address);
} }
// The final key only bounds the node; libhdf5 still checks it. }
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; // The final key only bounds the node; libhdf5 still checks it.
Ok(chunks) read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
let children = if node_level == 0 {
ChunkChildren::Chunks(chunks)
} else { } else {
// Internal node: recurse into children let mut nodes = Vec::with_capacity(child_addrs.len());
let needed = entries_used * (key_size + os) + key_size;
ensure_len(file_data, pos, needed)?;
let mut child_addrs = Vec::with_capacity(entries_used);
for _ in 0..entries_used {
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
let child_addr = read_offset(file_data, pos, offset_size)?;
child_addrs.push(child_addr);
pos += os;
}
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
let mut all_chunks = Vec::new();
for child_addr in child_addrs { for child_addr in child_addrs {
let child_chunks = collect_chunk_info_inner( nodes.push(parse_chunk_node(
file_data, file_data,
child_addr, child_addr,
ndims, ndims,
chunk_dimensions, chunk_dimensions,
offset_size, offset_size,
_length_size,
depth + 1, depth + 1,
)?; stored,
all_chunks.extend(child_chunks); )?);
} }
Ok(all_chunks) ChunkChildren::Nodes(nodes)
} };
Ok(ChunkNode {
ndims,
keys,
children,
})
} }
/// Generate ChunkInfo entries for an implicit index (v4 index type 2). /// Generate ChunkInfo entries for an implicit index (v4 index type 2).
@@ -1754,6 +1881,25 @@ mod tests {
/// Build a B-tree v1 type 1 leaf node with given chunk infos. /// Build a B-tree v1 type 1 leaf node with given chunk infos.
fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec<u8> { fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec<u8> {
build_chunk_btree_leaf_to(chunks, &vec![0; ndims], offset_size)
}
/// A leaf whose final key is the one libhdf5 writes past the last chunk:
/// each coordinate of the last chunk plus its chunk dimension (the
/// element size last).
fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u32], offset_size: u8) -> Vec<u8> {
let last = &chunks.last().expect("a chunk").offsets;
let end: Vec<u64> = dims
.iter()
.enumerate()
.map(|(d, &c)| last.get(d).copied().unwrap_or(0) + u64::from(c))
.collect();
build_chunk_btree_leaf_to(chunks, &end, offset_size)
}
/// A leaf holding `chunks`, with final key `end`.
fn build_chunk_btree_leaf_to(chunks: &[ChunkInfo], end: &[u64], offset_size: u8) -> Vec<u8> {
let ndims = end.len();
let _os = offset_size as usize; let _os = offset_size as usize;
let entries_used = chunks.len() as u16; let entries_used = chunks.len() as u16;
let mut buf = Vec::new(); let mut buf = Vec::new();
@@ -1795,8 +1941,8 @@ mod tests {
// checks; 0 always is) // checks; 0 always is)
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
for _ in 0..ndims { for &e in end {
write_offset(&mut buf, 0, 8); write_offset(&mut buf, e, 8);
} }
buf buf
@@ -1812,8 +1958,11 @@ mod tests {
offsets, offsets,
address, address,
}; };
let good = let good = build_chunk_btree_leaf_dims(
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8); &[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)],
&[10, 8],
8,
);
assert_eq!( assert_eq!(
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8) collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
.unwrap() .unwrap()
@@ -2044,7 +2193,6 @@ mod tests {
) -> (Vec<u8>, DataLayout, Dataspace) { ) -> (Vec<u8>, DataLayout, Dataspace) {
let os: u8 = 8; let os: u8 = 8;
let elem_size = 8usize; let elem_size = 8usize;
let ndims = 2; // rank(1) + 1
let total = values.len(); let total = values.len();
// Place chunk data starting at offset 0x2000 // Place chunk data starting at offset 0x2000
@@ -2075,12 +2223,13 @@ mod tests {
} }
// Build B-tree at offset 0x100 // Build B-tree at offset 0x100
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os); let dims = [chunk_size_elems as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize; let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree); file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked { let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_size_elems as u32, elem_size as u32], chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64), btree_address: Some(btree_addr as u64),
version: 3, version: 3,
chunk_index_type: None, chunk_index_type: None,
@@ -2218,7 +2367,6 @@ mod tests {
let os: u8 = 8; let os: u8 = 8;
let elem_size = 8usize; let elem_size = 8usize;
let ndims = 2;
let chunk_elems = 10usize; let chunk_elems = 10usize;
let total = 20usize; let total = 20usize;
@@ -2257,12 +2405,13 @@ mod tests {
data_offset += compressed.len() + 16; // some padding data_offset += compressed.len() + 16; // some padding
} }
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os); let dims = [chunk_elems as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize; let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree); file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked { let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_elems as u32, elem_size as u32], chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64), btree_address: Some(btree_addr as u64),
version: 3, version: 3,
chunk_index_type: None, chunk_index_type: None,
@@ -2300,7 +2449,6 @@ mod tests {
// 4x6 dataset with chunk size 2x3 => 4 chunks // 4x6 dataset with chunk size 2x3 => 4 chunks
let os: u8 = 8; let os: u8 = 8;
let elem_size = 4usize; // f32 let elem_size = 4usize; // f32
let ndims = 3; // rank(2) + 1
let ds_dims = [4usize, 6]; let ds_dims = [4usize, 6];
let chunk_dims = [2usize, 3]; let chunk_dims = [2usize, 3];
@@ -2340,12 +2488,13 @@ mod tests {
} }
} }
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os); let dims = [chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32];
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
let btree_addr = 0x100usize; let btree_addr = 0x100usize;
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree); file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
let layout = DataLayout::Chunked { let layout = DataLayout::Chunked {
chunk_dimensions: vec![chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32], chunk_dimensions: dims.to_vec(),
btree_address: Some(btree_addr as u64), btree_address: Some(btree_addr as u64),
version: 3, version: 3,
chunk_index_type: None, chunk_index_type: None,
@@ -629,17 +629,13 @@ save("mdc_past_eof", bad)
); );
} }
/// Chunk index entries HDF5 2.0 mis-reads, refused here. An unfiltered /// An unfiltered chunk the index records at less than the chunk's size
/// chunk the index records at less than the chunk's size (`cve-2025-44904`): /// (`cve-2025-44904`): HDF5 2.0 fills the rest of the chunk with whatever
/// HDF5 2.0 fills the rest of the chunk with whatever its buffer held, and /// its buffer held, and later libhdf5 releases refuse it ("incorrect chunk
/// later libhdf5 releases refuse it ("incorrect chunk size returned from /// size returned from index for unfiltered chunk"); we used to read the
/// index for unfiltered chunk"); we read the rest as zeros. A chunk keyed /// rest as zeros, and now refuse it.
/// with a non-zero element offset (`cve-2025-44905`): libhdf5's lookup
/// compares that coordinate too, so whether it finds the chunk depends on
/// where the key falls (in `cve-2025-44905` it does not, and h5py reads
/// fill values; in this file it does); we read the chunk.
#[test] #[test]
fn chunk_index_entries_libhdf5_misreads_are_refused() { fn unfiltered_chunk_of_the_wrong_size_is_refused() {
skip_if_no_python!(); skip_if_no_python!();
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
run_python( run_python(
@@ -658,8 +654,6 @@ second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address
assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0) assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0)
bad = bytearray(data); struct.pack_into("<I", bad, second, 100) bad = bytearray(data); struct.pack_into("<I", bad, second, 100)
open(os.path.join(d, "short_chunk.h5"), "wb").write(bad) open(os.path.join(d, "short_chunk.h5"), "wb").write(bad)
bad = bytearray(data); struct.pack_into("<Q", bad, second + 16, 4096)
open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
"#, "#,
); );
assert_eq!( assert_eq!(
@@ -667,8 +661,58 @@ open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
Ok(()), Ok(()),
"good" "good"
); );
for name in ["short_chunk", "element_offset"] { let err = clawhdf5_reads(&dir.path().join("short_chunk.h5"), "d").unwrap_err();
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err(); assert!(err.starts_with("read:"), "short_chunk: {err}");
assert!(err.starts_with("read:"), "{name}: {err}"); }
/// A v1 B-tree chunk key whose element-size coordinate is not 0. libhdf5
/// looks each chunk up with that coordinate set to 0 (`H5B_find`,
/// `H5D__btree_cmp3`, `H5D__btree_found`): in a 1-D dataset it still finds
/// the chunk, in a 2-D one it does not and reads fill values
/// (`cve-2025-44905` `/Shuffle_float_data_le`). Both must read exactly what
/// h5py reads: the 1-D file was refused, and before that the 2-D chunk was
/// read as if its key were well formed.
#[test]
fn chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
def corrupt(name, shape, chunks, which, element_offset):
path = os.path.join(d, name + ".h5")
with h5py.File(path, "w", libver="earliest") as f:
f.create_dataset("d", data=np.arange(np.prod(shape), dtype="<i4").reshape(shape),
chunks=chunks, fillvalue=-1)
data = bytearray(open(path, "rb").read())
tree = data.find(b"TREE")
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
tree = data.find(b"TREE", tree + 1)
assert tree > 0 and data[tree + 5] == 0
ndims = len(shape) + 1
key_size = 8 + 8 * ndims
key = tree + 8 + 16 + which * (key_size + 8)
last = key + 8 + 8 * (ndims - 1)
assert struct.unpack_from("<Q", data, last) == (0,)
struct.pack_into("<Q", data, last, element_offset)
open(path, "wb").write(data)
with h5py.File(path, "r") as f:
open(path + ".txt", "w").write(" ".join(map(str, f["d"][()].ravel())))
corrupt("one_d", (100,), (37,), 1, 4096)
corrupt("two_d", (4, 6), (2, 3), 1, 4)
corrupt("two_d_first", (4, 6), (2, 3), 0, 8)
"#,
);
for name in ["one_d", "two_d", "two_d_first"] {
let path = dir.path().join(format!("{name}.h5"));
let expected: Vec<i32> = std::fs::read_to_string(dir.path().join(format!("{name}.h5.txt")))
.unwrap()
.split_whitespace()
.map(|v| v.parse().unwrap())
.collect();
let file = File::open(&path).unwrap();
let got = file.dataset("d").unwrap().read_i32();
assert_eq!(got.as_deref().ok(), Some(&expected[..]), "{name}: {got:?}");
} }
} }