fix(format): read a v1 chunk B-tree where libhdf5's lookup finds chunks
libhdf5 does not walk the chunk B-tree to read a dataset: it looks each chunk up (H5B_find with H5D__btree_cmp3 and H5D__btree_found), asking for the element-size coordinate as 0. collect_chunk_info_checked now parses the tree with its keys and returns each stored chunk only when that lookup, replayed over the scaled keys, finds it. A key with a non-zero element-size coordinate is therefore found in a 1-D dataset (cmp3 compares only the first coordinate there, and found compares with <=) and missed in a dataset of rank 2 or more, which reads fill values. The previous commit refused every such key, which refused 1-D files libhdf5 reads correctly; before that, the rank-2 case read the chunk's data where h5py reads fill values (cve-2025-44905 /Shuffle_float_data_le, now identical to h5py, so it leaves the conformance report's list of libhdf5 bugs). Test: chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them compares 1-D and 2-D files against h5py's values. It fails on the previous commit (the 1-D file is refused) and with the refusal removed (the 2-D file reads 0..23 where h5py reads fill values). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
+9
-2
@@ -40,8 +40,15 @@ ZFP filters; the 2 mismatches are the known h5py big-endian VL bug.
|
|||||||
- shuffle uses its own parameter as the element size, as libhdf5 does
|
- shuffle uses its own parameter as the element size, as libhdf5 does
|
||||||
(`cve-2025-44905`);
|
(`cve-2025-44905`);
|
||||||
- an unfiltered chunk the index records at other than the chunk's size is
|
- an unfiltered chunk the index records at other than the chunk's size is
|
||||||
refused (it read with zeros for the missing bytes; `cve-2025-44904`),
|
refused (it read with zeros for the missing bytes; `cve-2025-44904`);
|
||||||
as is a chunk B-tree key with a non-zero element offset.
|
- a v1 B-tree chunk index is read as libhdf5 reads it: each chunk is
|
||||||
|
looked up the way `H5B_find` / `H5D__btree_cmp3` / `H5D__btree_found`
|
||||||
|
look it up, and a chunk that lookup does not find reads as fill values.
|
||||||
|
A key with a non-zero element-size coordinate is found in a 1-D dataset
|
||||||
|
and not in one of rank 2 or more (`cve-2025-44905`
|
||||||
|
`/Shuffle_float_data_le`, which read the chunk's data where h5py reads
|
||||||
|
fill values); an interim fix refused every such key, including 1-D
|
||||||
|
files libhdf5 reads correctly.
|
||||||
- **Refused as libhdf5 refuses them:** a v1 group with an empty link name
|
- **Refused as libhdf5 refuses them:** a v1 group with an empty link name
|
||||||
fails its listing (`FormatError::InvalidLinkName`; lookups still work,
|
fails its listing (`FormatError::InvalidLinkName`; lookups still work,
|
||||||
`cve-2021-46244`); dataspaces with more than 32 dimensions, a rank on a
|
`cve-2021-46244`); dataspaces with more than 32 dimensions, a rank on a
|
||||||
|
|||||||
@@ -100,8 +100,8 @@ def is_h5py_be_vlen(i):
|
|||||||
|
|
||||||
# Objects the reference (h5py 3.16 / HDF5 2.0) reads only because of an
|
# Objects the reference (h5py 3.16 / HDF5 2.0) reads only because of an
|
||||||
# HDF5 2.0 bug, and that clawhdf5 refuses: each one reads past a buffer or
|
# HDF5 2.0 bug, and that clawhdf5 refuses: each one reads past a buffer or
|
||||||
# returns bytes the file does not hold, and libhdf5's develop branch refuses the
|
# returns bytes the file does not hold, and libhdf5's develop branch refuses all
|
||||||
# first three. (file, object) -> why. Checked 2026-09-26 against HDF5 2.0.0
|
# three. (file, object) -> why. Checked 2026-09-26 against HDF5 2.0.0
|
||||||
# and HDFGroup/hdf5 develop sources; see docs/known-issues.md.
|
# and HDFGroup/hdf5 develop sources; see docs/known-issues.md.
|
||||||
LIBHDF5_BUGS = {
|
LIBHDF5_BUGS = {
|
||||||
("cve_hdf5/cvefiles/cve-2025-2308.h5", "/Scale_offset_long_long_data_le"):
|
("cve_hdf5/cvefiles/cve-2025-2308.h5", "/Scale_offset_long_long_data_le"):
|
||||||
@@ -114,9 +114,6 @@ LIBHDF5_BUGS = {
|
|||||||
("hdf5/test/testfiles/bad_nbit_parms_walk.h5", "/Nbit_int_data_le"):
|
("hdf5/test/testfiles/bad_nbit_parms_walk.h5", "/Nbit_int_data_le"):
|
||||||
"an N-Bit parameter list one value short: HDF5 2.0 reads past the list; libhdf5's own "
|
"an N-Bit parameter list one value short: HDF5 2.0 reads past the list; libhdf5's own "
|
||||||
"test (`test_filter_bad_params`, test/dsets.c) now requires the read to fail",
|
"test (`test_filter_bad_params`, test/dsets.c) now requires the read to fail",
|
||||||
("cve_hdf5/cvefiles/cve-2025-44905.h5", "/Shuffle_float_data_le"):
|
|
||||||
"a chunk B-tree key with element offset 4096: libhdf5's lookup misses the chunk and "
|
|
||||||
"returns fill values for data the file holds; clawhdf5 refuses the key",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -316,26 +316,41 @@ pub fn collect_chunk_info(
|
|||||||
offset_size: u8,
|
offset_size: u8,
|
||||||
length_size: u8,
|
length_size: u8,
|
||||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||||
collect_chunk_info_inner(
|
let _ = length_size;
|
||||||
|
let mut chunks = Vec::new();
|
||||||
|
parse_chunk_node(
|
||||||
file_data,
|
file_data,
|
||||||
btree_address,
|
btree_address,
|
||||||
ndims,
|
ndims,
|
||||||
None,
|
None,
|
||||||
offset_size,
|
offset_size,
|
||||||
length_size,
|
|
||||||
0,
|
0,
|
||||||
)
|
&mut chunks,
|
||||||
|
)?;
|
||||||
|
Ok(chunks)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
|
/// The chunks of a v1 B-tree chunk index as libhdf5 reads them, for a
|
||||||
/// layout message's list, element size last), checking every key of the
|
/// layout with these `chunk_dimensions` (the layout message's list, element
|
||||||
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
|
/// size last).
|
||||||
/// must be a multiple of its chunk dimension. That includes the keys that
|
///
|
||||||
/// only bound a node (internal-node keys and each node's final key), which
|
/// Every key of the B-tree is checked as libhdf5 checks it
|
||||||
/// is where a corrupt chunk dimension shows when the chunks themselves all
|
/// (`H5D__btree_decode_key`): each coordinate offset must be a multiple of
|
||||||
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
|
/// its chunk dimension. That includes the keys that only bound a node
|
||||||
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
|
/// (internal-node keys and each node's final key), which is where a
|
||||||
/// chunks were read at the wrong place, or the dataset read as fill values.
|
/// corrupt chunk dimension shows when the chunks themselves all start at
|
||||||
|
/// offset 0 in that dimension (`cve-2018-11205`). A key that fails ("bad
|
||||||
|
/// coordinate offset") means a corrupt index or chunk dimension.
|
||||||
|
///
|
||||||
|
/// libhdf5 does not read a chunk by walking the tree: it looks each chunk
|
||||||
|
/// up (`H5B_find` with `H5D__btree_cmp3` and `H5D__btree_found`), comparing
|
||||||
|
/// the element-size coordinate too, which it asks for as 0. So a chunk is
|
||||||
|
/// returned only where that lookup finds it: a key whose element-size
|
||||||
|
/// coordinate is not 0 is found in a 1-D dataset (the comparison looks at
|
||||||
|
/// that coordinate only against the next key) but not in a dataset of rank
|
||||||
|
/// 2 or more (`cve-2025-44905` `/Shuffle_float_data_le`), which then reads
|
||||||
|
/// as fill values, and a tree whose keys are out of order loses the chunks
|
||||||
|
/// libhdf5's binary search misses.
|
||||||
pub fn collect_chunk_info_checked(
|
pub fn collect_chunk_info_checked(
|
||||||
file_data: &[u8],
|
file_data: &[u8],
|
||||||
btree_address: u64,
|
btree_address: u64,
|
||||||
@@ -343,15 +358,141 @@ pub fn collect_chunk_info_checked(
|
|||||||
offset_size: u8,
|
offset_size: u8,
|
||||||
length_size: u8,
|
length_size: u8,
|
||||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||||
collect_chunk_info_inner(
|
let _ = length_size;
|
||||||
|
let ndims = chunk_dimensions.len();
|
||||||
|
if ndims == 0 {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"chunk layout has no dimensions".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut stored = Vec::new();
|
||||||
|
let mut root = parse_chunk_node(
|
||||||
file_data,
|
file_data,
|
||||||
btree_address,
|
btree_address,
|
||||||
chunk_dimensions.len(),
|
ndims,
|
||||||
Some(chunk_dimensions),
|
Some(chunk_dimensions),
|
||||||
offset_size,
|
offset_size,
|
||||||
length_size,
|
|
||||||
0,
|
0,
|
||||||
)
|
&mut stored,
|
||||||
|
)?;
|
||||||
|
// Keys were checked to be multiples of non-zero dimensions.
|
||||||
|
root.scale_keys(chunk_dimensions);
|
||||||
|
// Look every stored chunk's position up. A lookup finds a chunk only at
|
||||||
|
// that chunk's own position, so each is returned at most once.
|
||||||
|
let mut returned = vec![false; stored.len()];
|
||||||
|
let mut wanted = vec![0u64; ndims];
|
||||||
|
for chunk in &stored {
|
||||||
|
for (w, (&o, &d)) in wanted
|
||||||
|
.iter_mut()
|
||||||
|
.zip(chunk.offsets.iter().zip(chunk_dimensions))
|
||||||
|
{
|
||||||
|
*w = o / u64::from(d);
|
||||||
|
}
|
||||||
|
wanted[ndims - 1] = 0;
|
||||||
|
if let Some(i) = root.find(&wanted) {
|
||||||
|
returned[i] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(stored
|
||||||
|
.into_iter()
|
||||||
|
.zip(returned)
|
||||||
|
.filter_map(|(c, r)| r.then_some(c))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A node of a v1 B-tree chunk index: its `n + 1` keys (`ndims`
|
||||||
|
/// coordinates each, flattened; byte offsets as stored, or scaled by the
|
||||||
|
/// chunk dimensions after [`ChunkNode::scale_keys`]) and its `n` children,
|
||||||
|
/// a leaf's as indices into the list of stored chunks.
|
||||||
|
struct ChunkNode {
|
||||||
|
ndims: usize,
|
||||||
|
keys: Vec<u64>,
|
||||||
|
children: ChunkChildren,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ChunkChildren {
|
||||||
|
Nodes(Vec<ChunkNode>),
|
||||||
|
Chunks(Vec<usize>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChunkNode {
|
||||||
|
fn key(&self, i: usize) -> &[u64] {
|
||||||
|
&self.keys[i * self.ndims..(i + 1) * self.ndims]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn len(&self) -> usize {
|
||||||
|
match &self.children {
|
||||||
|
ChunkChildren::Nodes(n) => n.len(),
|
||||||
|
ChunkChildren::Chunks(c) => c.len(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scale_keys(&mut self, dims: &[u32]) {
|
||||||
|
for (k, &d) in self.keys.iter_mut().zip(dims.iter().cycle()) {
|
||||||
|
*k /= u64::from(d);
|
||||||
|
}
|
||||||
|
if let ChunkChildren::Nodes(nodes) = &mut self.children {
|
||||||
|
for n in nodes {
|
||||||
|
n.scale_keys(dims);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `H5B_find_helper` over scaled keys: binary search for the child
|
||||||
|
/// whose keys bracket `scaled`, then `H5D__btree_found` at the leaf.
|
||||||
|
/// Returns the index of the chunk found.
|
||||||
|
fn find(&self, scaled: &[u64]) -> Option<usize> {
|
||||||
|
let (mut lt, mut rt) = (0, self.len());
|
||||||
|
let mut idx = 0;
|
||||||
|
let mut cmp = core::cmp::Ordering::Greater;
|
||||||
|
while lt < rt && cmp != core::cmp::Ordering::Equal {
|
||||||
|
idx = (lt + rt) / 2;
|
||||||
|
cmp = btree_cmp3(self.key(idx), scaled, self.key(idx + 1));
|
||||||
|
if cmp == core::cmp::Ordering::Less {
|
||||||
|
rt = idx;
|
||||||
|
} else {
|
||||||
|
lt = idx + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cmp != core::cmp::Ordering::Equal {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
match &self.children {
|
||||||
|
ChunkChildren::Nodes(nodes) => nodes[idx].find(scaled),
|
||||||
|
ChunkChildren::Chunks(chunks) => {
|
||||||
|
// "Is this *really* the requested chunk?"
|
||||||
|
let lt_key = self.key(idx);
|
||||||
|
let found = scaled
|
||||||
|
.iter()
|
||||||
|
.zip(lt_key)
|
||||||
|
.all(|(&s, &k)| s < k.wrapping_add(1));
|
||||||
|
found.then_some(chunks[idx])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `H5D__btree_cmp3`: where `scaled` falls against a child's left and
|
||||||
|
/// right keys. `Less` is left of the child, `Greater` right of it. With a
|
||||||
|
/// rank-1 dataset (two coordinates, element size last) libhdf5 compares
|
||||||
|
/// only the first coordinate, and the second against the right key.
|
||||||
|
fn btree_cmp3(lt: &[u64], scaled: &[u64], rt: &[u64]) -> core::cmp::Ordering {
|
||||||
|
use core::cmp::Ordering;
|
||||||
|
if scaled.len() == 2 {
|
||||||
|
if scaled[0] > rt[0] || (scaled[0] == rt[0] && scaled[1] >= rt[1]) {
|
||||||
|
Ordering::Greater
|
||||||
|
} else if scaled[0] < lt[0] {
|
||||||
|
Ordering::Less
|
||||||
|
} else {
|
||||||
|
Ordering::Equal
|
||||||
|
}
|
||||||
|
} else if scaled >= rt {
|
||||||
|
Ordering::Greater
|
||||||
|
} else if scaled < lt {
|
||||||
|
Ordering::Less
|
||||||
|
} else {
|
||||||
|
Ordering::Equal
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check one v1 B-tree chunk key's offsets (see
|
/// Check one v1 B-tree chunk key's offsets (see
|
||||||
@@ -368,24 +509,25 @@ fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), Fo
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
|
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
|
||||||
/// chunk size and filter mask) and check them when `chunk_dimensions` is
|
/// chunk size and filter mask) into `out`, checking them when
|
||||||
/// given.
|
/// `chunk_dimensions` is given.
|
||||||
fn read_key_offsets(
|
fn read_key_offsets(
|
||||||
file_data: &[u8],
|
file_data: &[u8],
|
||||||
pos: usize,
|
pos: usize,
|
||||||
ndims: usize,
|
ndims: usize,
|
||||||
chunk_dimensions: Option<&[u32]>,
|
chunk_dimensions: Option<&[u32]>,
|
||||||
) -> Result<Vec<u64>, FormatError> {
|
out: &mut Vec<u64>,
|
||||||
let mut offsets = Vec::with_capacity(ndims);
|
) -> Result<(), FormatError> {
|
||||||
|
let start = out.len();
|
||||||
let mut kp = pos + 8;
|
let mut kp = pos + 8;
|
||||||
for _ in 0..ndims {
|
for _ in 0..ndims {
|
||||||
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
|
out.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
|
||||||
kp += CHUNK_KEY_OFFSET_SIZE as usize;
|
kp += CHUNK_KEY_OFFSET_SIZE as usize;
|
||||||
}
|
}
|
||||||
if let Some(dims) = chunk_dimensions {
|
if let Some(dims) = chunk_dimensions {
|
||||||
check_key_offsets(&offsets, dims)?;
|
check_key_offsets(&out[start..], dims)?;
|
||||||
}
|
}
|
||||||
Ok(offsets)
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
|
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
|
||||||
@@ -396,15 +538,17 @@ const CHUNK_KEY_OFFSET_SIZE: u8 = 8;
|
|||||||
/// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
|
/// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
|
||||||
const MAX_CHUNK_BTREE_DEPTH: usize = 64;
|
const MAX_CHUNK_BTREE_DEPTH: usize = 64;
|
||||||
|
|
||||||
fn collect_chunk_info_inner(
|
/// Parse the v1 B-tree chunk index node at `btree_address` and its
|
||||||
|
/// subtree, appending its chunks to `stored` in tree order.
|
||||||
|
fn parse_chunk_node(
|
||||||
file_data: &[u8],
|
file_data: &[u8],
|
||||||
btree_address: u64,
|
btree_address: u64,
|
||||||
ndims: usize,
|
ndims: usize,
|
||||||
chunk_dimensions: Option<&[u32]>,
|
chunk_dimensions: Option<&[u32]>,
|
||||||
offset_size: u8,
|
offset_size: u8,
|
||||||
_length_size: u8,
|
|
||||||
depth: usize,
|
depth: usize,
|
||||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
stored: &mut Vec<ChunkInfo>,
|
||||||
|
) -> Result<ChunkNode, FormatError> {
|
||||||
if depth > MAX_CHUNK_BTREE_DEPTH {
|
if depth > MAX_CHUNK_BTREE_DEPTH {
|
||||||
return Err(FormatError::NestingDepthExceeded);
|
return Err(FormatError::NestingDepthExceeded);
|
||||||
}
|
}
|
||||||
@@ -439,15 +583,14 @@ fn collect_chunk_info_inner(
|
|||||||
.and_then(|n| n.checked_add(8))
|
.and_then(|n| n.checked_add(8))
|
||||||
.ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
|
.ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
|
||||||
|
|
||||||
if node_level == 0 {
|
|
||||||
// Leaf node: keys and children interleaved
|
|
||||||
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
|
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
|
||||||
let needed = entries_used * (key_size + os) + key_size;
|
let needed = entries_used * (key_size + os) + key_size;
|
||||||
ensure_len(file_data, pos, needed)?;
|
ensure_len(file_data, pos, needed)?;
|
||||||
|
|
||||||
let mut chunks = Vec::with_capacity(entries_used);
|
let mut keys = Vec::with_capacity((entries_used + 1) * ndims);
|
||||||
|
let mut chunks = Vec::new();
|
||||||
|
let mut child_addrs = Vec::new();
|
||||||
for _ in 0..entries_used {
|
for _ in 0..entries_used {
|
||||||
// Parse key
|
|
||||||
let chunk_size = u32::from_le_bytes([
|
let chunk_size = u32::from_le_bytes([
|
||||||
file_data[pos],
|
file_data[pos],
|
||||||
file_data[pos + 1],
|
file_data[pos + 1],
|
||||||
@@ -460,64 +603,48 @@ fn collect_chunk_info_inner(
|
|||||||
file_data[pos + 6],
|
file_data[pos + 6],
|
||||||
file_data[pos + 7],
|
file_data[pos + 7],
|
||||||
]);
|
]);
|
||||||
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
let k = keys.len();
|
||||||
// A chunk's key carries 0 in the element-size dimension. libhdf5
|
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
|
||||||
// compares that coordinate too when it looks a chunk up
|
|
||||||
// (`H5D__btree_found`), so whether it finds a chunk keyed
|
|
||||||
// otherwise depends on where the key falls; in `cve-2025-44905`
|
|
||||||
// `/Shuffle_float_data_le` (offset 4096) it does not, and h5py
|
|
||||||
// reads fill values there. Such a key is refused here.
|
|
||||||
if chunk_dimensions.is_some() && offsets.last().is_some_and(|&o| o != 0) {
|
|
||||||
return Err(FormatError::ChunkedReadError(format!(
|
|
||||||
"chunk key {offsets:?} has a non-zero element offset"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
pos += key_size;
|
pos += key_size;
|
||||||
|
|
||||||
// Parse child address
|
|
||||||
let address = read_offset(file_data, pos, offset_size)?;
|
let address = read_offset(file_data, pos, offset_size)?;
|
||||||
pos += os;
|
pos += os;
|
||||||
|
if node_level == 0 {
|
||||||
chunks.push(ChunkInfo {
|
chunks.push(stored.len());
|
||||||
|
stored.push(ChunkInfo {
|
||||||
chunk_size,
|
chunk_size,
|
||||||
filter_mask,
|
filter_mask,
|
||||||
offsets,
|
offsets: keys[k..].to_vec(),
|
||||||
address,
|
address,
|
||||||
});
|
});
|
||||||
|
} else {
|
||||||
|
child_addrs.push(address);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// The final key only bounds the node; libhdf5 still checks it.
|
// The final key only bounds the node; libhdf5 still checks it.
|
||||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
|
||||||
Ok(chunks)
|
|
||||||
|
let children = if node_level == 0 {
|
||||||
|
ChunkChildren::Chunks(chunks)
|
||||||
} else {
|
} else {
|
||||||
// Internal node: recurse into children
|
let mut nodes = Vec::with_capacity(child_addrs.len());
|
||||||
let needed = entries_used * (key_size + os) + key_size;
|
|
||||||
ensure_len(file_data, pos, needed)?;
|
|
||||||
|
|
||||||
let mut child_addrs = Vec::with_capacity(entries_used);
|
|
||||||
for _ in 0..entries_used {
|
|
||||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
|
||||||
pos += key_size;
|
|
||||||
let child_addr = read_offset(file_data, pos, offset_size)?;
|
|
||||||
child_addrs.push(child_addr);
|
|
||||||
pos += os;
|
|
||||||
}
|
|
||||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
|
||||||
|
|
||||||
let mut all_chunks = Vec::new();
|
|
||||||
for child_addr in child_addrs {
|
for child_addr in child_addrs {
|
||||||
let child_chunks = collect_chunk_info_inner(
|
nodes.push(parse_chunk_node(
|
||||||
file_data,
|
file_data,
|
||||||
child_addr,
|
child_addr,
|
||||||
ndims,
|
ndims,
|
||||||
chunk_dimensions,
|
chunk_dimensions,
|
||||||
offset_size,
|
offset_size,
|
||||||
_length_size,
|
|
||||||
depth + 1,
|
depth + 1,
|
||||||
)?;
|
stored,
|
||||||
all_chunks.extend(child_chunks);
|
)?);
|
||||||
}
|
|
||||||
Ok(all_chunks)
|
|
||||||
}
|
}
|
||||||
|
ChunkChildren::Nodes(nodes)
|
||||||
|
};
|
||||||
|
Ok(ChunkNode {
|
||||||
|
ndims,
|
||||||
|
keys,
|
||||||
|
children,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generate ChunkInfo entries for an implicit index (v4 index type 2).
|
/// Generate ChunkInfo entries for an implicit index (v4 index type 2).
|
||||||
@@ -1754,6 +1881,25 @@ mod tests {
|
|||||||
|
|
||||||
/// Build a B-tree v1 type 1 leaf node with given chunk infos.
|
/// Build a B-tree v1 type 1 leaf node with given chunk infos.
|
||||||
fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec<u8> {
|
fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec<u8> {
|
||||||
|
build_chunk_btree_leaf_to(chunks, &vec![0; ndims], offset_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A leaf whose final key is the one libhdf5 writes past the last chunk:
|
||||||
|
/// each coordinate of the last chunk plus its chunk dimension (the
|
||||||
|
/// element size last).
|
||||||
|
fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u32], offset_size: u8) -> Vec<u8> {
|
||||||
|
let last = &chunks.last().expect("a chunk").offsets;
|
||||||
|
let end: Vec<u64> = dims
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(d, &c)| last.get(d).copied().unwrap_or(0) + u64::from(c))
|
||||||
|
.collect();
|
||||||
|
build_chunk_btree_leaf_to(chunks, &end, offset_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A leaf holding `chunks`, with final key `end`.
|
||||||
|
fn build_chunk_btree_leaf_to(chunks: &[ChunkInfo], end: &[u64], offset_size: u8) -> Vec<u8> {
|
||||||
|
let ndims = end.len();
|
||||||
let _os = offset_size as usize;
|
let _os = offset_size as usize;
|
||||||
let entries_used = chunks.len() as u16;
|
let entries_used = chunks.len() as u16;
|
||||||
let mut buf = Vec::new();
|
let mut buf = Vec::new();
|
||||||
@@ -1795,8 +1941,8 @@ mod tests {
|
|||||||
// checks; 0 always is)
|
// checks; 0 always is)
|
||||||
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
|
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
|
||||||
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
|
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
|
||||||
for _ in 0..ndims {
|
for &e in end {
|
||||||
write_offset(&mut buf, 0, 8);
|
write_offset(&mut buf, e, 8);
|
||||||
}
|
}
|
||||||
|
|
||||||
buf
|
buf
|
||||||
@@ -1812,8 +1958,11 @@ mod tests {
|
|||||||
offsets,
|
offsets,
|
||||||
address,
|
address,
|
||||||
};
|
};
|
||||||
let good =
|
let good = build_chunk_btree_leaf_dims(
|
||||||
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
|
&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)],
|
||||||
|
&[10, 8],
|
||||||
|
8,
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
|
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -2044,7 +2193,6 @@ mod tests {
|
|||||||
) -> (Vec<u8>, DataLayout, Dataspace) {
|
) -> (Vec<u8>, DataLayout, Dataspace) {
|
||||||
let os: u8 = 8;
|
let os: u8 = 8;
|
||||||
let elem_size = 8usize;
|
let elem_size = 8usize;
|
||||||
let ndims = 2; // rank(1) + 1
|
|
||||||
let total = values.len();
|
let total = values.len();
|
||||||
|
|
||||||
// Place chunk data starting at offset 0x2000
|
// Place chunk data starting at offset 0x2000
|
||||||
@@ -2075,12 +2223,13 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Build B-tree at offset 0x100
|
// Build B-tree at offset 0x100
|
||||||
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
|
let dims = [chunk_size_elems as u32, elem_size as u32];
|
||||||
|
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
|
||||||
let btree_addr = 0x100usize;
|
let btree_addr = 0x100usize;
|
||||||
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
||||||
|
|
||||||
let layout = DataLayout::Chunked {
|
let layout = DataLayout::Chunked {
|
||||||
chunk_dimensions: vec![chunk_size_elems as u32, elem_size as u32],
|
chunk_dimensions: dims.to_vec(),
|
||||||
btree_address: Some(btree_addr as u64),
|
btree_address: Some(btree_addr as u64),
|
||||||
version: 3,
|
version: 3,
|
||||||
chunk_index_type: None,
|
chunk_index_type: None,
|
||||||
@@ -2218,7 +2367,6 @@ mod tests {
|
|||||||
|
|
||||||
let os: u8 = 8;
|
let os: u8 = 8;
|
||||||
let elem_size = 8usize;
|
let elem_size = 8usize;
|
||||||
let ndims = 2;
|
|
||||||
let chunk_elems = 10usize;
|
let chunk_elems = 10usize;
|
||||||
let total = 20usize;
|
let total = 20usize;
|
||||||
|
|
||||||
@@ -2257,12 +2405,13 @@ mod tests {
|
|||||||
data_offset += compressed.len() + 16; // some padding
|
data_offset += compressed.len() + 16; // some padding
|
||||||
}
|
}
|
||||||
|
|
||||||
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
|
let dims = [chunk_elems as u32, elem_size as u32];
|
||||||
|
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
|
||||||
let btree_addr = 0x100usize;
|
let btree_addr = 0x100usize;
|
||||||
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
||||||
|
|
||||||
let layout = DataLayout::Chunked {
|
let layout = DataLayout::Chunked {
|
||||||
chunk_dimensions: vec![chunk_elems as u32, elem_size as u32],
|
chunk_dimensions: dims.to_vec(),
|
||||||
btree_address: Some(btree_addr as u64),
|
btree_address: Some(btree_addr as u64),
|
||||||
version: 3,
|
version: 3,
|
||||||
chunk_index_type: None,
|
chunk_index_type: None,
|
||||||
@@ -2300,7 +2449,6 @@ mod tests {
|
|||||||
// 4x6 dataset with chunk size 2x3 => 4 chunks
|
// 4x6 dataset with chunk size 2x3 => 4 chunks
|
||||||
let os: u8 = 8;
|
let os: u8 = 8;
|
||||||
let elem_size = 4usize; // f32
|
let elem_size = 4usize; // f32
|
||||||
let ndims = 3; // rank(2) + 1
|
|
||||||
let ds_dims = [4usize, 6];
|
let ds_dims = [4usize, 6];
|
||||||
let chunk_dims = [2usize, 3];
|
let chunk_dims = [2usize, 3];
|
||||||
|
|
||||||
@@ -2340,12 +2488,13 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
|
let dims = [chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32];
|
||||||
|
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
|
||||||
let btree_addr = 0x100usize;
|
let btree_addr = 0x100usize;
|
||||||
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
||||||
|
|
||||||
let layout = DataLayout::Chunked {
|
let layout = DataLayout::Chunked {
|
||||||
chunk_dimensions: vec![chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32],
|
chunk_dimensions: dims.to_vec(),
|
||||||
btree_address: Some(btree_addr as u64),
|
btree_address: Some(btree_addr as u64),
|
||||||
version: 3,
|
version: 3,
|
||||||
chunk_index_type: None,
|
chunk_index_type: None,
|
||||||
|
|||||||
@@ -629,17 +629,13 @@ save("mdc_past_eof", bad)
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Chunk index entries HDF5 2.0 mis-reads, refused here. An unfiltered
|
/// An unfiltered chunk the index records at less than the chunk's size
|
||||||
/// chunk the index records at less than the chunk's size (`cve-2025-44904`):
|
/// (`cve-2025-44904`): HDF5 2.0 fills the rest of the chunk with whatever
|
||||||
/// HDF5 2.0 fills the rest of the chunk with whatever its buffer held, and
|
/// its buffer held, and later libhdf5 releases refuse it ("incorrect chunk
|
||||||
/// later libhdf5 releases refuse it ("incorrect chunk size returned from
|
/// size returned from index for unfiltered chunk"); we used to read the
|
||||||
/// index for unfiltered chunk"); we read the rest as zeros. A chunk keyed
|
/// rest as zeros, and now refuse it.
|
||||||
/// with a non-zero element offset (`cve-2025-44905`): libhdf5's lookup
|
|
||||||
/// compares that coordinate too, so whether it finds the chunk depends on
|
|
||||||
/// where the key falls (in `cve-2025-44905` it does not, and h5py reads
|
|
||||||
/// fill values; in this file it does); we read the chunk.
|
|
||||||
#[test]
|
#[test]
|
||||||
fn chunk_index_entries_libhdf5_misreads_are_refused() {
|
fn unfiltered_chunk_of_the_wrong_size_is_refused() {
|
||||||
skip_if_no_python!();
|
skip_if_no_python!();
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
run_python(
|
run_python(
|
||||||
@@ -658,8 +654,6 @@ second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address
|
|||||||
assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0)
|
assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0)
|
||||||
bad = bytearray(data); struct.pack_into("<I", bad, second, 100)
|
bad = bytearray(data); struct.pack_into("<I", bad, second, 100)
|
||||||
open(os.path.join(d, "short_chunk.h5"), "wb").write(bad)
|
open(os.path.join(d, "short_chunk.h5"), "wb").write(bad)
|
||||||
bad = bytearray(data); struct.pack_into("<Q", bad, second + 16, 4096)
|
|
||||||
open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
|
|
||||||
"#,
|
"#,
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -667,8 +661,58 @@ open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
|
|||||||
Ok(()),
|
Ok(()),
|
||||||
"good"
|
"good"
|
||||||
);
|
);
|
||||||
for name in ["short_chunk", "element_offset"] {
|
let err = clawhdf5_reads(&dir.path().join("short_chunk.h5"), "d").unwrap_err();
|
||||||
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err();
|
assert!(err.starts_with("read:"), "short_chunk: {err}");
|
||||||
assert!(err.starts_with("read:"), "{name}: {err}");
|
}
|
||||||
|
|
||||||
|
/// A v1 B-tree chunk key whose element-size coordinate is not 0. libhdf5
|
||||||
|
/// looks each chunk up with that coordinate set to 0 (`H5B_find`,
|
||||||
|
/// `H5D__btree_cmp3`, `H5D__btree_found`): in a 1-D dataset it still finds
|
||||||
|
/// the chunk, in a 2-D one it does not and reads fill values
|
||||||
|
/// (`cve-2025-44905` `/Shuffle_float_data_le`). Both must read exactly what
|
||||||
|
/// h5py reads: the 1-D file was refused, and before that the 2-D chunk was
|
||||||
|
/// read as if its key were well formed.
|
||||||
|
#[test]
|
||||||
|
fn chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them() {
|
||||||
|
skip_if_no_python!();
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
run_python(
|
||||||
|
dir.path(),
|
||||||
|
r#"
|
||||||
|
def corrupt(name, shape, chunks, which, element_offset):
|
||||||
|
path = os.path.join(d, name + ".h5")
|
||||||
|
with h5py.File(path, "w", libver="earliest") as f:
|
||||||
|
f.create_dataset("d", data=np.arange(np.prod(shape), dtype="<i4").reshape(shape),
|
||||||
|
chunks=chunks, fillvalue=-1)
|
||||||
|
data = bytearray(open(path, "rb").read())
|
||||||
|
tree = data.find(b"TREE")
|
||||||
|
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
|
||||||
|
tree = data.find(b"TREE", tree + 1)
|
||||||
|
assert tree > 0 and data[tree + 5] == 0
|
||||||
|
ndims = len(shape) + 1
|
||||||
|
key_size = 8 + 8 * ndims
|
||||||
|
key = tree + 8 + 16 + which * (key_size + 8)
|
||||||
|
last = key + 8 + 8 * (ndims - 1)
|
||||||
|
assert struct.unpack_from("<Q", data, last) == (0,)
|
||||||
|
struct.pack_into("<Q", data, last, element_offset)
|
||||||
|
open(path, "wb").write(data)
|
||||||
|
with h5py.File(path, "r") as f:
|
||||||
|
open(path + ".txt", "w").write(" ".join(map(str, f["d"][()].ravel())))
|
||||||
|
|
||||||
|
corrupt("one_d", (100,), (37,), 1, 4096)
|
||||||
|
corrupt("two_d", (4, 6), (2, 3), 1, 4)
|
||||||
|
corrupt("two_d_first", (4, 6), (2, 3), 0, 8)
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
for name in ["one_d", "two_d", "two_d_first"] {
|
||||||
|
let path = dir.path().join(format!("{name}.h5"));
|
||||||
|
let expected: Vec<i32> = std::fs::read_to_string(dir.path().join(format!("{name}.h5.txt")))
|
||||||
|
.unwrap()
|
||||||
|
.split_whitespace()
|
||||||
|
.map(|v| v.parse().unwrap())
|
||||||
|
.collect();
|
||||||
|
let file = File::open(&path).unwrap();
|
||||||
|
let got = file.dataset("d").unwrap().read_i32();
|
||||||
|
assert_eq!(got.as_deref().ok(), Some(&expected[..]), "{name}: {got:?}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user