fix(format): read a v1 chunk B-tree where libhdf5's lookup finds chunks

libhdf5 does not walk the chunk B-tree to read a dataset: it looks each
chunk up (H5B_find with H5D__btree_cmp3 and H5D__btree_found), asking for
the element-size coordinate as 0. collect_chunk_info_checked now parses
the tree with its keys and returns each stored chunk only when that
lookup, replayed over the scaled keys, finds it.

A key with a non-zero element-size coordinate is therefore found in a
1-D dataset (cmp3 compares only the first coordinate there, and found
compares with <=) and missed in a dataset of rank 2 or more, which reads
fill values. The previous commit refused every such key, which refused
1-D files libhdf5 reads correctly; before that, the rank-2 case read the
chunk's data where h5py reads fill values (cve-2025-44905
/Shuffle_float_data_le, now identical to h5py, so it leaves the
conformance report's list of libhdf5 bugs).

Test: chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them
compares 1-D and 2-D files against h5py's values. It fails on the
previous commit (the 1-D file is refused) and with the refusal removed
(the 2-D file reads 0..23 where h5py reads fill values).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 11:35:13 -05:00
co-authored by Claude Opus 5.5
parent 378afa1584
commit 742ed4dfb8
4 changed files with 322 additions and 125 deletions
@@ -629,17 +629,13 @@ save("mdc_past_eof", bad)
);
}
/// Chunk index entries HDF5 2.0 mis-reads, refused here. An unfiltered
/// chunk the index records at less than the chunk's size (`cve-2025-44904`):
/// HDF5 2.0 fills the rest of the chunk with whatever its buffer held, and
/// later libhdf5 releases refuse it ("incorrect chunk size returned from
/// index for unfiltered chunk"); we read the rest as zeros. A chunk keyed
/// with a non-zero element offset (`cve-2025-44905`): libhdf5's lookup
/// compares that coordinate too, so whether it finds the chunk depends on
/// where the key falls (in `cve-2025-44905` it does not, and h5py reads
/// fill values; in this file it does); we read the chunk.
/// An unfiltered chunk the index records at less than the chunk's size
/// (`cve-2025-44904`): HDF5 2.0 fills the rest of the chunk with whatever
/// its buffer held, and later libhdf5 releases refuse it ("incorrect chunk
/// size returned from index for unfiltered chunk"); we used to read the
/// rest as zeros, and now refuse it.
#[test]
fn chunk_index_entries_libhdf5_misreads_are_refused() {
fn unfiltered_chunk_of_the_wrong_size_is_refused() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
@@ -658,8 +654,6 @@ second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address
assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0)
bad = bytearray(data); struct.pack_into("<I", bad, second, 100)
open(os.path.join(d, "short_chunk.h5"), "wb").write(bad)
bad = bytearray(data); struct.pack_into("<Q", bad, second + 16, 4096)
open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
"#,
);
assert_eq!(
@@ -667,8 +661,58 @@ open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
Ok(()),
"good"
);
for name in ["short_chunk", "element_offset"] {
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err();
assert!(err.starts_with("read:"), "{name}: {err}");
let err = clawhdf5_reads(&dir.path().join("short_chunk.h5"), "d").unwrap_err();
assert!(err.starts_with("read:"), "short_chunk: {err}");
}
/// A v1 B-tree chunk key whose element-size coordinate is not 0. libhdf5
/// looks each chunk up with that coordinate set to 0 (`H5B_find`,
/// `H5D__btree_cmp3`, `H5D__btree_found`): in a 1-D dataset it still finds
/// the chunk, in a 2-D one it does not and reads fill values
/// (`cve-2025-44905` `/Shuffle_float_data_le`). Both must read exactly what
/// h5py reads: the 1-D file was refused, and before that the 2-D chunk was
/// read as if its key were well formed.
#[test]
fn chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
def corrupt(name, shape, chunks, which, element_offset):
path = os.path.join(d, name + ".h5")
with h5py.File(path, "w", libver="earliest") as f:
f.create_dataset("d", data=np.arange(np.prod(shape), dtype="<i4").reshape(shape),
chunks=chunks, fillvalue=-1)
data = bytearray(open(path, "rb").read())
tree = data.find(b"TREE")
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
tree = data.find(b"TREE", tree + 1)
assert tree > 0 and data[tree + 5] == 0
ndims = len(shape) + 1
key_size = 8 + 8 * ndims
key = tree + 8 + 16 + which * (key_size + 8)
last = key + 8 + 8 * (ndims - 1)
assert struct.unpack_from("<Q", data, last) == (0,)
struct.pack_into("<Q", data, last, element_offset)
open(path, "wb").write(data)
with h5py.File(path, "r") as f:
open(path + ".txt", "w").write(" ".join(map(str, f["d"][()].ravel())))
corrupt("one_d", (100,), (37,), 1, 4096)
corrupt("two_d", (4, 6), (2, 3), 1, 4)
corrupt("two_d_first", (4, 6), (2, 3), 0, 8)
"#,
);
for name in ["one_d", "two_d", "two_d_first"] {
let path = dir.path().join(format!("{name}.h5"));
let expected: Vec<i32> = std::fs::read_to_string(dir.path().join(format!("{name}.h5.txt")))
.unwrap()
.split_whitespace()
.map(|v| v.parse().unwrap())
.collect();
let file = File::open(&path).unwrap();
let got = file.dataset("d").unwrap().read_i32();
assert_eq!(got.as_deref().ok(), Some(&expected[..]), "{name}: {got:?}");
}
}