fix(format): read a v1 chunk B-tree where libhdf5's lookup finds chunks
libhdf5 does not walk the chunk B-tree to read a dataset: it looks each chunk up (H5B_find with H5D__btree_cmp3 and H5D__btree_found), asking for the element-size coordinate as 0. collect_chunk_info_checked now parses the tree with its keys and returns each stored chunk only when that lookup, replayed over the scaled keys, finds it. A key with a non-zero element-size coordinate is therefore found in a 1-D dataset (cmp3 compares only the first coordinate there, and found compares with <=) and missed in a dataset of rank 2 or more, which reads fill values. The previous commit refused every such key, which refused 1-D files libhdf5 reads correctly; before that, the rank-2 case read the chunk's data where h5py reads fill values (cve-2025-44905 /Shuffle_float_data_le, now identical to h5py, so it leaves the conformance report's list of libhdf5 bugs). Test: chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them compares 1-D and 2-D files against h5py's values. It fails on the previous commit (the 1-D file is refused) and with the refusal removed (the 2-D file reads 0..23 where h5py reads fill values). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -629,17 +629,13 @@ save("mdc_past_eof", bad)
|
||||
);
|
||||
}
|
||||
|
||||
/// Chunk index entries HDF5 2.0 mis-reads, refused here. An unfiltered
|
||||
/// chunk the index records at less than the chunk's size (`cve-2025-44904`):
|
||||
/// HDF5 2.0 fills the rest of the chunk with whatever its buffer held, and
|
||||
/// later libhdf5 releases refuse it ("incorrect chunk size returned from
|
||||
/// index for unfiltered chunk"); we read the rest as zeros. A chunk keyed
|
||||
/// with a non-zero element offset (`cve-2025-44905`): libhdf5's lookup
|
||||
/// compares that coordinate too, so whether it finds the chunk depends on
|
||||
/// where the key falls (in `cve-2025-44905` it does not, and h5py reads
|
||||
/// fill values; in this file it does); we read the chunk.
|
||||
/// An unfiltered chunk the index records at less than the chunk's size
|
||||
/// (`cve-2025-44904`): HDF5 2.0 fills the rest of the chunk with whatever
|
||||
/// its buffer held, and later libhdf5 releases refuse it ("incorrect chunk
|
||||
/// size returned from index for unfiltered chunk"); we used to read the
|
||||
/// rest as zeros, and now refuse it.
|
||||
#[test]
|
||||
fn chunk_index_entries_libhdf5_misreads_are_refused() {
|
||||
fn unfiltered_chunk_of_the_wrong_size_is_refused() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
run_python(
|
||||
@@ -658,8 +654,6 @@ second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address
|
||||
assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0)
|
||||
bad = bytearray(data); struct.pack_into("<I", bad, second, 100)
|
||||
open(os.path.join(d, "short_chunk.h5"), "wb").write(bad)
|
||||
bad = bytearray(data); struct.pack_into("<Q", bad, second + 16, 4096)
|
||||
open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
|
||||
"#,
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -667,8 +661,58 @@ open(os.path.join(d, "element_offset.h5"), "wb").write(bad)
|
||||
Ok(()),
|
||||
"good"
|
||||
);
|
||||
for name in ["short_chunk", "element_offset"] {
|
||||
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err();
|
||||
assert!(err.starts_with("read:"), "{name}: {err}");
|
||||
let err = clawhdf5_reads(&dir.path().join("short_chunk.h5"), "d").unwrap_err();
|
||||
assert!(err.starts_with("read:"), "short_chunk: {err}");
|
||||
}
|
||||
|
||||
/// A v1 B-tree chunk key whose element-size coordinate is not 0. libhdf5
|
||||
/// looks each chunk up with that coordinate set to 0 (`H5B_find`,
|
||||
/// `H5D__btree_cmp3`, `H5D__btree_found`): in a 1-D dataset it still finds
|
||||
/// the chunk, in a 2-D one it does not and reads fill values
|
||||
/// (`cve-2025-44905` `/Shuffle_float_data_le`). Both must read exactly what
|
||||
/// h5py reads: the 1-D file was refused, and before that the 2-D chunk was
|
||||
/// read as if its key were well formed.
|
||||
#[test]
|
||||
fn chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
run_python(
|
||||
dir.path(),
|
||||
r#"
|
||||
def corrupt(name, shape, chunks, which, element_offset):
|
||||
path = os.path.join(d, name + ".h5")
|
||||
with h5py.File(path, "w", libver="earliest") as f:
|
||||
f.create_dataset("d", data=np.arange(np.prod(shape), dtype="<i4").reshape(shape),
|
||||
chunks=chunks, fillvalue=-1)
|
||||
data = bytearray(open(path, "rb").read())
|
||||
tree = data.find(b"TREE")
|
||||
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
|
||||
tree = data.find(b"TREE", tree + 1)
|
||||
assert tree > 0 and data[tree + 5] == 0
|
||||
ndims = len(shape) + 1
|
||||
key_size = 8 + 8 * ndims
|
||||
key = tree + 8 + 16 + which * (key_size + 8)
|
||||
last = key + 8 + 8 * (ndims - 1)
|
||||
assert struct.unpack_from("<Q", data, last) == (0,)
|
||||
struct.pack_into("<Q", data, last, element_offset)
|
||||
open(path, "wb").write(data)
|
||||
with h5py.File(path, "r") as f:
|
||||
open(path + ".txt", "w").write(" ".join(map(str, f["d"][()].ravel())))
|
||||
|
||||
corrupt("one_d", (100,), (37,), 1, 4096)
|
||||
corrupt("two_d", (4, 6), (2, 3), 1, 4)
|
||||
corrupt("two_d_first", (4, 6), (2, 3), 0, 8)
|
||||
"#,
|
||||
);
|
||||
for name in ["one_d", "two_d", "two_d_first"] {
|
||||
let path = dir.path().join(format!("{name}.h5"));
|
||||
let expected: Vec<i32> = std::fs::read_to_string(dir.path().join(format!("{name}.h5.txt")))
|
||||
.unwrap()
|
||||
.split_whitespace()
|
||||
.map(|v| v.parse().unwrap())
|
||||
.collect();
|
||||
let file = File::open(&path).unwrap();
|
||||
let got = file.dataset("d").unwrap().read_i32();
|
||||
assert_eq!(got.as_deref().ok(), Some(&expected[..]), "{name}: {got:?}");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user