fix(format): read layout v4 chunk dimensions of any width from 1 to 8 bytes

A version-4 layout stores every chunk dimension in the fewest bytes that
hold the largest one (H5D__chunk_set_sizes: (log2(dim) + 8) / 8), so a
chunk dimension of 65 536 to 16 777 215 takes 3 bytes. Only widths 1, 2,
4 and 8 were decoded; an h5py file with chunks=(70000,) and
libver='latest' failed with UnexpectedEof. Widths 1-8 are decoded now;
0 and more than 8 are refused with libhdf5's "encoded chunk dimension
size is too large", and a dimension past u32 is refused, not truncated.

The review asked for libhdf5's check that the stored width matches the
one computed from the dimensions. HDF5 2.0.0 (h5py 3.16) refuses any
mismatch, but HDFGroup/hdf5@e124c36 ("Allow reading of files with chunk
dimensions encoded using more bytes than necessary", 2026-06-05) relaxed
it to refusing only a width too small for the dimensions, which cannot
happen once the dimensions have been decoded from that width. Follow
current libhdf5: a wider-than-needed encoding is read. clawhdf5's own
writer produces such layouts (the next commit fixes that).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:25:43 -05:00
co-authored by Claude Opus 5.5
parent 9238605661
commit f713847e65
3 changed files with 114 additions and 34 deletions
@@ -343,3 +343,41 @@ print("OK")
let want: Vec<u8> = line[990..].iter().flat_map(|v| v.to_le_bytes()).collect();
assert_eq!(tail, want);
}
// ---------------------------------------------------------------------------
// Chunk dimension widths in layout version 4
// ---------------------------------------------------------------------------
/// A version-4 layout stores every chunk dimension in the fewest bytes that
/// hold the largest one (the element size included). A chunk dimension of
/// 70 000 takes 3 bytes and 2^32 + 1 elements would take 5; widths other
/// than 1, 2, 4 and 8 were refused, so these h5py files did not open.
#[test]
fn h5py_layout_v4_chunk_dimensions_of_3_bytes_read() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("enc3.h5");
let p = path.display().to_string();
run_python(&format!(
r#"
import h5py, numpy as np
with h5py.File("{p}", "w", libver="latest") as f:
f.create_dataset("single", data=np.arange(70000, dtype="<u4"), chunks=(70000,))
ds = f.create_dataset("ea", shape=(10,), maxshape=(None,), chunks=(70000,), dtype="<f8")
ds[:] = np.arange(10.0)
f.create_dataset("fa", data=np.arange(3 * 70000, dtype="<i4").reshape(3, 70000),
chunks=(1, 70000), compression="gzip")
with h5py.File("{p}", "r") as f:
raw = open("{p}", "rb").read()
# version 4, class 2 (chunked), flags, 2 or 3 dims, then 3 bytes each
assert raw.find(bytes([4, 2, 0, 2, 3])) > 0 or raw.find(bytes([4, 2, 1, 2, 3])) > 0
"#
));
let file = File::open(&path).unwrap();
let single = file.dataset("single").unwrap().read_u64().unwrap();
assert!(single.iter().copied().eq(0..70000), "single");
let ea = file.dataset("ea").unwrap().read_f64().unwrap();
assert_eq!(ea, (0..10).map(f64::from).collect::<Vec<_>>());
let fa = file.dataset("fa").unwrap().read_i32().unwrap();
assert!(fa.iter().copied().eq(0..3 * 70000), "fa");
}