diff --git a/CHANGELOG.md b/CHANGELOG.md index a658a30..e4ea2a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,12 @@ Covered now by interop tests at 4, 37, 400, 5 000 and 200 000 chunks (the last large enough for paged data blocks), plus sparse, gzip-filtered and 2-D cases. Writing is unaffected; this is a read-path bug. +- `clawhdf5-format`: the sibling Fixed Array index (fixed dimensions written + with `libver='latest'`) was checked against the same range and is correct, + including paged data blocks and sparse datasets — it really does keep its + page-init bitmap in the data block, where the Extensible Array does not. + It had no real-file coverage above the inline sizes either, so it now has + the same tests. ### Security - `clawhdf5-format`: **a crafted file could crash any reader through B-tree v2 diff --git a/crates/clawhdf5/tests/h5py_interop_tests.rs b/crates/clawhdf5/tests/h5py_interop_tests.rs index ca458ab..7082a06 100644 --- a/crates/clawhdf5/tests/h5py_interop_tests.rs +++ b/crates/clawhdf5/tests/h5py_interop_tests.rs @@ -1241,3 +1241,80 @@ with h5py.File("{two_d_str}", "w", libver="latest") as f: 0 ); } + +#[test] +fn h5py_fixed_array_chunk_index_clawhdf5_reads() { + // Fixed dimensions plus libver='latest' give a Fixed Array chunk index. + // Its data blocks are paged above 2^page_bits elements (1024 by default), + // and unlike the Extensible Array it keeps the page-init bitmap in the + // data block itself — a difference worth pinning down, since assuming + // otherwise is exactly what made the Extensible Array reader wrong. The + // sparse case leaves whole pages uninitialised so the bitmap is actually + // consulted rather than being all ones. + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + + for n in [100usize, 5_000, 200_000] { + let path = dir.path().join(format!("fa_{n}.h5")); + let path_str = path.display().to_string(); + run_python(&format!( + r#" +import h5py, numpy as np +with h5py.File("{path_str}", "w", libver="latest") as f: + d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4") + d[...] = np.arange({n}, dtype="i4") +"# + )); + let bytes = std::fs::read(&path).unwrap(); + assert!( + bytes.windows(4).any(|w| w == b"FAHD"), + "n={n}: fixture is not indexed by a Fixed Array" + ); + let values = File::open(&path) + .unwrap() + .dataset("x") + .unwrap() + .read_i32() + .unwrap(); + assert_eq!(values.len(), n, "n={n}"); + let wrong = values + .iter() + .enumerate() + .filter(|&(i, &v)| v != i as i32) + .count(); + assert_eq!(wrong, 0, "n={n}: {wrong} elements read back wrong"); + } + + let sparse = dir.path().join("fa_sparse.h5"); + let sparse_str = sparse.display().to_string(); + let (n, step) = (200_000usize, 997usize); + run_python(&format!( + r#" +import h5py +with h5py.File("{sparse_str}", "w", libver="latest") as f: + d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4", fillvalue=-1) + for i in list(range(0, {n}, {step})) + list(range(0, 40)): + d[i] = i +"# + )); + let values = File::open(&sparse) + .unwrap() + .dataset("x") + .unwrap() + .read_i32() + .unwrap(); + assert_eq!(values.len(), n); + let wrong = values + .iter() + .enumerate() + .filter(|&(i, &v)| { + let expected = if i % step == 0 || i < 40 { + i as i32 + } else { + -1 + }; + v != expected + }) + .count(); + assert_eq!(wrong, 0, "sparse: {wrong} of {n} elements read back wrong"); +}