test: cover Fixed Array chunk indexes against real files

The Extensible Array bug reached a release because no fixture had more
chunks than fit inline, so its data blocks were never read. The Fixed
Array index had the same blind spot: nothing exercised it above a
handful of chunks, and nothing reached the paged layout at all.

Checked at 100, 5 000 and 200 000 chunks plus a sparse dataset that
leaves whole pages uninitialised. It is correct throughout — it does
keep its page-init bitmap inside the data block, which is the
difference from the Extensible Array that made assuming otherwise a
bug. Adding the tests so that stays true.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-20 17:20:12 -07:00
co-authored by Claude Opus 5
parent 367faad7f7
commit eb196e824f
2 changed files with 83 additions and 0 deletions
+6
View File
@@ -27,6 +27,12 @@
Covered now by interop tests at 4, 37, 400, 5 000 and 200 000 chunks (the Covered now by interop tests at 4, 37, 400, 5 000 and 200 000 chunks (the
last large enough for paged data blocks), plus sparse, gzip-filtered and last large enough for paged data blocks), plus sparse, gzip-filtered and
2-D cases. Writing is unaffected; this is a read-path bug. 2-D cases. Writing is unaffected; this is a read-path bug.
- `clawhdf5-format`: the sibling Fixed Array index (fixed dimensions written
with `libver='latest'`) was checked against the same range and is correct,
including paged data blocks and sparse datasets — it really does keep its
page-init bitmap in the data block, where the Extensible Array does not.
It had no real-file coverage above the inline sizes either, so it now has
the same tests.
### Security ### Security
- `clawhdf5-format`: **a crafted file could crash any reader through B-tree v2 - `clawhdf5-format`: **a crafted file could crash any reader through B-tree v2
@@ -1241,3 +1241,80 @@ with h5py.File("{two_d_str}", "w", libver="latest") as f:
0 0
); );
} }
#[test]
fn h5py_fixed_array_chunk_index_clawhdf5_reads() {
// Fixed dimensions plus libver='latest' give a Fixed Array chunk index.
// Its data blocks are paged above 2^page_bits elements (1024 by default),
// and unlike the Extensible Array it keeps the page-init bitmap in the
// data block itself — a difference worth pinning down, since assuming
// otherwise is exactly what made the Extensible Array reader wrong. The
// sparse case leaves whole pages uninitialised so the bitmap is actually
// consulted rather than being all ones.
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
for n in [100usize, 5_000, 200_000] {
let path = dir.path().join(format!("fa_{n}.h5"));
let path_str = path.display().to_string();
run_python(&format!(
r#"
import h5py, numpy as np
with h5py.File("{path_str}", "w", libver="latest") as f:
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4")
d[...] = np.arange({n}, dtype="i4")
"#
));
let bytes = std::fs::read(&path).unwrap();
assert!(
bytes.windows(4).any(|w| w == b"FAHD"),
"n={n}: fixture is not indexed by a Fixed Array"
);
let values = File::open(&path)
.unwrap()
.dataset("x")
.unwrap()
.read_i32()
.unwrap();
assert_eq!(values.len(), n, "n={n}");
let wrong = values
.iter()
.enumerate()
.filter(|&(i, &v)| v != i as i32)
.count();
assert_eq!(wrong, 0, "n={n}: {wrong} elements read back wrong");
}
let sparse = dir.path().join("fa_sparse.h5");
let sparse_str = sparse.display().to_string();
let (n, step) = (200_000usize, 997usize);
run_python(&format!(
r#"
import h5py
with h5py.File("{sparse_str}", "w", libver="latest") as f:
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4", fillvalue=-1)
for i in list(range(0, {n}, {step})) + list(range(0, 40)):
d[i] = i
"#
));
let values = File::open(&sparse)
.unwrap()
.dataset("x")
.unwrap()
.read_i32()
.unwrap();
assert_eq!(values.len(), n);
let wrong = values
.iter()
.enumerate()
.filter(|&(i, &v)| {
let expected = if i % step == 0 || i < 40 {
i as i32
} else {
-1
};
v != expected
})
.count();
assert_eq!(wrong, 0, "sparse: {wrong} of {n} elements read back wrong");
}