test: cover Fixed Array chunk indexes against real files
The Extensible Array bug reached a release because no fixture had more chunks than fit inline, so its data blocks were never read. The Fixed Array index had the same blind spot: nothing exercised it above a handful of chunks, and nothing reached the paged layout at all. Checked at 100, 5 000 and 200 000 chunks plus a sparse dataset that leaves whole pages uninitialised. It is correct throughout — it does keep its page-init bitmap inside the data block, which is the difference from the Extensible Array that made assuming otherwise a bug. Adding the tests so that stays true. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
@@ -27,6 +27,12 @@
|
|||||||
Covered now by interop tests at 4, 37, 400, 5 000 and 200 000 chunks (the
|
Covered now by interop tests at 4, 37, 400, 5 000 and 200 000 chunks (the
|
||||||
last large enough for paged data blocks), plus sparse, gzip-filtered and
|
last large enough for paged data blocks), plus sparse, gzip-filtered and
|
||||||
2-D cases. Writing is unaffected; this is a read-path bug.
|
2-D cases. Writing is unaffected; this is a read-path bug.
|
||||||
|
- `clawhdf5-format`: the sibling Fixed Array index (fixed dimensions written
|
||||||
|
with `libver='latest'`) was checked against the same range and is correct,
|
||||||
|
including paged data blocks and sparse datasets — it really does keep its
|
||||||
|
page-init bitmap in the data block, where the Extensible Array does not.
|
||||||
|
It had no real-file coverage above the inline sizes either, so it now has
|
||||||
|
the same tests.
|
||||||
|
|
||||||
### Security
|
### Security
|
||||||
- `clawhdf5-format`: **a crafted file could crash any reader through B-tree v2
|
- `clawhdf5-format`: **a crafted file could crash any reader through B-tree v2
|
||||||
|
|||||||
@@ -1241,3 +1241,80 @@ with h5py.File("{two_d_str}", "w", libver="latest") as f:
|
|||||||
0
|
0
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn h5py_fixed_array_chunk_index_clawhdf5_reads() {
|
||||||
|
// Fixed dimensions plus libver='latest' give a Fixed Array chunk index.
|
||||||
|
// Its data blocks are paged above 2^page_bits elements (1024 by default),
|
||||||
|
// and unlike the Extensible Array it keeps the page-init bitmap in the
|
||||||
|
// data block itself — a difference worth pinning down, since assuming
|
||||||
|
// otherwise is exactly what made the Extensible Array reader wrong. The
|
||||||
|
// sparse case leaves whole pages uninitialised so the bitmap is actually
|
||||||
|
// consulted rather than being all ones.
|
||||||
|
skip_if_no_python!();
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|
||||||
|
for n in [100usize, 5_000, 200_000] {
|
||||||
|
let path = dir.path().join(format!("fa_{n}.h5"));
|
||||||
|
let path_str = path.display().to_string();
|
||||||
|
run_python(&format!(
|
||||||
|
r#"
|
||||||
|
import h5py, numpy as np
|
||||||
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
||||||
|
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4")
|
||||||
|
d[...] = np.arange({n}, dtype="i4")
|
||||||
|
"#
|
||||||
|
));
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
assert!(
|
||||||
|
bytes.windows(4).any(|w| w == b"FAHD"),
|
||||||
|
"n={n}: fixture is not indexed by a Fixed Array"
|
||||||
|
);
|
||||||
|
let values = File::open(&path)
|
||||||
|
.unwrap()
|
||||||
|
.dataset("x")
|
||||||
|
.unwrap()
|
||||||
|
.read_i32()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(values.len(), n, "n={n}");
|
||||||
|
let wrong = values
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter(|&(i, &v)| v != i as i32)
|
||||||
|
.count();
|
||||||
|
assert_eq!(wrong, 0, "n={n}: {wrong} elements read back wrong");
|
||||||
|
}
|
||||||
|
|
||||||
|
let sparse = dir.path().join("fa_sparse.h5");
|
||||||
|
let sparse_str = sparse.display().to_string();
|
||||||
|
let (n, step) = (200_000usize, 997usize);
|
||||||
|
run_python(&format!(
|
||||||
|
r#"
|
||||||
|
import h5py
|
||||||
|
with h5py.File("{sparse_str}", "w", libver="latest") as f:
|
||||||
|
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4", fillvalue=-1)
|
||||||
|
for i in list(range(0, {n}, {step})) + list(range(0, 40)):
|
||||||
|
d[i] = i
|
||||||
|
"#
|
||||||
|
));
|
||||||
|
let values = File::open(&sparse)
|
||||||
|
.unwrap()
|
||||||
|
.dataset("x")
|
||||||
|
.unwrap()
|
||||||
|
.read_i32()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(values.len(), n);
|
||||||
|
let wrong = values
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter(|&(i, &v)| {
|
||||||
|
let expected = if i % step == 0 || i < 40 {
|
||||||
|
i as i32
|
||||||
|
} else {
|
||||||
|
-1
|
||||||
|
};
|
||||||
|
v != expected
|
||||||
|
})
|
||||||
|
.count();
|
||||||
|
assert_eq!(wrong, 0, "sparse: {wrong} of {n} elements read back wrong");
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user