feat(format): read chunked datasets indexed by a version-2 B-tree

With libver='latest', a chunked dataset with two or more unlimited dimensions
indexes its chunks with a v2 B-tree (layout v4, index type 5). Reading one
failed with "unsupported chunked layout version=4, index_type=Some(5)".

read_btree_v2_chunks decodes record types 10 (address + scaled offsets) and 11
(address, stored size, filter mask, scaled offsets). The width of the
stored-size field is taken from the record size the tree header declares
rather than re-deriving the library's formula. Scaled offsets are multiplied
back by the chunk dimensions with overflow checks.

The chunk-index dispatch existed four times (uncached, cached, sweep and
indexed readers). The three copies outside list_chunks now call it, so every
read path — and fill-value handling and partial reads — supports every index
type from one place.

h5py interop test: plain, gzip+shuffle, a 2500-chunk tree with internal nodes,
a sparse dataset with a fill value, and a strided hyperslab.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
2026-09-19 13:59:18 -07:00
co-authored by Claude Fable 5.1
parent c6a7bbfc67
commit d668e45ab5
4 changed files with 229 additions and 256 deletions
@@ -913,3 +913,69 @@ with h5py.File("{dst_str}", "r") as f:
"[(1, 2.5), (3, 4.5)] ('a', 'b') [18446744073709551615, 0, 9223372036854775808] uint64"
);
}
// ---------------------------------------------------------------------------
// h5py writes datasets indexed by a version-2 B-tree -> clawhdf5 reads
// ---------------------------------------------------------------------------
/// With `libver='latest'`, a chunked dataset with two or more unlimited
/// dimensions indexes its chunks with a version-2 B-tree (layout v4, index
/// type 5). These used to fail with "unsupported chunked layout".
#[test]
fn h5py_btree_v2_chunk_index_clawhdf5_reads() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("bt2.h5");
let path_str = path.display().to_string();
let script = format!(
r#"
import h5py, numpy as np
with h5py.File("{path_str}", "w", libver="latest") as f:
a = np.arange(60 * 45, dtype="<i4").reshape(60, 45)
f.create_dataset("plain", data=a, chunks=(7, 8), maxshape=(None, None))
f.create_dataset("gz", data=a, chunks=(7, 8), maxshape=(None, None), compression="gzip", shuffle=True)
# Enough chunks (2500) that the tree has internal nodes.
big = np.arange(200 * 200, dtype="<i4").reshape(200, 200)
f.create_dataset("deep", data=big, chunks=(4, 4), maxshape=(None, None))
s = f.create_dataset("sparse", shape=(30, 30), dtype="<i4", chunks=(5, 5), maxshape=(None, None), fillvalue=-9)
s[10:15, 20:25] = 4
s[29, 29] = 1
with h5py.File("{path_str}", "r") as f:
print("sparse", f["sparse"][...].ravel().tolist())
print("slab", f["deep"][37:141:13, 5:190:31].ravel().tolist())
"#
);
let out = run_python_output(&script);
let expected: std::collections::HashMap<&str, Vec<i32>> = out
.lines()
.map(|l| {
let (name, list) = l.split_once(' ').unwrap();
(name, parse_int_list(list))
})
.collect();
let file = File::open(&path).unwrap();
let small: Vec<i32> = (0..60 * 45).collect();
assert_eq!(file.dataset("plain").unwrap().read_i32().unwrap(), small);
assert_eq!(file.dataset("gz").unwrap().read_i32().unwrap(), small);
let deep: Vec<i32> = (0..200 * 200).collect();
assert_eq!(file.dataset("deep").unwrap().read_i32().unwrap(), deep);
assert_eq!(
file.dataset("sparse").unwrap().read_i32().unwrap(),
expected["sparse"]
);
// Partial read through the same index: rows 37,50,..,128 x cols 5,36,..,160.
let slab = clawhdf5_format::selection::Selection::Hyperslab {
start: vec![37, 5],
stride: vec![13, 31],
count: vec![8, 6],
block: vec![1, 1],
};
assert_eq!(
file.dataset("deep")
.unwrap()
.read_i32_selection(&slab)
.unwrap(),
expected["slab"]
);
}