feat(format): read chunked datasets indexed by a version-2 B-tree
With libver='latest', a chunked dataset with two or more unlimited dimensions indexes its chunks with a v2 B-tree (layout v4, index type 5). Reading one failed with "unsupported chunked layout version=4, index_type=Some(5)". read_btree_v2_chunks decodes record types 10 (address + scaled offsets) and 11 (address, stored size, filter mask, scaled offsets). The width of the stored-size field is taken from the record size the tree header declares rather than re-deriving the library's formula. Scaled offsets are multiplied back by the chunk dimensions with overflow checks. The chunk-index dispatch existed four times (uncached, cached, sweep and indexed readers). The three copies outside list_chunks now call it, so every read path — and fill-value handling and partial reads — supports every index type from one place. h5py interop test: plain, gzip+shuffle, a 2500-chunk tree with internal nodes, a sparse dataset with a fill value, and a strided hyperslab. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
c6a7bbfc67
commit
d668e45ab5
@@ -913,3 +913,69 @@ with h5py.File("{dst_str}", "r") as f:
|
||||
"[(1, 2.5), (3, 4.5)] ('a', 'b') [18446744073709551615, 0, 9223372036854775808] uint64"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// h5py writes datasets indexed by a version-2 B-tree -> clawhdf5 reads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// With `libver='latest'`, a chunked dataset with two or more unlimited
|
||||
/// dimensions indexes its chunks with a version-2 B-tree (layout v4, index
|
||||
/// type 5). These used to fail with "unsupported chunked layout".
|
||||
#[test]
|
||||
fn h5py_btree_v2_chunk_index_clawhdf5_reads() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("bt2.h5");
|
||||
let path_str = path.display().to_string();
|
||||
let script = format!(
|
||||
r#"
|
||||
import h5py, numpy as np
|
||||
with h5py.File("{path_str}", "w", libver="latest") as f:
|
||||
a = np.arange(60 * 45, dtype="<i4").reshape(60, 45)
|
||||
f.create_dataset("plain", data=a, chunks=(7, 8), maxshape=(None, None))
|
||||
f.create_dataset("gz", data=a, chunks=(7, 8), maxshape=(None, None), compression="gzip", shuffle=True)
|
||||
# Enough chunks (2500) that the tree has internal nodes.
|
||||
big = np.arange(200 * 200, dtype="<i4").reshape(200, 200)
|
||||
f.create_dataset("deep", data=big, chunks=(4, 4), maxshape=(None, None))
|
||||
s = f.create_dataset("sparse", shape=(30, 30), dtype="<i4", chunks=(5, 5), maxshape=(None, None), fillvalue=-9)
|
||||
s[10:15, 20:25] = 4
|
||||
s[29, 29] = 1
|
||||
with h5py.File("{path_str}", "r") as f:
|
||||
print("sparse", f["sparse"][...].ravel().tolist())
|
||||
print("slab", f["deep"][37:141:13, 5:190:31].ravel().tolist())
|
||||
"#
|
||||
);
|
||||
let out = run_python_output(&script);
|
||||
let expected: std::collections::HashMap<&str, Vec<i32>> = out
|
||||
.lines()
|
||||
.map(|l| {
|
||||
let (name, list) = l.split_once(' ').unwrap();
|
||||
(name, parse_int_list(list))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
let small: Vec<i32> = (0..60 * 45).collect();
|
||||
assert_eq!(file.dataset("plain").unwrap().read_i32().unwrap(), small);
|
||||
assert_eq!(file.dataset("gz").unwrap().read_i32().unwrap(), small);
|
||||
let deep: Vec<i32> = (0..200 * 200).collect();
|
||||
assert_eq!(file.dataset("deep").unwrap().read_i32().unwrap(), deep);
|
||||
assert_eq!(
|
||||
file.dataset("sparse").unwrap().read_i32().unwrap(),
|
||||
expected["sparse"]
|
||||
);
|
||||
// Partial read through the same index: rows 37,50,..,128 x cols 5,36,..,160.
|
||||
let slab = clawhdf5_format::selection::Selection::Hyperslab {
|
||||
start: vec![37, 5],
|
||||
stride: vec![13, 31],
|
||||
count: vec![8, 6],
|
||||
block: vec![1, 1],
|
||||
};
|
||||
assert_eq!(
|
||||
file.dataset("deep")
|
||||
.unwrap()
|
||||
.read_i32_selection(&slab)
|
||||
.unwrap(),
|
||||
expected["slab"]
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user