quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

fix(format): index datasets with several unlimited dims by B-tree v2

Browse Source 
A dataset with more than one unlimited dimension got an Extensible Array
index, which libhdf5 refuses ("already found unlimited dimension"), so
the whole file failed to open in h5py and h5dump. The previous commit
turned that into a write error; this one writes what the library itself
uses there: a version-2 B-tree chunk index (record type 10/11), as a
single leaf of the library's 2048-byte node size, or a larger leaf when
the records do not fit. The root's record count is 16-bit, so more than
65535 chunks is still refused rather than written wrong.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:14:47 -05:00
co-authored by Claude Opus 5.5
parent 540fa08907
commit 1dba7b465a
2 changed files with 268 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5/tests/chunk_index_interop.rs
+26 -7
View File
@@ -515,16 +515,35 @@ fn we_write_maxshape_larger_than_shape() {
check_we_write(&cases);
}
/// More than one unlimited dimension needs a B-tree v2 chunk index; the
/// writer must not produce a file libhdf5 cannot open.
/// More than one unlimited dimension needs a version-2 B-tree chunk index,
/// as the library uses; an Extensible Array for `(None, None)` made libhdf5
/// refuse the whole file ("already found unlimited dimension").
#[test]
fn two_unlimited_dims_are_refused() {
fn we_write_btree_v2_for_several_unlimited_dims() {
const U: u64 = u64::MAX;
let mut cases = vec![
wcase("unl_unl", &[20, 30], &[5, 5], Some(&[U, U])),
wcase("unl_fin_unl", &[6, 7, 8], &[4, 3, 5], Some(&[U, 9, U])),
// More records than the library's 2048-byte node holds (84 here).
wcase("unl_unl_2400", &[40, 60], &[1, 1], Some(&[U, U])),
wcase("unl_unl_empty", &[0, 0], &[4, 4], Some(&[U, U])),
];
let mut filtered = wcase("unl_unl_deflate", &[6, 7, 8], &[4, 3, 5], Some(&[U, U, U]));
filtered.deflate = true;
cases.push(filtered);
check_we_write(&cases);
}
/// A single-leaf B-tree has a 16-bit record count; beyond it the writer
/// refuses rather than writing a tree libhdf5 would misread.
#[test]
fn btree_v2_index_past_one_leaf_is_refused() {
let mut b = FileBuilder::new();
b.create_dataset("d")
.with_i32_data(&(0..600).collect::<Vec<i32>>())
.with_shape(&[20, 30])
.with_chunks(&[5, 5])
.with_i32_data(&vec![0i32; 70_000])
.with_shape(&[70_000, 1])
.with_chunks(&[1, 1])
.with_maxshape(&[u64::MAX, u64::MAX]);
let dir = tempfile::tempdir().unwrap();
assert!(b.write(dir.path().join("unl_unl.h5")).is_err());
assert!(b.write(dir.path().join("too_many.h5")).is_err());
}