quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): write child indirect blocks in big fractal heaps

Browse Source 
Dense link and attribute storage keeps its messages in a fractal heap. Its
root indirect block holds direct blocks up to 64 KiB, 512 KiB in all; rows
past that are child indirect blocks. The writer kept adding rows of direct
blocks instead, and libhdf5 and h5rs read them as indirect blocks: a group
with 20 000 links of 20-byte names was written without error and could not
be listed ("incorrect metadata checksum"), and 150 dense attributes of up
to 56 KB could not be opened. The heap writer now follows the doubling
table: rows past the direct ones hold child indirect blocks, each with its
own rows, nested as deep as the heap needs.

Two more heap bugs are fixed on the way. An object bigger than the next
block's free space was written into it anyway and cut off; the block is
now left unallocated and the object goes in the first block big enough, as
libhdf5 skips blocks. And the header's next-block offset was 0, so libhdf5
adding a link to such a group overwrote the heap's first block ("bad
version number for message"); it is now the offset after the last block.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:57:01 -05:00
co-authored by Claude Opus 5.5
parent b0a1e4f9a6
commit 81a0e8685d
3 changed files with 467 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5/tests/writer_groups_interop.rs
+127
View File
@@ -638,3 +638,130 @@ fn a_group_attribute_set_again_takes_the_new_value() {
let f = File::open(&path).unwrap();
assert!(matches!(f.root().attrs().unwrap()["v"], AttrValue::I64(2)));
}
// ---- big dense storage: child indirect blocks in the fractal heap ----
/// A name `len` bytes long, unique per `i`.
fn long_name(i: usize, len: usize) -> String {
let n = format!("link_{i:06}_");
format!("{n}{}", "x".repeat(len - n.len()))
}
#[test]
fn dense_links_past_the_direct_blocks_of_the_root() {
skip_if_no_python!();
// A dense group's links live in a fractal heap whose root indirect
// block holds direct blocks up to 64 KiB: 512 KiB of link messages.
// Rows past that are child indirect blocks. The writer used to write
// them as direct blocks, which libhdf5 cannot read ("incorrect metadata
// checksum"), from about 17 000 links with 20-byte names.
// `g` crosses the first boundary (0.6 MB of links); `deep` has 65 535
// links of about 110 bytes (7 MB), so its heap reaches the child indirect
// blocks that hold indirect blocks themselves.
let dir = tempfile::tempdir().unwrap();
let mut b = FileBuilder::new();
b.create_dataset("x").with_i32_data(&[7]);
let mut g = b.create_group("g");
for i in 0..20_000 {
g.create_dataset(&format!("dataset_number_{i:06}"))
.with_i32_data(&[i]);
}
b.add_group(g.finish());
let mut g = b.create_group("deep");
g.track_order(true);
for i in 0..usize::from(u16::MAX) {
g.add_hard_link(&long_name(i, 100), "/x");
}
b.add_group(g.finish());
let path = write(&dir, "big_links.h5", b);
let out = h5py(
&path,
"with h5py.File(path, 'r') as f:\n\
\x20 g, d = f['g'], f['deep']\n\
\x20 names = list(g)\n\
\x20 dn = list(d)\n\
\x20 print(json.dumps([len(names), names[-1], int(g[names[-1]][0]),\n\
\x20 sum(int(g[n][0]) for n in names), len(dn), dn[0][:12], dn[-1][:12],\n\
\x20 int(d[dn[-1]][0]), h5py.h5o.get_info(f['x'].id).rc]))",
);
assert_eq!(
out,
r#"[20000, "dataset_number_019999", 19999, 199990000, 65535, "link_000000_", "link_065534_", 7, 65536]"#
);
h5dump_ok(&path);
let f = File::open(&path).unwrap();
let g = f.group("g").unwrap();
assert_eq!(g.datasets().unwrap().len(), 20_000);
assert_eq!(
g.dataset("dataset_number_019999")
.unwrap()
.read_i32()
.unwrap(),
[19999]
);
let d = f.group("deep").unwrap();
assert_eq!(d.datasets().unwrap().len(), usize::from(u16::MAX));
assert_eq!(
d.dataset(&long_name(65_534, 100))
.unwrap()
.read_i32()
.unwrap(),
[7]
);
// libhdf5 can add to and delete from the heap. It could not when the
// header's block allocation offset was 0: its next block overwrote the
// first ("bad version number for message").
let out = h5py(
&path,
"with h5py.File(path, 'r+') as f:\n\
\x20 f['g']['zz_new'] = np.arange(3)\n\
\x20 del f['g/dataset_number_000005']\n\
with h5py.File(path, 'r') as f:\n\
\x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]),\n\
\x20 int(f['g/dataset_number_019998'][0])]))",
);
assert_eq!(out, r#"[20000, 2, 19998]"#);
h5dump_ok(&path);
}
#[test]
fn dense_attributes_past_the_direct_blocks_of_the_root() {
skip_if_no_python!();
// Dense attributes share the heap writer. 150 attributes of up to 56 KB
// (8 MB) need child indirect blocks, and a big attribute after small
// ones must skip the small blocks rather than overrun one.
let dir = tempfile::tempdir().unwrap();
let mut b = FileBuilder::new();
let ds = b.create_dataset("x");
ds.with_i32_data(&[1]);
for i in 0..150usize {
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
let v: Vec<f64> = (0..len).map(|k| (i * 100_000 + k) as f64).collect();
ds.set_attr(&format!("a{i:03}"), AttrValue::F64Array(v));
}
let path = write(&dir, "big_attrs.h5", b);
let out = h5py(
&path,
"with h5py.File(path, 'r') as f:\n\
\x20 a = f['x'].attrs\n\
\x20 ok = all(np.array_equal(a['a%03d' % i],\n\
\x20 np.arange(7000 if i % 3 == 0 else 1 + i) + i * 100000) for i in range(150))\n\
\x20 print(json.dumps([len(a), ok]))",
);
assert_eq!(out, "[150, true]");
h5dump_ok(&path);
let f = File::open(&path).unwrap();
let attrs = f.dataset("x").unwrap().attrs().unwrap();
assert_eq!(attrs.len(), 150);
for i in [0usize, 1, 147, 149] {
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
let want: Vec<f64> = (0..len).map(|k| (i * 100_000 + k) as f64).collect();
match &attrs[&format!("a{i:03}")] {
AttrValue::F64Array(v) => assert_eq!(*v, want, "a{i:03}"),
other => panic!("a{i:03}: {other:?}"),
}
}
}