fix(format): keep a dense index leaf within 65 535 records
The link and attribute name indexes are one v2 B-tree leaf, sized to the
next power of two. libhdf5 takes a leaf's capacity from that node size,
but a leaf's record count is a 2-byte field. From about 47 700 links the
node had room for more than 65 535 records, so adding a link in h5py
overflowed the count: a group of 65 535 links crashed h5py, or could no
longer be listed ("unknown link class"). The node is now capped at a full
leaf of 65 535 records, so libhdf5 splits it instead.
Dense attributes now go through the same index builder. Their record
count was written modulo 65 536, without error; more than 65 535
attributes on one object are now refused, like links.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -538,7 +538,23 @@ fn more_links_than_one_index_leaf_holds_is_an_error() {
|
||||
b.add_soft_link(&format!("s{i}"), "/x");
|
||||
}
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(err.contains("at most 65535 links"), "{err}");
|
||||
assert!(
|
||||
err.contains("70000 links in one group: at most 65535"),
|
||||
"{err}"
|
||||
);
|
||||
// Dense attributes have the same one-leaf index. Their count used to
|
||||
// be written modulo 65 536.
|
||||
let mut b = FileBuilder::new();
|
||||
let x = b.create_dataset("x");
|
||||
x.with_i32_data(&[1]);
|
||||
for i in 0..70_000 {
|
||||
x.set_attr(&format!("a{i}"), AttrValue::I64(i));
|
||||
}
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("70000 attributes on one object: at most 65535"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -712,17 +728,21 @@ fn dense_links_past_the_direct_blocks_of_the_root() {
|
||||
|
||||
// libhdf5 can add to and delete from the heap. It could not when the
|
||||
// header's block allocation offset was 0: its next block overwrote the
|
||||
// first ("bad version number for message").
|
||||
// first ("bad version number for message"). Adding to `deep` also
|
||||
// needs its index's leaf node to have room for at most 65 535 records:
|
||||
// a bigger node made libhdf5 overflow the leaf's 2-byte record count
|
||||
// (a crash, or "unknown link class" when listing).
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 f['g']['zz_new'] = np.arange(3)\n\
|
||||
\x20 f['deep']['zz_new'] = np.arange(4)\n\
|
||||
\x20 del f['g/dataset_number_000005']\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]),\n\
|
||||
\x20 int(f['g/dataset_number_019998'][0])]))",
|
||||
\x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]), len(f['deep']),\n\
|
||||
\x20 list(f['deep'])[-1], int(f['g/dataset_number_019998'][0])]))",
|
||||
);
|
||||
assert_eq!(out, r#"[20000, 2, 19998]"#);
|
||||
assert_eq!(out, r#"[20000, 2, 65536, "zz_new", 19998]"#);
|
||||
h5dump_ok(&path);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user