quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): read v2 B-tree internal nodes with libhdf5's pointer widths

Browse Source 
An internal node's child pointer is an address, the child's record count
and (below the first internal level) the child subtree's total record
count. libhdf5 (H5B2__hdr_init) encodes the record count in the width of
a leaf's maximum and the subtree total in the width of cum_max_nrec for
that depth, computed level by level from the node size. The reader
guessed 2 * leaf_max and leaf_max^depth, which agree at depth 2 but not
at depth 3: a 24 000-link group's name index has depth 3, its root's
pointers were read 3 bytes wide instead of 2, and listing failed with a
garbage heap offset.

Regression tests: dense_group_with_a_three_level_name_index (h5py writes
24 000 links; listing compared with h5py) and
subtree_capacity_matches_libhdf5.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:53:00 -05:00
co-authored by Claude Opus 5.5
parent 8ebd488d9e
commit 8196fab72a
4 changed files with 94 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/btree_v2.rs
+64 -40
View File
@@ -323,39 +323,21 @@ fn collect_internal_records(
let records_start = pos;
pos += records_total;
// Compute sizes for child pointers
// max_records at child depth - for variable-width nrec encoding
// Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
// child's record count is always encoded in the width needed for a
// *leaf's* maximum, and — below the first internal level — the child
// subtree's total record count in the width needed for the most records
// a subtree of that depth can hold.
let child_depth = depth - 1;
let max_nrec_child = if child_depth == 0 {
max_leaf_nrec
} else {
// For internal nodes at child_depth, the true max_nrec depends on the
// node size, record size, and the recursive width of child pointer
// entries (which themselves depend on max_nrec at deeper levels).
// Computing the exact value requires iterating from the leaf level
// upward, as described in the HDF5 spec (III.A.2 "Computing the Size
// of B-tree Nodes").
//
// We use `max_leaf_nrec * 2` as a conservative upper bound. This
// over-estimates the nrec encoding width, which means we may read
// slightly more bytes per child pointer than strictly necessary, but
// never fewer. The over-read bytes are harmless because we only
// decode `num_records` entries (the actual count from the node header).
//
// Known limitation: for very deep trees (depth > 3) with small record
// sizes, the true max could exceed this estimate, causing us to
// under-allocate the nrec encoding width and misparse child pointers.
// In practice, HDF5 B-tree v2 depths rarely exceed 2-3.
max_leaf_nrec * 2
};
let nrec_width = bytes_for_max_records(max_nrec_child);
// Total records in subtree width (only if depth > 1)
let nrec_width = bytes_for_max_records(max_leaf_nrec);
let total_nrec_width = if depth > 1 {
// Width to hold total records in a subtree
// We compute max possible total records at this subtree depth
let max_total = header_max_total_records(max_leaf_nrec, depth - 1);
bytes_for_max_records(max_total)
bytes_for_max_records(cum_max_records(
node_size,
record_size,
offset_size,
max_leaf_nrec,
child_depth,
))
} else {
0
};
@@ -435,14 +417,36 @@ fn collect_internal_records(
Ok(())
}
/// Estimate maximum total records at a given depth (for variable-width encoding).
fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 {
// Conservative: branching factor * max_leaf at each level
let mut total = max_leaf_nrec;
for _ in 0..depth {
total = total.saturating_mul(max_leaf_nrec.max(2));
/// Most records a subtree whose root is at `depth` can hold (libhdf5's
/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth
/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth
/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is
/// paired with a child pointer of the width depth `d` needs.
fn cum_max_records(
node_size: u32,
record_size: u16,
offset_size: u8,
max_leaf_nrec: u64,
depth: u16,
) -> u64 {
// Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
const PREFIX: u64 = 10;
let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
let mut cum = max_leaf_nrec;
let mut cum_width = 0u64;
for d in 1..=depth {
let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 };
let max_nrec = u64::from(node_size)
.saturating_sub(PREFIX)
.saturating_sub(ptr)
/ (u64::from(record_size) + ptr).max(1);
cum = max_nrec
.saturating_add(1)
.saturating_mul(cum)
.saturating_add(max_nrec);
cum_width = bytes_for_max_records(cum) as u64;
}
total
cum
}
#[cfg(test)]
@@ -512,9 +516,15 @@ mod tests {
child_nrec: u64,
) -> Vec<u8> {
let max_leaf = max_records_leaf(node_size, record_size);
let nrec_width = bytes_for_max_records(if depth == 1 { max_leaf } else { max_leaf * 2 });
let nrec_width = bytes_for_max_records(max_leaf);
let total_width = if depth > 1 {
bytes_for_max_records(header_max_total_records(max_leaf, depth - 1))
bytes_for_max_records(cum_max_records(
node_size,
record_size,
8,
max_leaf,
depth - 1,
))
} else {
0
};
@@ -673,4 +683,18 @@ mod tests {
let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap();
assert!(records.is_empty());
}
#[test]
fn subtree_capacity_matches_libhdf5() {
// A link-name index (11-byte records, 512-byte nodes, 8-byte
// addresses): libhdf5's H5B2__hdr_init gives 45 records per leaf,
// then cum_max_nrec 1 149 at depth 1 and 26 449 at depth 2 — two
// bytes of subtree count in a depth-3 root's child pointers, where
// leaf_max^3 = 91 125 would need three.
let leaf = max_records_leaf(512, 11);
assert_eq!(leaf, 45);
assert_eq!(cum_max_records(512, 11, 8, leaf, 0), 45);
assert_eq!(cum_max_records(512, 11, 8, leaf, 1), 1_149);
assert_eq!(cum_max_records(512, 11, 8, leaf, 2), 26_449);
}
}