quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): size fractal heap child indirect blocks by their row's span

Browse Source 
A child indirect block in row r of a fractal heap's doubling table spans
that row's block size of heap space, so it has
log2(size) - log2(start_block_size * width) + 1 rows (libhdf5's
H5HF__dtable_size_to_rows). The reader used row - first_indirect_row + 1,
which undercounts, so every object stored past the root block's direct
rows (512 KiB with libhdf5's defaults) was unreachable: dense groups with
a few thousand long link names, or ~20 000 short ones, could not be listed.

Regression test: dense_group_whose_heap_outgrows_the_root_direct_rows
(h5py writes 2 500 links with 248-byte names; listing compared with h5py).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:52:07 -05:00
co-authored by Claude Opus 5.5
parent 42b81d9f1c
commit 8ebd488d9e
4 changed files with 183 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/fractal_heap.rs
+32 -33
View File
@@ -418,35 +418,35 @@ impl FractalHeapHeader {
}
// If we have indirect block rows
// A child indirect block in row r spans exactly that row's block size
// of heap space, so it has as many rows as a table of that total size
// needs (not `row - start_indirect + 1`, which undercounts and makes
// every object past the root's direct rows unreachable).
for row in start_indirect..nrows_usize {
let _block_size = self.block_size_for_row(row);
let child_nrows = row - start_indirect + 1;
let child_space = self.block_size_for_row(row);
let child_nrows = self.rows_for_size(child_space);
for _col in 0..tw {
let child_addr = read_offset(file_data, pos, offset_size)?;
pos += offset_size as usize;
if !is_undefined(child_addr, offset_size) {
// Calculate total heap space covered by this indirect block child
let total_child_space = self.indirect_block_heap_size(child_nrows);
let block_end = current_heap_offset + total_child_space;
if target_offset >= current_heap_offset && target_offset < block_end {
return self.read_from_indirect_block(
file_data,
child_addr as usize,
child_nrows as u16,
current_heap_offset,
target_offset,
length,
offset_size,
depth_remaining - 1,
);
}
current_heap_offset += total_child_space;
} else {
let total_child_space = self.indirect_block_heap_size(child_nrows);
current_heap_offset += total_child_space;
let block_end = current_heap_offset.saturating_add(child_space);
if !is_undefined(child_addr, offset_size)
&& target_offset >= current_heap_offset
&& target_offset < block_end
{
return self.read_from_indirect_block(
file_data,
child_addr as usize,
child_nrows,
current_heap_offset,
target_offset,
length,
offset_size,
depth_remaining - 1,
);
}
current_heap_offset = block_end;
}
}
@@ -475,25 +475,24 @@ impl FractalHeapHeader {
log2 + 2
}
/// Rows an indirect block needs to span `size` bytes of heap space:
/// `log2(size) - log2(starting_block_size * table_width) + 1`, as
/// libhdf5's `H5HF__dtable_size_to_rows`.
fn rows_for_size(&self, size: u64) -> u16 {
let log2 = |v: u64| 63u32.saturating_sub(v.max(1).leading_zeros());
let first_row_bits = log2(self.starting_block_size) + log2(u64::from(self.table_width));
(log2(size).saturating_sub(first_row_bits) + 1) as u16
}
/// Get block size for a given row in the doubling table.
fn block_size_for_row(&self, row: usize) -> u64 {
let sbs = self.starting_block_size;
if row <= 1 {
sbs
} else {
sbs * (1u64 << (row - 1))
sbs.saturating_mul(1u64.checked_shl((row - 1) as u32).unwrap_or(u64::MAX))
}
}
/// Total heap space covered by an indirect block with the given number of rows.
fn indirect_block_heap_size(&self, nrows: usize) -> u64 {
let tw = self.table_width as u64;
let mut total = 0u64;
for row in 0..nrows {
total += self.block_size_for_row(row) * tw;
}
total
}
}
#[cfg(test)]