fix: read multi-direct-block fractal heaps (root indirect block)
The fractal-heap reader split direct vs indirect block rows using the FRHP "Starting # of Rows in Root Indirect Block" field (a constant, typically 1), mislabeled as starting_row_of_indirect_blocks. For any heap whose data spans more than one direct block — common in libhdf5 files with a large group or many dense attributes — this treated direct blocks as indirect and walked into garbage, failing with InvalidFractalHeapSignature. Derive the split from the heap geometry instead: max_direct_rows = log2(max_direct_block_size / starting_block_size) + 2. Rows below it hold direct blocks; rows at/above hold child indirect blocks. Validated against an h5py-written group with 400 dense attributes (root indirect block, 4 rows, 13 direct blocks): all values now read correctly. Regression fixture covers an 80-attribute multi-block heap. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
@@ -379,8 +379,9 @@ impl FractalHeapHeader {
|
|||||||
// Build table of (block_size, heap_offset) for each child entry
|
// Build table of (block_size, heap_offset) for each child entry
|
||||||
let mut current_heap_offset = iblock_heap_offset;
|
let mut current_heap_offset = iblock_heap_offset;
|
||||||
|
|
||||||
// Count direct block entries vs indirect block entries
|
// Rows below max_direct_rows hold direct blocks; rows at/above hold
|
||||||
let start_indirect = self.starting_row_of_indirect_blocks as usize;
|
// child indirect blocks. (NOT the FRHP "starting rows" field.)
|
||||||
|
let start_indirect = self.max_direct_rows();
|
||||||
|
|
||||||
// Read child addresses for direct block rows
|
// Read child addresses for direct block rows
|
||||||
let max_direct_rows = nrows_usize.min(start_indirect);
|
let max_direct_rows = nrows_usize.min(start_indirect);
|
||||||
@@ -455,6 +456,25 @@ impl FractalHeapHeader {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Number of rows in the doubling table whose block size is at most the
|
||||||
|
/// maximum *direct* block size. Rows below this hold direct blocks; rows at
|
||||||
|
/// or above it hold child indirect blocks.
|
||||||
|
///
|
||||||
|
/// This is derived from the heap geometry, NOT the FRHP
|
||||||
|
/// "Starting # of Rows in Root Indirect Block" field (a constant, often 1)
|
||||||
|
/// — confusing the two makes a multi-direct-block heap unreadable.
|
||||||
|
fn max_direct_rows(&self) -> usize {
|
||||||
|
if self.starting_block_size == 0 {
|
||||||
|
return usize::MAX;
|
||||||
|
}
|
||||||
|
// Rows 0 and 1 share the starting block size; row r (r >= 1) is
|
||||||
|
// starting_block_size * 2^(r-1). The largest direct row reaches
|
||||||
|
// max_direct_block_size, giving log2(max/start) + 2 direct rows.
|
||||||
|
let ratio = (self.max_direct_block_size / self.starting_block_size).max(1);
|
||||||
|
let log2 = 63 - ratio.leading_zeros() as usize;
|
||||||
|
log2 + 2
|
||||||
|
}
|
||||||
|
|
||||||
/// Get block size for a given row in the doubling table.
|
/// Get block size for a given row in the doubling table.
|
||||||
fn block_size_for_row(&self, row: usize) -> u64 {
|
fn block_size_for_row(&self, row: usize) -> u64 {
|
||||||
let sbs = self.starting_block_size;
|
let sbs = self.starting_block_size;
|
||||||
|
|||||||
Binary file not shown.
@@ -853,3 +853,21 @@ fn dense_group_links_roundtrip() {
|
|||||||
assert_eq!(file.dataset("small/a").unwrap().read_f64().unwrap(), vec![1.0]);
|
assert_eq!(file.dataset("small/a").unwrap().read_f64().unwrap(), vec![1.0]);
|
||||||
assert_eq!(file.dataset("small/b").unwrap().read_f64().unwrap(), vec![2.0]);
|
assert_eq!(file.dataset("small/b").unwrap().read_f64().unwrap(), vec![2.0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reads_libhdf5_multiblock_fractal_heap() {
|
||||||
|
// A group whose dense attributes overflow a single fractal-heap direct
|
||||||
|
// block, so libhdf5 stored them under a root indirect block (FHIB) with
|
||||||
|
// multiple direct blocks. Reading requires deriving the direct/indirect row
|
||||||
|
// split from the heap geometry, not the FRHP "starting rows" field.
|
||||||
|
let bytes = include_bytes!("../../clawhdf5-format/tests/fixtures/fractal_heap_multiblock.h5");
|
||||||
|
let file = File::from_bytes(bytes.to_vec()).unwrap();
|
||||||
|
let attrs = file.group("g").unwrap().attrs().unwrap();
|
||||||
|
for i in 0..80i64 {
|
||||||
|
let name = format!("a{i:03}");
|
||||||
|
match attrs.get(&name) {
|
||||||
|
Some(AttrValue::I64(v)) => assert_eq!(*v, i * 3, "{name}"),
|
||||||
|
other => panic!("{name} = {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user