quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

read: look names up through the dense name indexes

Browse Source 
Finding one link or attribute by name read every entry: Group::dataset and
Group::group (File, MmapFile, LazyFile) listed the whole group per call, and
path resolution scanned each group's links. Opening every child of a
35 001-link group by name decoded ~1.2e9 links.

Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the
name) is descended to the records with the name's hash
(btree_v2::find_btree_v2_records reads only the nodes whose key interval
overlaps), and only those links are read and compared; all hash-equal
records are compared, so libhdf5's tie order does not matter. Dense
attributes the same through their type 8 index
(attribute::find_attribute_in_file, facade attr(name)); huge heap objects
through their ID-ordered index. group_v2::resolve_child returns what the
listing has under a name (soft links followed, dangling/external ones not
found). Group::entries and File::group_at hand out a listing's addresses.

The lookup-stats feature counts heap objects read. Tests: one lookup in
an h5py-written 35 001-link group with colliding hashes reads at most two
links (before: 35 001, failing), attribute lookups likewise (before: 3 000,
failing), every child opens through all three readers and matches h5py,
every link kind resolves as h5py resolves it in dense and compact groups,
300 huge attributes are found, and a range search matches a full scan at
every tree depth.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:33:19 -05:00
co-authored by Claude Opus 5.5
parent 6248b411f0
commit 02e89c1d2d
15 changed files with 1245 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/fractal_heap.rs
+20 -14
View File
@@ -6,7 +6,8 @@ use alloc::{format, vec::Vec};
#[cfg(feature = "checksum")]
use byteorder::{ByteOrder, LittleEndian};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, find_btree_v2_records};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
@@ -354,6 +355,7 @@ impl FractalHeapHeader {
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
crate::lookup_stats::heap_object_read();
let Some(&first) = id_bytes.first() else {
return Err(FormatError::UnexpectedEof {
expected: 1,
@@ -448,7 +450,7 @@ impl FractalHeapHeader {
}
let hdr = BTreeV2Header::parse(
file_data,
self.huge_btree_address as usize,
to_usize(self.huge_btree_address)?,
self.offset_size,
self.length_size,
)?;
@@ -463,8 +465,12 @@ impl FractalHeapHeader {
if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
return Err(heap_error("unexpected huge-object B-tree record type"));
}
let records =
collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
// Records are ordered by ID (the last field): descend to the ones
// equal to `key` instead of reading the whole index.
let id_at = rec_len - ls;
let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| {
le_uint(&r[id_at..id_at + ls]).cmp(&key)
})?;
for rec in &records {
let d = &rec.data;
if d.len() < rec_len {
@@ -533,24 +539,24 @@ impl FractalHeapHeader {
self.read_from_direct_block(
file_data,
DirectBlock {
addr: self.root_block_address as usize,
addr: to_usize(self.root_block_address)?,
size: self.starting_block_size,
heap_offset: 0,
filtered_size: self.root_direct_block_filtered_size,
filter_mask: self.root_direct_block_filter_mask,
},
heap_offset,
obj_len as usize,
to_usize(obj_len)?,
)
} else {
// Root is an indirect block — limit recursion to 64 levels
self.read_from_indirect_block(
file_data,
self.root_block_address as usize,
to_usize(self.root_block_address)?,
self.current_rows_in_root_indirect_block,
0, // block offset
heap_offset,
obj_len as usize,
to_usize(obj_len)?,
offset_size,
64, // max recursion depth
)
@@ -572,11 +578,11 @@ impl FractalHeapHeader {
) -> Result<Vec<u8>, FormatError> {
if target_offset < block.heap_offset {
return Err(FormatError::UnexpectedEof {
expected: block.heap_offset as usize,
available: target_offset as usize,
expected: to_usize(block.heap_offset)?,
available: to_usize(target_offset)?,
});
}
let local_offset = (target_offset - block.heap_offset) as usize;
let local_offset = to_usize(target_offset - block.heap_offset)?;
if let Some(pipeline) = &self.filter_pipeline {
let stored_len = usize::try_from(block.filtered_size)
.map_err(|_| heap_error("direct block size"))?;
@@ -673,7 +679,7 @@ impl FractalHeapHeader {
return self.read_from_direct_block(
file_data,
DirectBlock {
addr: child_addr as usize,
addr: to_usize(child_addr)?,
size: block_size,
heap_offset: current_heap_offset,
filtered_size,
@@ -705,7 +711,7 @@ impl FractalHeapHeader {
{
return self.read_from_indirect_block(
file_data,
child_addr as usize,
to_usize(child_addr)?,
child_nrows,
current_heap_offset,
target_offset,
@@ -719,7 +725,7 @@ impl FractalHeapHeader {
}
Err(FormatError::UnexpectedEof {
expected: target_offset as usize + length,
expected: to_usize(target_offset)?.saturating_add(length),
available: file_data.len(),
})
}