quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 1 Wiki Activity

format: group walks go on past a failed node and hint what they read next

Browse Source 
Listing a large group over openUrl still took 6-11 passes (network round
trips) for the reviewer's 3000-dataset h5py file: each pass only found
the structures the walk reached before its first miss.

- The v1 and v2 B-tree collectors descend into every child of a node
  after one fails (they only read the siblings before, so a sibling's
  subtree came a pass later), then return the first error: results and
  errors unchanged. The v2 walk stops once its record budget is spent,
  so a shared-subtree tree still cannot multiply the work.
- Hints (`Storage::hint`, a no-op for every backend but the lazy one):
  a group B-tree node's and a symbol table node's body (read once their
  header gives a length, a round trip later when the body is in the
  next block), an object header's first chunk and its continuation
  chunks, the symbol table nodes a B-tree leaf names, a dense group's
  name index header and the heap's root block (both read right after
  the heap header). A listing also hints every child's object header as
  its entry is read, even after a failure, and every direct block of a
  dense group's heap (reading the indirect blocks, at most 4096 entries
  and 4 levels deep); a lookup does not.
- The fractal heap's indirect-block layout (entry sizes, where the first
  n entries end) is one helper used by the object reads and the hints.

Measured with tests/lazy.rs listing_cost_of_a_given_file on an h5py file
like the reviewer's (3000 datasets of 64 KiB, 198 MB), list('/'),
passes/requests/bytes, before -> after:
  earliest, 1 MiB:  6/73/192.5 MB -> 4/68/192.5 MB
  earliest, 64 KiB: 8/531/35.2 MB -> 5/530/35.3 MB
  latest,   1 MiB:  9/98/196.5 MB -> 5/86/196.5 MB
  latest,   64 KiB: 11/452/29.6 MB -> 6/454/30.5 MB
listing_a_large_group_takes_a_few_passes (512-byte blocks), budgets
tightened to the new counts: FileBuilder 600 children 5 -> 4 passes,
h5py 2000 children earliest 8 -> 5, latest 11 -> 6.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 22:47:44 -05:00
co-authored by Claude Opus 5.5
parent ca81c3ebfa
commit 6f5d14fd62
8 changed files with 443 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/group_v2.rs
+75 -11
View File
@@ -101,18 +101,50 @@ fn resolve_compact_entries(
Ok(entries)
}
/// What a version-2 B-tree header takes with 8-byte offsets and lengths
/// (22 bytes of fields, the root node's address and record count, and the
/// checksum), rounded up: hinted before one is read.
const BTREE_V2_HEADER_HINT_LEN: usize = 64;
/// The fractal heap of a dense group. The name index's header and the
/// heap's root block are read next, whatever the lookup: they are hinted
/// (see [`Storage::hint`]) so that a storage fetching between attempts
/// gets them in the same round trip as the heap's header.
fn dense_heap<S: Storage + ?Sized>(
file_data: &S,
link_info: &LinkInfoMessage,
fh_addr: u64,
offset_size: u8,
length_size: u8,
) -> Result<FractalHeapHeader, FormatError> {
if let Some(btree_addr) = link_info.btree_name_index_address {
file_data.hint(btree_addr, BTREE_V2_HEADER_HINT_LEN);
}
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
fh.hint_root_block(file_data);
Ok(fh)
}
/// Visit every link in dense storage (fractal heap + B-tree v2 name index).
///
/// With `hint_headers` (a listing, whose children are usually opened
/// next), the object header of every hard link is hinted (see
/// [`Storage::hint`]) as soon as the link is read, even after a failure.
fn for_each_dense_link<S: Storage + ?Sized>(
file_data: &S,
link_info: &LinkInfoMessage,
fh_addr: u64,
offset_size: u8,
length_size: u8,
hint_headers: bool,
mut visit: impl FnMut(LinkMessage),
) -> Result<(), FormatError> {
// Parse fractal heap
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
let fh = dense_heap(file_data, link_info, fh_addr, offset_size, length_size)?;
if hint_headers {
// Every link is read: so is every block of the heap.
fh.hint_managed_blocks(file_data);
}
// Parse B-tree v2 for name index
let btree_addr = link_info
@@ -144,11 +176,27 @@ fn for_each_dense_link<S: Storage + ?Sized>(
let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
// Read managed object from fractal heap
let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size);
let link = fh
.read_managed_object_in(file_data, id_bytes, offset_size)
.and_then(|d| parse_link(&d, offset_size));
if hint_headers
&& let Ok(Some(LinkMessage {
link_target:
LinkTarget::Hard {
object_header_address,
},
..
})) = &link
{
file_data.hint(
*object_header_address,
crate::object_header::OBJECT_HEADER_HINT_LEN,
);
}
if failed.is_some() {
continue;
}
match link_data.and_then(|d| parse_link(&d, offset_size)) {
match link {
Ok(Some(link)) => visit(link),
Ok(None) => {}
Err(e) => failed = Some(e),
@@ -175,6 +223,7 @@ fn resolve_dense_entries<S: Storage + ?Sized>(
fh_addr,
offset_size,
length_size,
true,
|link| {
if let LinkTarget::Hard {
object_header_address,
@@ -247,8 +296,7 @@ fn links_named<S: Storage + ?Sized>(
return Ok(found);
};
let fh =
FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
let fh = dense_heap(file_data, &link_info, fh_addr, offset_size, length_size)?;
let btree_addr = link_info
.btree_name_index_address
.ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
@@ -265,6 +313,7 @@ fn links_named<S: Storage + ?Sized>(
fh_addr,
offset_size,
length_size,
false,
|link| {
if link.name == name {
found.push(link);
@@ -409,7 +458,7 @@ fn resolve_child_core<S: Storage + ?Sized>(
let not_found = || FormatError::PathNotFound(String::from(name));
let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?;
if !is_v2_group(&header) || is_v1_group(&header) {
return resolve_group_children_in(file_data, superblock, group_address)?
return group_children(file_data, superblock, group_address, false)?
.into_iter()
.find(|e| e.name == name)
.map(|e| e.object_header_address)
@@ -575,6 +624,18 @@ fn resolve_group_children_core<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
group_address: u64,
) -> Result<Vec<GroupEntry>, FormatError> {
group_children(file_data, superblock, group_address, true)
}
/// [`resolve_group_children`]; with `hint_headers`, every child's object
/// header is hinted (see [`Storage::hint`]) as soon as its address is
/// known, for a listing whose children are opened next.
fn group_children<S: Storage + ?Sized>(
file_data: &S,
superblock: &Superblock,
group_address: u64,
hint_headers: bool,
) -> Result<Vec<GroupEntry>, FormatError> {
let os = superblock.offset_size;
let ls = superblock.length_size;
@@ -589,7 +650,10 @@ fn resolve_group_children_core<S: Storage + ?Sized>(
.find(|m| m.msg_type == MessageType::SymbolTable)
.ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
let stm = SymbolTableMessage::parse(&sym_msg.data, os)?;
let all = group_v1::resolve_v1_group_entries_in(file_data, &stm, os, ls)?;
let all = group_v1::v1_group_entries(file_data, &stm, os, ls, hint_headers)?;
if all.iter().any(|e| e.name.is_empty()) {
return Err(FormatError::InvalidLinkName);
}
if all.iter().any(group_v1::is_v1_soft_link) {
soft = group_v1::v1_soft_links_in(file_data, &stm, os, ls)?;
}
@@ -615,7 +679,7 @@ fn resolve_group_children_core<S: Storage + ?Sized>(
};
let link_info = find_link_info(&header, os)?;
if let Some(fh_addr) = link_info.fractal_heap_address {
for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?;
for_each_dense_link(file_data, &link_info, fh_addr, os, ls, hint_headers, visit)?;
} else {
for msg in &header.messages {
if msg.msg_type == MessageType::Link
@@ -737,7 +801,7 @@ fn resolve_group_entries<S: Storage + ?Sized>(
let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
// A lookup: an entry with an empty name (which fails a listing) is
// skipped by the name comparison, as in libhdf5.
group_v1::v1_group_entries(file_data, &stm, offset_size, length_size)
group_v1::v1_group_entries(file_data, &stm, offset_size, length_size, false)
} else if is_v2_group(object_header) {
resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size)
} else {