quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

wasm, format: listing a group asks for all its missing blocks per pass

Browse Source 
Listing a group read every child's object header and stopped at the
first that was not fetched yet, and so did the traversals of the group's
index (v1 B-tree and symbol table nodes, the local heap's names, v2
B-tree nodes and fractal heap objects). Over openUrl's restartable
reader each block cost its own pass and round trip: 184 serial requests
to list 3000 datasets at 1 MiB blocks, 536 at 64 KiB.

- core::Reader::list reads every child's header before returning the
  first error (the same error, in listing order, Group::groups/datasets
  return), classifying them as those do.
- clawhdf5-format: after the first sibling that fails, the B-tree v1
  and v2 collectors, the symbol table node loop and the dense-link loop
  go on reading (not using) the remaining siblings, then return that
  first error: results and errors are unchanged, only failing
  traversals read more, and in memory that is free (storage::touch).
  A v1 group's local heap segment (names) is read at once, up to 1 MiB.
- LazyStorage no longer fills a one-block hole that is already cached
  (it was fetched again: 215 MB fetched from a 198 MB file).

Measured with tests/lazy.rs listing_cost_of_a_given_file on the
reviewer's file (h5py, 3000 datasets of 64 KiB, 198 MB), list('/'):
  libver earliest, 1 MiB blocks: 185 passes/184 requests -> 6/73
  libver earliest, 64 KiB:       537/536 -> 8/531 (6 in flight)
  libver latest,   1 MiB:        189/188 -> 9/98
  libver latest,   64 KiB:       453/452 -> 11/452
Bytes fetched are unchanged (the headers are spread through the file).
New test listing_a_large_group_takes_a_few_passes (512-byte blocks):
FileBuilder 600 children 102 -> 5 passes; h5py earliest/latest 2000
children 8 and 11 passes. Conformance 600 of 697 (baseline 600);
check-32bit-casts, check-nostd and h5rs-fuzz over the CVE corpus clean.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 07:48:49 -05:00
co-authored by Claude Opus 5.5
parent 1d065adf8b
commit e553153e48
8 changed files with 317 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/btree_v1.rs
+18 -5
View File
@@ -199,19 +199,32 @@ fn collect_symbol_table_nodes_inner<S: Storage + ?Sized>(
// Leaf: children are SNOD addresses
Ok(node.children)
} else {
// Internal: recurse into children
// Internal: recurse into children. After the first child that
// fails, the others are only read (as `storage::touch` does), not
// descended into; that error is returned.
let mut result = Vec::new();
let mut failed = None;
for &child_addr in &node.children {
let child_snods = collect_symbol_table_nodes_inner(
if failed.is_some() {
// Parsing reads the node's header, then its body.
let _ = BTreeV1Node::parse_in(file, child_addr, offset_size, length_size);
continue;
}
match collect_symbol_table_nodes_inner(
file,
child_addr,
offset_size,
length_size,
depth + 1,
)?;
result.extend(child_snods);
) {
Ok(child_snods) => result.extend(child_snods),
Err(e) => failed = Some(e),
}
}
match failed {
Some(e) => Err(e),
None => Ok(result),
}
Ok(result)
}
}
crates/clawhdf5-format/src/btree_v2.rs
+51 -33
View File
@@ -504,45 +504,63 @@ fn collect_internal_records<S: Storage + ?Sized>(
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
// We collect child[0] records, then record[0], then child[1], etc.
// After the first child that fails, the others are only touched (see
// `storage::touch`); that error is returned.
let mut failed = None;
for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
if child_depth == 0 {
// Before parsing, so a refused tree is not also a large allocation.
spend(budget, usize::from(child_nrec))?;
let leaf_recs = parse_leaf_records(
file,
to_usize(child_addr)?,
child_nrec,
record_size,
node_size,
)?;
out.extend(leaf_recs);
} else {
collect_internal_records(
file,
to_usize(child_addr)?,
child_nrec,
child_depth,
record_size,
node_size,
offset_size,
length_size,
max_leaf_nrec,
budget,
out,
)?;
if failed.is_some() {
let len = usize::try_from(node_size)
.unwrap_or(usize::MAX)
.min(1 << 16);
crate::storage::touch(file, child_addr, len);
continue;
}
if let Err(e) = (|| -> Result<(), FormatError> {
if child_depth == 0 {
// Before parsing, so a refused tree is not also a large allocation.
spend(budget, usize::from(child_nrec))?;
let leaf_recs = parse_leaf_records(
file,
to_usize(child_addr)?,
child_nrec,
record_size,
node_size,
)?;
out.extend(leaf_recs);
} else {
collect_internal_records(
file,
to_usize(child_addr)?,
child_nrec,
child_depth,
record_size,
node_size,
offset_size,
length_size,
max_leaf_nrec,
budget,
out,
)?;
}
// Add record[i] (except after the last child)
if i < nr {
let data = node.record(i, rs)?;
spend(budget, 1)?;
out.push(BTreeV2Record {
data: data.to_vec(),
});
// Add record[i] (except after the last child)
if i < nr {
let data = node.record(i, rs)?;
spend(budget, 1)?;
out.push(BTreeV2Record {
data: data.to_vec(),
});
}
Ok(())
})() {
failed = Some(e);
}
}
Ok(())
match failed {
Some(e) => Err(e),
None => Ok(()),
}
}
/// The records of a B-tree v2 that fall in one key range, found by
crates/clawhdf5-format/src/group_v1.rs
+38 -14
View File
@@ -79,27 +79,51 @@ pub(crate) fn v1_group_entries<S: Storage + ?Sized>(
length_size,
)?;
// The names are read one by one from the heap's data segment; read
// (up to 1 MiB of) it first, so a storage that records what it lacks
// asks for it at once (see `storage::touch`).
if !snod_addrs.is_empty() {
let len = usize::try_from(heap.data_segment_size).map_or(1 << 20, |n| n.min(1 << 20));
crate::storage::touch(file_data, heap.data_segment_address, len);
}
let mut entries = Vec::new();
let mut heap_checked = false;
// After the first node that fails, the others are only read (as
// `storage::touch` does); that error is returned.
let mut failed = None;
for snod_addr in snod_addrs {
let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?;
for entry in &snod.entries {
// Like libhdf5, look at the heap's free list only once a name is
// needed: an empty group with a damaged heap still lists.
if !heap_checked {
heap.validate_free_list_in(file_data, length_size)?;
heap_checked = true;
if failed.is_some() {
let _ = SymbolTableNode::parse_in(file_data, snod_addr, offset_size);
continue;
}
let mut node = || -> Result<(), FormatError> {
let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?;
for entry in &snod.entries {
// Like libhdf5, look at the heap's free list only once a name
// is needed: an empty group with a damaged heap still lists.
if !heap_checked {
heap.validate_free_list_in(file_data, length_size)?;
heap_checked = true;
}
let name = heap.read_string_in(file_data, entry.link_name_offset)?;
entries.push(GroupEntry {
name,
object_header_address: entry.object_header_address,
cache_type: entry.cache_type,
});
}
let name = heap.read_string_in(file_data, entry.link_name_offset)?;
entries.push(GroupEntry {
name,
object_header_address: entry.object_header_address,
cache_type: entry.cache_type,
});
Ok(())
};
if let Err(e) = node() {
failed = Some(e);
}
}
Ok(entries)
match failed {
Some(e) => Err(e),
None => Ok(entries),
}
}
/// Symbol table cache type for a soft link: the scratch pad's first four bytes
crates/clawhdf5-format/src/group_v2.rs
+15 -4
View File
@@ -126,6 +126,9 @@ fn for_each_dense_link<S: Storage + ?Sized>(
)?;
let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?;
// After the first link that fails, the others are only read, not
// visited (a touch, see `storage::touch`); that error is returned.
let mut failed = None;
for record in &records {
// For type 5 (name index): hash(4) + heap_id(heap_id_length)
// For type 6 (creation order): creation_order(8) + heap_id(heap_id_length)
@@ -141,12 +144,20 @@ fn for_each_dense_link<S: Storage + ?Sized>(
let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
// Read managed object from fractal heap
let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?;
if let Some(link) = parse_link(&link_data, offset_size)? {
visit(link);
let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size);
if failed.is_some() {
continue;
}
match link_data.and_then(|d| parse_link(&d, offset_size)) {
Ok(Some(link)) => visit(link),
Ok(None) => {}
Err(e) => failed = Some(e),
}
}
Ok(())
match failed {
Some(e) => Err(e),
None => Ok(()),
}
}
/// Resolve entries from dense storage (fractal heap + B-tree v2).
crates/clawhdf5-format/src/storage.rs
+13
View File
@@ -202,6 +202,19 @@ pub(crate) fn len_usize<S: Storage + ?Sized>(file: &S) -> usize {
usize::try_from(file.len()).unwrap_or(usize::MAX)
}
/// Read `len` bytes at `offset` and drop them, ignoring any error.
///
/// For a traversal that has failed on one sibling (a B-tree child, a
/// symbol table node, a heap object) and would stop there: it first
/// touches the siblings it did not get to, so a storage that records what
/// it lacks — the browser's restartable reader, which fetches over the
/// network between attempts — learns about all of them in one attempt
/// instead of one per attempt. Results and errors are unchanged (the first
/// error is still the one returned); an in-memory read is free.
pub fn touch<S: Storage + ?Sized>(file: &S, offset: u64, len: usize) {
let _ = file.read_at(offset, len);
}
/// Bytes `[offset, offset + len)`, all of them.
///
/// A range that runs past the end of the storage is