wasm, format: listing a group asks for all its missing blocks per pass
Listing a group read every child's object header and stopped at the
first that was not fetched yet, and so did the traversals of the group's
index (v1 B-tree and symbol table nodes, the local heap's names, v2
B-tree nodes and fractal heap objects). Over openUrl's restartable
reader each block cost its own pass and round trip: 184 serial requests
to list 3000 datasets at 1 MiB blocks, 536 at 64 KiB.
- core::Reader::list reads every child's header before returning the
first error (the same error, in listing order, Group::groups/datasets
return), classifying them as those do.
- clawhdf5-format: after the first sibling that fails, the B-tree v1
and v2 collectors, the symbol table node loop and the dense-link loop
go on reading (not using) the remaining siblings, then return that
first error: results and errors are unchanged, only failing
traversals read more, and in memory that is free (storage::touch).
A v1 group's local heap segment (names) is read at once, up to 1 MiB.
- LazyStorage no longer fills a one-block hole that is already cached
(it was fetched again: 215 MB fetched from a 198 MB file).
Measured with tests/lazy.rs listing_cost_of_a_given_file on the
reviewer's file (h5py, 3000 datasets of 64 KiB, 198 MB), list('/'):
libver earliest, 1 MiB blocks: 185 passes/184 requests -> 6/73
libver earliest, 64 KiB: 537/536 -> 8/531 (6 in flight)
libver latest, 1 MiB: 189/188 -> 9/98
libver latest, 64 KiB: 453/452 -> 11/452
Bytes fetched are unchanged (the headers are spread through the file).
New test listing_a_large_group_takes_a_few_passes (512-byte blocks):
FileBuilder 600 children 102 -> 5 passes; h5py earliest/latest 2000
children 8 and 11 passes. Conformance 600 of 697 (baseline 600);
check-32bit-casts, check-nostd and h5rs-fuzz over the CVE corpus clean.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -504,45 +504,63 @@ fn collect_internal_records<S: Storage + ?Sized>(
|
||||
|
||||
// Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
|
||||
// We collect child[0] records, then record[0], then child[1], etc.
|
||||
// After the first child that fails, the others are only touched (see
|
||||
// `storage::touch`); that error is returned.
|
||||
let mut failed = None;
|
||||
for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
|
||||
if child_depth == 0 {
|
||||
// Before parsing, so a refused tree is not also a large allocation.
|
||||
spend(budget, usize::from(child_nrec))?;
|
||||
let leaf_recs = parse_leaf_records(
|
||||
file,
|
||||
to_usize(child_addr)?,
|
||||
child_nrec,
|
||||
record_size,
|
||||
node_size,
|
||||
)?;
|
||||
out.extend(leaf_recs);
|
||||
} else {
|
||||
collect_internal_records(
|
||||
file,
|
||||
to_usize(child_addr)?,
|
||||
child_nrec,
|
||||
child_depth,
|
||||
record_size,
|
||||
node_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
max_leaf_nrec,
|
||||
budget,
|
||||
out,
|
||||
)?;
|
||||
if failed.is_some() {
|
||||
let len = usize::try_from(node_size)
|
||||
.unwrap_or(usize::MAX)
|
||||
.min(1 << 16);
|
||||
crate::storage::touch(file, child_addr, len);
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = (|| -> Result<(), FormatError> {
|
||||
if child_depth == 0 {
|
||||
// Before parsing, so a refused tree is not also a large allocation.
|
||||
spend(budget, usize::from(child_nrec))?;
|
||||
let leaf_recs = parse_leaf_records(
|
||||
file,
|
||||
to_usize(child_addr)?,
|
||||
child_nrec,
|
||||
record_size,
|
||||
node_size,
|
||||
)?;
|
||||
out.extend(leaf_recs);
|
||||
} else {
|
||||
collect_internal_records(
|
||||
file,
|
||||
to_usize(child_addr)?,
|
||||
child_nrec,
|
||||
child_depth,
|
||||
record_size,
|
||||
node_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
max_leaf_nrec,
|
||||
budget,
|
||||
out,
|
||||
)?;
|
||||
}
|
||||
|
||||
// Add record[i] (except after the last child)
|
||||
if i < nr {
|
||||
let data = node.record(i, rs)?;
|
||||
spend(budget, 1)?;
|
||||
out.push(BTreeV2Record {
|
||||
data: data.to_vec(),
|
||||
});
|
||||
// Add record[i] (except after the last child)
|
||||
if i < nr {
|
||||
let data = node.record(i, rs)?;
|
||||
spend(budget, 1)?;
|
||||
out.push(BTreeV2Record {
|
||||
data: data.to_vec(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
})() {
|
||||
failed = Some(e);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
match failed {
|
||||
Some(e) => Err(e),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The records of a B-tree v2 that fall in one key range, found by
|
||||
|
||||
Reference in New Issue
Block a user