quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

wasm, format: listing a group asks for all its missing blocks per pass

Browse Source 
Listing a group read every child's object header and stopped at the
first that was not fetched yet, and so did the traversals of the group's
index (v1 B-tree and symbol table nodes, the local heap's names, v2
B-tree nodes and fractal heap objects). Over openUrl's restartable
reader each block cost its own pass and round trip: 184 serial requests
to list 3000 datasets at 1 MiB blocks, 536 at 64 KiB.

- core::Reader::list reads every child's header before returning the
  first error (the same error, in listing order, Group::groups/datasets
  return), classifying them as those do.
- clawhdf5-format: after the first sibling that fails, the B-tree v1
  and v2 collectors, the symbol table node loop and the dense-link loop
  go on reading (not using) the remaining siblings, then return that
  first error: results and errors are unchanged, only failing
  traversals read more, and in memory that is free (storage::touch).
  A v1 group's local heap segment (names) is read at once, up to 1 MiB.
- LazyStorage no longer fills a one-block hole that is already cached
  (it was fetched again: 215 MB fetched from a 198 MB file).

Measured with tests/lazy.rs listing_cost_of_a_given_file on the
reviewer's file (h5py, 3000 datasets of 64 KiB, 198 MB), list('/'):
  libver earliest, 1 MiB blocks: 185 passes/184 requests -> 6/73
  libver earliest, 64 KiB:       537/536 -> 8/531 (6 in flight)
  libver latest,   1 MiB:        189/188 -> 9/98
  libver latest,   64 KiB:       453/452 -> 11/452
Bytes fetched are unchanged (the headers are spread through the file).
New test listing_a_large_group_takes_a_few_passes (512-byte blocks):
FileBuilder 600 children 102 -> 5 passes; h5py earliest/latest 2000
children 8 and 11 passes. Conformance 600 of 697 (baseline 600);
check-32bit-casts, check-nostd and h5rs-fuzz over the CVE corpus clean.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 07:48:49 -05:00
co-authored by Claude Opus 5.5
parent 1d065adf8b
commit e553153e48
8 changed files with 317 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-wasm/src/core.rs
+45 -20
View File
@@ -11,6 +11,8 @@ use std::sync::Arc;
use clawhdf5::{AttrValue, File, Selection};
use clawhdf5_format::data_read;
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::storage::Storage;
use clawhdf5_format::vl_data::{VlResolver, check_element_size};
@@ -166,31 +168,54 @@ impl Reader {
/// The groups, then the datasets, in the group at `path` (`/` is the
/// root). Soft links are listed as their targets; external and dangling
/// links, and named datatypes, are left out.
///
/// What [`Group::groups`](clawhdf5::Group::groups) and `datasets` list,
/// but every child's object header is read before an error ends the
/// listing (the first error, in listing order, is the one returned, as
/// there). Over a [`LazyStorage`](crate::lazy::LazyStorage) that makes
/// one pass ask for all the headers it is missing at once, instead of
/// one pass, and one round trip, per header.
pub fn list(&self, path: &str) -> Result<Vec<Child>> {
if self.kind(path)? != Kind::Group {
return Err(format!("not a group: {path}"));
}
let group = self.file.group(path).map_err(err)?;
let mut out: Vec<Child> = group
.groups()
.map_err(err)?
.into_iter()
.map(|name| Child {
name,
kind: Kind::Group,
})
.collect();
out.extend(
group
.datasets()
.map_err(err)?
.into_iter()
.map(|name| Child {
name,
kind: Kind::Dataset,
}),
);
Ok(out)
let entries = group.entries().map_err(err)?;
let sb = self.file.superblock();
let storage = self.file.storage();
let mut groups = Vec::new();
let mut datasets = Vec::new();
let mut first_error = None;
for (name, address) in entries {
match ObjectHeader::parse_in(storage, address, sb.offset_size, sb.length_size) {
Ok(header) => {
let has = |t: MessageType| header.messages.iter().any(|m| m.msg_type == t);
if has(MessageType::LinkInfo)
|| has(MessageType::Link)
|| has(MessageType::SymbolTable)
{
groups.push(Child {
name: name.clone(),
kind: Kind::Group,
});
}
if has(MessageType::DataLayout) {
datasets.push(Child {
name,
kind: Kind::Dataset,
});
}
}
Err(e) => {
first_error.get_or_insert(e);
}
}
}
if let Some(e) = first_error {
return Err(err(clawhdf5::Error::from(e)));
}
groups.extend(datasets);
Ok(groups)
}
/// Shape, max shape and datatype of the dataset at `path`.
crates/clawhdf5-wasm/src/lazy.rs
+34 -13
View File
@@ -347,32 +347,38 @@ impl LazyStorage {
}
/// Byte ranges covering the missing blocks: runs of consecutive
/// blocks, a one-block hole between two runs filled so they merge,
/// each at most `max_request` long.
/// blocks, a one-block hole between two runs filled so they merge
/// (unless the hole is cached: it would be fetched again), each at
/// most `max_request` long.
fn runs(&self, missing: HashMap<u64, bool>) -> Vec<Range<u64>> {
let bs = self.config.block_size;
let mut wanted: Vec<u64> = missing.keys().copied().collect();
wanted.sort_unstable();
{
// Remember which blocks only bulk reads asked for: they are
// kept as bulk once supplied.
let mut st = lock(&self.state);
for (&i, &metadata) in &missing {
if metadata {
st.bulk_pending.remove(&i);
} else {
st.bulk_pending.insert(i);
}
let mut st = lock(&self.state);
// Remember which blocks only bulk reads asked for: they are kept
// as bulk once supplied.
for (&i, &metadata) in &missing {
if metadata {
st.bulk_pending.remove(&i);
} else {
st.bulk_pending.insert(i);
}
}
let per_request = self.config.max_request / bs;
let mut runs: Vec<(u64, u64)> = Vec::new();
for i in wanted {
match runs.last_mut() {
Some((first, last)) if i <= *last + 2 && i - *first < per_request => *last = i,
Some((first, last))
if (i == *last + 1
|| (i == *last + 2 && !st.blocks.contains_key(&(i - 1))))
&& i - *first < per_request =>
{
*last = i
}
_ => runs.push((i, i)),
}
}
drop(st);
runs.into_iter()
.map(|(a, b)| a * bs..((b + 1) * bs).min(self.len))
.collect()
@@ -650,6 +656,21 @@ mod tests {
);
}
#[test]
fn a_cached_hole_is_not_fetched_again() {
let data = file(8 * 1024);
let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
serve(&s, &data, &[1024..2048]);
// Blocks 0 and 2 missing, 1 cached: two requests, not 0..3072.
let Step::Need(need) = s.attempt(|| {
let _ = s.read_at(0, 10);
s.read_at(2048, 10).map(|_| ())
}) else {
panic!("blocks 0 and 2 are missing");
};
assert_eq!(need, vec![0..1024, 2048..3072]);
}
#[test]
fn supply_refuses_what_was_not_asked_for() {
let data = file(10_000);