quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

wasm, format: listing a group asks for all its missing blocks per pass

Browse Source 
Listing a group read every child's object header and stopped at the
first that was not fetched yet, and so did the traversals of the group's
index (v1 B-tree and symbol table nodes, the local heap's names, v2
B-tree nodes and fractal heap objects). Over openUrl's restartable
reader each block cost its own pass and round trip: 184 serial requests
to list 3000 datasets at 1 MiB blocks, 536 at 64 KiB.

- core::Reader::list reads every child's header before returning the
  first error (the same error, in listing order, Group::groups/datasets
  return), classifying them as those do.
- clawhdf5-format: after the first sibling that fails, the B-tree v1
  and v2 collectors, the symbol table node loop and the dense-link loop
  go on reading (not using) the remaining siblings, then return that
  first error: results and errors are unchanged, only failing
  traversals read more, and in memory that is free (storage::touch).
  A v1 group's local heap segment (names) is read at once, up to 1 MiB.
- LazyStorage no longer fills a one-block hole that is already cached
  (it was fetched again: 215 MB fetched from a 198 MB file).

Measured with tests/lazy.rs listing_cost_of_a_given_file on the
reviewer's file (h5py, 3000 datasets of 64 KiB, 198 MB), list('/'):
  libver earliest, 1 MiB blocks: 185 passes/184 requests -> 6/73
  libver earliest, 64 KiB:       537/536 -> 8/531 (6 in flight)
  libver latest,   1 MiB:        189/188 -> 9/98
  libver latest,   64 KiB:       453/452 -> 11/452
Bytes fetched are unchanged (the headers are spread through the file).
New test listing_a_large_group_takes_a_few_passes (512-byte blocks):
FileBuilder 600 children 102 -> 5 passes; h5py earliest/latest 2000
children 8 and 11 passes. Conformance 600 of 697 (baseline 600);
check-32bit-casts, check-nostd and h5rs-fuzz over the CVE corpus clean.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 07:48:49 -05:00
co-authored by Claude Opus 5.5
parent 1d065adf8b
commit e553153e48
8 changed files with 317 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-wasm/src/core.rs
+45 -20
View File
@@ -11,6 +11,8 @@ use std::sync::Arc;
use clawhdf5::{AttrValue, File, Selection};
use clawhdf5_format::data_read;
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::storage::Storage;
use clawhdf5_format::vl_data::{VlResolver, check_element_size};
@@ -166,31 +168,54 @@ impl Reader {
/// The groups, then the datasets, in the group at `path` (`/` is the
/// root). Soft links are listed as their targets; external and dangling
/// links, and named datatypes, are left out.
///
/// What [`Group::groups`](clawhdf5::Group::groups) and `datasets` list,
/// but every child's object header is read before an error ends the
/// listing (the first error, in listing order, is the one returned, as
/// there). Over a [`LazyStorage`](crate::lazy::LazyStorage) that makes
/// one pass ask for all the headers it is missing at once, instead of
/// one pass, and one round trip, per header.
pub fn list(&self, path: &str) -> Result<Vec<Child>> {
if self.kind(path)? != Kind::Group {
return Err(format!("not a group: {path}"));
}
let group = self.file.group(path).map_err(err)?;
let mut out: Vec<Child> = group
.groups()
.map_err(err)?
.into_iter()
.map(|name| Child {
name,
kind: Kind::Group,
})
.collect();
out.extend(
group
.datasets()
.map_err(err)?
.into_iter()
.map(|name| Child {
name,
kind: Kind::Dataset,
}),
);
Ok(out)
let entries = group.entries().map_err(err)?;
let sb = self.file.superblock();
let storage = self.file.storage();
let mut groups = Vec::new();
let mut datasets = Vec::new();
let mut first_error = None;
for (name, address) in entries {
match ObjectHeader::parse_in(storage, address, sb.offset_size, sb.length_size) {
Ok(header) => {
let has = |t: MessageType| header.messages.iter().any(|m| m.msg_type == t);
if has(MessageType::LinkInfo)
|| has(MessageType::Link)
|| has(MessageType::SymbolTable)
{
groups.push(Child {
name: name.clone(),
kind: Kind::Group,
});
}
if has(MessageType::DataLayout) {
datasets.push(Child {
name,
kind: Kind::Dataset,
});
}
}
Err(e) => {
first_error.get_or_insert(e);
}
}
}
if let Some(e) = first_error {
return Err(err(clawhdf5::Error::from(e)));
}
groups.extend(datasets);
Ok(groups)
}
/// Shape, max shape and datatype of the dataset at `path`.
crates/clawhdf5-wasm/src/lazy.rs
+34 -13
View File
@@ -347,32 +347,38 @@ impl LazyStorage {
}
/// Byte ranges covering the missing blocks: runs of consecutive
/// blocks, a one-block hole between two runs filled so they merge,
/// each at most `max_request` long.
/// blocks, a one-block hole between two runs filled so they merge
/// (unless the hole is cached: it would be fetched again), each at
/// most `max_request` long.
fn runs(&self, missing: HashMap<u64, bool>) -> Vec<Range<u64>> {
let bs = self.config.block_size;
let mut wanted: Vec<u64> = missing.keys().copied().collect();
wanted.sort_unstable();
{
// Remember which blocks only bulk reads asked for: they are
// kept as bulk once supplied.
let mut st = lock(&self.state);
for (&i, &metadata) in &missing {
if metadata {
st.bulk_pending.remove(&i);
} else {
st.bulk_pending.insert(i);
}
let mut st = lock(&self.state);
// Remember which blocks only bulk reads asked for: they are kept
// as bulk once supplied.
for (&i, &metadata) in &missing {
if metadata {
st.bulk_pending.remove(&i);
} else {
st.bulk_pending.insert(i);
}
}
let per_request = self.config.max_request / bs;
let mut runs: Vec<(u64, u64)> = Vec::new();
for i in wanted {
match runs.last_mut() {
Some((first, last)) if i <= *last + 2 && i - *first < per_request => *last = i,
Some((first, last))
if (i == *last + 1
|| (i == *last + 2 && !st.blocks.contains_key(&(i - 1))))
&& i - *first < per_request =>
{
*last = i
}
_ => runs.push((i, i)),
}
}
drop(st);
runs.into_iter()
.map(|(a, b)| a * bs..((b + 1) * bs).min(self.len))
.collect()
@@ -650,6 +656,21 @@ mod tests {
);
}
#[test]
fn a_cached_hole_is_not_fetched_again() {
let data = file(8 * 1024);
let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
serve(&s, &data, &[1024..2048]);
// Blocks 0 and 2 missing, 1 cached: two requests, not 0..3072.
let Step::Need(need) = s.attempt(|| {
let _ = s.read_at(0, 10);
s.read_at(2048, 10).map(|_| ())
}) else {
panic!("blocks 0 and 2 are missing");
};
assert_eq!(need, vec![0..1024, 2048..3072]);
}
#[test]
fn supply_refuses_what_was_not_asked_for() {
let data = file(10_000);
crates/clawhdf5-wasm/tests/lazy.rs
+103
View File
@@ -473,3 +473,106 @@ fn corpus_files_read_the_same_lazily() {
files.len()
);
}
/// Passes and requests `list(path)` takes on a file opened lazily at
/// `block`-byte blocks (the open not counted), checking the listing against
/// the in-memory one.
fn listing_cost(data: &[u8], path: &str, block: u64) -> (u64, u64) {
let want = Reader::open(data.to_vec()).unwrap().list(path).unwrap();
let lazy = Lazy::open(
data.to_vec(),
LazyConfig {
block_size: block,
..LazyConfig::default()
},
)
.unwrap();
let before = lazy.storage.stats();
assert_eq!(lazy.call(|r| r.list(path)).unwrap(), want);
let after = lazy.storage.stats();
(
after.passes - before.passes,
after.requests - before.requests,
)
}
/// Listing a group reads every child's object header, and its index (B-tree
/// and symbol table nodes, or B-tree v2 and heap blocks) before that. Each
/// pass asks for every node of a level it is missing, not the first one
/// only, so the passes (network round trips) grow with the depth of the
/// index, not with the number of children: 2000 children with headers
/// scattered over 512-byte blocks list in a handful of passes, where each
/// header block used to cost its own.
#[test]
fn listing_a_large_group_takes_a_few_passes() {
let mut b = FileBuilder::new();
let mut g = b.create_group("many");
for i in 0..600 {
g.create_dataset(&format!("d{i}")).with_i32_data(&[i; 64]);
}
b.add_group(g.finish());
let data = b.finish().unwrap();
let (passes, requests) = listing_cost(&data, "/many", 512);
eprintln!("FileBuilder, 600 children: {passes} passes, {requests} requests");
assert!(passes <= 6, "{passes} passes");
if !python_available() {
return;
}
let dir = tempfile::tempdir().unwrap();
for libver in ["earliest", "latest"] {
let path = dir.path().join(format!("{libver}.h5"));
let script = format!(
"import h5py, numpy as np\n\
with h5py.File({:?}, 'w', libver='{libver}') as f:\n\
\x20 for i in range(2000):\n\
\x20 f.create_dataset('d%d' % i, data=np.full(256, i, np.float32))\n",
path.display().to_string()
);
let out = Command::new(python())
.args(["-c", &script])
.output()
.unwrap();
assert!(
out.status.success(),
"{}",
String::from_utf8_lossy(&out.stderr)
);
let data = std::fs::read(&path).unwrap();
let (passes, requests) = listing_cost(&data, "/", 512);
eprintln!("h5py libver={libver}, 2000 children: {passes} passes, {requests} requests");
assert!(passes <= 12, "{libver}: {passes} passes");
}
}
/// `CLAWHDF5_WASM_LIST_FILE=file.h5`: what listing the root group of that
/// file costs lazily, at 1 MiB and 64 KiB blocks (a measurement, printed).
#[test]
fn listing_cost_of_a_given_file() {
let Ok(path) = std::env::var("CLAWHDF5_WASM_LIST_FILE") else {
return;
};
let data = std::fs::read(&path).unwrap();
for block in [1 << 20, 64 << 10] {
let lazy = Lazy::open(
data.clone(),
LazyConfig {
block_size: block,
..LazyConfig::default()
},
)
.unwrap();
let open = lazy.storage.stats();
let n = lazy.call(|r| r.list("/")).unwrap().len();
let st = lazy.storage.stats();
eprintln!(
"{path} ({} bytes), {block}-byte blocks: open {} requests / {} passes; list('/') of {n}: {} passes, {} requests, {} bytes",
data.len(),
open.requests,
open.passes,
st.passes - open.passes,
st.requests - open.requests,
st.bytes_fetched - open.bytes_fetched
);
}
}