wasm: restartable NeedBytes storage for lazy reads (range-read M4, core)

LazyStorage holds the blocks of a remote file fetched so far. An
operation runs as passes over it: a read that misses records the
missing blocks and fails, the pass's result is dropped whatever it is
(a parser may have caught the error and carried on), and the caller
fetches the reported ranges and re-runs the pass. No block is evicted
while an operation is in flight, so every pass that does not finish
asks for at least one new block and the operation ends. Blocks sit in
an LRU with a byte budget, trimmed between operations, bulk (raw data)
blocks first.

Reader::open_storage opens a file through any Storage, and variable-
length strings resolve through the file's storage instead of
File::as_bytes, which panics for a file not in memory.

tests/lazy.rs compares, file by file, what the viewer can show (kinds,
listings, attributes, info, whole reads and hyperslabs) read lazily
with the facade's range-storage path and the in-memory reader: files
built here at 512 B to 1 MiB blocks, the h5py/netCDF4 fixture, and with
CLAWHDF5_WASM_CORPUS the conformance corpus (656 files agree). A
listing plus a small read of a 48 MB file fetches 3 ranges.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-wasm/src/core.rs

@@ -6,9 +6,12 @@
 //! with no typed-array mapping (compound, reference, opaque, ...) is refused
 //! with a message naming it, never returned as reinterpreted bytes.
 
+use std::sync::Arc;
+
 use clawhdf5::{AttrValue, File, Selection};
 use clawhdf5_format::data_read;
 use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
+use clawhdf5_format::storage::Storage;
 use clawhdf5_format::vl_data::{VlResolver, check_element_size};
 
 /// Errors are reported to JavaScript as messages.
@@ -119,7 +122,9 @@ pub struct Attr {
     pub value: AttrValue,
 }
 
-/// An open file, held in memory.
+/// An open file: held in memory ([`Reader::open`]) or read through a
+/// [`Storage`] ([`Reader::open_storage`], such as a
+/// [`LazyStorage`](crate::lazy::LazyStorage)).
 pub struct Reader {
     file: File,
 }
@@ -132,6 +137,14 @@ impl Reader {
         })
     }
 
+    /// Open a file read through `storage` (the file's bytes from offset 0,
+    /// user block included, as [`File::open_storage`] takes them).
+    pub fn open_storage(storage: Arc<dyn Storage + Send + Sync>) -> Result<Self> {
+        Ok(Self {
+            file: File::open_storage(storage).map_err(err)?,
+        })
+    }
+
     /// Whether `path` names a group or a dataset.
     pub fn kind(&self, path: &str) -> Result<Kind> {
         match self.file.dataset(path) {
@@ -281,11 +294,14 @@ impl Reader {
                 // string ends at its first NUL and a heap object of the
                 // wrong size is an error, as in libhdf5 and h5py.
                 let sb = self.file.superblock();
-                Data::Strings(
+                let strings = match self.file.contiguous_bytes() {
-                    VlResolver::new(self.file.as_bytes(), sb.offset_size, sb.length_size)
+                    Some(bytes) => {
-                        .strings(raw)
+                        VlResolver::new(bytes, sb.offset_size, sb.length_size).strings(raw)
-                        .map_err(err)?,
+                    }
-                )
+                    None => VlResolver::new_in(self.file.storage(), sb.offset_size, sb.length_size)
+                        .strings(raw),
+                };
+                Data::Strings(strings.map_err(err)?)
             }
             Datatype::Enumeration { .. } if !is_array => {
                 Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?)

crates/clawhdf5-wasm/src/lazy.rs

@@ -0,0 +1,672 @@
+//! Reading a file that is not all here, when no read may wait for the
+//! network: the restartable "NeedBytes" mode of `docs/design/range-reads.md`
+//! (milestone M4).
+//!
+//! A browser's main thread cannot block on `fetch`, and the parsers are
+//! synchronous. So an operation (open, list a group, read a dataset) runs
+//! as a *pass* over a [`LazyStorage`] that holds the blocks fetched so far:
+//!
+//! 1. [`LazyStorage::attempt`] runs the operation. A read whose blocks are
+//!    all present is served; a read that misses records the missing blocks
+//!    and fails with a storage error.
+//! 2. If the pass missed anything, its result is thrown away — whatever it
+//!    is, since a parser may have caught the error and carried on (a
+//!    listing skips a link it cannot resolve) — and the caller gets the
+//!    byte ranges to fetch ([`Step::Need`]).
+//! 3. The caller fetches them (asynchronously, with HTTP `Range` requests),
+//!    hands them over with [`LazyStorage::supply`] and runs the operation
+//!    again.
+//!
+//! A pass is pure over the storage: the facade only caches what completed
+//! reads decoded (its chunk cache), so re-running it is safe. Every pass
+//! that does not finish asks for at least one block not yet present, and no
+//! block is evicted while an operation is in flight
+//! ([`LazyStorage::operation`]), so an operation finishes after at most one
+//! pass per block it needs. In practice it is one pass per *wave* of
+//! misses: a chunked read asks for all the chunks of a batch at once.
+//!
+//! Blocks are kept in an LRU cache with a byte budget, trimmed only when no
+//! operation is in flight. Blocks fetched for bulk reads (raw data: a
+//! `read_ranges` call, or a read longer than a block) go first, so reading
+//! a large dataset does not evict the metadata.
+
+use std::borrow::Cow;
+use std::collections::{BTreeSet, HashMap};
+use std::ops::Range;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use clawhdf5_format::error::FormatError;
+use clawhdf5_format::storage::Storage;
+
+/// Default block size: 1 MiB, as `clawhdf5-remote`'s block cache (the size
+/// `docs/design/range-reads.md` §2 measured).
+pub const DEFAULT_BLOCK_SIZE: u64 = 1 << 20;
+
+/// The message of the error a read that misses returns. It never reaches
+/// the caller of [`LazyStorage::attempt`]: a pass that missed is re-run.
+pub const NEED_BYTES: &str = "bytes not fetched yet (restartable read)";
+
+/// Settings of a [`LazyStorage`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct LazyConfig {
+    /// Size of a block in bytes (at least 512); fetches are whole, aligned
+    /// blocks (the file's last block is shorter).
+    pub block_size: u64,
+    /// Byte budget of cached blocks between operations. An operation keeps
+    /// every block it needs until it finishes, whatever the budget.
+    pub capacity: u64,
+    /// Largest single range asked for, in bytes (whole blocks, at least
+    /// one); longer runs are split so they can be fetched in parallel.
+    pub max_request: u64,
+}
+
+impl Default for LazyConfig {
+    fn default() -> Self {
+        LazyConfig {
+            block_size: DEFAULT_BLOCK_SIZE,
+            capacity: 64 << 20,
+            max_request: 8 << 20,
+        }
+    }
+}
+
+/// What a [`LazyStorage`] has done so far.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct LazyStats {
+    /// Passes run by [`LazyStorage::attempt`].
+    pub passes: u64,
+    /// Ranges handed to [`LazyStorage::supply`]: one HTTP request each.
+    pub requests: u64,
+    /// Bytes handed to [`LazyStorage::supply`].
+    pub bytes_fetched: u64,
+    /// Blocks evicted to stay within the budget.
+    pub evictions: u64,
+    /// Bytes cached now.
+    pub cached_bytes: u64,
+}
+
+/// The outcome of one pass.
+#[derive(Debug)]
+pub enum Step<T> {
+    /// The pass read only bytes that were present: its result stands.
+    Done(T),
+    /// The pass missed: fetch these byte ranges (sorted, disjoint, block
+    /// aligned), [`supply`](LazyStorage::supply) them and run it again.
+    Need(Vec<Range<u64>>),
+}
+
+struct Block {
+    data: Arc<[u8]>,
+    /// Eviction order: bulk blocks (`false`) before metadata (`true`),
+    /// then least recently used first.
+    key: (bool, u64),
+}
+
+#[derive(Default)]
+struct State {
+    blocks: HashMap<u64, Block>,
+    /// `(metadata?, tick, block index)`, in eviction order.
+    order: BTreeSet<(bool, u64, u64)>,
+    tick: u64,
+    bytes: u64,
+    /// Blocks the current pass missed, and whether a small read wanted
+    /// them (metadata).
+    missing: HashMap<u64, bool>,
+    /// Blocks a bulk read missed that have not been supplied yet: kept
+    /// as bulk when they arrive.
+    bulk_pending: BTreeSet<u64>,
+    /// Operations in flight: no eviction while any is.
+    active: u32,
+    stats: LazyStats,
+}
+
+/// A [`Storage`] over the blocks of a file fetched so far; a read of
+/// anything else fails and is recorded, so the pass can be re-run once the
+/// bytes arrive. See the [module documentation](self).
+pub struct LazyStorage {
+    len: u64,
+    config: LazyConfig,
+    state: Mutex<State>,
+}
+
+fn lock(m: &Mutex<State>) -> MutexGuard<'_, State> {
+    m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
+}
+
+/// Keeps an operation's blocks cached until it is dropped; see
+/// [`LazyStorage::operation`].
+pub struct Operation<'a> {
+    storage: &'a LazyStorage,
+}
+
+impl Drop for Operation<'_> {
+    fn drop(&mut self) {
+        let mut st = lock(&self.storage.state);
+        st.active = st.active.saturating_sub(1);
+        if st.active == 0 {
+            self.storage.evict(&mut st);
+        }
+    }
+}
+
+impl LazyStorage {
+    /// An empty cache for a file of `len` bytes.
+    pub fn new(len: u64, mut config: LazyConfig) -> Self {
+        config.block_size = config.block_size.max(512);
+        config.max_request = (config.max_request / config.block_size).max(1) * config.block_size;
+        LazyStorage {
+            len,
+            config,
+            state: Mutex::new(State::default()),
+        }
+    }
+
+    /// The settings in use (after rounding).
+    pub fn config(&self) -> &LazyConfig {
+        &self.config
+    }
+
+    /// Counters since the storage was made.
+    pub fn stats(&self) -> LazyStats {
+        let st = lock(&self.state);
+        LazyStats {
+            cached_bytes: st.bytes,
+            ..st.stats
+        }
+    }
+
+    /// Mark an operation in flight until the guard is dropped: no block is
+    /// evicted meanwhile, so re-running its passes always makes progress.
+    /// Hold it across every pass of one operation.
+    pub fn operation(&self) -> Operation<'_> {
+        lock(&self.state).active += 1;
+        Operation { storage: self }
+    }
+
+    /// Run one pass of `f` over this storage. `Done` when `f` read nothing
+    /// that is missing; otherwise `Need` with the ranges to fetch, and `f`'s
+    /// result is dropped (it may be an error caused by the miss, or a
+    /// result built around one).
+    pub fn attempt<T>(&self, f: impl FnOnce() -> T) -> Step<T> {
+        {
+            let mut st = lock(&self.state);
+            st.missing.clear();
+            st.stats.passes += 1;
+        }
+        let out = f();
+        let missing = std::mem::take(&mut lock(&self.state).missing);
+        if missing.is_empty() {
+            return Step::Done(out);
+        }
+        drop(out);
+        Step::Need(self.runs(missing))
+    }
+
+    /// The bytes of the file at `offset`, fetched for a range a pass asked
+    /// for. `offset` must be block aligned and the bytes whole blocks (the
+    /// last block of the file may be short) inside the file, or this is an
+    /// error and nothing is kept. Blocks already present are left alone.
+    pub fn supply(&self, offset: u64, bytes: &[u8]) -> Result<(), String> {
+        let bs = self.config.block_size;
+        let end = offset
+            .checked_add(bytes.len() as u64)
+            .filter(|&e| e <= self.len)
+            .ok_or_else(|| {
+                format!(
+                    "{} bytes at offset {offset} run past the end of the {}-byte file",
+                    bytes.len(),
+                    self.len
+                )
+            })?;
+        if !offset.is_multiple_of(bs) || (!end.is_multiple_of(bs) && end != self.len) {
+            return Err(format!(
+                "{} bytes at offset {offset} are not whole {bs}-byte blocks",
+                bytes.len()
+            ));
+        }
+        let mut st = lock(&self.state);
+        st.stats.requests += 1;
+        st.stats.bytes_fetched += bytes.len() as u64;
+        let mut start = offset;
+        while start < end {
+            let i = start / bs;
+            let stop = (start + bs).min(end);
+            if !st.blocks.contains_key(&i) {
+                let rel = (start - offset) as usize..(stop - offset) as usize;
+                let metadata = !st.bulk_pending.remove(&i);
+                self.keep(&mut st, i, Arc::from(&bytes[rel]), metadata);
+            }
+            start = stop;
+        }
+        if st.active == 0 {
+            self.evict(&mut st);
+        }
+        Ok(())
+    }
+
+    /// [`supply`](Self::supply) the bytes fetched for `range`, one of the
+    /// ranges a [`Step::Need`] asked for: anything but exactly its length
+    /// (a server that answered with more or less) is an error.
+    pub fn supply_range(&self, range: &Range<u64>, bytes: &[u8]) -> Result<(), String> {
+        let want = range.end.saturating_sub(range.start);
+        if bytes.len() as u64 != want {
+            return Err(format!(
+                "asked for {want} bytes at offset {}, got {}",
+                range.start,
+                bytes.len()
+            ));
+        }
+        self.supply(range.start, bytes)
+    }
+
+    /// Run `f` to completion, fetching what its passes miss with `fetch`
+    /// (a byte range to its bytes). The blocking driver, for native code
+    /// and tests; the browser's is the same loop with an `await` between
+    /// passes.
+    pub fn run_blocking<T>(
+        &self,
+        mut f: impl FnMut() -> T,
+        mut fetch: impl FnMut(Range<u64>) -> Result<Vec<u8>, String>,
+    ) -> Result<T, String> {
+        let _op = self.operation();
+        loop {
+            match self.attempt(&mut f) {
+                Step::Done(v) => return Ok(v),
+                Step::Need(ranges) => {
+                    for r in ranges {
+                        let bytes = fetch(r.clone())?;
+                        self.supply_range(&r, &bytes)?;
+                    }
+                }
+            }
+        }
+    }
+
+    /// Cache block `i`.
+    fn keep(&self, st: &mut State, i: u64, data: Arc<[u8]>, metadata: bool) {
+        st.tick += 1;
+        let key = (metadata, st.tick);
+        st.bytes += <[u8]>::len(&data) as u64;
+        st.order.insert((key.0, key.1, i));
+        if let Some(old) = st.blocks.insert(i, Block { data, key }) {
+            st.order.remove(&(old.key.0, old.key.1, i));
+            st.bytes -= <[u8]>::len(&old.data) as u64;
+        }
+    }
+
+    fn evict(&self, st: &mut State) {
+        while st.bytes > self.config.capacity {
+            let Some((_, _, i)) = st.order.pop_first() else {
+                break;
+            };
+            if let Some(b) = st.blocks.remove(&i) {
+                st.bytes -= <[u8]>::len(&b.data) as u64;
+                st.stats.evictions += 1;
+            }
+        }
+    }
+
+    /// Byte ranges covering the missing blocks: runs of consecutive
+    /// blocks, a one-block hole between two runs filled so they merge,
+    /// each at most `max_request` long.
+    fn runs(&self, missing: HashMap<u64, bool>) -> Vec<Range<u64>> {
+        let bs = self.config.block_size;
+        let mut wanted: Vec<u64> = missing.keys().copied().collect();
+        wanted.sort_unstable();
+        {
+            // Remember which blocks only bulk reads asked for: they are
+            // kept as bulk once supplied.
+            let mut st = lock(&self.state);
+            for (&i, &metadata) in &missing {
+                if metadata {
+                    st.bulk_pending.remove(&i);
+                } else {
+                    st.bulk_pending.insert(i);
+                }
+            }
+        }
+        let per_request = self.config.max_request / bs;
+        let mut runs: Vec<(u64, u64)> = Vec::new();
+        for i in wanted {
+            match runs.last_mut() {
+                Some((first, last)) if i <= *last + 2 && i - *first < per_request => *last = i,
+                _ => runs.push((i, i)),
+            }
+        }
+        runs.into_iter()
+            .map(|(a, b)| a * bs..((b + 1) * bs).min(self.len))
+            .collect()
+    }
+
+    /// Block indices covering `[offset, offset + len)`, clamped to the file.
+    fn span(&self, offset: u64, len: u64) -> Option<Range<u64>> {
+        let end = offset.saturating_add(len).min(self.len);
+        if offset >= end {
+            return None;
+        }
+        let bs = self.config.block_size;
+        Some(offset / bs..(end - 1) / bs + 1)
+    }
+
+    /// The blocks of `spans` if all are present (touching them), else
+    /// record the missing ones and fail.
+    fn blocks(
+        &self,
+        spans: &[Range<u64>],
+        metadata: bool,
+    ) -> Result<HashMap<u64, Arc<[u8]>>, FormatError> {
+        let mut st = lock(&self.state);
+        let mut have = HashMap::new();
+        let mut missed = false;
+        for span in spans {
+            for i in span.clone() {
+                if have.contains_key(&i) {
+                    continue;
+                }
+                match st.blocks.get(&i) {
+                    Some(b) => {
+                        have.insert(i, b.data.clone());
+                    }
+                    None => {
+                        missed = true;
+                        let m = st.missing.entry(i).or_insert(metadata);
+                        *m |= metadata;
+                    }
+                }
+            }
+        }
+        if missed {
+            return Err(FormatError::Storage(NEED_BYTES.into()));
+        }
+        // Touch: most recently used last; a small read promotes a bulk
+        // block to metadata.
+        for &i in have.keys() {
+            st.tick += 1;
+            let tick = st.tick;
+            let Some(b) = st.blocks.get_mut(&i) else {
+                continue;
+            };
+            let old = b.key;
+            b.key = (old.0 || metadata, tick);
+            let new = b.key;
+            st.order.remove(&(old.0, old.1, i));
+            st.order.insert((new.0, new.1, i));
+        }
+        Ok(have)
+    }
+
+    fn assemble(&self, offset: u64, end: u64, blocks: &HashMap<u64, Arc<[u8]>>) -> Vec<u8> {
+        let bs = self.config.block_size;
+        let mut out = Vec::with_capacity(usize::try_from(end - offset).unwrap_or(0));
+        let mut pos = offset;
+        while pos < end {
+            let i = pos / bs;
+            let block = &blocks[&i];
+            let from = (pos - i * bs) as usize;
+            let to = ((end - i * bs) as usize).min(<[u8]>::len(block));
+            out.extend_from_slice(&block[from..to]);
+            pos = i * bs + to as u64;
+        }
+        out
+    }
+}
+
+impl Storage for LazyStorage {
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        let Some(span) = self.span(offset, len as u64) else {
+            return Ok(Cow::Owned(Vec::new()));
+        };
+        let metadata = len as u64 <= self.config.block_size;
+        let blocks = self.blocks(std::slice::from_ref(&span), metadata)?;
+        let end = offset.saturating_add(len as u64).min(self.len);
+        Ok(Cow::Owned(self.assemble(offset, end, &blocks)))
+    }
+
+    fn len(&self) -> u64 {
+        self.len
+    }
+
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        let mut spans = Vec::with_capacity(ranges.len());
+        for r in ranges {
+            if r.end < r.start {
+                return Err(FormatError::Storage(
+                    "read range ends before it starts".into(),
+                ));
+            }
+            spans.extend(self.span(r.start, r.end - r.start));
+        }
+        let blocks = self.blocks(&spans, false)?;
+        Ok(ranges
+            .iter()
+            .map(|r| {
+                let end = r.end.min(self.len);
+                if r.start >= end {
+                    Cow::Owned(Vec::new())
+                } else {
+                    Cow::Owned(self.assemble(r.start, end, &blocks))
+                }
+            })
+            .collect())
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::single_range_in_vec_init)]
+mod tests {
+    use super::*;
+
+    fn file(n: usize) -> Vec<u8> {
+        (0..n).map(|i| (i * 7 + i / 251) as u8).collect()
+    }
+
+    fn config(block: u64, capacity: u64) -> LazyConfig {
+        LazyConfig {
+            block_size: block,
+            capacity,
+            max_request: 4 * block,
+        }
+    }
+
+    /// Supply every range of `need` from `data`.
+    fn serve(s: &LazyStorage, data: &[u8], need: &[Range<u64>]) {
+        for r in need {
+            s.supply_range(r, &data[r.start as usize..r.end as usize])
+                .unwrap();
+        }
+    }
+
+    fn owned(r: Result<Cow<'_, [u8]>, FormatError>) -> Result<Vec<u8>, FormatError> {
+        r.map(Cow::into_owned)
+    }
+
+    #[test]
+    fn a_miss_asks_for_whole_blocks_then_the_rerun_reads_them() {
+        let data = file(10_000);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
+        let Step::Need(need) = s.attempt(|| owned(s.read_at(1500, 1000))) else {
+            panic!("nothing is cached yet");
+        };
+        assert_eq!(need, vec![1024..3072]);
+        serve(&s, &data, &need);
+        let Step::Done(got) = s.attempt(|| owned(s.read_at(1500, 1000))) else {
+            panic!("the blocks were supplied");
+        };
+        assert_eq!(got.unwrap(), &data[1500..2500]);
+        // Past the end: short, then empty, as for a slice.
+        serve(&s, &data, &[9216..10_000]);
+        let Step::Done(tail) = s.attempt(|| owned(s.read_at(9_990, 100))) else {
+            panic!("the last block was supplied");
+        };
+        assert_eq!(tail.unwrap(), &data[9_990..]);
+        assert!(matches!(
+            s.attempt(|| s.read_at(20_000, 10).map(|c| c.len())),
+            Step::Done(Ok(0))
+        ));
+        let st = s.stats();
+        assert_eq!(
+            (st.passes, st.requests, st.bytes_fetched),
+            (4, 2, 2048 + 784)
+        );
+    }
+
+    #[test]
+    fn a_pass_that_swallowed_the_miss_is_still_rerun() {
+        // A parser that catches the error and returns something anyway
+        // (a listing skipping a link it cannot resolve) must not have its
+        // result used.
+        let data = file(4096);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
+        let step = s.attempt(|| s.read_at(0, 4).map(|b| b.len()).unwrap_or(0));
+        assert!(
+            matches!(step, Step::Need(ref n) if n == &vec![0..1024]),
+            "{step:?}"
+        );
+    }
+
+    #[test]
+    fn read_ranges_asks_for_every_missing_block_in_one_pass() {
+        let data = file(64 * 1024);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
+        let ranges = [
+            100..200,
+            5000..5100,
+            5200..5300,
+            30_000..33_000,
+            60_000..60_010,
+        ];
+        let read = || {
+            s.read_ranges(&ranges)
+                .map(|v| v.into_iter().map(Cow::into_owned).collect::<Vec<_>>())
+        };
+        let Step::Need(need) = s.attempt(read) else {
+            panic!("nothing is cached yet");
+        };
+        // 5000..5300 is blocks 4 and 5; 30_000..33_000 is blocks 29..=32.
+        assert_eq!(
+            need,
+            vec![
+                0..1024,
+                4096..6144,
+                29 * 1024..33 * 1024,
+                58 * 1024..59 * 1024
+            ]
+        );
+        serve(&s, &data, &need);
+        let Step::Done(got) = s.attempt(read) else {
+            panic!("one pass fetched everything");
+        };
+        for (r, g) in ranges.iter().zip(&got.unwrap()) {
+            assert_eq!(g, &data[r.start as usize..r.end as usize]);
+        }
+    }
+
+    #[test]
+    fn runs_merge_one_block_holes_and_split_long_runs() {
+        let data = file(32 * 1024);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
+        // Blocks 0 and 2 (a hole of one: merged), 5..=14 (split in fours).
+        let Step::Need(need) = s.attempt(|| {
+            let _ = s.read_at(0, 10);
+            let _ = s.read_at(2048, 10);
+            s.read_ranges(&[5120..15 * 1024]).map(|_| ())
+        }) else {
+            panic!("nothing is cached yet");
+        };
+        assert_eq!(
+            need,
+            vec![0..3072, 5120..9216, 9216..13_312, 13_312..15_360]
+        );
+    }
+
+    #[test]
+    fn supply_refuses_what_was_not_asked_for() {
+        let data = file(10_000);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
+        assert!(s.supply(1, &data[1..1025]).unwrap_err().contains("whole"));
+        assert!(s.supply(0, &data[..1000]).unwrap_err().contains("whole"));
+        assert!(
+            s.supply(9216, &[0u8; 1024])
+                .unwrap_err()
+                .contains("past the end")
+        );
+        for wrong in [&data[..1023], &data[..2048]] {
+            assert!(
+                s.supply_range(&(0..1024), wrong)
+                    .unwrap_err()
+                    .contains("asked for 1024 bytes")
+            );
+        }
+        assert_eq!(s.stats().cached_bytes, 0);
+        // The file's last, short block is whole.
+        s.supply(9216, &data[9216..]).unwrap();
+        assert_eq!(s.stats().cached_bytes, 784);
+    }
+
+    #[test]
+    fn an_operation_keeps_its_blocks_whatever_the_budget() {
+        // A budget of one block, an operation that needs eight: without
+        // the operation guard every pass would evict what the last one
+        // fetched and never finish.
+        let data = file(8 * 1024);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 1024));
+        let got = s
+            .run_blocking(
+                || {
+                    (0..8)
+                        .map(|i| owned(s.read_at(i * 1024 + 3, 10)))
+                        .collect::<Result<Vec<_>, _>>()
+                },
+                |r| Ok(data[r.start as usize..r.end as usize].to_vec()),
+            )
+            .unwrap()
+            .unwrap();
+        for (i, g) in got.iter().enumerate() {
+            assert_eq!(g, &data[i * 1024 + 3..i * 1024 + 13]);
+        }
+        // Trimmed to the budget once the operation is over.
+        let st = s.stats();
+        assert_eq!(st.cached_bytes, 1024);
+        assert_eq!(st.evictions, 7);
+        assert_eq!(st.passes, 9, "one pass per block, then the one that ends");
+    }
+
+    #[test]
+    fn bulk_blocks_are_evicted_before_metadata() {
+        let data = file(16 * 1024);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 4 * 1024));
+        let fetch = |r: Range<u64>| Ok(data[r.start as usize..r.end as usize].to_vec());
+        // Metadata: a small read of block 0.
+        s.run_blocking(|| s.read_at(0, 16).map(|_| ()), fetch)
+            .unwrap()
+            .unwrap();
+        // Bulk: raw data over blocks 4..12, more than the budget.
+        s.run_blocking(|| s.read_ranges(&[4096..12 * 1024]).map(|_| ()), fetch)
+            .unwrap()
+            .unwrap();
+        // The metadata block survived: reading it again is a hit.
+        let before = s.stats();
+        assert!(before.cached_bytes <= 4 * 1024);
+        assert!(matches!(
+            s.attempt(|| s.read_at(0, 16).map(|_| ())),
+            Step::Done(Ok(()))
+        ));
+        assert_eq!(s.stats().requests, before.requests);
+    }
+
+    #[test]
+    fn a_failed_fetch_is_an_error_not_data() {
+        let data = file(4096);
+        let s = LazyStorage::new(data.len() as u64, config(1024, 1 << 20));
+        let e = s
+            .run_blocking(|| owned(s.read_at(0, 8)), |_| Err("HTTP 500".into()))
+            .unwrap_err();
+        assert_eq!(e, "HTTP 500");
+        let e = s
+            .run_blocking(|| owned(s.read_at(0, 8)), |_| Ok(vec![0; 10]))
+            .unwrap_err();
+        assert!(e.contains("asked for 1024 bytes"), "{e}");
+        assert_eq!(s.stats().cached_bytes, 0);
+        drop(data);
+    }
+}

crates/clawhdf5-wasm/src/lib.rs

@@ -21,6 +21,7 @@
 //! The logic lives in [`core`], which is plain Rust and tested natively.
 
 pub mod core;
+pub mod lazy;
 
 use clawhdf5::AttrValue;
 use js_sys::{Array, Object, Reflect};

crates/clawhdf5-wasm/tests/lazy.rs

@@ -0,0 +1,391 @@
+//! The restartable ("NeedBytes") reader against the in-memory one: every
+//! file must list, describe and read the same through a [`LazyStorage`]
+//! that starts empty and is fed only the ranges its passes ask for, as the
+//! browser's `openUrl` feeds it from HTTP range requests.
+//!
+//! - Files written here with `FileBuilder`, at several block sizes (512 B
+//!   blocks make almost every structure read a miss).
+//! - The h5py/netCDF4 fixture of `examples/wasm-viewer/test/make_fixture.py`
+//!   (skipped without h5py, unless `CLAWHDF5_REQUIRE_INTEROP=1`;
+//!   `CLAWHDF5_PYTHON` names the interpreter).
+//! - `CLAWHDF5_WASM_CORPUS=dir[:dir...]`: every HDF5 file under those
+//!   directories up to 64 MiB (e.g. `conformance/.cache/corpus`).
+//!
+//! Also the request budget: listing and reading one small dataset of a large
+//! file fetches a few blocks, not the file.
+
+use std::ops::Range;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::Arc;
+
+use clawhdf5::{AttrValue, FileBuilder};
+use clawhdf5_format::storage::CountingStorage;
+use clawhdf5_wasm::core::{Hyperslab, Kind, Reader};
+use clawhdf5_wasm::lazy::{LazyConfig, LazyStorage};
+
+/// The API the JavaScript side calls, one operation at a time.
+trait Api {
+    fn call<T>(&self, op: impl Fn(&Reader) -> T) -> T;
+}
+
+struct Local(Reader);
+
+impl Api for Local {
+    fn call<T>(&self, op: impl Fn(&Reader) -> T) -> T {
+        op(&self.0)
+    }
+}
+
+/// A lazily read file and the "server" it fetches from.
+struct Lazy {
+    data: Arc<Vec<u8>>,
+    storage: Arc<LazyStorage>,
+    reader: Reader,
+}
+
+fn fetch(data: &[u8], r: Range<u64>) -> Result<Vec<u8>, String> {
+    Ok(data[r.start as usize..r.end as usize].to_vec())
+}
+
+impl Lazy {
+    /// Open as `openUrl` does: the first block comes with the probe that
+    /// learns the length, then the open is run until it has its bytes.
+    fn open(data: Vec<u8>, config: LazyConfig) -> Result<Lazy, String> {
+        let data = Arc::new(data);
+        let storage = Arc::new(LazyStorage::new(data.len() as u64, config));
+        let first = (storage.config().block_size as usize).min(data.len());
+        storage.supply(0, &data[..first])?;
+        let s = storage.clone();
+        let reader =
+            storage.run_blocking(|| Reader::open_storage(s.clone()), |r| fetch(&data, r))??;
+        Ok(Lazy {
+            data,
+            storage,
+            reader,
+        })
+    }
+}
+
+impl Api for Lazy {
+    fn call<T>(&self, op: impl Fn(&Reader) -> T) -> T {
+        self.storage
+            .run_blocking(|| op(&self.reader), |r| fetch(&self.data, r))
+            .expect("serving from memory cannot fail")
+    }
+}
+
+/// Everything the viewer can show of a file, as text: each object's kind,
+/// listing, attributes (and attribute errors), dataset info, whole value
+/// and a hyperslab — or the error each gives.
+fn transcript(api: &impl Api) -> Vec<String> {
+    let mut out = Vec::new();
+    let mut todo = vec![("/".to_string(), 0usize)];
+    while let Some((path, depth)) = todo.pop() {
+        if out.len() > 4000 {
+            out.push("... (truncated)".into());
+            break;
+        }
+        let kind = api.call(|r| r.kind(&path));
+        out.push(format!("{path}: {kind:?}"));
+        out.push(format!("{path} attrs: {:?}", api.call(|r| r.attrs(&path))));
+        match kind {
+            Ok(Kind::Group) => {
+                let list = api.call(|r| r.list(&path));
+                out.push(format!("{path} list: {list:?}"));
+                if let Ok(children) = list
+                    && depth < 12
+                {
+                    for c in children.into_iter().rev() {
+                        let child = if path == "/" {
+                            format!("/{}", c.name)
+                        } else {
+                            format!("{path}/{}", c.name)
+                        };
+                        todo.push((child, depth + 1));
+                    }
+                }
+            }
+            Ok(Kind::Dataset) => {
+                let info = api.call(|r| r.info(&path));
+                out.push(format!("{path} info: {info:?}"));
+                let Ok(info) = info else { continue };
+                let n = info
+                    .shape
+                    .iter()
+                    .chain(&info.element_shape)
+                    .try_fold(1u64, |a, &d| a.checked_mul(d));
+                if n.is_none_or(|n| n > 4_000_000) {
+                    out.push(format!("{path}: not read ({n:?} values)"));
+                    continue;
+                }
+                out.push(format!(
+                    "{path} read: {:?}",
+                    api.call(|r| r.read(&path, None))
+                ));
+                if !info.shape.is_empty() && info.shape.iter().all(|&d| d > 1) {
+                    let slab = Hyperslab {
+                        start: info.shape.iter().map(|_| 1).collect(),
+                        count: info.shape.iter().map(|&d| d / 2).collect(),
+                        stride: None,
+                        block: None,
+                    };
+                    let part = api.call(|r| r.read(&path, Some(&slab)));
+                    out.push(format!("{path} slab: {part:?}"));
+                }
+            }
+            Err(_) => {}
+        }
+    }
+    out
+}
+
+/// The lazy transcript of `data` at `block` bytes per block equals the
+/// transcript of the same file through a range storage that has every byte
+/// (`CountingStorage`: the facade's `Storage` path, the one the lazy reader
+/// takes), and agrees with the in-memory one: the same values, and an error
+/// wherever it has one (a malformed file can fail at a different check,
+/// with a different message, when read by ranges). Returns what the lazy
+/// reader fetched and its transcript.
+fn check_equal(name: &str, data: &[u8], block: u64) -> (u64, u64, Vec<String>) {
+    let ctx = format!("{name} (blocks of {block} B)");
+    let ranged = Reader::open_storage(Arc::new(CountingStorage::new(data.to_vec())));
+    let local = Reader::open(data.to_vec());
+    let lazy = Lazy::open(data.to_vec(), config(block));
+    let (ranged, local, lazy) = match (ranged, local, lazy) {
+        (Ok(r), Ok(l), Ok(z)) => (r, l, z),
+        (Err(r), Err(_), Err(z)) => {
+            assert_eq!(z, r, "{ctx}: open error");
+            return (0, 0, Vec::new());
+        }
+        (r, l, z) => panic!(
+            "{ctx}: opens differently: ranged {:?}, in memory {:?}, lazily {:?}",
+            r.err(),
+            l.err(),
+            z.err()
+        ),
+    };
+    let got = transcript(&lazy);
+    let want = transcript(&Local(ranged));
+    for (i, (w, g)) in want.iter().zip(&got).enumerate() {
+        assert_eq!(g, w, "{ctx}, line {i}");
+    }
+    assert_eq!(got.len(), want.len(), "{ctx}: transcript length");
+    let local = transcript(&Local(local));
+    for (i, (l, g)) in local.iter().zip(&got).enumerate() {
+        let both_errors = match (l.split_once("Err("), g.split_once("Err(")) {
+            (Some((a, _)), Some((b, _))) => a == b,
+            _ => false,
+        };
+        assert!(
+            l == g || both_errors,
+            "{ctx}, line {i}: in memory\n  {l}\nlazily\n  {g}"
+        );
+    }
+    assert_eq!(got.len(), local.len(), "{ctx}: transcript length");
+    let st = lazy.storage.stats();
+    (st.requests, st.bytes_fetched, got)
+}
+
+fn config(block: u64) -> LazyConfig {
+    LazyConfig {
+        block_size: block,
+        // A small budget, so eviction between operations is exercised.
+        capacity: 16 * block,
+        max_request: 8 * block,
+    }
+}
+
+fn builder_file() -> Vec<u8> {
+    let mut b = FileBuilder::new();
+    b.create_dataset("grid")
+        .with_f64_data(&(0..20_000).map(f64::from).collect::<Vec<_>>())
+        .with_shape(&[100, 200])
+        .with_chunks(&[10, 25])
+        .with_deflate(4);
+    b.create_dataset("contiguous")
+        .with_i32_data(&(0..50_000).collect::<Vec<_>>());
+    b.create_dataset("bytes").with_u8_data(&[1, 2, 250]);
+    let mut g = b.create_group("sensors");
+    for i in 0..40 {
+        g.create_dataset(&format!("t{i}"))
+            .with_f32_data(&[i as f32, 1.5, -2.25]);
+    }
+    g.set_attr("location", AttrValue::String("lab".into()));
+    b.add_group(g.finish());
+    b.set_attr("version", AttrValue::I64(3));
+    b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0]));
+    b.finish().unwrap()
+}
+
+#[test]
+fn builder_files_read_the_same_at_every_block_size() {
+    let data = builder_file();
+    for block in [512, 4096, 1 << 20] {
+        let (requests, _, lines) = check_equal("builder", &data, block);
+        assert!(requests > 0);
+        // The transcript covers every object, values included.
+        assert!(lines.iter().any(|l| l.starts_with("/grid read: Ok")));
+        assert!(lines.iter().any(|l| l.starts_with("/grid slab: Ok")));
+        assert!(lines.iter().any(|l| l.starts_with("/sensors/t39 read: Ok")));
+    }
+}
+
+#[test]
+fn garbage_fails_to_open_as_in_memory() {
+    check_equal("zeros", &[0u8; 5000], 512);
+    check_equal("empty", &[], 512);
+    let mut cut = builder_file();
+    cut.truncate(cut.len() / 3);
+    check_equal("truncated", &cut, 512);
+}
+
+/// Listing a large file and reading one small dataset fetches a few blocks,
+/// not the file.
+#[test]
+fn a_small_read_of_a_large_file_fetches_a_few_blocks() {
+    let mut b = FileBuilder::new();
+    b.create_dataset("small").with_f64_data(&[1.0, 2.0, 3.0]);
+    // 48 MB of raw data, written after the small dataset's metadata.
+    b.create_dataset("big")
+        .with_f64_data(&(0..6_000_000).map(f64::from).collect::<Vec<_>>());
+    let mut g = b.create_group("group");
+    g.create_dataset("inner").with_i32_data(&[7, 8]);
+    b.add_group(g.finish());
+    let data = b.finish().unwrap();
+    let lazy = Lazy::open(data.clone(), LazyConfig::default()).unwrap();
+    let list = lazy.call(|r| r.list("/")).unwrap();
+    assert_eq!(list.len(), 3);
+    assert_eq!(
+        format!("{:?}", lazy.call(|r| r.read("/small", None)).unwrap().data),
+        "F64([1.0, 2.0, 3.0])"
+    );
+    assert_eq!(
+        format!(
+            "{:?}",
+            lazy.call(|r| r.read("/group/inner", None)).unwrap().data
+        ),
+        "I32([7, 8])"
+    );
+    // A window of the big dataset reads only its block(s).
+    let slab = Hyperslab {
+        start: vec![3_000_000],
+        count: vec![4],
+        stride: None,
+        block: None,
+    };
+    assert_eq!(
+        format!(
+            "{:?}",
+            lazy.call(|r| r.read("/big", Some(&slab))).unwrap().data
+        ),
+        "F64([3000000.0, 3000001.0, 3000002.0, 3000003.0])"
+    );
+    let st = lazy.storage.stats();
+    eprintln!("{} bytes: {st:?}", data.len());
+    assert!(st.requests <= 6, "{st:?}");
+    assert!(st.bytes_fetched <= 6 << 20, "{st:?}");
+    assert!(st.bytes_fetched * 8 < data.len() as u64, "{st:?}");
+}
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, netCDF4, numpy"])
+        .output()
+        .is_ok_and(|o| o.status.success())
+}
+
+#[test]
+fn h5py_and_netcdf4_files_read_the_same_lazily() {
+    if !python_available() {
+        assert!(
+            !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but {} lacks h5py/netCDF4/numpy",
+            python()
+        );
+        eprintln!("skipping: {} lacks h5py/netCDF4/numpy", python());
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let generator = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../examples/wasm-viewer/test/make_fixture.py");
+    let out = Command::new(python())
+        .arg(&generator)
+        .arg(dir.path())
+        .output()
+        .unwrap();
+    assert!(
+        out.status.success(),
+        "{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    for name in ["fixture.h5", "fixture.nc"] {
+        let data = std::fs::read(dir.path().join(name)).unwrap();
+        for block in [512, 64 * 1024] {
+            let (_, _, lines) = check_equal(name, &data, block);
+            assert!(lines.iter().filter(|l| l.contains(" read: Ok")).count() >= 2);
+        }
+    }
+}
+
+fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    for e in entries.flatten() {
+        let p = e.path();
+        if p.is_dir() {
+            hdf5_files(&p, out);
+        } else if std::fs::read(&p)
+            .ok()
+            .is_some_and(|b| b.len() <= 64 << 20 && is_hdf5(&b))
+        {
+            out.push(p);
+        }
+    }
+}
+
+/// The HDF5 signature at 0 or a power-of-two user-block offset.
+fn is_hdf5(b: &[u8]) -> bool {
+    const SIG: &[u8] = b"\x89HDF\r\n\x1a\n";
+    let mut at = 0usize;
+    loop {
+        if b.get(at..at + 8) == Some(SIG) {
+            return true;
+        }
+        at = if at == 0 { 512 } else { at * 2 };
+        if at >= b.len() {
+            return false;
+        }
+    }
+}
+
+#[test]
+fn corpus_files_read_the_same_lazily() {
+    let Ok(dirs) = std::env::var("CLAWHDF5_WASM_CORPUS") else {
+        eprintln!("CLAWHDF5_WASM_CORPUS not set; skipping the corpus");
+        return;
+    };
+    let mut files = Vec::new();
+    for d in std::env::split_paths(&dirs) {
+        hdf5_files(&d, &mut files);
+    }
+    files.sort();
+    assert!(!files.is_empty(), "no HDF5 files under {dirs}");
+    let (mut requests, mut bytes, mut total) = (0u64, 0u64, 0u64);
+    for f in &files {
+        let data = std::fs::read(f).unwrap();
+        total += data.len() as u64;
+        let (r, b, _) = check_equal(&f.display().to_string(), &data, 64 * 1024);
+        requests += r;
+        bytes += b;
+    }
+    eprintln!(
+        "{} files ({total} bytes): {requests} requests, {bytes} bytes fetched",
+        files.len()
+    );
+}