format: raw data, VDS and VL data over Storage

Every raw-data path has a generic *_in core, with the &[u8] functions as
thin wrappers: data_read (read_raw_data*, read_raw_data_selection,
read_chunked_native), chunked_read (the v1 B-tree chunk index, list_chunks,
the full, cached, sweep and indexed reads), parallel_read, partial_read,
fill_value (read_full_with_fill, apply_to_unallocated_chunks; and
dataset_fill_value_from_storage is now generic), vds (the virtual file
through Storage, external sources still through the resolver),
vl_data (VlResolver<'a, S = [u8]>, read_vl_strings_in, read_vl_bytes_in),
AttributeMessage::read_vl_strings_in and provenance::verify_dataset_in.

With the whole file in memory nothing changes: chunks and contiguous data
are sliced from it as before. Otherwise a chunked read lists its chunks,
fetches their stored bytes with one Storage::read_ranges call per 64 MiB
batch (chunks the cache already holds are not fetched), then decodes as
today; a selection fetches only the chunks it overlaps, and a contiguous
selection only its runs. Each extent's bounds error is the one the slice
code gave, reported when that extent is reached, so errors keep their
order.

Tests: the equivalence harness now reads every dataset's values (whole,
fill-aware, cached, indexed, three selections, VDS, VL strings and
sequences) through the read_at-only storage and requires the slice
results (all 653 corpus files agree); a misbehaving storage (a failing
Nth read, short reads) only ever yields errors or the right values; and
chunked reads are checked to use one read_ranges call.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/gather.rs

@@ -13,6 +13,7 @@ use alloc::{vec, vec::Vec};
 use crate::data_read::NativeElement;
 use crate::error::FormatError;
 use crate::selection::Selection;
+use crate::storage::Storage;
 
 /// Row-major element strides of `dims` (the last dimension has stride 1).
 fn strides(dims: &[u64]) -> Vec<u64> {
@@ -261,6 +262,134 @@ pub(crate) fn gather<T: NativeElement>(
     Ok(out)
 }
 
+/// [`gather`] of bytes (`T = u8`) from a dataset that is not in memory: the
+/// dataset's `src_len` bytes start at `base` in `file`, which must hold all
+/// of them (the caller checks). The selection's runs are collected first,
+/// adjacent ones merged, and fetched with one [`Storage::read_ranges`] call,
+/// so only the selected bytes are read. Same checks and errors as
+/// [`gather`].
+pub(crate) fn gather_storage<S: Storage + ?Sized>(
+    file: &S,
+    base: u64,
+    src_len: usize,
+    dims: &[u64],
+    elem_size: usize,
+    selection: &Selection,
+) -> Result<Vec<u8>, FormatError> {
+    if elem_size == 0 {
+        return Err(FormatError::DataSizeMismatch {
+            expected: 1,
+            actual: elem_size,
+        });
+    }
+    let n_elements = match selection {
+        Selection::None => 0,
+        Selection::Hyperslab { count, block, .. } => count
+            .iter()
+            .zip(block)
+            .try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?))
+            .ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?,
+        Selection::Points(points) => points.len() as u64,
+        Selection::All => {
+            return Err(FormatError::SelectionOutOfBounds(
+                "gather does not take Selection::All".into(),
+            ));
+        }
+    };
+    let out_bytes = crate::chunked_read::checked_byte_len(n_elements, elem_size)?;
+    // Runs as (byte offset in the dataset, byte length), in output order.
+    let mut runs: Vec<(usize, usize)> = Vec::new();
+    let mut total = 0usize;
+    let mut failed = false;
+    let mut collect = |first: u64, n: u64| {
+        if failed {
+            return;
+        }
+        let range = usize::try_from(first)
+            .ok()
+            .and_then(|f| f.checked_mul(elem_size))
+            .zip(
+                usize::try_from(n)
+                    .ok()
+                    .and_then(|n| n.checked_mul(elem_size)),
+            )
+            .and_then(|(at, len)| Some((at, len, at.checked_add(len)?)));
+        match range {
+            Some((at, len, end)) if end <= src_len && total + len <= out_bytes => {
+                match runs.last_mut() {
+                    Some((a, l)) if *a + *l == at => *l += len,
+                    _ => runs.push((at, len)),
+                }
+                total += len;
+            }
+            _ => failed = true,
+        }
+    };
+    let mut bad_point = false;
+    match selection {
+        Selection::Hyperslab {
+            start,
+            stride,
+            count,
+            block,
+        } => {
+            let rank = dims.len();
+            if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] {
+                return Err(FormatError::SelectionOutOfBounds(
+                    "hyperslab rank does not match dataset rank".into(),
+                ));
+            }
+            hyperslab_runs(dims, start, stride, count, block, &mut collect);
+        }
+        Selection::Points(points) => {
+            let strides = strides(dims);
+            let mut coalesce = Coalesce {
+                start: 0,
+                len: 0,
+                emit: &mut collect,
+            };
+            for p in points {
+                if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) {
+                    bad_point = true;
+                    break;
+                }
+                let at = p
+                    .iter()
+                    .zip(&strides)
+                    .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s)));
+                coalesce.push(at, 1);
+            }
+            coalesce.flush();
+        }
+        Selection::None | Selection::All => {}
+    }
+    if failed || bad_point || total != out_bytes {
+        return Err(FormatError::SelectionOutOfBounds(
+            "selection addresses elements outside the dataset".into(),
+        ));
+    }
+    let ranges: Vec<core::ops::Range<u64>> = runs
+        .iter()
+        .map(|&(at, len)| base + at as u64..base + (at + len) as u64)
+        .collect();
+    let fetched = file.read_ranges(&ranges)?;
+    if fetched.len() != ranges.len() {
+        return Err(FormatError::Storage(
+            "read_ranges returned the wrong number of ranges".into(),
+        ));
+    }
+    let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes);
+    for (bytes, &(_, len)) in fetched.iter().zip(&runs) {
+        let Some(b) = bytes.get(..len) else {
+            return Err(FormatError::Storage(
+                "short read inside the file (the storage shrank or the backend failed)".into(),
+            ));
+        };
+        out.extend_from_slice(b);
+    }
+    Ok(out)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;