CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+### Range reads, milestone M1: the `Storage` trait (2026-09-26)
+- **Breaking: `clawhdf5_format::error::FormatError` and `clawhdf5::Error`
+  are now `#[non_exhaustive]`.** An exhaustive `match` on either needs a
+  wildcard arm. `FormatError` has two new variants: `Storage(String)` (a
+  storage backend failed to serve a read) and
+  `ContiguousStorageRequired(&'static str)` (an operation not yet converted
+  to range reads was asked of a backend without the whole file in memory).
+- New `clawhdf5_format::storage::Storage`, the synchronous, `no_std` read
+  interface of `docs/design/range-reads.md` option (a): `read_at(offset:
+  u64, len) -> Cow<[u8]>`, `read_ranges`, `len()` and an `as_contiguous()`
+  fast path; implemented for `[u8]`, `Vec<u8>`, and references, `Box`es
+  and (with `std`) `Arc`s of a `Storage`. Slices and `Vec`s serve borrowed
+  bytes, so parsing an in-memory file costs no copy.
+
 ### Chunked full reads (2026-09-26)
 - **Chunks are decoded straight into the output, into reused buffers.** A
   full read of a chunked dataset faulted in about three times its size in

crates/clawhdf5-format/src/error.rs

@@ -12,7 +12,11 @@ use std::string::String;
 use core::fmt;
 
 /// Errors that can occur when parsing HDF5 binary format structures.
+///
+/// Non-exhaustive: new failure modes (new storage backends, new file
+/// features) add variants, so a `match` needs a wildcard arm.
 #[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
 pub enum FormatError {
     /// The HDF5 magic signature was not found at any valid offset.
     SignatureNotFound,
@@ -243,6 +247,13 @@ pub enum FormatError {
     /// A metadata cache image block libhdf5 refuses to load (the reason is
     /// libhdf5's own error text).
     InvalidCacheImage(&'static str),
+    /// The [`Storage`](crate::storage::Storage) backend failed to serve a
+    /// read (an I/O or network error, or a short read inside the file).
+    Storage(String),
+    /// The operation still needs the whole file as one slice and the
+    /// [`Storage`](crate::storage::Storage) backend has no contiguous view
+    /// (`as_contiguous()` is `None`); the text names the operation.
+    ContiguousStorageRequired(&'static str),
 }
 
 impl fmt::Display for FormatError {
@@ -529,6 +540,16 @@ impl fmt::Display for FormatError {
             FormatError::InvalidCacheImage(why) => {
                 write!(f, "invalid metadata cache image: {why}")
             }
+            FormatError::Storage(why) => {
+                write!(f, "storage read failed: {why}")
+            }
+            FormatError::ContiguousStorageRequired(what) => {
+                write!(
+                    f,
+                    "{what} needs the whole file in memory, which this storage backend does \
+                     not provide"
+                )
+            }
         }
     }
 }

crates/clawhdf5-format/src/lib.rs

@@ -120,6 +120,7 @@ pub mod property_list;
 pub mod selection;
 pub mod shared_message;
 pub mod signature;
+pub mod storage;
 pub mod superblock;
 pub mod superblock_ext;
 pub mod symbol_table;

crates/clawhdf5-format/src/storage.rs

@@ -0,0 +1,357 @@
+//! Where the parsers read the file from: the [`Storage`] trait.
+//!
+//! Every parser used to take the whole file as one `&[u8]`. [`Storage`] is
+//! the abstraction that replaces it (see `docs/design/range-reads.md`,
+//! option (a)): a parser asks for the bytes it needs, `[offset, offset +
+//! len)`, with 64-bit offsets, and gets them back as a [`Cow`] — borrowed
+//! when the backend holds the file in memory (a `Vec`, an mmap), owned when
+//! it had to fetch them (a range request, a block cache).
+//!
+//! `impl Storage for [u8]` serves the in-memory case with no copy, and
+//! [`Storage::as_contiguous`] lets a hot loop borrow the whole file at once
+//! when the backend has it. Modules are converted one at a time: a converted
+//! parser has an `*_in(file: &dyn Storage, ..)` core and keeps its old
+//! `&[u8]` signature as a thin wrapper, so callers do not change.
+//!
+//! The trait is synchronous and `no_std`: parsing is CPU work, and a remote
+//! backend bridges to its own I/O.
+
+#[cfg(not(feature = "std"))]
+use alloc::{borrow::Cow, boxed::Box, vec::Vec};
+#[cfg(feature = "std")]
+use std::{borrow::Cow, boxed::Box, vec::Vec};
+
+use core::ops::Range;
+
+use crate::error::FormatError;
+
+/// A random-access source of file bytes.
+///
+/// Offsets are relative to the start of the HDF5 data (the superblock), like
+/// every address in the file.
+pub trait Storage {
+    /// Bytes `[offset, offset + len)`.
+    ///
+    /// The result is shorter than `len` only when the range runs past the
+    /// end of the storage (and empty when `offset` is at or past the end);
+    /// a backend that cannot serve a range returns an error instead of a
+    /// short read.
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError>;
+
+    /// Current length of the storage in bytes.
+    fn len(&self) -> u64;
+
+    /// Whether the storage holds no bytes.
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Several reads at once, in the order given. Backends that talk to a
+    /// remote store coalesce and parallelise these; the default reads them
+    /// one by one with [`Storage::read_at`].
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        ranges
+            .iter()
+            .map(|r| {
+                let len = usize::try_from(r.end.saturating_sub(r.start)).map_err(|_| {
+                    FormatError::Overflow("read range longer than the address space".into())
+                })?;
+                self.read_at(r.start, len)
+            })
+            .collect()
+    }
+
+    /// The whole storage as one slice, when the backend has it in memory
+    /// (a `Vec`, an mmap). Hot loops use this to keep their zero-copy path;
+    /// `None` means every byte has to go through [`Storage::read_at`].
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        None
+    }
+}
+
+impl Storage for [u8] {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        let n = self.len();
+        let start = usize::try_from(offset).map_or(n, |o| o.min(n));
+        let end = start.saturating_add(len).min(n);
+        Ok(Cow::Borrowed(&self[start..end]))
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        <[u8]>::len(self) as u64
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        Some(self)
+    }
+}
+
+impl Storage for Vec<u8> {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        self.as_slice().read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        Vec::len(self) as u64
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        Some(self.as_slice())
+    }
+}
+
+impl<T: Storage + ?Sized> Storage for &T {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        (**self).read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        (**self).len()
+    }
+
+    #[inline]
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        (**self).read_ranges(ranges)
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        (**self).as_contiguous()
+    }
+}
+
+impl<T: Storage + ?Sized> Storage for Box<T> {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        (**self).read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        (**self).len()
+    }
+
+    #[inline]
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        (**self).read_ranges(ranges)
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        (**self).as_contiguous()
+    }
+}
+
+#[cfg(feature = "std")]
+impl<T: Storage + ?Sized> Storage for std::sync::Arc<T> {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        (**self).read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        (**self).len()
+    }
+
+    #[inline]
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        (**self).read_ranges(ranges)
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        (**self).as_contiguous()
+    }
+}
+
+/// `storage.len()` as the `usize` the parsers' end-of-file errors report
+/// (saturating on targets where the file is larger than the address space).
+#[inline]
+pub(crate) fn len_usize(file: &dyn Storage) -> usize {
+    usize::try_from(file.len()).unwrap_or(usize::MAX)
+}
+
+/// Bytes `[offset, offset + len)`, all of them.
+///
+/// A range that runs past the end of the storage is
+/// [`FormatError::UnexpectedEof`] with `expected = offset + len` and
+/// `available = storage length` — the error the `&[u8]` parsers give for
+/// the same bounds check (`offset + len > file_data.len()`).
+#[inline]
+pub fn read_exact_at(
+    file: &dyn Storage,
+    offset: u64,
+    len: usize,
+) -> Result<Cow<'_, [u8]>, FormatError> {
+    let eof = || FormatError::UnexpectedEof {
+        expected: usize::try_from(offset)
+            .unwrap_or(usize::MAX)
+            .saturating_add(len),
+        available: len_usize(file),
+    };
+    match offset.checked_add(len as u64) {
+        Some(end) if end <= file.len() => {}
+        _ => return Err(eof()),
+    }
+    let bytes = file.read_at(offset, len)?;
+    if bytes.len() < len {
+        // The storage shrank or the backend served a short read inside the
+        // file: never parse a partial structure.
+        return Err(eof());
+    }
+    Ok(bytes)
+}
+
+/// Up to `max` bytes from `offset` on: fewer only at the end of the
+/// storage. For structures whose size is only known once their prefix has
+/// been parsed and whose parsers bound-check what they are given.
+#[inline]
+pub fn read_upto(file: &dyn Storage, offset: u64, max: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+    let avail = file.len().saturating_sub(offset);
+    let len = usize::try_from(avail).map_or(max, |a| a.min(max));
+    let bytes = file.read_at(offset, len)?;
+    if bytes.len() < len {
+        return Err(FormatError::Storage(
+            "short read inside the file (the storage shrank or the backend failed)".into(),
+        ));
+    }
+    Ok(bytes)
+}
+
+/// Borrow the whole file for a code path that has not been converted to
+/// [`Storage`] yet. On a backend without a contiguous view this is the
+/// clean [`FormatError::ContiguousStorageRequired`] error, never a guess.
+#[inline]
+pub fn require_contiguous<'a>(
+    file: &'a dyn Storage,
+    what: &'static str,
+) -> Result<&'a [u8], FormatError> {
+    file.as_contiguous()
+        .ok_or(FormatError::ContiguousStorageRequired(what))
+}
+
+/// A [`Storage`] over an in-memory buffer that serves every byte through
+/// [`Storage::read_at`] (its [`Storage::as_contiguous`] is `None`, so no
+/// parser can take the whole-slice shortcut), copies what it serves (as a
+/// remote backend would), and counts the reads and bytes.
+///
+/// It is the equivalence harness of the range-read migration: parsing a
+/// file through it must give exactly what parsing the `&[u8]` gives, and
+/// the counters are the request counts a cacheless range reader would make.
+#[derive(Debug)]
+pub struct CountingStorage {
+    data: Vec<u8>,
+    reads: portable_atomic::AtomicU64,
+    bytes: portable_atomic::AtomicU64,
+}
+
+impl CountingStorage {
+    /// Serve `data` (the file from the superblock on).
+    pub fn new(data: Vec<u8>) -> Self {
+        CountingStorage {
+            data,
+            reads: portable_atomic::AtomicU64::new(0),
+            bytes: portable_atomic::AtomicU64::new(0),
+        }
+    }
+
+    /// Number of `read_at` calls served so far.
+    pub fn reads(&self) -> u64 {
+        self.reads.load(portable_atomic::Ordering::Relaxed)
+    }
+
+    /// Number of bytes served so far.
+    pub fn bytes_read(&self) -> u64 {
+        self.bytes.load(portable_atomic::Ordering::Relaxed)
+    }
+
+    /// Reset both counters.
+    pub fn reset(&self) {
+        self.reads.store(0, portable_atomic::Ordering::Relaxed);
+        self.bytes.store(0, portable_atomic::Ordering::Relaxed);
+    }
+}
+
+impl Storage for CountingStorage {
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        let got = self.data.as_slice().read_at(offset, len)?;
+        self.reads.fetch_add(1, portable_atomic::Ordering::Relaxed);
+        self.bytes
+            .fetch_add(got.len() as u64, portable_atomic::Ordering::Relaxed);
+        Ok(Cow::Owned(got.into_owned()))
+    }
+
+    fn len(&self) -> u64 {
+        self.data.len() as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn slice_reads_are_borrowed_and_clamped() {
+        let data: Vec<u8> = (0u8..10).collect();
+        let s: &[u8] = &data;
+        let dynamic: &dyn Storage = &s;
+        assert_eq!(dynamic.len(), 10);
+        let r = dynamic.read_at(2, 3).unwrap();
+        assert!(matches!(r, Cow::Borrowed(_)));
+        assert_eq!(&*r, &[2, 3, 4]);
+        assert_eq!(&*dynamic.read_at(8, 5).unwrap(), &[8, 9]);
+        assert!(dynamic.read_at(10, 5).unwrap().is_empty());
+        assert!(dynamic.read_at(u64::MAX, 5).unwrap().is_empty());
+        assert_eq!(dynamic.as_contiguous(), Some(&data[..]));
+        let v: &dyn Storage = &data;
+        assert_eq!(v.as_contiguous(), Some(&data[..]));
+    }
+
+    #[test]
+    fn read_exact_matches_slice_bounds_errors() {
+        let data = [0u8; 10];
+        let s: &[u8] = &data;
+        assert_eq!(&*read_exact_at(&s, 4, 6).unwrap(), &[0; 6]);
+        assert_eq!(
+            read_exact_at(&s, 4, 7).unwrap_err(),
+            FormatError::UnexpectedEof {
+                expected: 11,
+                available: 10
+            }
+        );
+        assert!(read_exact_at(&s, u64::MAX, 1).is_err());
+        assert_eq!(read_upto(&s, 7, 100).unwrap().len(), 3);
+        assert_eq!(read_upto(&s, 70, 100).unwrap().len(), 0);
+    }
+
+    #[test]
+    fn counting_storage_counts_and_hides_the_slice() {
+        let c = CountingStorage::new((0u8..10).collect());
+        assert!(c.as_contiguous().is_none());
+        let r = c.read_at(3, 4).unwrap();
+        assert!(matches!(r, Cow::Owned(_)));
+        assert_eq!(&*r, &[3, 4, 5, 6]);
+        c.read_at(8, 4).unwrap();
+        assert_eq!((c.reads(), c.bytes_read()), (2, 6));
+        c.reset();
+        assert_eq!((c.reads(), c.bytes_read()), (0, 0));
+    }
+
+    #[test]
+    fn read_ranges_default_loops() {
+        let data: Vec<u8> = (0u8..10).collect();
+        let s: &[u8] = &data;
+        let got = s.read_ranges(&[1..3, 5..9]).unwrap();
+        assert_eq!(&*got[0], &[1, 2]);
+        assert_eq!(&*got[1], &[5, 6, 7, 8]);
+    }
+}

crates/clawhdf5-py/src/lib.rs

@@ -82,6 +82,7 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr {
         | Error::ZeroCopyUnaligned { .. } => {
             PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string())
         }
+        _ => PyErr::new::<pyo3::exceptions::PyOSError, _>(e.to_string()),
     }
 }
 

crates/clawhdf5/src/error.rs

@@ -6,7 +6,12 @@ use clawhdf5_format::error::FormatError;
 use clawhdf5_format::message_type::MessageType;
 
 /// Errors that can occur when using the high-level API.
+///
+/// Non-exhaustive: new failure modes add variants, so a `match` needs a
+/// wildcard arm. Failures of a [`Storage`](clawhdf5_format::storage::Storage)
+/// backend arrive as `Error::Format(FormatError::Storage(..))`.
 #[derive(Debug)]
+#[non_exhaustive]
 pub enum Error {
     /// I/O error from the filesystem.
     Io(std::io::Error),