//! Where the parsers read the file from: the [`Storage`] trait. //! //! Every parser used to take the whole file as one `&[u8]`. [`Storage`] is //! the abstraction that replaces it (see `docs/design/range-reads.md`, //! option (a)): a parser asks for the bytes it needs, `[offset, offset + //! len)`, with 64-bit offsets, and gets them back as a [`Cow`] — borrowed //! when the backend holds the file in memory (a `Vec`, an mmap), owned when //! it had to fetch them (a range request, a block cache). //! //! `impl Storage for [u8]` serves the in-memory case with no copy, and //! [`Storage::as_contiguous`] lets a hot loop borrow the whole file at once //! when the backend has it. Modules are converted one at a time: a converted //! parser has an `*_in(file: &S, ..)` core and keeps //! its old `&[u8]` signature as a thin wrapper, so callers do not change. //! //! The cores are generic rather than taking `&dyn Storage` so that the //! wrappers monomorphise for `[u8]`: the bounds check of each structure read //! inlines to what the slice code did, with no indirect call and no copy, //! which keeps local files as fast as before the migration. A `&dyn Storage` //! still works (`S = dyn Storage`), and a remote backend pays one indirect //! call per structure read. //! //! The trait is synchronous and `no_std`: parsing is CPU work, and a remote //! backend bridges to its own I/O. #[cfg(not(feature = "std"))] use alloc::{borrow::Cow, boxed::Box, vec::Vec}; #[cfg(feature = "std")] use std::{borrow::Cow, boxed::Box, vec::Vec}; use core::ops::Range; use crate::error::FormatError; /// A random-access source of file bytes. /// /// Offsets are relative to the start of the HDF5 data (the superblock), like /// every address in the file. pub trait Storage { /// Bytes `[offset, offset + len)`. /// /// The result is shorter than `len` only when the range runs past the /// end of the storage (and empty when `offset` is at or past the end); /// a backend that cannot serve a range returns an error instead of a /// short read. fn read_at(&self, offset: u64, len: usize) -> Result, FormatError>; /// Current length of the storage in bytes. fn len(&self) -> u64; /// Whether the storage holds no bytes. fn is_empty(&self) -> bool { self.len() == 0 } /// Several reads at once, in the order given. Backends that talk to a /// remote store coalesce and parallelise these; the default reads them /// one by one with [`Storage::read_at`]. fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { ranges .iter() .map(|r| { let len = usize::try_from(r.end.saturating_sub(r.start)).map_err(|_| { FormatError::Overflow("read range longer than the address space".into()) })?; self.read_at(r.start, len) }) .collect() } /// The whole storage as one slice, when the backend has it in memory /// (a `Vec`, an mmap). Hot loops use this to keep their zero-copy path; /// `None` means every byte has to go through [`Storage::read_at`]. fn as_contiguous(&self) -> Option<&[u8]> { None } } impl Storage for [u8] { #[inline] fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { let n = self.len(); let start = usize::try_from(offset).map_or(n, |o| o.min(n)); let end = start.saturating_add(len).min(n); Ok(Cow::Borrowed(&self[start..end])) } #[inline] fn len(&self) -> u64 { <[u8]>::len(self) as u64 } #[inline] fn as_contiguous(&self) -> Option<&[u8]> { Some(self) } } impl Storage for Vec { #[inline] fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { self.as_slice().read_at(offset, len) } #[inline] fn len(&self) -> u64 { Vec::len(self) as u64 } #[inline] fn as_contiguous(&self) -> Option<&[u8]> { Some(self.as_slice()) } } impl Storage for &T { #[inline] fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { (**self).read_at(offset, len) } #[inline] fn len(&self) -> u64 { (**self).len() } #[inline] fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { (**self).read_ranges(ranges) } #[inline] fn as_contiguous(&self) -> Option<&[u8]> { (**self).as_contiguous() } } impl Storage for Box { #[inline] fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { (**self).read_at(offset, len) } #[inline] fn len(&self) -> u64 { (**self).len() } #[inline] fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { (**self).read_ranges(ranges) } #[inline] fn as_contiguous(&self) -> Option<&[u8]> { (**self).as_contiguous() } } #[cfg(feature = "std")] impl Storage for std::sync::Arc { #[inline] fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { (**self).read_at(offset, len) } #[inline] fn len(&self) -> u64 { (**self).len() } #[inline] fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { (**self).read_ranges(ranges) } #[inline] fn as_contiguous(&self) -> Option<&[u8]> { (**self).as_contiguous() } } /// `storage.len()` as the `usize` the parsers' end-of-file errors report /// (saturating on targets where the file is larger than the address space). #[inline] pub(crate) fn len_usize(file: &S) -> usize { usize::try_from(file.len()).unwrap_or(usize::MAX) } /// Bytes `[offset, offset + len)`, all of them. /// /// A range that runs past the end of the storage is /// [`FormatError::UnexpectedEof`] with `expected = offset + len` and /// `available = storage length` — the error the `&[u8]` parsers give for /// the same bounds check (`offset + len > file_data.len()`). #[inline] pub fn read_exact_at( file: &S, offset: u64, len: usize, ) -> Result, FormatError> { let eof = || FormatError::UnexpectedEof { expected: usize::try_from(offset) .unwrap_or(usize::MAX) .saturating_add(len), available: len_usize(file), }; // In-memory fast path: plain slicing (for `S = [u8]` this inlines to // the slice code's bounds check). if let Some(all) = file.as_contiguous() { return usize::try_from(offset) .ok() .and_then(|start| all.get(start..start.checked_add(len)?)) .map(Cow::Borrowed) .ok_or_else(eof); } match offset.checked_add(len as u64) { Some(end) if end <= file.len() => {} _ => return Err(eof()), } let bytes = file.read_at(offset, len)?; if bytes.len() < len { // The storage shrank or the backend served a short read inside the // file: never parse a partial structure. return Err(short_read()); } Ok(bytes) } #[cold] #[inline(never)] fn short_read() -> FormatError { FormatError::Storage( "short read inside the file (the storage shrank or the backend failed)".into(), ) } /// Largest paged data block (fixed or extensible array) read in one piece. /// A bigger one is read as its prefix and then page by page, only the pages /// in use, so a block whose size fields claim more than the file holds /// costs no more than the pages it really has. pub(crate) const PAGED_BLOCK_ONE_READ_MAX: usize = 1 << 20; /// A window of the file: up to `max` bytes read at `base`, fewer only at /// the end of the file. Its [`Window::ensure`] reports a bounds failure /// exactly as the whole-file check `ensure_len(file_data, base + rel, n)` /// did — with the absolute position and the file's length — as long as /// every position checked lies within the `max` bytes the window was asked /// for: then a position past the window is past the end of the file. pub(crate) struct Window<'a> { /// The bytes, from `base` on. pub bytes: Cow<'a, [u8]>, base: usize, file_len: usize, } impl<'a> Window<'a> { /// Read up to `max` bytes at `base`. pub fn read( file: &'a S, base: u64, max: usize, ) -> Result { Ok(Window { bytes: read_upto(file, base, max)?, base: usize::try_from(base).unwrap_or(usize::MAX), file_len: len_usize(file), }) } /// A whole in-memory file as one window (base 0). #[cfg(test)] pub fn whole(bytes: &'a [u8]) -> Self { Window { bytes: Cow::Borrowed(bytes), base: 0, file_len: bytes.len(), } } /// [`Window::ensure`] for a window at `base` that has not been read: /// whether `[rel, rel + needed)` lies in the file, with the same error. /// Lets a parser whose first step is to check a structure's whole extent /// (a checksum at its end) fail before reading a structure that a /// hostile size field has stretched past the end of the file. pub fn check_extent( file: &S, base: u64, rel: usize, needed: usize, ) -> Result<(), FormatError> { let base = usize::try_from(base).unwrap_or(usize::MAX); let file_len = len_usize(file); match base.checked_add(rel).and_then(|p| p.checked_add(needed)) { Some(end) if end <= file_len => Ok(()), _ => Err(FormatError::UnexpectedEof { expected: base.saturating_add(rel).saturating_add(needed), available: file_len, }), } } /// Check that `[rel, rel + needed)` (relative to `base`) is in the file. #[inline] pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> { match rel.checked_add(needed) { Some(end) if end <= self.bytes.len() => Ok(()), _ => Err(FormatError::UnexpectedEof { expected: self.base.saturating_add(rel).saturating_add(needed), available: self.file_len, }), } } } /// Up to `max` bytes from `offset` on: fewer only at the end of the /// storage. For structures whose size is only known once their prefix has /// been parsed and whose parsers bound-check what they are given. #[inline] pub fn read_upto( file: &S, offset: u64, max: usize, ) -> Result, FormatError> { if let Some(all) = file.as_contiguous() { let start = usize::try_from(offset).map_or(all.len(), |o| o.min(all.len())); let end = start.saturating_add(max).min(all.len()); return Ok(Cow::Borrowed(&all[start..end])); } let avail = file.len().saturating_sub(offset); let len = usize::try_from(avail).map_or(max, |a| a.min(max)); let bytes = file.read_at(offset, len)?; if bytes.len() < len { return Err(short_read()); } Ok(bytes) } /// Most stored bytes fetched by one [`Storage::read_ranges`] call when a /// read gathers many extents (a chunked dataset's chunks, a selection's /// runs): a larger read is fetched and decoded batch by batch, so a backend /// without the file in memory never holds more than this much undecoded /// data per read (or one extent, when a single one is larger — and every /// chunk's extent is bounded by what the chunk can need, see /// [`crate::filters::stored_chunk_limit`]). pub const RAW_BATCH_BYTES: usize = 64 << 20; /// One extent of a raw-data read: `len` bytes stored at `addr`, whose /// bounds are checked against the file, of which the first `fetch` bytes /// are read (`None`: only checked, not read — its bytes are not needed). /// /// `fetch` below `len` bounds what a crafted size field can make a read /// fetch: a chunk never needs more of its stored bytes than its decoded /// size allows, however large its index entry says it is. #[derive(Debug, Clone, Copy)] pub(crate) struct ExtentReq { pub addr: u64, pub len: usize, pub fetch: Option, } impl ExtentReq { /// How many bytes are read for this extent. #[inline] fn fetch_len(&self) -> usize { self.fetch.map_or(0, |f| f.min(self.len)) } } /// One extent's bytes on their own (see [`ExtentReq`]): the whole extent's /// bounds checked as [`read_exact_at`] checks them, and its first /// `req.fetch` bytes read (none when `fetch` is `None`). pub(crate) fn read_extent<'a, S: Storage + ?Sized>( file: &'a S, req: &ExtentReq, ) -> Result, FormatError> { let start = usize::try_from(req.addr).unwrap_or(usize::MAX); match start.checked_add(req.len) { Some(end) if end <= len_usize(file) => read_exact_at(file, req.addr, req.fetch_len()), _ => Err(FormatError::UnexpectedEof { expected: start.saturating_add(req.len), available: len_usize(file), }), } } /// The stored bytes of one batch of extents (chunks, contiguous runs), /// fetched together: [`Storage::read_ranges`] is called once for the batch, /// so a remote backend can coalesce and parallelise the requests. See /// [`for_each_extent_batch`], which is how every raw-data read gets them. /// /// With the whole file in memory nothing is fetched: [`Self::get`] slices /// it, as the slice readers did. Either way an extent that does not lie in /// the file is the error the slice readers gave for it /// ([`FormatError::UnexpectedEof`] with its end and the file length, or /// [`FormatError::Overflow`] for an address past this platform's `usize`), /// reported when that extent is asked for — so a read reports the first /// failing extent in its own order, whatever fails after it. pub(crate) enum ExtentBytes<'a> { /// The whole file. Contiguous(&'a [u8]), /// Each extent's bytes, or its bounds error; the first is extent /// `base` of the read. Fetched { base: usize, extents: Vec>, }, } /// One extent of [`ExtentBytes::Fetched`]. pub(crate) enum Extent<'a> { /// Its bytes. Bytes(Cow<'a, [u8]>), /// In the file, but not fetched (the caller did not want its bytes). NotFetched, /// The error reading it gives. Err(FormatError), } impl<'a> ExtentBytes<'a> { /// Fetch `reqs`, extents `base..base + reqs.len()` of the read: the /// bytes of those wanted, and the bounds check of all of them. fn fetch( file: &'a S, reqs: &[ExtentReq], base: usize, ) -> Result { if let Some(all) = file.as_contiguous() { return Ok(ExtentBytes::Contiguous(all)); } let file_len = len_usize(file); let mut ranges = Vec::new(); let mut out = Vec::with_capacity(reqs.len()); // Positions in `out` of the extents being read, in `ranges` order. let mut slots = Vec::new(); for req in reqs { let checked = crate::addr::to_usize(req.addr).and_then(|start| { match start.checked_add(req.len) { Some(end) if end <= file_len => Ok(()), _ => Err(FormatError::UnexpectedEof { expected: start.saturating_add(req.len), available: file_len, }), } }); match checked { Ok(()) if req.fetch.is_some() => { slots.push(out.len()); ranges.push(req.addr..req.addr + req.fetch_len() as u64); out.push(Extent::NotFetched); } Ok(()) => out.push(Extent::NotFetched), Err(e) => out.push(Extent::Err(e)), } } if !ranges.is_empty() { let got = file.read_ranges(&ranges)?; if got.len() != ranges.len() { return Err(FormatError::Storage( "read_ranges returned the wrong number of ranges".into(), )); } for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) { if (bytes.len() as u64) < r.end - r.start { return Err(short_read()); } out[slot] = Extent::Bytes(bytes); } } Ok(ExtentBytes::Fetched { base, extents: out }) } /// Whether extent `i` of the read (`req`) lies in the file: its bounds /// error if not. pub(crate) fn check(&self, i: usize, req: &ExtentReq) -> Result<(), FormatError> { match self { ExtentBytes::Contiguous(_) => self.get(i, req).map(|_| ()), ExtentBytes::Fetched { base, extents } => { match i.checked_sub(*base).and_then(|j| extents.get(j)) { Some(Extent::Err(e)) => Err(e.clone()), Some(_) => Ok(()), None => Err(not_fetched()), } } } } /// Extent `i` of the read (`req`): its first `req.fetch` bytes, the /// same whether the file is in memory or not. pub(crate) fn get(&self, i: usize, req: &ExtentReq) -> Result<&[u8], FormatError> { match self { ExtentBytes::Contiguous(all) => { let start = crate::addr::to_usize(req.addr)?; start .checked_add(req.len) .and_then(|end| all.get(start..end)) .map(|b| &b[..req.fetch_len()]) .ok_or(FormatError::UnexpectedEof { expected: start.saturating_add(req.len), available: <[u8]>::len(all), }) } ExtentBytes::Fetched { base, extents } => { match i.checked_sub(*base).and_then(|j| extents.get(j)) { Some(Extent::Bytes(b)) => Ok(b), Some(Extent::Err(e)) => Err(e.clone()), _ => Err(not_fetched()), } } } } } #[cold] fn not_fetched() -> FormatError { FormatError::Storage("an extent that was not fetched was asked for".into()) } /// The one way raw-data reads fetch stored bytes: `reqs` are split into /// consecutive batches of at most [`RAW_BATCH_BYTES`] of fetched bytes (at /// least one extent each — and no extent fetches more than its /// [`ExtentReq::fetch`]), and for each batch in turn its bytes are fetched /// with one [`Storage::read_ranges`] call and `f(batch, &bytes)` is called, /// with `bytes` indexed by the extent's position in `reqs`. A batch's bytes /// are dropped before the next batch is fetched, and an error from `f` /// stops the read before anything more is fetched. /// /// With the whole file in memory there is nothing to fetch: one call, over /// all of `reqs`, that slices the file. pub(crate) fn for_each_extent_batch<'a, S: Storage + ?Sized>( file: &'a S, reqs: &[ExtentReq], mut f: impl FnMut(Range, &ExtentBytes<'a>) -> Result<(), FormatError>, ) -> Result<(), FormatError> { let contiguous = file.as_contiguous().is_some(); for batch in raw_batches(reqs.len(), contiguous, |i| reqs[i].fetch_len()) { let bytes = ExtentBytes::fetch(file, &reqs[batch.clone()], batch.start)?; f(batch, &bytes)?; } Ok(()) } /// Split `n` extents, whose sizes `size(i)` gives, into consecutive batches /// of at most [`RAW_BATCH_BYTES`] (at least one extent each): the ranges of /// `0..n` to fetch together. With the whole file in memory (`contiguous`) /// there is nothing to fetch, and one batch. pub(crate) fn raw_batches( n: usize, contiguous: bool, size: impl Fn(usize) -> usize, ) -> Vec> { if contiguous || n == 0 { return core::iter::once(0..n).collect(); } let mut out = Vec::new(); let (mut start, mut bytes) = (0, 0usize); for i in 0..n { let s = size(i); if i > start && bytes.saturating_add(s) > RAW_BATCH_BYTES { out.push(start..i); start = i; bytes = 0; } bytes = bytes.saturating_add(s); } out.push(start..n); out } /// Borrow the whole file for a code path that has not been converted to /// [`Storage`] yet. On a backend without a contiguous view this is the /// clean [`FormatError::ContiguousStorageRequired`] error, never a guess. #[inline] pub fn require_contiguous<'a, S: Storage + ?Sized>( file: &'a S, what: &'static str, ) -> Result<&'a [u8], FormatError> { file.as_contiguous() .ok_or(FormatError::ContiguousStorageRequired(what)) } /// A [`Storage`] over an in-memory buffer that serves every byte through /// [`Storage::read_at`] (its [`Storage::as_contiguous`] is `None`, so no /// parser can take the whole-slice shortcut), copies what it serves (as a /// remote backend would), and counts the reads and bytes. /// /// It is the equivalence harness of the range-read migration: parsing a /// file through it must give exactly what parsing the `&[u8]` gives, and /// the counters are the request counts a cacheless range reader would make. #[derive(Debug)] pub struct CountingStorage { data: Vec, reads: portable_atomic::AtomicU64, bytes: portable_atomic::AtomicU64, } impl CountingStorage { /// Serve `data` (the file from the superblock on). pub fn new(data: Vec) -> Self { CountingStorage { data, reads: portable_atomic::AtomicU64::new(0), bytes: portable_atomic::AtomicU64::new(0), } } /// Number of `read_at` calls served so far. pub fn reads(&self) -> u64 { self.reads.load(portable_atomic::Ordering::Relaxed) } /// Number of bytes served so far. pub fn bytes_read(&self) -> u64 { self.bytes.load(portable_atomic::Ordering::Relaxed) } /// Reset both counters. pub fn reset(&self) { self.reads.store(0, portable_atomic::Ordering::Relaxed); self.bytes.store(0, portable_atomic::Ordering::Relaxed); } } impl Storage for CountingStorage { fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { let got = self.data.as_slice().read_at(offset, len)?; self.reads.fetch_add(1, portable_atomic::Ordering::Relaxed); self.bytes .fetch_add(got.len() as u64, portable_atomic::Ordering::Relaxed); Ok(Cow::Owned(got.into_owned())) } fn len(&self) -> u64 { self.data.len() as u64 } } #[cfg(test)] mod tests { use super::*; #[test] fn slice_reads_are_borrowed_and_clamped() { let data: Vec = (0u8..10).collect(); let s: &[u8] = &data; let dynamic: &dyn Storage = &s; assert_eq!(dynamic.len(), 10); let r = dynamic.read_at(2, 3).unwrap(); assert!(matches!(r, Cow::Borrowed(_))); assert_eq!(&*r, &[2, 3, 4]); assert_eq!(&*dynamic.read_at(8, 5).unwrap(), &[8, 9]); assert!(dynamic.read_at(10, 5).unwrap().is_empty()); assert!(dynamic.read_at(u64::MAX, 5).unwrap().is_empty()); assert_eq!(dynamic.as_contiguous(), Some(&data[..])); let v: &dyn Storage = &data; assert_eq!(v.as_contiguous(), Some(&data[..])); } #[test] fn read_exact_matches_slice_bounds_errors() { let data = [0u8; 10]; let s: &[u8] = &data; assert_eq!(&*read_exact_at(&s, 4, 6).unwrap(), &[0; 6]); assert_eq!( read_exact_at(&s, 4, 7).unwrap_err(), FormatError::UnexpectedEof { expected: 11, available: 10 } ); assert!(read_exact_at(&s, u64::MAX, 1).is_err()); assert_eq!(read_upto(&s, 7, 100).unwrap().len(), 3); assert_eq!(read_upto(&s, 70, 100).unwrap().len(), 0); } #[test] fn counting_storage_counts_and_hides_the_slice() { let c = CountingStorage::new((0u8..10).collect()); assert!(c.as_contiguous().is_none()); let r = c.read_at(3, 4).unwrap(); assert!(matches!(r, Cow::Owned(_))); assert_eq!(&*r, &[3, 4, 5, 6]); c.read_at(8, 4).unwrap(); assert_eq!((c.reads(), c.bytes_read()), (2, 6)); c.reset(); assert_eq!((c.reads(), c.bytes_read()), (0, 0)); } #[test] fn read_ranges_default_loops() { let data: Vec = (0u8..10).collect(); let s: &[u8] = &data; let got = s.read_ranges(&[1..3, 5..9]).unwrap(); assert_eq!(&*got[0], &[1, 2]); assert_eq!(&*got[1], &[5, 6, 7, 8]); } }