diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 28bc4f2..40826c1 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -956,6 +956,78 @@ pub fn read_selection_native( crate::gather::gather::(raw, dims, elem_size, selection).map(Some) } +/// [`read_selection_native`] of a contiguous dataset in any [`Storage`], +/// reading only the selected elements' runs (adjacent ones merged, one +/// [`Storage::read_ranges`] call) instead of the whole dataset. +/// +/// `Ok(None)` wherever the in-memory fast path does not apply and the +/// caller converts through the byte readers instead: `datatype` is not +/// `T`'s native representation, the layout is not contiguous, or the +/// dataset's bytes cannot be located in the file (no address, storage too +/// small, past the end of file: the cases [`read_raw_data_zerocopy`] +/// fails). Otherwise the result and errors are [`read_selection_native`]'s +/// over those bytes. +pub fn read_selection_native_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + selection: &crate::selection::Selection, +) -> Result>, FormatError> { + if !T::is_native(datatype) { + return Ok(None); + } + let DataLayout::Contiguous { + address: Some(address), + size, + } = layout + else { + return Ok(None); + }; + // Where the dataset's bytes are, as `read_raw_data_zerocopy` finds them. + let located = to_usize(dataspace.num_elements()) + .ok() + .and_then(|n| n.checked_mul(datatype.type_size() as usize)) + .filter(|&len| contiguous_read_len(*size, len).is_ok()) + .filter(|&len| { + address + .checked_add(len as u64) + .is_some_and(|end| end <= file_data.len()) + }); + let Some(len) = located else { + return Ok(None); + }; + if let Some(all) = file_data.as_contiguous() { + let start = to_usize(*address)?; + return read_selection_native( + &all[start..start + len], + &dataspace.dimensions, + datatype, + selection, + ); + } + let dims = &dataspace.dimensions; + let elem_size = core::mem::size_of::(); + let total = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?; + let expected = crate::chunked_read::checked_byte_len(total, elem_size)?; + if len != expected { + return Err(FormatError::DataSizeMismatch { + expected, + actual: len, + }); + } + let bytes = if let crate::selection::Selection::All = selection { + read_exact_at(file_data, *address, len)?.into_owned() + } else { + crate::partial_read::validate(selection, dims)?; + crate::gather::gather_storage(file_data, *address, len, dims, elem_size, selection)? + }; + Ok(Some(native_to_vec(&bytes, bytes.len() / elem_size))) +} + /// The bytes of a slice of [`NativeElement`]s. #[cfg(feature = "std")] fn bytes_of_mut(values: &mut [T]) -> &mut [u8] { diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index 4a66473..7edbda6 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -490,6 +490,23 @@ impl CacheImage { image_block_in(file, self.location) } + /// Every entry as `(file address, its bytes)`, taken from `block` (the + /// image block, see [`Self::block_in`]), in the order [`Self::apply`] + /// writes them: for a reader that cannot write the image over the + /// file's bytes and lays the entries over each read instead. + pub fn entries<'b>(&self, block: &'b [u8]) -> Result, FormatError> { + let short = || FormatError::InvalidCacheImage("image applied to the wrong file"); + self.entries + .iter() + .map(|e| { + let src = block + .get(e.image_offset..e.image_offset + e.len) + .ok_or_else(short)?; + Ok((e.address, src)) + }) + .collect() + } + /// Write every entry over `dst`, the file's bytes from the superblock /// on (as long as the `data` the image was decoded from), taking the /// entries from `block` (the image block, see [`Self::block`]). `block` diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 280f64b..5ceec87 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -56,7 +56,7 @@ pub use error::Error; pub use lazy::{LazyDataset, LazyFile, LazyGroup}; #[cfg(feature = "mmap")] pub use mmap_file::{MmapDataset, MmapFile, MmapGroup}; -pub use reader::{Dataset, File, Group}; +pub use reader::{Dataset, File, Group, SharedStorage, VdsResolver}; pub use types::{AttrValue, DType}; pub use vlen::VlenValue; pub use writer::FileBuilder; @@ -72,6 +72,7 @@ pub use clawhdf5_format::property_list::{ #[cfg(feature = "provenance")] pub use clawhdf5_format::provenance; pub use clawhdf5_format::selection::Selection; +pub use clawhdf5_format::storage::Storage; pub use clawhdf5_format::superblock::swmr_flags; pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime}; diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index fcd3b67..6bd7b35 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -5,7 +5,10 @@ //! the traditional read-into-`Vec` fallback. [`File::from_bytes`] remains //! available for in-memory usage (tests, etc.). +use std::borrow::Cow; use std::collections::HashMap; +use std::ops::Range; +use std::sync::Arc; use clawhdf5_format::chunk_cache::ChunkCache; use clawhdf5_format::data_layout::DataLayout; @@ -19,29 +22,44 @@ use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; +use clawhdf5_format::storage::Storage; use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext::{self, CacheImageState}; use crate::cache_image::{self, ImageView}; use crate::error::Error; use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; // --------------------------------------------------------------------------- -// FileData — internal storage for either owned bytes or an mmap +// FileData — internal storage for owned bytes, an mmap, or any Storage // --------------------------------------------------------------------------- -/// Internal storage: either an owned `Vec` or a memory-mapped region. +/// A [`Storage`] a [`File`] can be opened over: any backend that can be +/// shared between threads (see [`File::open_storage`]). +pub type SharedStorage = Arc; + +/// Resolves an external Virtual Dataset source file name to that file's +/// bytes (`Ok(None)`: the file does not exist, its mappings read as the +/// fill value). See [`File::set_vds_resolver`]. +pub type VdsResolver = Arc Result>, FormatError> + Send + Sync>; + +/// Internal storage: an owned `Vec`, a memory-mapped region, or a +/// [`Storage`] backend. enum Backing { Owned(Vec), #[cfg(feature = "mmap")] Mmap(clawhdf5_io::MmapReader), + Storage(SharedStorage), } impl Backing { - fn whole_file(&self) -> &[u8] { + /// The whole file, when it is in memory. + fn whole_file(&self) -> Option<&[u8]> { match self { - Backing::Owned(v) => v, + Backing::Owned(v) => Some(v), #[cfg(feature = "mmap")] - Backing::Mmap(r) => r.as_bytes(), + Backing::Mmap(r) => Some(r.as_bytes()), + Backing::Storage(s) => s.as_contiguous(), } } } @@ -49,13 +67,16 @@ impl Backing { /// The file's bytes, viewed from the superblock on and up to the end of /// file the superblock records. A file may start with a user block (the /// superblock at 512, 1024, …); every HDF5 address is relative to the -/// superblock, so all parsing goes through [`Self::as_bytes`]. +/// superblock, so all parsing goes through this view, which is a +/// [`Storage`]: in memory (a `Vec`, an mmap) its reads are slices of the +/// file, as before; over any other storage they are reads of that storage, +/// shifted by the user block and bounded by the end of file. struct FileData { backing: Backing, /// Offset of the superblock in the file (the user-block size). - base: usize, + base: u64, /// End of the HDF5 data in the file (`Superblock::data_end`, absolute). - end: usize, + end: u64, /// A mapped file that holds a metadata cache image, with the image /// written in: a private copy-on-write mapping of the whole file, so /// only the pages the image's entries land on are copied (see @@ -63,6 +84,10 @@ struct FileData { /// an image is read straight from the mapping, and an owned buffer has /// the image written into it in place. patched: Option, + /// A [`Storage`]-backed file's metadata cache image: its entries + /// (address relative to the superblock, bytes), laid over every read in + /// order, as libhdf5 reads them instead of the file's own bytes. + overlay: Vec<(u64, Vec)>, /// The file has a metadata cache image libhdf5 cannot load. libhdf5 /// opens such a file and fails its first metadata read (the image loads /// then); every object lookup here fails with this error, and no @@ -74,7 +99,10 @@ impl FileData { /// Locate the superblock and parse it. A truncated file is refused, and /// bytes past the recorded end of file are not read, as in libhdf5. fn new(mut backing: Backing) -> Result<(Self, Superblock), Error> { - let whole = backing.whole_file(); + if let Backing::Storage(storage) = backing { + return Self::new_storage(storage); + } + let whole = backing.whole_file().unwrap_or_default(); let (user_block, hdf5) = signature::split_user_block(whole)?; let base = user_block.len(); let superblock = Superblock::parse(hdf5, 0)?; @@ -92,6 +120,7 @@ impl FileData { clawhdf5_io::HDF5Read::private_copy(r) })? } + Backing::Storage(_) => unreachable!("handled above"), }; let (patched, image_error) = match view { ImageView::Plain => (None, None), @@ -101,34 +130,140 @@ impl FileData { Ok(( Self { backing, - base, - end, + base: base as u64, + end: end as u64, patched, + overlay: Vec::new(), image_error, }, superblock, )) } - fn as_bytes(&self) -> &[u8] { - match &self.patched { - Some(p) => &p[self.base..self.end], - None => &self.backing.whole_file()[self.base..self.end], + /// [`Self::new`] for a [`Storage`] backend: the same checks, through + /// reads of the storage. + fn new_storage(storage: SharedStorage) -> Result<(Self, Superblock), Error> { + let file_len = storage.len(); + let base = signature::find_signature_in(&*storage)?; + let mut data = Self { + backing: Backing::Storage(storage), + base, + end: file_len, + patched: None, + overlay: Vec::new(), + image_error: None, + }; + let superblock = Superblock::parse_in(&data, 0)?; + data.end = base + superblock.data_end(base, file_len)?; + match superblock_ext::cache_image_state_in(&data, &superblock)? { + CacheImageState::Absent => {} + CacheImageState::Unloadable(e) => data.image_error = Some(e), + CacheImageState::Loaded(image) => { + let block = image.block_in(&data)?.into_owned(); + data.overlay = image + .entries(&block)? + .into_iter() + .map(|(addr, bytes)| (addr, bytes.to_vec())) + .collect(); + } } + Ok((data, superblock)) } - fn len(&self) -> usize { - self.as_bytes().len() + /// The HDF5 data as one slice, when the file is in memory (a `Vec`, an + /// mmap, or a storage that holds it all and has no cache image to lay + /// over it). + fn contiguous(&self) -> Option<&[u8]> { + if let Some(p) = &self.patched { + return p.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?); + } + if !self.overlay.is_empty() { + return None; + } + self.backing + .whole_file()? + .get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?) } /// The bytes to read metadata from; fails for a file whose cache image /// cannot be loaded (see [`Self::image_error`]). - fn meta(&self) -> Result<&[u8], FormatError> { + fn meta(&self) -> Result<&Self, FormatError> { match &self.image_error { Some(e) => Err(e.clone()), - None => Ok(self.as_bytes()), + None => Ok(self), } } + + /// The backend of a file that is not in memory. + fn remote(&self) -> Result<&SharedStorage, FormatError> { + match &self.backing { + Backing::Storage(s) => Ok(s), + _ => Err(FormatError::Storage( + "in-memory file has no contiguous view".into(), + )), + } + } + + /// `bytes`, read at `offset`, with the cache image entries they overlap + /// written over them. + fn with_overlay<'a>(&self, offset: u64, mut bytes: Cow<'a, [u8]>) -> Cow<'a, [u8]> { + let end = offset + bytes.len() as u64; + for (addr, entry) in &self.overlay { + let entry_end = addr + entry.len() as u64; + if *addr >= end || entry_end <= offset { + continue; + } + let from = (*addr).max(offset); + let to = entry_end.min(end); + let dst = bytes.to_mut(); + dst[(from - offset) as usize..(to - offset) as usize] + .copy_from_slice(&entry[(from - addr) as usize..(to - addr) as usize]); + } + bytes + } +} + +impl Storage for FileData { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if let Some(all) = self.contiguous() { + return all.read_at(offset, len); + } + let size = self.end - self.base; + let len = usize::try_from(size.saturating_sub(offset)).map_or(len, |avail| avail.min(len)); + if len == 0 { + return Ok(Cow::Borrowed(&[])); + } + let bytes = self.remote()?.read_at(self.base + offset, len)?; + Ok(self.with_overlay(offset, bytes)) + } + + fn len(&self) -> u64 { + self.end - self.base + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if let Some(all) = self.contiguous() { + return all.read_ranges(ranges); + } + let size = self.end - self.base; + let shifted: Vec> = ranges + .iter() + .map(|r| { + let (s, e) = (r.start.min(size), r.end.min(size)); + self.base + s..self.base + e.max(s) + }) + .collect(); + let got = self.remote()?.read_ranges(&shifted)?; + Ok(got + .into_iter() + .zip(ranges) + .map(|(bytes, r)| self.with_overlay(r.start, bytes)) + .collect()) + } + + fn as_contiguous(&self) -> Option<&[u8]> { + self.contiguous() + } } // --------------------------------------------------------------------------- @@ -149,6 +284,9 @@ pub struct File { /// Directory the file was opened from, used to resolve external Virtual /// Dataset source files relative to this file. `None` for in-memory files. base_dir: Option, + /// Resolves external Virtual Dataset source files instead of + /// `base_dir` (see [`File::set_vds_resolver`]). + vds_resolver: Option, } impl File { @@ -167,6 +305,7 @@ impl File { superblock, chunk_cache: ChunkCache::new(), base_dir, + vds_resolver: None, }) } #[cfg(not(feature = "mmap"))] @@ -200,9 +339,53 @@ impl File { superblock, chunk_cache: ChunkCache::new(), base_dir: None, + vds_resolver: None, }) } + /// Open an HDF5 file served by any [`Storage`]: a range-reading remote + /// backend, a block cache, or an in-memory buffer — through the same + /// read API as [`File::open`] (groups, datasets, attributes, `read_*`, + /// selections, variable-length data, virtual datasets). + /// + /// Every read goes through the storage's [`Storage::read_at`] and + /// [`Storage::read_ranges`] (a chunked read fetches all the chunks it + /// needs with one `read_ranges` call per 64 MiB), so nothing is read that + /// the operation does not need. A storage that has the whole file in + /// memory ([`Storage::as_contiguous`]) is read as [`File::from_bytes`] + /// reads its buffer. The storage holds the whole file: a user block is + /// found and skipped, and bytes past the end of file the superblock + /// records are not read. A metadata cache image is laid over the reads + /// it covers, as libhdf5 loads it. + /// + /// The zero-copy methods ([`Dataset::read_raw_ref`], + /// [`Dataset::read_as_slice`], `read_*_zerocopy`) borrow the file's bytes + /// and return [`FormatError::ContiguousStorageRequired`] over a storage + /// that does not hold them; [`File::as_bytes`] panics there (use + /// [`File::contiguous_bytes`]). External virtual-dataset source files + /// are read through [`File::set_vds_resolver`]; without one they cannot + /// be resolved. + pub fn open_storage(storage: SharedStorage) -> Result { + let (data, superblock) = FileData::new(Backing::Storage(storage))?; + Ok(Self { + data, + superblock, + chunk_cache: ChunkCache::new(), + base_dir: None, + vds_resolver: None, + }) + } + + /// Resolve external Virtual Dataset source files (their names as the + /// mappings store them) with `resolver`, instead of reading them from + /// the directory of the file (for [`File::open`]) or refusing them (for + /// in-memory and [`Storage`]-backed files). `Ok(None)` means the source + /// file does not exist, and its mappings read as the fill value, as in + /// libhdf5; `Err` fails the read. + pub fn set_vds_resolver(&mut self, resolver: VdsResolver) { + self.vds_resolver = Some(resolver); + } + /// Returns a handle to the root group. pub fn root(&self) -> Group<'_> { Group { @@ -216,7 +399,7 @@ impl File { /// The path uses `/` separators (e.g., `"group1/values"`). pub fn dataset(&self, path: &str) -> Result, Error> { let data = self.data.meta()?; - let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; + let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); @@ -263,7 +446,7 @@ impl File { /// Use `"/"` or `""` for the root group. pub fn group(&self, path: &str) -> Result, Error> { let data = self.data.meta()?; - let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; + let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; Ok(Group { file: self, address: addr, @@ -322,8 +505,22 @@ impl File { /// with a metadata cache image these are the bytes with the image /// applied; when the image cannot be loaded they are the file's own /// bytes, whose metadata may be stale (every object lookup fails then). + /// + /// # Panics + /// + /// For a file opened with [`File::open_storage`] over a storage that + /// does not hold the whole file in memory; use + /// [`contiguous_bytes`](Self::contiguous_bytes) there. pub fn as_bytes(&self) -> &[u8] { - self.data.as_bytes() + self.data + .contiguous() + .expect("File::as_bytes: the file is not in memory (see File::contiguous_bytes)") + } + + /// [`as_bytes`](Self::as_bytes), or `None` for a file whose bytes are + /// not all in memory ([`File::open_storage`]). + pub fn contiguous_bytes(&self) -> Option<&[u8]> { + self.data.contiguous() } /// The error of a metadata cache image libhdf5 cannot load, when the @@ -338,7 +535,7 @@ impl File { /// Size of the user block before the superblock (0 for most files). /// Matches h5py's `File.userblock_size`. pub fn user_block_size(&self) -> u64 { - self.data.base as u64 + self.data.base } /// Returns a reference to the parsed superblock. @@ -349,7 +546,7 @@ impl File { /// Returns `true` when the file is backed by memory-mapped I/O. pub fn is_mmap(&self) -> bool { match &self.data.backing { - Backing::Owned(_) => false, + Backing::Owned(_) | Backing::Storage(_) => false, #[cfg(feature = "mmap")] Backing::Mmap(_) => true, } @@ -362,7 +559,7 @@ impl File { /// this file's global heap; see [`Dataset::read_string`] for the values. pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result, Error> { crate::vlen::decode_strings( - self.as_bytes(), + &self.data, datatype, raw, self.offset_size(), @@ -406,9 +603,9 @@ impl File { } fn parse_header(&self, address: u64) -> Result { - ObjectHeader::parse( + ObjectHeader::parse_in( self.data.meta()?, - address as usize, + address, self.superblock.offset_size, self.superblock.length_size, ) @@ -426,7 +623,7 @@ impl File { impl std::fmt::Debug for File { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("File") - .field("size", &self.data.len()) + .field("size", &Storage::len(&self.data)) .field("superblock_version", &self.superblock.version) .field("mmap", &self.is_mmap()) .finish() @@ -489,7 +686,7 @@ impl<'f> Group<'f> { &self, ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.parse_header(self.address)?; - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) } @@ -520,7 +717,7 @@ impl<'f> Group<'f> { /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { let hdr = self.file.parse_header(self.address)?; - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attr( data, &hdr, @@ -536,7 +733,7 @@ impl<'f> Group<'f> { /// [`group_v2::resolve_child`]). fn child_address(&self, name: &str) -> Result { let data = self.file.data.meta()?; - group_v2::resolve_child(data, &self.file.superblock, self.address, name) + group_v2::resolve_child_in(data, &self.file.superblock, self.address, name) .map_err(Error::Format) } @@ -559,7 +756,7 @@ impl<'f> Group<'f> { /// user-defined links are left out. fn children(&self) -> Result, Error> { let data = self.file.data.meta()?; - group_v2::resolve_group_children(data, &self.file.superblock, self.address) + group_v2::resolve_group_children_in(data, &self.file.superblock, self.address) .map_err(Error::Format) } } @@ -589,7 +786,7 @@ impl<'f> Dataset<'f> { Ok((self.datatype()?, ds, self.data_layout()?)) })(); if let Ok((dt, ds, dl)) = decoded { - data_read::check_dataset_storage(&dl, &ds, &dt, self.file.data.len() as u64)?; + data_read::check_dataset_storage(&dl, &ds, &dt, Storage::len(&self.file.data))?; } Ok(self) } @@ -629,8 +826,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_f64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_f64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -649,8 +846,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_f32(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_f32(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -664,8 +861,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_i32(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_i32(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -679,8 +876,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_i64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_i64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -694,8 +891,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_u64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_u64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -793,8 +990,8 @@ impl<'f> Dataset<'f> { // sparse) dataset — select from a fill-aware full read instead. (The // selection reader currently decodes the full dataset too, so this // costs nothing extra.) - let fill = clawhdf5_format::fill_value::dataset_fill_value_in( - self.file.data.as_bytes(), + let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage( + &self.file.data, &self.header.messages, self.file.offset_size(), self.file.length_size(), @@ -813,8 +1010,8 @@ impl<'f> Dataset<'f> { selection, )?); } - Ok(data_read::read_raw_data_selection( - self.file.data.as_bytes(), + Ok(data_read::read_raw_data_selection_in( + &self.file.data, &dl, &ds, &dt, @@ -871,12 +1068,26 @@ impl<'f> Dataset<'f> { return full(); } let dt = self.datatype()?; - if T::is_native(&dt) - && let Ok(Some(raw)) = self.read_raw_ref() + if T::is_native(&dt) && self.file.data.contiguous().is_some() { + if let Ok(Some(raw)) = self.read_raw_ref() { + let dims = self.dataspace()?.dimensions; + if let Some(values) = + data_read::read_selection_native::(raw, &dims, &dt, selection)? + { + return Ok(values); + } + } + } else if T::is_native(&dt) + && let (Ok(dl), Ok(ds)) = (self.data_layout(), self.dataspace()) { - let dims = self.dataspace()?.dimensions; - if let Some(values) = data_read::read_selection_native::(raw, &dims, &dt, selection)? - { + // Not in memory: only the selected runs are read. + if let Some(values) = data_read::read_selection_native_in::( + &self.file.data, + &dl, + &ds, + &dt, + selection, + )? { return Ok(values); } } @@ -893,10 +1104,46 @@ impl<'f> Dataset<'f> { let dl = self.data_layout()?; let ds = self.dataspace()?; let dt = self.datatype()?; - let slice = data_read::read_raw_data_zerocopy(self.file.data.as_bytes(), &dl, &ds, &dt)?; + let Some(bytes) = self.file.data.contiguous() else { + // The bytes are not in memory to borrow. + return match dl { + DataLayout::Contiguous { .. } => { + Err(Error::Format(FormatError::ContiguousStorageRequired( + "a zero-copy read (the file is not in memory)", + ))) + } + _ => Ok(None), + }; + }; + let slice = data_read::read_raw_data_zerocopy(bytes, &dl, &ds, &dt)?; Ok(slice) } + /// A contiguous dataset's stored bytes, for the typed readers' fast + /// path: borrowed from the file when it is in memory + /// ([`read_raw_ref`](Self::read_raw_ref)), read in one piece otherwise. + /// `Ok(None)` for other layouts; an error where `read_raw_ref` fails. + fn contiguous_raw(&self) -> Result>, Error> { + if self.file.data.contiguous().is_some() { + return Ok(self.read_raw_ref()?.map(Cow::Borrowed)); + } + let dl = self.data_layout()?; + if !matches!(dl, DataLayout::Contiguous { .. }) { + return Ok(None); + } + let ds = self.dataspace()?; + let dt = self.datatype()?; + Ok(Some(Cow::Owned(data_read::read_raw_data_full_in( + &self.file.data, + &dl, + &ds, + &dt, + None, + self.file.offset_size(), + self.file.length_size(), + )?))) + } + /// Zero-copy typed read of contiguous data as `&[T]`. /// /// Returns a borrowed slice of `T` directly from the file buffer with @@ -1083,7 +1330,7 @@ impl<'f> Dataset<'f> { pub fn attrs_with_errors( &self, ) -> Result<(HashMap, Vec), Error> { - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attrs( data, &self.header, @@ -1097,7 +1344,7 @@ impl<'f> Dataset<'f> { /// that name, found without reading the other attributes when they are /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attr( data, &self.header, @@ -1124,8 +1371,8 @@ impl<'f> Dataset<'f> { /// result is not a tamper-evidence or authenticity guarantee. #[cfg(feature = "provenance")] pub fn verify_provenance(&self) -> Result { - Ok(clawhdf5_format::provenance::verify_dataset( - self.file.as_bytes(), + Ok(clawhdf5_format::provenance::verify_dataset_in( + &self.file.data, &self.header, self.file.offset_size(), self.file.length_size(), @@ -1144,8 +1391,8 @@ impl<'f> Dataset<'f> { .iter() .find(|m| m.msg_type == msg_type) .map(|msg| { - clawhdf5_format::shared_message::message_data( - self.file.as_bytes(), + clawhdf5_format::shared_message::message_data_in( + &self.file.data, msg, self.file.offset_size(), self.file.length_size(), @@ -1179,8 +1426,8 @@ impl<'f> Dataset<'f> { // one (`H5Dget_space`). if let Ok(dl @ DataLayout::Virtual { .. }) = self.data_layout() { let resolver = self.vds_resolver(); - ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent( - self.file.data.as_bytes(), + ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent_in( + &self.file.data, &dl, &ds, self.file.offset_size(), @@ -1225,9 +1472,9 @@ impl<'f> Dataset<'f> { } let ds = self.dataspace()?; let pipeline = self.filter_pipeline()?; - Ok(data_read::read_chunked_native::( + Ok(data_read::read_chunked_native_in::( &self.header.messages, - self.file.data.as_bytes(), + &self.file.data, &dl, &ds, &dt, @@ -1251,17 +1498,17 @@ impl<'f> Dataset<'f> { } // Unallocated storage reads as the dataset's fill value. - clawhdf5_format::fill_value::read_full_with_fill( + clawhdf5_format::fill_value::read_full_with_fill_in( &self.header.messages, - self.file.data.as_bytes(), + &self.file.data, &dl, &ds, dt.type_size() as usize, self.file.offset_size(), self.file.length_size(), || { - Ok(data_read::read_raw_data_cached( - self.file.data.as_bytes(), + Ok(data_read::read_raw_data_cached_in( + &self.file.data, &dl, &ds, &dt, @@ -1281,7 +1528,11 @@ impl<'f> Dataset<'f> { /// refused with an error rather than read as fill. fn vds_resolver(&self) -> impl Fn(&str) -> Result>, FormatError> + use<> { let base_dir = self.file.base_dir.clone(); + let custom = self.file.vds_resolver.clone(); move |name: &str| { + if let Some(resolve) = &custom { + return resolve(name); + } let Some(dir) = base_dir.as_ref() else { return Err(FormatError::ChunkedReadError(format!( "virtual dataset source file {name:?} cannot be resolved for an in-memory file" @@ -1310,15 +1561,15 @@ impl<'f> Dataset<'f> { ds: &Dataspace, dt: &Datatype, ) -> Result, Error> { - let fill = clawhdf5_format::fill_value::dataset_fill_value_in( - self.file.data.as_bytes(), + let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage( + &self.file.data, &self.header.messages, self.file.offset_size(), self.file.length_size(), )?; let resolver = self.vds_resolver(); - let v = clawhdf5_format::vds::read_virtual_dataset( - self.file.data.as_bytes(), + let v = clawhdf5_format::vds::read_virtual_dataset_in( + &self.file.data, dl, ds, dt, @@ -1439,7 +1690,7 @@ mod zero_copy_tests { let Backing::Mmap(r) = &f.data.backing else { return None; }; - let mapped = r.as_bytes()[f.data.base..].as_ptr(); + let mapped = r.as_bytes()[f.data.base as usize..].as_ptr(); match &f.data.patched { None => Some(std::ptr::eq(f.as_bytes().as_ptr(), mapped)), Some(p) => { diff --git a/crates/clawhdf5/src/types.rs b/crates/clawhdf5/src/types.rs index acae280..b3f7823 100644 --- a/crates/clawhdf5/src/types.rs +++ b/crates/clawhdf5/src/types.rs @@ -158,8 +158,8 @@ pub(crate) fn classify_datatype(dt: &clawhdf5_format::datatype::Datatype) -> DTy /// The attributes of the object with header `header` that could be read, /// and one error for each that could not (see /// [`extract_attributes_tolerant`](clawhdf5_format::attribute::extract_attributes_tolerant)). -pub(crate) fn read_attrs( - file_data: &[u8], +pub(crate) fn read_attrs( + file_data: &S, header: &clawhdf5_format::object_header::ObjectHeader, offset_size: u8, length_size: u8, @@ -170,7 +170,7 @@ pub(crate) fn read_attrs( ), crate::Error, > { - let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant( + let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant_in( file_data, header, offset_size, @@ -185,14 +185,14 @@ pub(crate) fn read_attrs( /// The attribute called `name` on the object with header `header`, decoded /// as [`read_attrs`] decodes it, or `None` (see /// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)). -pub(crate) fn read_attr( - file_data: &[u8], +pub(crate) fn read_attr( + file_data: &S, header: &clawhdf5_format::object_header::ObjectHeader, name: &str, offset_size: u8, length_size: u8, ) -> Result, crate::Error> { - let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file( + let Some(msg) = clawhdf5_format::attribute::find_attribute_in( file_data, header, name, @@ -211,9 +211,9 @@ pub(crate) fn read_attr( .remove(name)) } -pub(crate) fn attrs_to_map( +pub(crate) fn attrs_to_map( attrs: &[clawhdf5_format::attribute::AttributeMessage], - file_data: &[u8], + file_data: &S, offset_size: u8, length_size: u8, ) -> HashMap { @@ -266,9 +266,9 @@ fn decode_bool_enum(attr: &clawhdf5_format::attribute::AttributeMessage) -> Opti values.iter().all(|v| *v == 0 || *v == 1).then_some(values) } -fn decode_attr_value( +fn decode_attr_value( attr: &clawhdf5_format::attribute::AttributeMessage, - file_data: &[u8], + file_data: &S, offset_size: u8, length_size: u8, ) -> Option { @@ -311,7 +311,7 @@ fn decode_attr_value( is_string: true, .. } => { let strings = attr - .read_vl_strings(file_data, offset_size, length_size) + .read_vl_strings_in(file_data, offset_size, length_size) .ok()?; if strings.len() == 1 { Some(AttrValue::String(strings[0].clone())) diff --git a/crates/clawhdf5/src/vlen.rs b/crates/clawhdf5/src/vlen.rs index 01ba02d..f471b24 100644 --- a/crates/clawhdf5/src/vlen.rs +++ b/crates/clawhdf5/src/vlen.rs @@ -14,6 +14,7 @@ use clawhdf5_format::data_read; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; use clawhdf5_format::vl_data::{VlResolver, check_element_size}; use crate::error::Error; @@ -68,8 +69,8 @@ fn class_name(dt: &Datatype) -> &'static str { /// The strings in `raw`, elements of `dt`: fixed-length strings decoded as /// `read_string` always has, variable-length strings resolved in the heap. -pub(crate) fn decode_strings( - file_data: &[u8], +pub(crate) fn decode_strings( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -82,15 +83,15 @@ pub(crate) fn decode_strings( .. } => { check_element_size(*size, offset_size)?; - Ok(VlResolver::new(file_data, offset_size, length_size).strings(raw)?) + Ok(VlResolver::new_in(file_data, offset_size, length_size).strings(raw)?) } _ => Ok(data_read::read_as_strings(raw, dt)?), } } /// The exact bytes of the variable-length strings in `raw`. -pub(crate) fn decode_string_bytes( - file_data: &[u8], +pub(crate) fn decode_string_bytes( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -103,7 +104,7 @@ pub(crate) fn decode_string_bytes( .. } => { check_element_size(*size, offset_size)?; - Ok(VlResolver::new(file_data, offset_size, length_size).string_bytes(raw)?) + Ok(VlResolver::new_in(file_data, offset_size, length_size).string_bytes(raw)?) } other => Err(Error::Format(FormatError::TypeMismatch { expected: "variable-length string", @@ -114,8 +115,8 @@ pub(crate) fn decode_string_bytes( /// The sequences in `raw`, elements of the variable-length sequence type /// `dt`, converted to `T`. -pub(crate) fn decode_vlen( - file_data: &[u8], +pub(crate) fn decode_vlen( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -135,7 +136,7 @@ pub(crate) fn decode_vlen( }; check_element_size(*size, offset_size)?; let base_size = base_type.type_size() as usize; - VlResolver::new(file_data, offset_size, length_size) + VlResolver::new_in(file_data, offset_size, length_size) .sequences(raw, base_size)? .iter() .map(|bytes| Ok(T::decode(bytes, base_type)?)) diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs new file mode 100644 index 0000000..7a2749b --- /dev/null +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -0,0 +1,434 @@ +//! `File::open_storage` over a read_at-only storage reads every file as +//! `File::open` does (range reads, milestone M2 in +//! `docs/design/range-reads.md`). +//! +//! Each file is read end to end twice — through `File::open` (the mmap +//! fast path) and through `File::open_storage` over a +//! [`CountingStorage`], which serves the file through `read_at` only +//! (`as_contiguous()` is `None`, so no reader can fall back to a slice of +//! the whole file) — and the two transcripts must be identical: the tree +//! (every group's entries, followed by address), every object's attributes +//! (the whole map and each one by name), and every dataset's shape, types +//! and values (all bytes, as `f64`, a hyperslab of them, strings and +//! variable-length sequences). +//! +//! The storage also counts its `read_at` calls and bytes: what a remote +//! backend without a cache would be asked for. The totals and the files +//! that cost most are printed. +//! +//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every HDF5 file under those +//! directories (the conformance corpus is `conformance/.cache/corpus`); +//! `CLAWHDF5_STORAGE_REPORT=1` prints every file's counts. + +use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clawhdf5::{DType, File, Selection}; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::CountingStorage; + +/// Objects visited per file. +const MAX_OBJECTS: usize = 2000; +/// Datasets with more bytes than this are not read (their metadata is). +const MAX_DATA_BYTES: u64 = 64 << 20; + +/// A short, stable digest of a value's `Debug` form. +fn digest(v: &T) -> String { + let s = format!("{v:?}"); + if s.len() <= 200 { + return s; + } + let mut h = 0xcbf2_9ce4_8422_2325u64; + for b in s.bytes() { + h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); + } + format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len()) +} + +/// A data read's result: its value, or just `Err` — a full read goes +/// through the file's chunk cache, which lists a damaged dataset's chunks +/// in hash-map order, so which failing chunk it reports varies from one +/// `File` to the next (the format crate's harness compares these errors on +/// the uncached path). +fn value(r: &Result) -> String { + match r { + Ok(v) => digest(v), + Err(_) => "Err".into(), + } +} + +fn transcript(file: &File) -> String { + let mut out = String::new(); + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]); + while let Some((path, addr)) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let entries = group.entries(); + writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap(); + match file.dataset_at(addr) { + Ok(ds) => dataset(&mut out, &path, &ds), + Err(e) => writeln!(out, "{path} dataset_at {e:?}").unwrap(), + } + let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap(); + if let Ok((attrs, _)) = &attrs { + for name in attrs.keys().take(50) { + writeln!(out, "{path} attr {name:?} {}", digest(&group.attr(name))).unwrap(); + } + } + if let Ok(entries) = entries { + for (name, child) in entries { + queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child)); + // Name lookups (through the name index of a dense group). + if queue.len() < 64 { + writeln!( + out, + "{path} group({name:?}) {}", + digest(&group.group(&name).map(|_| ())) + ) + .unwrap(); + } + } + } + } + out +} + +fn sorted(m: std::collections::HashMap) -> BTreeMap { + m.into_iter().collect() +} + +fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { + let shape = ds.shape(); + let dtype = ds.dtype(); + writeln!( + out, + "{path} shape {} max {} dtype {} raw {}", + digest(&shape), + digest(&ds.max_dimensions()), + digest(&dtype), + digest(&ds.raw_datatype()) + ) + .unwrap(); + let attrs = ds.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} dataset attrs {}", digest(&attrs)).unwrap(); + let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else { + return; + }; + let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d)); + let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size()))); + if bytes.is_none_or(|b| b > MAX_DATA_BYTES) { + writeln!(out, "{path} too large to read").unwrap(); + return; + } + writeln!( + out, + "{path} all {}", + value(&ds.read_selection(&Selection::All)) + ) + .unwrap(); + let numeric = matches!( + dtype, + DType::F32 + | DType::F64 + | DType::I8 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::U8 + | DType::U16 + | DType::U32 + | DType::U64 + ); + if numeric { + writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap(); + writeln!(out, "{path} f32 {}", value(&ds.read_f32())).unwrap(); + writeln!(out, "{path} i64 {}", value(&ds.read_i64())).unwrap(); + if let Some(&d0) = shape.first() { + let rank = shape.len(); + let sel = Selection::Hyperslab { + start: std::iter::once(d0 / 3) + .chain(std::iter::repeat_n(0, rank - 1)) + .collect(), + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(3)) + .chain(shape[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + writeln!( + out, + "{path} f64 third {}", + value(&ds.read_f64_selection(&sel)) + ) + .unwrap(); + writeln!( + out, + "{path} bytes third {}", + value(&ds.read_selection(&sel)) + ) + .unwrap(); + } + } + match &raw_dt { + clawhdf5_format::datatype::Datatype::String { .. } + | clawhdf5_format::datatype::Datatype::VariableLength { + is_string: true, .. + } => { + writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap(); + writeln!(out, "{path} string {}", value(&ds.read_string())).unwrap(); + } + clawhdf5_format::datatype::Datatype::VariableLength { .. } => { + writeln!(out, "{path} vlen {}", value(&ds.read_vlen::())).unwrap(); + } + _ => {} + } +} + +/// External virtual-dataset sources as `File::open` finds them: files in +/// the same directory. +fn sibling_resolver(dir: Option) -> clawhdf5::VdsResolver { + Arc::new(move |name: &str| { + let Some(dir) = dir.as_ref() else { + return Err(FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} cannot be resolved for an in-memory file" + ))); + }; + let p = Path::new(name); + if name.is_empty() + || !p.components().all(|c| { + matches!( + c, + std::path::Component::Normal(_) | std::path::Component::CurDir + ) + }) + { + return Err(FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} is outside the virtual file's \ + directory and is not followed" + ))); + } + match std::fs::read(dir.join(p)) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(FormatError::ChunkedReadError(format!( + "cannot read virtual dataset source file {name:?}: {e}" + ))), + } + }) +} + +#[derive(Default)] +struct Totals { + files: usize, + opened: usize, + /// The comparison's reads (each dataset read several ways). + reads: u64, + bytes: u64, + /// One pass: open, list every group, read every attribute and every + /// dataset once (`read_selection(All)`). + pass_reads: u64, + pass_bytes: u64, + file_bytes: u64, + /// (one-pass reads, one-pass bytes, file size, name) per file. + per_file: Vec<(u64, u64, u64, String)>, +} + +/// Open the file and read everything once, as a tree viewer that then +/// shows every value would. +fn one_pass(file: &File) { + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([file.superblock().root_group_address]); + while let Some(addr) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let _ = group.attrs(); + if let Ok(ds) = file.dataset_at(addr) { + let small = ds.shape().ok().and_then(|s| { + let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; + let size = u64::from(ds.raw_datatype().ok()?.type_size()); + n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) + }); + if small.is_some() { + let _ = ds.read_selection(&Selection::All); + } + } + if let Ok(entries) = group.entries() { + queue.extend(entries.into_iter().map(|(_, a)| a)); + } + } +} + +fn check(path: &Path, totals: &mut Totals) { + let Ok(bytes) = std::fs::read(path) else { + return; + }; + let name = path.display().to_string(); + let local = File::open(path); + let storage = Arc::new(CountingStorage::new(bytes.clone())); + let resolver = sibling_resolver(path.parent().map(Path::to_path_buf)); + let remote = File::open_storage(storage.clone()).map(|mut f| { + f.set_vds_resolver(resolver.clone()); + f + }); + totals.files += 1; + let (local, remote) = match (local, remote) { + (Ok(l), Ok(r)) => (l, r), + (l, r) => { + // Both refuse the file, with the same error. + assert_eq!( + format!("{:?}", l.map(|_| ())), + format!("{:?}", r.map(|_| ())), + "{name}: open" + ); + return; + } + }; + totals.opened += 1; + assert!(remote.contiguous_bytes().is_none(), "{name}"); + assert_eq!(local.user_block_size(), remote.user_block_size(), "{name}"); + let want = transcript(&local); + let got = transcript(&remote); + if want != got { + let first = want + .lines() + .zip(got.lines()) + .find(|(w, g)| w != g) + .map(|(w, g)| format!("\n local: {w}\n storage: {g}")) + .unwrap_or_else(|| { + format!( + "\n {} vs {} lines", + want.lines().count(), + got.lines().count() + ) + }); + panic!("{name}: File::open_storage differs from File::open{first}"); + } + totals.reads += storage.reads(); + totals.bytes += storage.bytes_read(); + totals.file_bytes += bytes.len() as u64; + + let pass = Arc::new(CountingStorage::new(bytes.clone())); + if let Ok(mut f) = File::open_storage(pass.clone()) { + f.set_vds_resolver(resolver); + one_pass(&f); + } + let (reads, read_bytes) = (pass.reads(), pass.bytes_read()); + totals.pass_reads += reads; + totals.pass_bytes += read_bytes; + if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") { + eprintln!( + "{reads:>9} reads {read_bytes:>12} bytes {:>12} file {name}", + bytes.len() + ); + } + totals + .per_file + .push((reads, read_bytes, bytes.len() as u64, name)); +} + +fn report(what: &str, totals: &mut Totals) { + eprintln!( + "{what}: {} files ({} open, {} bytes); comparison: {} read_at calls, {} bytes; \ + one pass (list, attributes, every dataset once): {} read_at calls, {} bytes", + totals.files, + totals.opened, + totals.file_bytes, + totals.reads, + totals.bytes, + totals.pass_reads, + totals.pass_bytes + ); + totals.per_file.sort_by_key(|a| std::cmp::Reverse(a.0)); + for (reads, bytes, size, name) in totals.per_file.iter().take(10) { + eprintln!(" {reads:>9} reads {bytes:>12} bytes (file {size:>11}) {name}"); + } +} +fn hdf5_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + hdf5_files(&p, out); + } else if p + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf")) + { + out.push(p); + } + } +} + +#[test] +fn fixtures_read_identically_through_open_storage() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + files.sort(); + assert!(files.len() >= 45, "{} fixtures", files.len()); + let mut totals = Totals::default(); + for f in &files { + check(f, &mut totals); + } + report("fixtures", &mut totals); + assert!(totals.opened >= 40, "{}", totals.opened); + assert!(totals.reads > 0); +} + +#[test] +fn corpus_reads_identically_through_open_storage() { + let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else { + eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus"); + return; + }; + let mut files = Vec::new(); + for d in std::env::split_paths(&dirs) { + hdf5_files(&d, &mut files); + } + files.sort(); + let mut totals = Totals::default(); + for f in &files { + check(f, &mut totals); + } + report("corpus", &mut totals); + assert!(totals.files > 0); +} + +/// A user block, a metadata cache image and a Storage that is itself in +/// memory: the in-memory view of a storage that has one is used as is. +#[test] +fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let path = root.join("tests/fixtures/h5clear_mdc_image.h5"); + let bytes = std::fs::read(&path).unwrap(); + let local = File::open(&path).unwrap(); + // A Vec is a Storage with a contiguous view; the image still has + // to be laid over it, so the view is not used. + let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap(); + assert!(in_memory.contiguous_bytes().is_none()); + assert_eq!(transcript(&local), transcript(&in_memory)); + + let plain = root.join("../clawhdf5-format/tests/fixtures/chunked_2d.h5"); + let bytes = std::fs::read(&plain).unwrap(); + let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap(); + assert_eq!(in_memory.contiguous_bytes(), Some(&bytes[..])); + let counting = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap(); + assert!(counting.contiguous_bytes().is_none()); + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| counting.as_bytes().len())); + assert!( + result.is_err(), + "as_bytes over a range storage must not answer" + ); +}