clawhdf5: File::open_storage reads any Storage through the full API
File::open_storage(Arc<dyn Storage + Send + Sync>) opens a file served by any backend: groups, datasets, attributes, read_*, selections, VL data and virtual datasets (external sources through the new File::set_vds_resolver) all read through Storage::read_at/read_ranges. The file's view (user block skipped, bounded by the recorded end of file) is itself a Storage; File::open and File::from_bytes keep their mmap and in-memory paths, now as that view's as_contiguous() fast path. A storage-backed file's metadata cache image is laid over each read it covers (new CacheImage::entries), as libhdf5 loads it. The typed readers keep their fast paths over any storage: a contiguous dataset is read in one piece and converted (read_f64 and friends), and a contiguous native selection reads only its runs (data_read::read_selection_native_in). Zero-copy methods answer ContiguousStorageRequired when the bytes are not in memory, and File::as_bytes panics there (File::contiguous_bytes is the fallible form). Test: tests/storage_equivalence.rs reads every fixture, and with CLAWHDF5_STORAGE_CORPUS every conformance-corpus file, through File::open and through open_storage over a read_at-only CountingStorage — tree, attributes, and every dataset's values several ways — and requires identical transcripts; it prints the read_at calls and bytes a one-pass read costs per file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -956,6 +956,78 @@ pub fn read_selection_native<T: NativeElement>(
|
||||
crate::gather::gather::<T>(raw, dims, elem_size, selection).map(Some)
|
||||
}
|
||||
|
||||
/// [`read_selection_native`] of a contiguous dataset in any [`Storage`],
|
||||
/// reading only the selected elements' runs (adjacent ones merged, one
|
||||
/// [`Storage::read_ranges`] call) instead of the whole dataset.
|
||||
///
|
||||
/// `Ok(None)` wherever the in-memory fast path does not apply and the
|
||||
/// caller converts through the byte readers instead: `datatype` is not
|
||||
/// `T`'s native representation, the layout is not contiguous, or the
|
||||
/// dataset's bytes cannot be located in the file (no address, storage too
|
||||
/// small, past the end of file: the cases [`read_raw_data_zerocopy`]
|
||||
/// fails). Otherwise the result and errors are [`read_selection_native`]'s
|
||||
/// over those bytes.
|
||||
pub fn read_selection_native_in<T: NativeElement, S: Storage + ?Sized>(
|
||||
file_data: &S,
|
||||
layout: &DataLayout,
|
||||
dataspace: &Dataspace,
|
||||
datatype: &Datatype,
|
||||
selection: &crate::selection::Selection,
|
||||
) -> Result<Option<Vec<T>>, FormatError> {
|
||||
if !T::is_native(datatype) {
|
||||
return Ok(None);
|
||||
}
|
||||
let DataLayout::Contiguous {
|
||||
address: Some(address),
|
||||
size,
|
||||
} = layout
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Where the dataset's bytes are, as `read_raw_data_zerocopy` finds them.
|
||||
let located = to_usize(dataspace.num_elements())
|
||||
.ok()
|
||||
.and_then(|n| n.checked_mul(datatype.type_size() as usize))
|
||||
.filter(|&len| contiguous_read_len(*size, len).is_ok())
|
||||
.filter(|&len| {
|
||||
address
|
||||
.checked_add(len as u64)
|
||||
.is_some_and(|end| end <= file_data.len())
|
||||
});
|
||||
let Some(len) = located else {
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(all) = file_data.as_contiguous() {
|
||||
let start = to_usize(*address)?;
|
||||
return read_selection_native(
|
||||
&all[start..start + len],
|
||||
&dataspace.dimensions,
|
||||
datatype,
|
||||
selection,
|
||||
);
|
||||
}
|
||||
let dims = &dataspace.dimensions;
|
||||
let elem_size = core::mem::size_of::<T>();
|
||||
let total = dims
|
||||
.iter()
|
||||
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
||||
.ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?;
|
||||
let expected = crate::chunked_read::checked_byte_len(total, elem_size)?;
|
||||
if len != expected {
|
||||
return Err(FormatError::DataSizeMismatch {
|
||||
expected,
|
||||
actual: len,
|
||||
});
|
||||
}
|
||||
let bytes = if let crate::selection::Selection::All = selection {
|
||||
read_exact_at(file_data, *address, len)?.into_owned()
|
||||
} else {
|
||||
crate::partial_read::validate(selection, dims)?;
|
||||
crate::gather::gather_storage(file_data, *address, len, dims, elem_size, selection)?
|
||||
};
|
||||
Ok(Some(native_to_vec(&bytes, bytes.len() / elem_size)))
|
||||
}
|
||||
|
||||
/// The bytes of a slice of [`NativeElement`]s.
|
||||
#[cfg(feature = "std")]
|
||||
fn bytes_of_mut<T: NativeElement>(values: &mut [T]) -> &mut [u8] {
|
||||
|
||||
@@ -490,6 +490,23 @@ impl CacheImage {
|
||||
image_block_in(file, self.location)
|
||||
}
|
||||
|
||||
/// Every entry as `(file address, its bytes)`, taken from `block` (the
|
||||
/// image block, see [`Self::block_in`]), in the order [`Self::apply`]
|
||||
/// writes them: for a reader that cannot write the image over the
|
||||
/// file's bytes and lays the entries over each read instead.
|
||||
pub fn entries<'b>(&self, block: &'b [u8]) -> Result<Vec<(u64, &'b [u8])>, FormatError> {
|
||||
let short = || FormatError::InvalidCacheImage("image applied to the wrong file");
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|e| {
|
||||
let src = block
|
||||
.get(e.image_offset..e.image_offset + e.len)
|
||||
.ok_or_else(short)?;
|
||||
Ok((e.address, src))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Write every entry over `dst`, the file's bytes from the superblock
|
||||
/// on (as long as the `data` the image was decoded from), taking the
|
||||
/// entries from `block` (the image block, see [`Self::block`]). `block`
|
||||
|
||||
@@ -56,7 +56,7 @@ pub use error::Error;
|
||||
pub use lazy::{LazyDataset, LazyFile, LazyGroup};
|
||||
#[cfg(feature = "mmap")]
|
||||
pub use mmap_file::{MmapDataset, MmapFile, MmapGroup};
|
||||
pub use reader::{Dataset, File, Group};
|
||||
pub use reader::{Dataset, File, Group, SharedStorage, VdsResolver};
|
||||
pub use types::{AttrValue, DType};
|
||||
pub use vlen::VlenValue;
|
||||
pub use writer::FileBuilder;
|
||||
@@ -72,6 +72,7 @@ pub use clawhdf5_format::property_list::{
|
||||
#[cfg(feature = "provenance")]
|
||||
pub use clawhdf5_format::provenance;
|
||||
pub use clawhdf5_format::selection::Selection;
|
||||
pub use clawhdf5_format::storage::Storage;
|
||||
pub use clawhdf5_format::superblock::swmr_flags;
|
||||
pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime};
|
||||
|
||||
|
||||
+323
-72
@@ -5,7 +5,10 @@
|
||||
//! the traditional read-into-`Vec<u8>` fallback. [`File::from_bytes`] remains
|
||||
//! available for in-memory usage (tests, etc.).
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
|
||||
use clawhdf5_format::chunk_cache::ChunkCache;
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
@@ -19,29 +22,44 @@ use clawhdf5_format::group_v2;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::signature;
|
||||
use clawhdf5_format::storage::Storage;
|
||||
use clawhdf5_format::superblock::Superblock;
|
||||
use clawhdf5_format::superblock_ext::{self, CacheImageState};
|
||||
|
||||
use crate::cache_image::{self, ImageView};
|
||||
use crate::error::Error;
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileData — internal storage for either owned bytes or an mmap
|
||||
// FileData — internal storage for owned bytes, an mmap, or any Storage
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Internal storage: either an owned `Vec<u8>` or a memory-mapped region.
|
||||
/// A [`Storage`] a [`File`] can be opened over: any backend that can be
|
||||
/// shared between threads (see [`File::open_storage`]).
|
||||
pub type SharedStorage = Arc<dyn Storage + Send + Sync>;
|
||||
|
||||
/// Resolves an external Virtual Dataset source file name to that file's
|
||||
/// bytes (`Ok(None)`: the file does not exist, its mappings read as the
|
||||
/// fill value). See [`File::set_vds_resolver`].
|
||||
pub type VdsResolver = Arc<dyn Fn(&str) -> Result<Option<Vec<u8>>, FormatError> + Send + Sync>;
|
||||
|
||||
/// Internal storage: an owned `Vec<u8>`, a memory-mapped region, or a
|
||||
/// [`Storage`] backend.
|
||||
enum Backing {
|
||||
Owned(Vec<u8>),
|
||||
#[cfg(feature = "mmap")]
|
||||
Mmap(clawhdf5_io::MmapReader),
|
||||
Storage(SharedStorage),
|
||||
}
|
||||
|
||||
impl Backing {
|
||||
fn whole_file(&self) -> &[u8] {
|
||||
/// The whole file, when it is in memory.
|
||||
fn whole_file(&self) -> Option<&[u8]> {
|
||||
match self {
|
||||
Backing::Owned(v) => v,
|
||||
Backing::Owned(v) => Some(v),
|
||||
#[cfg(feature = "mmap")]
|
||||
Backing::Mmap(r) => r.as_bytes(),
|
||||
Backing::Mmap(r) => Some(r.as_bytes()),
|
||||
Backing::Storage(s) => s.as_contiguous(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,13 +67,16 @@ impl Backing {
|
||||
/// The file's bytes, viewed from the superblock on and up to the end of
|
||||
/// file the superblock records. A file may start with a user block (the
|
||||
/// superblock at 512, 1024, …); every HDF5 address is relative to the
|
||||
/// superblock, so all parsing goes through [`Self::as_bytes`].
|
||||
/// superblock, so all parsing goes through this view, which is a
|
||||
/// [`Storage`]: in memory (a `Vec`, an mmap) its reads are slices of the
|
||||
/// file, as before; over any other storage they are reads of that storage,
|
||||
/// shifted by the user block and bounded by the end of file.
|
||||
struct FileData {
|
||||
backing: Backing,
|
||||
/// Offset of the superblock in the file (the user-block size).
|
||||
base: usize,
|
||||
base: u64,
|
||||
/// End of the HDF5 data in the file (`Superblock::data_end`, absolute).
|
||||
end: usize,
|
||||
end: u64,
|
||||
/// A mapped file that holds a metadata cache image, with the image
|
||||
/// written in: a private copy-on-write mapping of the whole file, so
|
||||
/// only the pages the image's entries land on are copied (see
|
||||
@@ -63,6 +84,10 @@ struct FileData {
|
||||
/// an image is read straight from the mapping, and an owned buffer has
|
||||
/// the image written into it in place.
|
||||
patched: Option<clawhdf5_io::PrivateCopy>,
|
||||
/// A [`Storage`]-backed file's metadata cache image: its entries
|
||||
/// (address relative to the superblock, bytes), laid over every read in
|
||||
/// order, as libhdf5 reads them instead of the file's own bytes.
|
||||
overlay: Vec<(u64, Vec<u8>)>,
|
||||
/// The file has a metadata cache image libhdf5 cannot load. libhdf5
|
||||
/// opens such a file and fails its first metadata read (the image loads
|
||||
/// then); every object lookup here fails with this error, and no
|
||||
@@ -74,7 +99,10 @@ impl FileData {
|
||||
/// Locate the superblock and parse it. A truncated file is refused, and
|
||||
/// bytes past the recorded end of file are not read, as in libhdf5.
|
||||
fn new(mut backing: Backing) -> Result<(Self, Superblock), Error> {
|
||||
let whole = backing.whole_file();
|
||||
if let Backing::Storage(storage) = backing {
|
||||
return Self::new_storage(storage);
|
||||
}
|
||||
let whole = backing.whole_file().unwrap_or_default();
|
||||
let (user_block, hdf5) = signature::split_user_block(whole)?;
|
||||
let base = user_block.len();
|
||||
let superblock = Superblock::parse(hdf5, 0)?;
|
||||
@@ -92,6 +120,7 @@ impl FileData {
|
||||
clawhdf5_io::HDF5Read::private_copy(r)
|
||||
})?
|
||||
}
|
||||
Backing::Storage(_) => unreachable!("handled above"),
|
||||
};
|
||||
let (patched, image_error) = match view {
|
||||
ImageView::Plain => (None, None),
|
||||
@@ -101,34 +130,140 @@ impl FileData {
|
||||
Ok((
|
||||
Self {
|
||||
backing,
|
||||
base,
|
||||
end,
|
||||
base: base as u64,
|
||||
end: end as u64,
|
||||
patched,
|
||||
overlay: Vec::new(),
|
||||
image_error,
|
||||
},
|
||||
superblock,
|
||||
))
|
||||
}
|
||||
|
||||
fn as_bytes(&self) -> &[u8] {
|
||||
match &self.patched {
|
||||
Some(p) => &p[self.base..self.end],
|
||||
None => &self.backing.whole_file()[self.base..self.end],
|
||||
/// [`Self::new`] for a [`Storage`] backend: the same checks, through
|
||||
/// reads of the storage.
|
||||
fn new_storage(storage: SharedStorage) -> Result<(Self, Superblock), Error> {
|
||||
let file_len = storage.len();
|
||||
let base = signature::find_signature_in(&*storage)?;
|
||||
let mut data = Self {
|
||||
backing: Backing::Storage(storage),
|
||||
base,
|
||||
end: file_len,
|
||||
patched: None,
|
||||
overlay: Vec::new(),
|
||||
image_error: None,
|
||||
};
|
||||
let superblock = Superblock::parse_in(&data, 0)?;
|
||||
data.end = base + superblock.data_end(base, file_len)?;
|
||||
match superblock_ext::cache_image_state_in(&data, &superblock)? {
|
||||
CacheImageState::Absent => {}
|
||||
CacheImageState::Unloadable(e) => data.image_error = Some(e),
|
||||
CacheImageState::Loaded(image) => {
|
||||
let block = image.block_in(&data)?.into_owned();
|
||||
data.overlay = image
|
||||
.entries(&block)?
|
||||
.into_iter()
|
||||
.map(|(addr, bytes)| (addr, bytes.to_vec()))
|
||||
.collect();
|
||||
}
|
||||
}
|
||||
Ok((data, superblock))
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.as_bytes().len()
|
||||
/// The HDF5 data as one slice, when the file is in memory (a `Vec`, an
|
||||
/// mmap, or a storage that holds it all and has no cache image to lay
|
||||
/// over it).
|
||||
fn contiguous(&self) -> Option<&[u8]> {
|
||||
if let Some(p) = &self.patched {
|
||||
return p.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?);
|
||||
}
|
||||
if !self.overlay.is_empty() {
|
||||
return None;
|
||||
}
|
||||
self.backing
|
||||
.whole_file()?
|
||||
.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?)
|
||||
}
|
||||
|
||||
/// The bytes to read metadata from; fails for a file whose cache image
|
||||
/// cannot be loaded (see [`Self::image_error`]).
|
||||
fn meta(&self) -> Result<&[u8], FormatError> {
|
||||
fn meta(&self) -> Result<&Self, FormatError> {
|
||||
match &self.image_error {
|
||||
Some(e) => Err(e.clone()),
|
||||
None => Ok(self.as_bytes()),
|
||||
None => Ok(self),
|
||||
}
|
||||
}
|
||||
|
||||
/// The backend of a file that is not in memory.
|
||||
fn remote(&self) -> Result<&SharedStorage, FormatError> {
|
||||
match &self.backing {
|
||||
Backing::Storage(s) => Ok(s),
|
||||
_ => Err(FormatError::Storage(
|
||||
"in-memory file has no contiguous view".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `bytes`, read at `offset`, with the cache image entries they overlap
|
||||
/// written over them.
|
||||
fn with_overlay<'a>(&self, offset: u64, mut bytes: Cow<'a, [u8]>) -> Cow<'a, [u8]> {
|
||||
let end = offset + bytes.len() as u64;
|
||||
for (addr, entry) in &self.overlay {
|
||||
let entry_end = addr + entry.len() as u64;
|
||||
if *addr >= end || entry_end <= offset {
|
||||
continue;
|
||||
}
|
||||
let from = (*addr).max(offset);
|
||||
let to = entry_end.min(end);
|
||||
let dst = bytes.to_mut();
|
||||
dst[(from - offset) as usize..(to - offset) as usize]
|
||||
.copy_from_slice(&entry[(from - addr) as usize..(to - addr) as usize]);
|
||||
}
|
||||
bytes
|
||||
}
|
||||
}
|
||||
|
||||
impl Storage for FileData {
|
||||
fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
|
||||
if let Some(all) = self.contiguous() {
|
||||
return all.read_at(offset, len);
|
||||
}
|
||||
let size = self.end - self.base;
|
||||
let len = usize::try_from(size.saturating_sub(offset)).map_or(len, |avail| avail.min(len));
|
||||
if len == 0 {
|
||||
return Ok(Cow::Borrowed(&[]));
|
||||
}
|
||||
let bytes = self.remote()?.read_at(self.base + offset, len)?;
|
||||
Ok(self.with_overlay(offset, bytes))
|
||||
}
|
||||
|
||||
fn len(&self) -> u64 {
|
||||
self.end - self.base
|
||||
}
|
||||
|
||||
fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
|
||||
if let Some(all) = self.contiguous() {
|
||||
return all.read_ranges(ranges);
|
||||
}
|
||||
let size = self.end - self.base;
|
||||
let shifted: Vec<Range<u64>> = ranges
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let (s, e) = (r.start.min(size), r.end.min(size));
|
||||
self.base + s..self.base + e.max(s)
|
||||
})
|
||||
.collect();
|
||||
let got = self.remote()?.read_ranges(&shifted)?;
|
||||
Ok(got
|
||||
.into_iter()
|
||||
.zip(ranges)
|
||||
.map(|(bytes, r)| self.with_overlay(r.start, bytes))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn as_contiguous(&self) -> Option<&[u8]> {
|
||||
self.contiguous()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -149,6 +284,9 @@ pub struct File {
|
||||
/// Directory the file was opened from, used to resolve external Virtual
|
||||
/// Dataset source files relative to this file. `None` for in-memory files.
|
||||
base_dir: Option<std::path::PathBuf>,
|
||||
/// Resolves external Virtual Dataset source files instead of
|
||||
/// `base_dir` (see [`File::set_vds_resolver`]).
|
||||
vds_resolver: Option<VdsResolver>,
|
||||
}
|
||||
|
||||
impl File {
|
||||
@@ -167,6 +305,7 @@ impl File {
|
||||
superblock,
|
||||
chunk_cache: ChunkCache::new(),
|
||||
base_dir,
|
||||
vds_resolver: None,
|
||||
})
|
||||
}
|
||||
#[cfg(not(feature = "mmap"))]
|
||||
@@ -200,9 +339,53 @@ impl File {
|
||||
superblock,
|
||||
chunk_cache: ChunkCache::new(),
|
||||
base_dir: None,
|
||||
vds_resolver: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Open an HDF5 file served by any [`Storage`]: a range-reading remote
|
||||
/// backend, a block cache, or an in-memory buffer — through the same
|
||||
/// read API as [`File::open`] (groups, datasets, attributes, `read_*`,
|
||||
/// selections, variable-length data, virtual datasets).
|
||||
///
|
||||
/// Every read goes through the storage's [`Storage::read_at`] and
|
||||
/// [`Storage::read_ranges`] (a chunked read fetches all the chunks it
|
||||
/// needs with one `read_ranges` call per 64 MiB), so nothing is read that
|
||||
/// the operation does not need. A storage that has the whole file in
|
||||
/// memory ([`Storage::as_contiguous`]) is read as [`File::from_bytes`]
|
||||
/// reads its buffer. The storage holds the whole file: a user block is
|
||||
/// found and skipped, and bytes past the end of file the superblock
|
||||
/// records are not read. A metadata cache image is laid over the reads
|
||||
/// it covers, as libhdf5 loads it.
|
||||
///
|
||||
/// The zero-copy methods ([`Dataset::read_raw_ref`],
|
||||
/// [`Dataset::read_as_slice`], `read_*_zerocopy`) borrow the file's bytes
|
||||
/// and return [`FormatError::ContiguousStorageRequired`] over a storage
|
||||
/// that does not hold them; [`File::as_bytes`] panics there (use
|
||||
/// [`File::contiguous_bytes`]). External virtual-dataset source files
|
||||
/// are read through [`File::set_vds_resolver`]; without one they cannot
|
||||
/// be resolved.
|
||||
pub fn open_storage(storage: SharedStorage) -> Result<Self, Error> {
|
||||
let (data, superblock) = FileData::new(Backing::Storage(storage))?;
|
||||
Ok(Self {
|
||||
data,
|
||||
superblock,
|
||||
chunk_cache: ChunkCache::new(),
|
||||
base_dir: None,
|
||||
vds_resolver: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve external Virtual Dataset source files (their names as the
|
||||
/// mappings store them) with `resolver`, instead of reading them from
|
||||
/// the directory of the file (for [`File::open`]) or refusing them (for
|
||||
/// in-memory and [`Storage`]-backed files). `Ok(None)` means the source
|
||||
/// file does not exist, and its mappings read as the fill value, as in
|
||||
/// libhdf5; `Err` fails the read.
|
||||
pub fn set_vds_resolver(&mut self, resolver: VdsResolver) {
|
||||
self.vds_resolver = Some(resolver);
|
||||
}
|
||||
|
||||
/// Returns a handle to the root group.
|
||||
pub fn root(&self) -> Group<'_> {
|
||||
Group {
|
||||
@@ -216,7 +399,7 @@ impl File {
|
||||
/// The path uses `/` separators (e.g., `"group1/values"`).
|
||||
pub fn dataset(&self, path: &str) -> Result<Dataset<'_>, Error> {
|
||||
let data = self.data.meta()?;
|
||||
let addr = group_v2::resolve_path_any(data, &self.superblock, path)?;
|
||||
let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?;
|
||||
let hdr = self.parse_header(addr)?;
|
||||
if !has_message(&hdr, MessageType::DataLayout) {
|
||||
return Err(Error::NotADataset(path.to_string()));
|
||||
@@ -263,7 +446,7 @@ impl File {
|
||||
/// Use `"/"` or `""` for the root group.
|
||||
pub fn group(&self, path: &str) -> Result<Group<'_>, Error> {
|
||||
let data = self.data.meta()?;
|
||||
let addr = group_v2::resolve_path_any(data, &self.superblock, path)?;
|
||||
let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?;
|
||||
Ok(Group {
|
||||
file: self,
|
||||
address: addr,
|
||||
@@ -322,8 +505,22 @@ impl File {
|
||||
/// with a metadata cache image these are the bytes with the image
|
||||
/// applied; when the image cannot be loaded they are the file's own
|
||||
/// bytes, whose metadata may be stale (every object lookup fails then).
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// For a file opened with [`File::open_storage`] over a storage that
|
||||
/// does not hold the whole file in memory; use
|
||||
/// [`contiguous_bytes`](Self::contiguous_bytes) there.
|
||||
pub fn as_bytes(&self) -> &[u8] {
|
||||
self.data.as_bytes()
|
||||
self.data
|
||||
.contiguous()
|
||||
.expect("File::as_bytes: the file is not in memory (see File::contiguous_bytes)")
|
||||
}
|
||||
|
||||
/// [`as_bytes`](Self::as_bytes), or `None` for a file whose bytes are
|
||||
/// not all in memory ([`File::open_storage`]).
|
||||
pub fn contiguous_bytes(&self) -> Option<&[u8]> {
|
||||
self.data.contiguous()
|
||||
}
|
||||
|
||||
/// The error of a metadata cache image libhdf5 cannot load, when the
|
||||
@@ -338,7 +535,7 @@ impl File {
|
||||
/// Size of the user block before the superblock (0 for most files).
|
||||
/// Matches h5py's `File.userblock_size`.
|
||||
pub fn user_block_size(&self) -> u64 {
|
||||
self.data.base as u64
|
||||
self.data.base
|
||||
}
|
||||
|
||||
/// Returns a reference to the parsed superblock.
|
||||
@@ -349,7 +546,7 @@ impl File {
|
||||
/// Returns `true` when the file is backed by memory-mapped I/O.
|
||||
pub fn is_mmap(&self) -> bool {
|
||||
match &self.data.backing {
|
||||
Backing::Owned(_) => false,
|
||||
Backing::Owned(_) | Backing::Storage(_) => false,
|
||||
#[cfg(feature = "mmap")]
|
||||
Backing::Mmap(_) => true,
|
||||
}
|
||||
@@ -362,7 +559,7 @@ impl File {
|
||||
/// this file's global heap; see [`Dataset::read_string`] for the values.
|
||||
pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result<Vec<String>, Error> {
|
||||
crate::vlen::decode_strings(
|
||||
self.as_bytes(),
|
||||
&self.data,
|
||||
datatype,
|
||||
raw,
|
||||
self.offset_size(),
|
||||
@@ -406,9 +603,9 @@ impl File {
|
||||
}
|
||||
|
||||
fn parse_header(&self, address: u64) -> Result<ObjectHeader, FormatError> {
|
||||
ObjectHeader::parse(
|
||||
ObjectHeader::parse_in(
|
||||
self.data.meta()?,
|
||||
address as usize,
|
||||
address,
|
||||
self.superblock.offset_size,
|
||||
self.superblock.length_size,
|
||||
)
|
||||
@@ -426,7 +623,7 @@ impl File {
|
||||
impl std::fmt::Debug for File {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("File")
|
||||
.field("size", &self.data.len())
|
||||
.field("size", &Storage::len(&self.data))
|
||||
.field("superblock_version", &self.superblock.version)
|
||||
.field("mmap", &self.is_mmap())
|
||||
.finish()
|
||||
@@ -489,7 +686,7 @@ impl<'f> Group<'f> {
|
||||
&self,
|
||||
) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
|
||||
let hdr = self.file.parse_header(self.address)?;
|
||||
let data = self.file.data.as_bytes();
|
||||
let data = &self.file.data;
|
||||
read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size())
|
||||
}
|
||||
|
||||
@@ -520,7 +717,7 @@ impl<'f> Group<'f> {
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let hdr = self.file.parse_header(self.address)?;
|
||||
let data = self.file.data.as_bytes();
|
||||
let data = &self.file.data;
|
||||
read_attr(
|
||||
data,
|
||||
&hdr,
|
||||
@@ -536,7 +733,7 @@ impl<'f> Group<'f> {
|
||||
/// [`group_v2::resolve_child`]).
|
||||
fn child_address(&self, name: &str) -> Result<u64, Error> {
|
||||
let data = self.file.data.meta()?;
|
||||
group_v2::resolve_child(data, &self.file.superblock, self.address, name)
|
||||
group_v2::resolve_child_in(data, &self.file.superblock, self.address, name)
|
||||
.map_err(Error::Format)
|
||||
}
|
||||
|
||||
@@ -559,7 +756,7 @@ impl<'f> Group<'f> {
|
||||
/// user-defined links are left out.
|
||||
fn children(&self) -> Result<Vec<GroupEntry>, Error> {
|
||||
let data = self.file.data.meta()?;
|
||||
group_v2::resolve_group_children(data, &self.file.superblock, self.address)
|
||||
group_v2::resolve_group_children_in(data, &self.file.superblock, self.address)
|
||||
.map_err(Error::Format)
|
||||
}
|
||||
}
|
||||
@@ -589,7 +786,7 @@ impl<'f> Dataset<'f> {
|
||||
Ok((self.datatype()?, ds, self.data_layout()?))
|
||||
})();
|
||||
if let Ok((dt, ds, dl)) = decoded {
|
||||
data_read::check_dataset_storage(&dl, &ds, &dt, self.file.data.len() as u64)?;
|
||||
data_read::check_dataset_storage(&dl, &ds, &dt, Storage::len(&self.file.data))?;
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
@@ -629,8 +826,8 @@ impl<'f> Dataset<'f> {
|
||||
let dt = self.datatype()?;
|
||||
// A contiguous dataset is converted straight from the file bytes; going
|
||||
// through `read_raw` first copied the whole dataset an extra time.
|
||||
if let Ok(Some(bytes)) = self.read_raw_ref() {
|
||||
return Ok(data_read::read_as_f64(bytes, &dt)?);
|
||||
if let Ok(Some(bytes)) = self.contiguous_raw() {
|
||||
return Ok(data_read::read_as_f64(&bytes, &dt)?);
|
||||
}
|
||||
if let Some(values) = self.read_chunked_native::<f64>()? {
|
||||
return Ok(values);
|
||||
@@ -649,8 +846,8 @@ impl<'f> Dataset<'f> {
|
||||
let dt = self.datatype()?;
|
||||
// A contiguous dataset is converted straight from the file bytes; going
|
||||
// through `read_raw` first copied the whole dataset an extra time.
|
||||
if let Ok(Some(bytes)) = self.read_raw_ref() {
|
||||
return Ok(data_read::read_as_f32(bytes, &dt)?);
|
||||
if let Ok(Some(bytes)) = self.contiguous_raw() {
|
||||
return Ok(data_read::read_as_f32(&bytes, &dt)?);
|
||||
}
|
||||
if let Some(values) = self.read_chunked_native::<f32>()? {
|
||||
return Ok(values);
|
||||
@@ -664,8 +861,8 @@ impl<'f> Dataset<'f> {
|
||||
let dt = self.datatype()?;
|
||||
// A contiguous dataset is converted straight from the file bytes; going
|
||||
// through `read_raw` first copied the whole dataset an extra time.
|
||||
if let Ok(Some(bytes)) = self.read_raw_ref() {
|
||||
return Ok(data_read::read_as_i32(bytes, &dt)?);
|
||||
if let Ok(Some(bytes)) = self.contiguous_raw() {
|
||||
return Ok(data_read::read_as_i32(&bytes, &dt)?);
|
||||
}
|
||||
if let Some(values) = self.read_chunked_native::<i32>()? {
|
||||
return Ok(values);
|
||||
@@ -679,8 +876,8 @@ impl<'f> Dataset<'f> {
|
||||
let dt = self.datatype()?;
|
||||
// A contiguous dataset is converted straight from the file bytes; going
|
||||
// through `read_raw` first copied the whole dataset an extra time.
|
||||
if let Ok(Some(bytes)) = self.read_raw_ref() {
|
||||
return Ok(data_read::read_as_i64(bytes, &dt)?);
|
||||
if let Ok(Some(bytes)) = self.contiguous_raw() {
|
||||
return Ok(data_read::read_as_i64(&bytes, &dt)?);
|
||||
}
|
||||
if let Some(values) = self.read_chunked_native::<i64>()? {
|
||||
return Ok(values);
|
||||
@@ -694,8 +891,8 @@ impl<'f> Dataset<'f> {
|
||||
let dt = self.datatype()?;
|
||||
// A contiguous dataset is converted straight from the file bytes; going
|
||||
// through `read_raw` first copied the whole dataset an extra time.
|
||||
if let Ok(Some(bytes)) = self.read_raw_ref() {
|
||||
return Ok(data_read::read_as_u64(bytes, &dt)?);
|
||||
if let Ok(Some(bytes)) = self.contiguous_raw() {
|
||||
return Ok(data_read::read_as_u64(&bytes, &dt)?);
|
||||
}
|
||||
if let Some(values) = self.read_chunked_native::<u64>()? {
|
||||
return Ok(values);
|
||||
@@ -793,8 +990,8 @@ impl<'f> Dataset<'f> {
|
||||
// sparse) dataset — select from a fill-aware full read instead. (The
|
||||
// selection reader currently decodes the full dataset too, so this
|
||||
// costs nothing extra.)
|
||||
let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
|
||||
self.file.data.as_bytes(),
|
||||
let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage(
|
||||
&self.file.data,
|
||||
&self.header.messages,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
@@ -813,8 +1010,8 @@ impl<'f> Dataset<'f> {
|
||||
selection,
|
||||
)?);
|
||||
}
|
||||
Ok(data_read::read_raw_data_selection(
|
||||
self.file.data.as_bytes(),
|
||||
Ok(data_read::read_raw_data_selection_in(
|
||||
&self.file.data,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
@@ -871,12 +1068,26 @@ impl<'f> Dataset<'f> {
|
||||
return full();
|
||||
}
|
||||
let dt = self.datatype()?;
|
||||
if T::is_native(&dt)
|
||||
&& let Ok(Some(raw)) = self.read_raw_ref()
|
||||
if T::is_native(&dt) && self.file.data.contiguous().is_some() {
|
||||
if let Ok(Some(raw)) = self.read_raw_ref() {
|
||||
let dims = self.dataspace()?.dimensions;
|
||||
if let Some(values) =
|
||||
data_read::read_selection_native::<T>(raw, &dims, &dt, selection)?
|
||||
{
|
||||
return Ok(values);
|
||||
}
|
||||
}
|
||||
} else if T::is_native(&dt)
|
||||
&& let (Ok(dl), Ok(ds)) = (self.data_layout(), self.dataspace())
|
||||
{
|
||||
let dims = self.dataspace()?.dimensions;
|
||||
if let Some(values) = data_read::read_selection_native::<T>(raw, &dims, &dt, selection)?
|
||||
{
|
||||
// Not in memory: only the selected runs are read.
|
||||
if let Some(values) = data_read::read_selection_native_in::<T, _>(
|
||||
&self.file.data,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
selection,
|
||||
)? {
|
||||
return Ok(values);
|
||||
}
|
||||
}
|
||||
@@ -893,10 +1104,46 @@ impl<'f> Dataset<'f> {
|
||||
let dl = self.data_layout()?;
|
||||
let ds = self.dataspace()?;
|
||||
let dt = self.datatype()?;
|
||||
let slice = data_read::read_raw_data_zerocopy(self.file.data.as_bytes(), &dl, &ds, &dt)?;
|
||||
let Some(bytes) = self.file.data.contiguous() else {
|
||||
// The bytes are not in memory to borrow.
|
||||
return match dl {
|
||||
DataLayout::Contiguous { .. } => {
|
||||
Err(Error::Format(FormatError::ContiguousStorageRequired(
|
||||
"a zero-copy read (the file is not in memory)",
|
||||
)))
|
||||
}
|
||||
_ => Ok(None),
|
||||
};
|
||||
};
|
||||
let slice = data_read::read_raw_data_zerocopy(bytes, &dl, &ds, &dt)?;
|
||||
Ok(slice)
|
||||
}
|
||||
|
||||
/// A contiguous dataset's stored bytes, for the typed readers' fast
|
||||
/// path: borrowed from the file when it is in memory
|
||||
/// ([`read_raw_ref`](Self::read_raw_ref)), read in one piece otherwise.
|
||||
/// `Ok(None)` for other layouts; an error where `read_raw_ref` fails.
|
||||
fn contiguous_raw(&self) -> Result<Option<Cow<'f, [u8]>>, Error> {
|
||||
if self.file.data.contiguous().is_some() {
|
||||
return Ok(self.read_raw_ref()?.map(Cow::Borrowed));
|
||||
}
|
||||
let dl = self.data_layout()?;
|
||||
if !matches!(dl, DataLayout::Contiguous { .. }) {
|
||||
return Ok(None);
|
||||
}
|
||||
let ds = self.dataspace()?;
|
||||
let dt = self.datatype()?;
|
||||
Ok(Some(Cow::Owned(data_read::read_raw_data_full_in(
|
||||
&self.file.data,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
None,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)?)))
|
||||
}
|
||||
|
||||
/// Zero-copy typed read of contiguous data as `&[T]`.
|
||||
///
|
||||
/// Returns a borrowed slice of `T` directly from the file buffer with
|
||||
@@ -1083,7 +1330,7 @@ impl<'f> Dataset<'f> {
|
||||
pub fn attrs_with_errors(
|
||||
&self,
|
||||
) -> Result<(HashMap<String, AttrValue>, Vec<FormatError>), Error> {
|
||||
let data = self.file.data.as_bytes();
|
||||
let data = &self.file.data;
|
||||
read_attrs(
|
||||
data,
|
||||
&self.header,
|
||||
@@ -1097,7 +1344,7 @@ impl<'f> Dataset<'f> {
|
||||
/// that name, found without reading the other attributes when they are
|
||||
/// stored densely.
|
||||
pub fn attr(&self, name: &str) -> Result<Option<AttrValue>, Error> {
|
||||
let data = self.file.data.as_bytes();
|
||||
let data = &self.file.data;
|
||||
read_attr(
|
||||
data,
|
||||
&self.header,
|
||||
@@ -1124,8 +1371,8 @@ impl<'f> Dataset<'f> {
|
||||
/// result is not a tamper-evidence or authenticity guarantee.
|
||||
#[cfg(feature = "provenance")]
|
||||
pub fn verify_provenance(&self) -> Result<clawhdf5_format::provenance::VerifyResult, Error> {
|
||||
Ok(clawhdf5_format::provenance::verify_dataset(
|
||||
self.file.as_bytes(),
|
||||
Ok(clawhdf5_format::provenance::verify_dataset_in(
|
||||
&self.file.data,
|
||||
&self.header,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
@@ -1144,8 +1391,8 @@ impl<'f> Dataset<'f> {
|
||||
.iter()
|
||||
.find(|m| m.msg_type == msg_type)
|
||||
.map(|msg| {
|
||||
clawhdf5_format::shared_message::message_data(
|
||||
self.file.as_bytes(),
|
||||
clawhdf5_format::shared_message::message_data_in(
|
||||
&self.file.data,
|
||||
msg,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
@@ -1179,8 +1426,8 @@ impl<'f> Dataset<'f> {
|
||||
// one (`H5Dget_space`).
|
||||
if let Ok(dl @ DataLayout::Virtual { .. }) = self.data_layout() {
|
||||
let resolver = self.vds_resolver();
|
||||
ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent(
|
||||
self.file.data.as_bytes(),
|
||||
ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent_in(
|
||||
&self.file.data,
|
||||
&dl,
|
||||
&ds,
|
||||
self.file.offset_size(),
|
||||
@@ -1225,9 +1472,9 @@ impl<'f> Dataset<'f> {
|
||||
}
|
||||
let ds = self.dataspace()?;
|
||||
let pipeline = self.filter_pipeline()?;
|
||||
Ok(data_read::read_chunked_native::<T>(
|
||||
Ok(data_read::read_chunked_native_in::<T, _>(
|
||||
&self.header.messages,
|
||||
self.file.data.as_bytes(),
|
||||
&self.file.data,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
@@ -1251,17 +1498,17 @@ impl<'f> Dataset<'f> {
|
||||
}
|
||||
|
||||
// Unallocated storage reads as the dataset's fill value.
|
||||
clawhdf5_format::fill_value::read_full_with_fill(
|
||||
clawhdf5_format::fill_value::read_full_with_fill_in(
|
||||
&self.header.messages,
|
||||
self.file.data.as_bytes(),
|
||||
&self.file.data,
|
||||
&dl,
|
||||
&ds,
|
||||
dt.type_size() as usize,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
|| {
|
||||
Ok(data_read::read_raw_data_cached(
|
||||
self.file.data.as_bytes(),
|
||||
Ok(data_read::read_raw_data_cached_in(
|
||||
&self.file.data,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
@@ -1281,7 +1528,11 @@ impl<'f> Dataset<'f> {
|
||||
/// refused with an error rather than read as fill.
|
||||
fn vds_resolver(&self) -> impl Fn(&str) -> Result<Option<Vec<u8>>, FormatError> + use<> {
|
||||
let base_dir = self.file.base_dir.clone();
|
||||
let custom = self.file.vds_resolver.clone();
|
||||
move |name: &str| {
|
||||
if let Some(resolve) = &custom {
|
||||
return resolve(name);
|
||||
}
|
||||
let Some(dir) = base_dir.as_ref() else {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"virtual dataset source file {name:?} cannot be resolved for an in-memory file"
|
||||
@@ -1310,15 +1561,15 @@ impl<'f> Dataset<'f> {
|
||||
ds: &Dataspace,
|
||||
dt: &Datatype,
|
||||
) -> Result<Vec<u8>, Error> {
|
||||
let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
|
||||
self.file.data.as_bytes(),
|
||||
let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage(
|
||||
&self.file.data,
|
||||
&self.header.messages,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)?;
|
||||
let resolver = self.vds_resolver();
|
||||
let v = clawhdf5_format::vds::read_virtual_dataset(
|
||||
self.file.data.as_bytes(),
|
||||
let v = clawhdf5_format::vds::read_virtual_dataset_in(
|
||||
&self.file.data,
|
||||
dl,
|
||||
ds,
|
||||
dt,
|
||||
@@ -1439,7 +1690,7 @@ mod zero_copy_tests {
|
||||
let Backing::Mmap(r) = &f.data.backing else {
|
||||
return None;
|
||||
};
|
||||
let mapped = r.as_bytes()[f.data.base..].as_ptr();
|
||||
let mapped = r.as_bytes()[f.data.base as usize..].as_ptr();
|
||||
match &f.data.patched {
|
||||
None => Some(std::ptr::eq(f.as_bytes().as_ptr(), mapped)),
|
||||
Some(p) => {
|
||||
|
||||
@@ -158,8 +158,8 @@ pub(crate) fn classify_datatype(dt: &clawhdf5_format::datatype::Datatype) -> DTy
|
||||
/// The attributes of the object with header `header` that could be read,
|
||||
/// and one error for each that could not (see
|
||||
/// [`extract_attributes_tolerant`](clawhdf5_format::attribute::extract_attributes_tolerant)).
|
||||
pub(crate) fn read_attrs(
|
||||
file_data: &[u8],
|
||||
pub(crate) fn read_attrs<S: clawhdf5_format::storage::Storage + ?Sized>(
|
||||
file_data: &S,
|
||||
header: &clawhdf5_format::object_header::ObjectHeader,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
@@ -170,7 +170,7 @@ pub(crate) fn read_attrs(
|
||||
),
|
||||
crate::Error,
|
||||
> {
|
||||
let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant(
|
||||
let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant_in(
|
||||
file_data,
|
||||
header,
|
||||
offset_size,
|
||||
@@ -185,14 +185,14 @@ pub(crate) fn read_attrs(
|
||||
/// The attribute called `name` on the object with header `header`, decoded
|
||||
/// as [`read_attrs`] decodes it, or `None` (see
|
||||
/// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)).
|
||||
pub(crate) fn read_attr(
|
||||
file_data: &[u8],
|
||||
pub(crate) fn read_attr<S: clawhdf5_format::storage::Storage + ?Sized>(
|
||||
file_data: &S,
|
||||
header: &clawhdf5_format::object_header::ObjectHeader,
|
||||
name: &str,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Option<AttrValue>, crate::Error> {
|
||||
let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file(
|
||||
let Some(msg) = clawhdf5_format::attribute::find_attribute_in(
|
||||
file_data,
|
||||
header,
|
||||
name,
|
||||
@@ -211,9 +211,9 @@ pub(crate) fn read_attr(
|
||||
.remove(name))
|
||||
}
|
||||
|
||||
pub(crate) fn attrs_to_map(
|
||||
pub(crate) fn attrs_to_map<S: clawhdf5_format::storage::Storage + ?Sized>(
|
||||
attrs: &[clawhdf5_format::attribute::AttributeMessage],
|
||||
file_data: &[u8],
|
||||
file_data: &S,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> HashMap<String, AttrValue> {
|
||||
@@ -266,9 +266,9 @@ fn decode_bool_enum(attr: &clawhdf5_format::attribute::AttributeMessage) -> Opti
|
||||
values.iter().all(|v| *v == 0 || *v == 1).then_some(values)
|
||||
}
|
||||
|
||||
fn decode_attr_value(
|
||||
fn decode_attr_value<S: clawhdf5_format::storage::Storage + ?Sized>(
|
||||
attr: &clawhdf5_format::attribute::AttributeMessage,
|
||||
file_data: &[u8],
|
||||
file_data: &S,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Option<AttrValue> {
|
||||
@@ -311,7 +311,7 @@ fn decode_attr_value(
|
||||
is_string: true, ..
|
||||
} => {
|
||||
let strings = attr
|
||||
.read_vl_strings(file_data, offset_size, length_size)
|
||||
.read_vl_strings_in(file_data, offset_size, length_size)
|
||||
.ok()?;
|
||||
if strings.len() == 1 {
|
||||
Some(AttrValue::String(strings[0].clone()))
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
use clawhdf5_format::data_read;
|
||||
use clawhdf5_format::datatype::Datatype;
|
||||
use clawhdf5_format::error::FormatError;
|
||||
use clawhdf5_format::storage::Storage;
|
||||
use clawhdf5_format::vl_data::{VlResolver, check_element_size};
|
||||
|
||||
use crate::error::Error;
|
||||
@@ -68,8 +69,8 @@ fn class_name(dt: &Datatype) -> &'static str {
|
||||
|
||||
/// The strings in `raw`, elements of `dt`: fixed-length strings decoded as
|
||||
/// `read_string` always has, variable-length strings resolved in the heap.
|
||||
pub(crate) fn decode_strings(
|
||||
file_data: &[u8],
|
||||
pub(crate) fn decode_strings<S: Storage + ?Sized>(
|
||||
file_data: &S,
|
||||
dt: &Datatype,
|
||||
raw: &[u8],
|
||||
offset_size: u8,
|
||||
@@ -82,15 +83,15 @@ pub(crate) fn decode_strings(
|
||||
..
|
||||
} => {
|
||||
check_element_size(*size, offset_size)?;
|
||||
Ok(VlResolver::new(file_data, offset_size, length_size).strings(raw)?)
|
||||
Ok(VlResolver::new_in(file_data, offset_size, length_size).strings(raw)?)
|
||||
}
|
||||
_ => Ok(data_read::read_as_strings(raw, dt)?),
|
||||
}
|
||||
}
|
||||
|
||||
/// The exact bytes of the variable-length strings in `raw`.
|
||||
pub(crate) fn decode_string_bytes(
|
||||
file_data: &[u8],
|
||||
pub(crate) fn decode_string_bytes<S: Storage + ?Sized>(
|
||||
file_data: &S,
|
||||
dt: &Datatype,
|
||||
raw: &[u8],
|
||||
offset_size: u8,
|
||||
@@ -103,7 +104,7 @@ pub(crate) fn decode_string_bytes(
|
||||
..
|
||||
} => {
|
||||
check_element_size(*size, offset_size)?;
|
||||
Ok(VlResolver::new(file_data, offset_size, length_size).string_bytes(raw)?)
|
||||
Ok(VlResolver::new_in(file_data, offset_size, length_size).string_bytes(raw)?)
|
||||
}
|
||||
other => Err(Error::Format(FormatError::TypeMismatch {
|
||||
expected: "variable-length string",
|
||||
@@ -114,8 +115,8 @@ pub(crate) fn decode_string_bytes(
|
||||
|
||||
/// The sequences in `raw`, elements of the variable-length sequence type
|
||||
/// `dt`, converted to `T`.
|
||||
pub(crate) fn decode_vlen<T: VlenValue>(
|
||||
file_data: &[u8],
|
||||
pub(crate) fn decode_vlen<T: VlenValue, S: Storage + ?Sized>(
|
||||
file_data: &S,
|
||||
dt: &Datatype,
|
||||
raw: &[u8],
|
||||
offset_size: u8,
|
||||
@@ -135,7 +136,7 @@ pub(crate) fn decode_vlen<T: VlenValue>(
|
||||
};
|
||||
check_element_size(*size, offset_size)?;
|
||||
let base_size = base_type.type_size() as usize;
|
||||
VlResolver::new(file_data, offset_size, length_size)
|
||||
VlResolver::new_in(file_data, offset_size, length_size)
|
||||
.sequences(raw, base_size)?
|
||||
.iter()
|
||||
.map(|bytes| Ok(T::decode(bytes, base_type)?))
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
//! `File::open_storage` over a read_at-only storage reads every file as
|
||||
//! `File::open` does (range reads, milestone M2 in
|
||||
//! `docs/design/range-reads.md`).
|
||||
//!
|
||||
//! Each file is read end to end twice — through `File::open` (the mmap
|
||||
//! fast path) and through `File::open_storage` over a
|
||||
//! [`CountingStorage`], which serves the file through `read_at` only
|
||||
//! (`as_contiguous()` is `None`, so no reader can fall back to a slice of
|
||||
//! the whole file) — and the two transcripts must be identical: the tree
|
||||
//! (every group's entries, followed by address), every object's attributes
|
||||
//! (the whole map and each one by name), and every dataset's shape, types
|
||||
//! and values (all bytes, as `f64`, a hyperslab of them, strings and
|
||||
//! variable-length sequences).
|
||||
//!
|
||||
//! The storage also counts its `read_at` calls and bytes: what a remote
|
||||
//! backend without a cache would be asked for. The totals and the files
|
||||
//! that cost most are printed.
|
||||
//!
|
||||
//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every HDF5 file under those
|
||||
//! directories (the conformance corpus is `conformance/.cache/corpus`);
|
||||
//! `CLAWHDF5_STORAGE_REPORT=1` prints every file's counts.
|
||||
|
||||
use std::collections::{BTreeMap, HashSet, VecDeque};
|
||||
use std::fmt::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use clawhdf5::{DType, File, Selection};
|
||||
use clawhdf5_format::error::FormatError;
|
||||
use clawhdf5_format::storage::CountingStorage;
|
||||
|
||||
/// Objects visited per file.
|
||||
const MAX_OBJECTS: usize = 2000;
|
||||
/// Datasets with more bytes than this are not read (their metadata is).
|
||||
const MAX_DATA_BYTES: u64 = 64 << 20;
|
||||
|
||||
/// A short, stable digest of a value's `Debug` form.
|
||||
fn digest<T: std::fmt::Debug>(v: &T) -> String {
|
||||
let s = format!("{v:?}");
|
||||
if s.len() <= 200 {
|
||||
return s;
|
||||
}
|
||||
let mut h = 0xcbf2_9ce4_8422_2325u64;
|
||||
for b in s.bytes() {
|
||||
h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3);
|
||||
}
|
||||
format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len())
|
||||
}
|
||||
|
||||
/// A data read's result: its value, or just `Err` — a full read goes
|
||||
/// through the file's chunk cache, which lists a damaged dataset's chunks
|
||||
/// in hash-map order, so which failing chunk it reports varies from one
|
||||
/// `File` to the next (the format crate's harness compares these errors on
|
||||
/// the uncached path).
|
||||
fn value<T: std::fmt::Debug, E>(r: &Result<T, E>) -> String {
|
||||
match r {
|
||||
Ok(v) => digest(v),
|
||||
Err(_) => "Err".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn transcript(file: &File) -> String {
|
||||
let mut out = String::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]);
|
||||
while let Some((path, addr)) = queue.pop_front() {
|
||||
if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
|
||||
continue;
|
||||
}
|
||||
let group = file.group_at(addr);
|
||||
let entries = group.entries();
|
||||
writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap();
|
||||
match file.dataset_at(addr) {
|
||||
Ok(ds) => dataset(&mut out, &path, &ds),
|
||||
Err(e) => writeln!(out, "{path} dataset_at {e:?}").unwrap(),
|
||||
}
|
||||
let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e));
|
||||
writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap();
|
||||
if let Ok((attrs, _)) = &attrs {
|
||||
for name in attrs.keys().take(50) {
|
||||
writeln!(out, "{path} attr {name:?} {}", digest(&group.attr(name))).unwrap();
|
||||
}
|
||||
}
|
||||
if let Ok(entries) = entries {
|
||||
for (name, child) in entries {
|
||||
queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child));
|
||||
// Name lookups (through the name index of a dense group).
|
||||
if queue.len() < 64 {
|
||||
writeln!(
|
||||
out,
|
||||
"{path} group({name:?}) {}",
|
||||
digest(&group.group(&name).map(|_| ()))
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn sorted<V: std::fmt::Debug>(m: std::collections::HashMap<String, V>) -> BTreeMap<String, V> {
|
||||
m.into_iter().collect()
|
||||
}
|
||||
|
||||
fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) {
|
||||
let shape = ds.shape();
|
||||
let dtype = ds.dtype();
|
||||
writeln!(
|
||||
out,
|
||||
"{path} shape {} max {} dtype {} raw {}",
|
||||
digest(&shape),
|
||||
digest(&ds.max_dimensions()),
|
||||
digest(&dtype),
|
||||
digest(&ds.raw_datatype())
|
||||
)
|
||||
.unwrap();
|
||||
let attrs = ds.attrs_with_errors().map(|(a, e)| (sorted(a), e));
|
||||
writeln!(out, "{path} dataset attrs {}", digest(&attrs)).unwrap();
|
||||
let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else {
|
||||
return;
|
||||
};
|
||||
let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d));
|
||||
let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size())));
|
||||
if bytes.is_none_or(|b| b > MAX_DATA_BYTES) {
|
||||
writeln!(out, "{path} too large to read").unwrap();
|
||||
return;
|
||||
}
|
||||
writeln!(
|
||||
out,
|
||||
"{path} all {}",
|
||||
value(&ds.read_selection(&Selection::All))
|
||||
)
|
||||
.unwrap();
|
||||
let numeric = matches!(
|
||||
dtype,
|
||||
DType::F32
|
||||
| DType::F64
|
||||
| DType::I8
|
||||
| DType::I16
|
||||
| DType::I32
|
||||
| DType::I64
|
||||
| DType::U8
|
||||
| DType::U16
|
||||
| DType::U32
|
||||
| DType::U64
|
||||
);
|
||||
if numeric {
|
||||
writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap();
|
||||
writeln!(out, "{path} f32 {}", value(&ds.read_f32())).unwrap();
|
||||
writeln!(out, "{path} i64 {}", value(&ds.read_i64())).unwrap();
|
||||
if let Some(&d0) = shape.first() {
|
||||
let rank = shape.len();
|
||||
let sel = Selection::Hyperslab {
|
||||
start: std::iter::once(d0 / 3)
|
||||
.chain(std::iter::repeat_n(0, rank - 1))
|
||||
.collect(),
|
||||
stride: vec![1; rank],
|
||||
count: std::iter::once(d0.div_ceil(3))
|
||||
.chain(shape[1..].iter().copied())
|
||||
.collect(),
|
||||
block: vec![1; rank],
|
||||
};
|
||||
writeln!(
|
||||
out,
|
||||
"{path} f64 third {}",
|
||||
value(&ds.read_f64_selection(&sel))
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
out,
|
||||
"{path} bytes third {}",
|
||||
value(&ds.read_selection(&sel))
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
match &raw_dt {
|
||||
clawhdf5_format::datatype::Datatype::String { .. }
|
||||
| clawhdf5_format::datatype::Datatype::VariableLength {
|
||||
is_string: true, ..
|
||||
} => {
|
||||
writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap();
|
||||
writeln!(out, "{path} string {}", value(&ds.read_string())).unwrap();
|
||||
}
|
||||
clawhdf5_format::datatype::Datatype::VariableLength { .. } => {
|
||||
writeln!(out, "{path} vlen {}", value(&ds.read_vlen::<f64>())).unwrap();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// External virtual-dataset sources as `File::open` finds them: files in
|
||||
/// the same directory.
|
||||
fn sibling_resolver(dir: Option<PathBuf>) -> clawhdf5::VdsResolver {
|
||||
Arc::new(move |name: &str| {
|
||||
let Some(dir) = dir.as_ref() else {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"virtual dataset source file {name:?} cannot be resolved for an in-memory file"
|
||||
)));
|
||||
};
|
||||
let p = Path::new(name);
|
||||
if name.is_empty()
|
||||
|| !p.components().all(|c| {
|
||||
matches!(
|
||||
c,
|
||||
std::path::Component::Normal(_) | std::path::Component::CurDir
|
||||
)
|
||||
})
|
||||
{
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"virtual dataset source file {name:?} is outside the virtual file's \
|
||||
directory and is not followed"
|
||||
)));
|
||||
}
|
||||
match std::fs::read(dir.join(p)) {
|
||||
Ok(bytes) => Ok(Some(bytes)),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(FormatError::ChunkedReadError(format!(
|
||||
"cannot read virtual dataset source file {name:?}: {e}"
|
||||
))),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Totals {
|
||||
files: usize,
|
||||
opened: usize,
|
||||
/// The comparison's reads (each dataset read several ways).
|
||||
reads: u64,
|
||||
bytes: u64,
|
||||
/// One pass: open, list every group, read every attribute and every
|
||||
/// dataset once (`read_selection(All)`).
|
||||
pass_reads: u64,
|
||||
pass_bytes: u64,
|
||||
file_bytes: u64,
|
||||
/// (one-pass reads, one-pass bytes, file size, name) per file.
|
||||
per_file: Vec<(u64, u64, u64, String)>,
|
||||
}
|
||||
|
||||
/// Open the file and read everything once, as a tree viewer that then
|
||||
/// shows every value would.
|
||||
fn one_pass(file: &File) {
|
||||
let mut seen = HashSet::new();
|
||||
let mut queue = VecDeque::from([file.superblock().root_group_address]);
|
||||
while let Some(addr) = queue.pop_front() {
|
||||
if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
|
||||
continue;
|
||||
}
|
||||
let group = file.group_at(addr);
|
||||
let _ = group.attrs();
|
||||
if let Ok(ds) = file.dataset_at(addr) {
|
||||
let small = ds.shape().ok().and_then(|s| {
|
||||
let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?;
|
||||
let size = u64::from(ds.raw_datatype().ok()?.type_size());
|
||||
n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES)
|
||||
});
|
||||
if small.is_some() {
|
||||
let _ = ds.read_selection(&Selection::All);
|
||||
}
|
||||
}
|
||||
if let Ok(entries) = group.entries() {
|
||||
queue.extend(entries.into_iter().map(|(_, a)| a));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check(path: &Path, totals: &mut Totals) {
|
||||
let Ok(bytes) = std::fs::read(path) else {
|
||||
return;
|
||||
};
|
||||
let name = path.display().to_string();
|
||||
let local = File::open(path);
|
||||
let storage = Arc::new(CountingStorage::new(bytes.clone()));
|
||||
let resolver = sibling_resolver(path.parent().map(Path::to_path_buf));
|
||||
let remote = File::open_storage(storage.clone()).map(|mut f| {
|
||||
f.set_vds_resolver(resolver.clone());
|
||||
f
|
||||
});
|
||||
totals.files += 1;
|
||||
let (local, remote) = match (local, remote) {
|
||||
(Ok(l), Ok(r)) => (l, r),
|
||||
(l, r) => {
|
||||
// Both refuse the file, with the same error.
|
||||
assert_eq!(
|
||||
format!("{:?}", l.map(|_| ())),
|
||||
format!("{:?}", r.map(|_| ())),
|
||||
"{name}: open"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
totals.opened += 1;
|
||||
assert!(remote.contiguous_bytes().is_none(), "{name}");
|
||||
assert_eq!(local.user_block_size(), remote.user_block_size(), "{name}");
|
||||
let want = transcript(&local);
|
||||
let got = transcript(&remote);
|
||||
if want != got {
|
||||
let first = want
|
||||
.lines()
|
||||
.zip(got.lines())
|
||||
.find(|(w, g)| w != g)
|
||||
.map(|(w, g)| format!("\n local: {w}\n storage: {g}"))
|
||||
.unwrap_or_else(|| {
|
||||
format!(
|
||||
"\n {} vs {} lines",
|
||||
want.lines().count(),
|
||||
got.lines().count()
|
||||
)
|
||||
});
|
||||
panic!("{name}: File::open_storage differs from File::open{first}");
|
||||
}
|
||||
totals.reads += storage.reads();
|
||||
totals.bytes += storage.bytes_read();
|
||||
totals.file_bytes += bytes.len() as u64;
|
||||
|
||||
let pass = Arc::new(CountingStorage::new(bytes.clone()));
|
||||
if let Ok(mut f) = File::open_storage(pass.clone()) {
|
||||
f.set_vds_resolver(resolver);
|
||||
one_pass(&f);
|
||||
}
|
||||
let (reads, read_bytes) = (pass.reads(), pass.bytes_read());
|
||||
totals.pass_reads += reads;
|
||||
totals.pass_bytes += read_bytes;
|
||||
if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") {
|
||||
eprintln!(
|
||||
"{reads:>9} reads {read_bytes:>12} bytes {:>12} file {name}",
|
||||
bytes.len()
|
||||
);
|
||||
}
|
||||
totals
|
||||
.per_file
|
||||
.push((reads, read_bytes, bytes.len() as u64, name));
|
||||
}
|
||||
|
||||
fn report(what: &str, totals: &mut Totals) {
|
||||
eprintln!(
|
||||
"{what}: {} files ({} open, {} bytes); comparison: {} read_at calls, {} bytes; \
|
||||
one pass (list, attributes, every dataset once): {} read_at calls, {} bytes",
|
||||
totals.files,
|
||||
totals.opened,
|
||||
totals.file_bytes,
|
||||
totals.reads,
|
||||
totals.bytes,
|
||||
totals.pass_reads,
|
||||
totals.pass_bytes
|
||||
);
|
||||
totals.per_file.sort_by_key(|a| std::cmp::Reverse(a.0));
|
||||
for (reads, bytes, size, name) in totals.per_file.iter().take(10) {
|
||||
eprintln!(" {reads:>9} reads {bytes:>12} bytes (file {size:>11}) {name}");
|
||||
}
|
||||
}
|
||||
fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for e in entries.flatten() {
|
||||
let p = e.path();
|
||||
if p.is_dir() {
|
||||
hdf5_files(&p, out);
|
||||
} else if p
|
||||
.extension()
|
||||
.and_then(|x| x.to_str())
|
||||
.is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf"))
|
||||
{
|
||||
out.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixtures_read_identically_through_open_storage() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let mut files = Vec::new();
|
||||
hdf5_files(&root.join("tests/fixtures"), &mut files);
|
||||
hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files);
|
||||
files.sort();
|
||||
assert!(files.len() >= 45, "{} fixtures", files.len());
|
||||
let mut totals = Totals::default();
|
||||
for f in &files {
|
||||
check(f, &mut totals);
|
||||
}
|
||||
report("fixtures", &mut totals);
|
||||
assert!(totals.opened >= 40, "{}", totals.opened);
|
||||
assert!(totals.reads > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_reads_identically_through_open_storage() {
|
||||
let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else {
|
||||
eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus");
|
||||
return;
|
||||
};
|
||||
let mut files = Vec::new();
|
||||
for d in std::env::split_paths(&dirs) {
|
||||
hdf5_files(&d, &mut files);
|
||||
}
|
||||
files.sort();
|
||||
let mut totals = Totals::default();
|
||||
for f in &files {
|
||||
check(f, &mut totals);
|
||||
}
|
||||
report("corpus", &mut totals);
|
||||
assert!(totals.files > 0);
|
||||
}
|
||||
|
||||
/// A user block, a metadata cache image and a Storage that is itself in
|
||||
/// memory: the in-memory view of a storage that has one is used as is.
|
||||
#[test]
|
||||
fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let path = root.join("tests/fixtures/h5clear_mdc_image.h5");
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
let local = File::open(&path).unwrap();
|
||||
// A Vec<u8> is a Storage with a contiguous view; the image still has
|
||||
// to be laid over it, so the view is not used.
|
||||
let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap();
|
||||
assert!(in_memory.contiguous_bytes().is_none());
|
||||
assert_eq!(transcript(&local), transcript(&in_memory));
|
||||
|
||||
let plain = root.join("../clawhdf5-format/tests/fixtures/chunked_2d.h5");
|
||||
let bytes = std::fs::read(&plain).unwrap();
|
||||
let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap();
|
||||
assert_eq!(in_memory.contiguous_bytes(), Some(&bytes[..]));
|
||||
let counting = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap();
|
||||
assert!(counting.contiguous_bytes().is_none());
|
||||
let result =
|
||||
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| counting.as_bytes().len()));
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"as_bytes over a range storage must not answer"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user