quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

format: raw data, VDS and VL data over Storage

Browse Source 
Every raw-data path has a generic *_in core, with the &[u8] functions as
thin wrappers: data_read (read_raw_data*, read_raw_data_selection,
read_chunked_native), chunked_read (the v1 B-tree chunk index, list_chunks,
the full, cached, sweep and indexed reads), parallel_read, partial_read,
fill_value (read_full_with_fill, apply_to_unallocated_chunks; and
dataset_fill_value_from_storage is now generic), vds (the virtual file
through Storage, external sources still through the resolver),
vl_data (VlResolver<'a, S = [u8]>, read_vl_strings_in, read_vl_bytes_in),
AttributeMessage::read_vl_strings_in and provenance::verify_dataset_in.

With the whole file in memory nothing changes: chunks and contiguous data
are sliced from it as before. Otherwise a chunked read lists its chunks,
fetches their stored bytes with one Storage::read_ranges call per 64 MiB
batch (chunks the cache already holds are not fetched), then decodes as
today; a selection fetches only the chunks it overlaps, and a contiguous
selection only its runs. Each extent's bounds error is the one the slice
code gave, reported when that extent is reached, so errors keep their
order.

Tests: the equivalence harness now reads every dataset's values (whole,
fill-aware, cached, indexed, three selections, VDS, VL strings and
sequences) through the read_at-only storage and requires the slice
results (all 653 corpus files agree); a misbehaving storage (a failing
Nth read, short reads) only ever yields errors or the right values; and
chunked reads are checked to use one read_ranges call.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 16:28:01 -05:00
co-authored by Claude Opus 5.5
parent 42894bf93b
commit 3fa5ed1dda
13 changed files with 1803 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_read.rs
+190 -25
View File
@@ -1,7 +1,9 @@
//! Raw data reading and typed conversion for HDF5 datasets.
#[cfg(not(feature = "std"))]
use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
use alloc::{borrow::Cow, collections::BTreeMap, format, string::String, vec, vec::Vec};
#[cfg(feature = "std")]
use std::borrow::Cow;
#[cfg(feature = "std")]
use std::collections::BTreeMap;
@@ -9,14 +11,15 @@ use std::collections::BTreeMap;
use crate::addr::to_usize;
#[cfg(feature = "std")]
use crate::chunk_cache::ChunkCache;
use crate::chunked_read::read_chunked_data;
use crate::chunked_read::read_chunked_data_in;
#[cfg(feature = "std")]
use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed};
use crate::chunked_read::{read_chunked_data_cached_in, read_chunked_data_indexed_in};
use crate::data_layout::DataLayout;
use crate::dataspace::Dataspace;
use crate::datatype::{Datatype, DatatypeByteOrder};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
use crate::storage::{Storage, read_exact_at};
/// Checks that `[offset, offset + needed)` fits within `data`, guarding the
/// addition against `usize` overflow from a crafted near-`usize::MAX` offset.
@@ -149,7 +152,17 @@ pub fn read_raw_data(
dataspace: &Dataspace,
datatype: &Datatype,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8)
read_raw_data_in(file_data, layout, dataspace, datatype)
}
/// [`read_raw_data`] over any [`Storage`].
pub fn read_raw_data_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full_in(file_data, layout, dataspace, datatype, None, 8, 8)
}
/// Resolves a Virtual Dataset source **file name** (as stored in the mapping,
@@ -171,6 +184,27 @@ pub fn read_raw_data_full(
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full_in(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
)
}
/// [`read_raw_data_full`] over any [`Storage`].
pub fn read_raw_data_full_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full_impl(
file_data,
@@ -196,6 +230,30 @@ pub fn read_raw_data_full_with_resolver(
offset_size: u8,
length_size: u8,
resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full_with_resolver_in(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
resolver,
)
}
/// [`read_raw_data_full_with_resolver`] over any [`Storage`].
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_full_with_resolver_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full_impl(
file_data,
@@ -210,8 +268,8 @@ pub fn read_raw_data_full_with_resolver(
}
#[allow(clippy::too_many_arguments)]
fn read_raw_data_full_impl(
file_data: &[u8],
fn read_raw_data_full_impl<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
@@ -242,12 +300,17 @@ fn read_raw_data_full_impl(
let addr = address.ok_or(FormatError::NoDataAllocated)?;
let addr = to_usize(addr)?;
let sz = contiguous_read_len(*size, expected_size)?;
ensure_len(file_data, addr, sz)?;
let mut out = crate::bulk_alloc::vec_for_bulk(sz);
out.extend_from_slice(&file_data[addr..addr + sz]);
Ok(out)
match read_exact_at(file_data, addr as u64, sz)? {
Cow::Borrowed(bytes) => {
let mut out = crate::bulk_alloc::vec_for_bulk(sz);
out.extend_from_slice(bytes);
Ok(out)
}
// Fetched for this read: already the caller's copy.
Cow::Owned(out) => Ok(out),
}
}
DataLayout::Chunked { .. } => read_chunked_data(
DataLayout::Chunked { .. } => read_chunked_data_in(
file_data,
layout,
dataspace,
@@ -284,9 +347,34 @@ pub fn read_raw_data_cached(
offset_size: u8,
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_cached_in(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
cache,
)
}
/// [`read_raw_data_cached`] over any [`Storage`].
#[cfg(feature = "std")]
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_cached_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
match layout {
DataLayout::Chunked { .. } => read_chunked_data_cached(
DataLayout::Chunked { .. } => read_chunked_data_cached_in(
file_data,
layout,
dataspace,
@@ -296,7 +384,7 @@ pub fn read_raw_data_cached(
length_size,
cache,
),
_ => read_raw_data_full(
_ => read_raw_data_full_in(
file_data,
layout,
dataspace,
@@ -325,9 +413,34 @@ pub fn read_raw_data_indexed(
offset_size: u8,
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_indexed_in(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
cache,
)
}
/// [`read_raw_data_indexed`] over any [`Storage`].
#[cfg(feature = "std")]
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_indexed_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
match layout {
DataLayout::Chunked { .. } => read_chunked_data_indexed(
DataLayout::Chunked { .. } => read_chunked_data_indexed_in(
file_data,
layout,
dataspace,
@@ -337,7 +450,7 @@ pub fn read_raw_data_indexed(
length_size,
cache,
),
_ => read_raw_data_full(
_ => read_raw_data_full_in(
file_data,
layout,
dataspace,
@@ -366,6 +479,30 @@ pub fn read_raw_data_selection(
offset_size: u8,
length_size: u8,
selection: &crate::selection::Selection,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_selection_in(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
selection,
)
}
/// [`read_raw_data_selection`] over any [`Storage`].
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_selection_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
selection: &crate::selection::Selection,
) -> Result<Vec<u8>, FormatError> {
use crate::selection::Selection;
@@ -375,7 +512,7 @@ pub fn read_raw_data_selection(
// Read only what the selection's bounding box touches when that is
// possible; everything below is the decode-everything-then-pick path,
// kept for the cases `partial_read` declines.
if let Some(selected) = crate::partial_read::read_selection(
if let Some(selected) = crate::partial_read::read_selection_in(
file_data,
layout,
dataspace,
@@ -390,7 +527,7 @@ pub fn read_raw_data_selection(
match selection {
Selection::All => {
return read_raw_data_full(
return read_raw_data_full_in(
file_data,
layout,
dataspace,
@@ -410,7 +547,7 @@ pub fn read_raw_data_selection(
match layout {
DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => {
// Read all data, then extract the selection
let full_data = read_raw_data_full(
let full_data = read_raw_data_full_in(
file_data,
layout,
dataspace,
@@ -434,7 +571,7 @@ pub fn read_raw_data_selection(
// implicit-index generator, which then indexed past the rank and
// panicked — only to decode the full dataset anyway.
crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?;
let full_data = read_raw_data_full(
let full_data = read_raw_data_full_in(
file_data,
layout,
dataspace,
@@ -447,7 +584,7 @@ pub fn read_raw_data_selection(
}
DataLayout::Virtual { .. } => {
// Assemble the full virtual dataset, then apply the read selection.
let full_data = read_raw_data_full(
let full_data = read_raw_data_full_in(
file_data,
layout,
dataspace,
@@ -471,8 +608,8 @@ pub fn read_raw_data_selection(
/// would report differently from the stored dataspace (unlimited mappings).
/// Use [`crate::vds::read_virtual_dataset`] to read those.
#[allow(clippy::too_many_arguments)]
fn read_virtual_data(
file_data: &[u8],
fn read_virtual_data<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
@@ -483,7 +620,7 @@ fn read_virtual_data(
let wrapped =
resolver.map(|r| move |name: &str| -> Result<Option<Vec<u8>>, FormatError> { Ok(r(name)) });
let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver);
let v = crate::vds::read_virtual_dataset(
let v = crate::vds::read_virtual_dataset_in(
file_data,
layout,
dataspace,
@@ -885,6 +1022,33 @@ pub fn read_chunked_native<T: NativeElement>(
offset_size: u8,
length_size: u8,
cache: Option<&ChunkCache>,
) -> Result<Option<Vec<T>>, FormatError> {
read_chunked_native_in(
messages,
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
cache,
)
}
/// [`read_chunked_native`] over any [`Storage`].
#[cfg(feature = "std")]
#[allow(clippy::too_many_arguments)]
pub fn read_chunked_native_in<T: NativeElement, S: Storage + ?Sized>(
messages: &[crate::object_header::HeaderMessage],
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
cache: Option<&ChunkCache>,
) -> Result<Option<Vec<T>>, FormatError> {
use crate::fill_value;
use crate::message_type::MessageType;
@@ -919,8 +1083,9 @@ pub fn read_chunked_native<T: NativeElement>(
},
|values| bytes_of_mut(values),
)?;
let fill = fill_value::dataset_fill_value_in(file_data, messages, offset_size, length_size)?;
fill_value::apply_to_unallocated_chunks(
let fill =
fill_value::dataset_fill_value_from_storage(file_data, messages, offset_size, length_size)?;
fill_value::apply_to_unallocated_chunks_in(
bytes_of_mut(&mut values),
file_data,
layout,