quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 4 Actions 5 Packages Projects Releases 1 Wiki Activity

format: raw data, VDS and VL data over Storage

Browse Source 
Every raw-data path has a generic *_in core, with the &[u8] functions as
thin wrappers: data_read (read_raw_data*, read_raw_data_selection,
read_chunked_native), chunked_read (the v1 B-tree chunk index, list_chunks,
the full, cached, sweep and indexed reads), parallel_read, partial_read,
fill_value (read_full_with_fill, apply_to_unallocated_chunks; and
dataset_fill_value_from_storage is now generic), vds (the virtual file
through Storage, external sources still through the resolver),
vl_data (VlResolver<'a, S = [u8]>, read_vl_strings_in, read_vl_bytes_in),
AttributeMessage::read_vl_strings_in and provenance::verify_dataset_in.

With the whole file in memory nothing changes: chunks and contiguous data
are sliced from it as before. Otherwise a chunked read lists its chunks,
fetches their stored bytes with one Storage::read_ranges call per 64 MiB
batch (chunks the cache already holds are not fetched), then decodes as
today; a selection fetches only the chunks it overlaps, and a contiguous
selection only its runs. Each extent's bounds error is the one the slice
code gave, reported when that extent is reached, so errors keep their
order.

Tests: the equivalence harness now reads every dataset's values (whole,
fill-aware, cached, indexed, three selections, VDS, VL strings and
sequences) through the read_at-only storage and requires the slice
results (all 653 corpus files agree); a misbehaving storage (a failing
Nth read, short reads) only ever yields errors or the right values; and
chunked reads are checked to use one read_ranges call.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 16:28:01 -05:00
co-authored by Claude Opus 5.5
parent 42894bf93b
commit 3fa5ed1dda
13 changed files with 1803 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/vds.rs
+94 -29
View File
@@ -15,12 +15,13 @@
#[cfg(not(feature = "std"))]
use alloc::{format, string::String, vec, vec::Vec};
use crate::addr::to_usize;
use crate::addr::{checked_addr, to_usize};
use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings};
use crate::dataspace::Dataspace;
use crate::datatype::Datatype;
use crate::error::FormatError;
use crate::selection::{SerializedSelection, UNLIMITED};
use crate::storage::Storage;
/// Resolves the name of an external VDS source file, as stored in the
/// mapping, to that file's bytes.
@@ -192,8 +193,8 @@ fn non_unlimited_elements(sel: &SerializedSelection, skip: usize) -> Option<u64>
}
/// Load and decode the mapping list of a virtual layout.
fn load_mappings(
file_data: &[u8],
fn load_mappings<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
length_size: u8,
) -> Result<Vec<Mapping>, FormatError> {
@@ -208,8 +209,11 @@ fn load_mappings(
let Some(addr) = *global_heap_address else {
return Ok(Vec::new());
};
let coll =
crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?;
let coll = crate::global_heap::GlobalHeapCollection::parse_in(
file_data,
checked_addr(addr)?,
length_size,
)?;
let index = u16::try_from(*global_heap_index)
.map_err(|_| vds_err("VDS mapping heap index out of range"))?;
let obj = coll
@@ -330,7 +334,11 @@ enum Step {
/// Work out the extent libhdf5 gives the virtual dataset
/// (`H5D__virtual_set_extent_unlim`, default view `H5D_VDS_LAST_AVAILABLE`
/// with a printf gap of 0) and how much of each unlimited mapping is read.
fn plan(mappings: &[Mapping], stored: &[u64], sources: &mut Sources) -> Result<Plan, FormatError> {
fn plan<S: Storage + ?Sized>(
mappings: &[Mapping],
stored: &[u64],
sources: &mut Sources<'_, '_, S>,
) -> Result<Plan, FormatError> {
let overflow = || FormatError::Overflow("VDS extent overflow".into());
let rank = stored.len();
let mut new_dims: Vec<Option<u64>> = vec![None; rank];
@@ -472,6 +480,25 @@ pub fn virtual_dataset_extent(
_offset_size: u8,
length_size: u8,
resolver: Option<&VdsFileResolver>,
) -> Result<Vec<u64>, FormatError> {
virtual_dataset_extent_in(
file_data,
layout,
dataspace,
_offset_size,
length_size,
resolver,
)
}
/// [`virtual_dataset_extent`] over any [`Storage`].
pub fn virtual_dataset_extent_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
_offset_size: u8,
length_size: u8,
resolver: Option<&VdsFileResolver>,
) -> Result<Vec<u64>, FormatError> {
let mappings = load_mappings(file_data, layout, length_size)?;
if mappings.iter().all(|m| m.kind == Kind::Fixed) {
@@ -498,6 +525,30 @@ pub fn read_virtual_dataset(
_offset_size: u8,
length_size: u8,
resolver: Option<&VdsFileResolver>,
) -> Result<VirtualData, FormatError> {
read_virtual_dataset_in(
file_data,
layout,
dataspace,
datatype,
fill,
_offset_size,
length_size,
resolver,
)
}
/// [`read_virtual_dataset`] over any [`Storage`].
#[allow(clippy::too_many_arguments)]
pub fn read_virtual_dataset_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
fill: Option<&[u8]>,
_offset_size: u8,
length_size: u8,
resolver: Option<&VdsFileResolver>,
) -> Result<VirtualData, FormatError> {
let mappings = load_mappings(file_data, layout, length_size)?;
let mut sources = Sources::new(file_data, resolver);
@@ -511,7 +562,7 @@ pub fn read_virtual_dataset(
let mut data = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len(
total, elem_size,
)?)?;
if let Some(fill) = fill.filter(|f| f.len() == elem_size && f.iter().any(|&b| b != 0)) {
if let Some(fill) = fill.filter(|f| <[u8]>::len(f) == elem_size && f.iter().any(|&b| b != 0)) {
for element in data.chunks_exact_mut(elem_size) {
element.copy_from_slice(fill);
}
@@ -780,14 +831,17 @@ struct SourceData {
/// Source files and datasets, fetched on demand. The most recently used
/// external file is kept, since consecutive mappings usually share one.
struct Sources<'a, 'r> {
file_data: &'a [u8],
///
/// The virtual dataset's own file (`"."`) is read through its [`Storage`];
/// an external source file is loaded whole, through the resolver.
struct Sources<'a, 'r, S: Storage + ?Sized> {
file_data: &'a S,
resolver: Option<&'r VdsFileResolver<'r>>,
cached_file: Option<(String, Option<Vec<u8>>)>,
}
impl<'a, 'r> Sources<'a, 'r> {
fn new(file_data: &'a [u8], resolver: Option<&'r VdsFileResolver<'r>>) -> Self {
impl<'a, 'r, S: Storage + ?Sized> Sources<'a, 'r, S> {
fn new(file_data: &'a S, resolver: Option<&'r VdsFileResolver<'r>>) -> Self {
Sources {
file_data,
resolver,
@@ -795,11 +849,9 @@ impl<'a, 'r> Sources<'a, 'r> {
}
}
/// The bytes of source file `name`, or `None` if it does not exist.
fn file(&mut self, name: &str) -> Result<Option<&[u8]>, FormatError> {
if name == "." {
return Ok(Some(self.file_data));
}
/// The bytes of external source file `name` (not `"."`), or `None` if it
/// does not exist.
fn external(&mut self, name: &str) -> Result<Option<&[u8]>, FormatError> {
if self.cached_file.as_ref().is_none_or(|(n, _)| n != name) {
let resolver = self.resolver.ok_or_else(|| {
vds_err("external-file virtual dataset sources require a file resolver")
@@ -821,7 +873,10 @@ impl<'a, 'r> Sources<'a, 'r> {
/// The extent of source dataset `path` in file `file`, or `None` when
/// either does not exist.
fn dims(&mut self, file: &str, path: &str) -> Result<Option<Vec<u64>>, FormatError> {
let Some(bytes) = self.file(file)? else {
if file == "." {
return Ok(open_source(self.file_data, path)?.map(|s| s.dataspace.dimensions));
}
let Some(bytes) = self.external(file)? else {
return Ok(None);
};
Ok(open_source(bytes, path)?.map(|s| s.dataspace.dimensions))
@@ -846,7 +901,13 @@ impl<'a, 'r> Sources<'a, 'r> {
from another file is not supported"
)));
}
let Some(bytes) = self.file(file)? else {
if file == "." {
let Some(src) = open_source(self.file_data, path)? else {
return Ok(None);
};
return read_source(self.file_data, src, path, datatype).map(Some);
}
let Some(bytes) = self.external(file)? else {
return Ok(None);
};
let Some(src) = open_source(bytes, path)? else {
@@ -910,19 +971,23 @@ fn source_message<'h>(
/// Open source dataset `path` of the file in `file_data`, or `None` if there
/// is no such object (libhdf5 reads a missing source as fill).
fn open_source(file_data: &[u8], path: &str) -> Result<Option<OpenSource>, FormatError> {
fn open_source<S: Storage + ?Sized>(
file_data: &S,
path: &str,
) -> Result<Option<OpenSource>, FormatError> {
use crate::message_type::MessageType;
use crate::shared_message::message_data_with_sohm;
use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm;
// `file_data` starts at the superblock (see `Sources::file`).
let sb = crate::superblock::Superblock::parse(file_data, 0)?;
// `file_data` starts at the superblock (see `Sources::external`).
let sb = crate::superblock::Superblock::parse_in(file_data, 0)?;
let (os, ls) = (sb.offset_size, sb.length_size);
let addr = match crate::group_v2::resolve_path_any(file_data, &sb, path) {
let addr = match crate::group_v2::resolve_path_any_in(file_data, &sb, path) {
Ok(a) => a,
Err(FormatError::PathNotFound(_)) => return Ok(None),
Err(e) => return Err(e),
};
let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?;
let header =
crate::object_header::ObjectHeader::parse_in(file_data, checked_addr(addr)?, os, ls)?;
let mut src = OpenSource {
offset_size: os,
length_size: ls,
@@ -941,15 +1006,15 @@ fn open_source(file_data: &[u8], path: &str) -> Result<Option<OpenSource>, Forma
/// Read an opened source dataset in full (its own fill value applied to
/// unallocated chunks).
fn read_source(
file_data: &[u8],
fn read_source<S: Storage + ?Sized>(
file_data: &S,
src: OpenSource,
path: &str,
datatype: &Datatype,
) -> Result<SourceData, FormatError> {
use crate::filter_pipeline::FilterPipeline;
use crate::message_type::MessageType;
use crate::shared_message::message_data_with_sohm;
use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm;
let (os, ls) = (src.offset_size, src.length_size);
let dt_msg = source_message(&src, path, MessageType::Datatype)?;
@@ -987,7 +1052,7 @@ fn read_source(
message_data_with_sohm(file_data, m, os, ls).and_then(|d| FilterPipeline::parse(&d))
})
.transpose()?;
let raw = crate::fill_value::read_full_with_fill(
let raw = crate::fill_value::read_full_with_fill_in(
&src.header.messages,
file_data,
&layout,
@@ -996,7 +1061,7 @@ fn read_source(
os,
ls,
|| {
crate::data_read::read_raw_data_full(
crate::data_read::read_raw_data_full_in(
file_data,
&layout,
&src.dataspace,