quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

format: raw data, VDS and VL data over Storage

Browse Source 
Every raw-data path has a generic *_in core, with the &[u8] functions as
thin wrappers: data_read (read_raw_data*, read_raw_data_selection,
read_chunked_native), chunked_read (the v1 B-tree chunk index, list_chunks,
the full, cached, sweep and indexed reads), parallel_read, partial_read,
fill_value (read_full_with_fill, apply_to_unallocated_chunks; and
dataset_fill_value_from_storage is now generic), vds (the virtual file
through Storage, external sources still through the resolver),
vl_data (VlResolver<'a, S = [u8]>, read_vl_strings_in, read_vl_bytes_in),
AttributeMessage::read_vl_strings_in and provenance::verify_dataset_in.

With the whole file in memory nothing changes: chunks and contiguous data
are sliced from it as before. Otherwise a chunked read lists its chunks,
fetches their stored bytes with one Storage::read_ranges call per 64 MiB
batch (chunks the cache already holds are not fetched), then decodes as
today; a selection fetches only the chunks it overlaps, and a contiguous
selection only its runs. Each extent's bounds error is the one the slice
code gave, reported when that extent is reached, so errors keep their
order.

Tests: the equivalence harness now reads every dataset's values (whole,
fill-aware, cached, indexed, three selections, VDS, VL strings and
sequences) through the read_at-only storage and requires the slice
results (all 653 corpus files agree); a misbehaving storage (a failing
Nth read, short reads) only ever yields errors or the right values; and
chunked reads are checked to use one read_ranges call.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 16:28:01 -05:00
co-authored by Claude Opus 5.5
parent 42894bf93b
commit 3fa5ed1dda
13 changed files with 1803 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/partial_read.rs
+79 -27
View File
@@ -18,7 +18,7 @@ use alloc::{format, vec, vec::Vec};
#[cfg(feature = "std")]
use std::string as alloc_or_std;
use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read};
use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read_in};
use crate::data_layout::DataLayout;
use crate::data_read::extract_selection_from_buffer;
use crate::dataspace::Dataspace;
@@ -26,6 +26,7 @@ use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
use crate::filters::{all_filters_skipped, decompress_chunk_exact_with};
use crate::selection::Selection;
use crate::storage::{ExtentBytes, Storage};
/// The smallest axis-aligned box containing every selected element, as
/// `(start, extent)` per dimension. `None` when there is nothing to gain or
@@ -256,6 +257,30 @@ pub fn read_selection(
offset_size: u8,
length_size: u8,
selection: &Selection,
) -> Result<Option<Vec<u8>>, FormatError> {
read_selection_in(
file_data,
layout,
dataspace,
elem_size,
pipeline,
offset_size,
length_size,
selection,
)
}
/// [`read_selection`] over any [`Storage`].
#[allow(clippy::too_many_arguments)]
pub fn read_selection_in<S: Storage + ?Sized>(
file_data: &S,
layout: &DataLayout,
dataspace: &Dataspace,
elem_size: usize,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
selection: &Selection,
) -> Result<Option<Vec<u8>>, FormatError> {
let dims = &dataspace.dimensions;
if dims.is_empty() || elem_size == 0 {
@@ -276,14 +301,33 @@ pub fn read_selection(
validate(selection, dims)?;
let base = usize::try_from(*address)
.map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?;
let data = file_data
.get(base..)
.and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?))
.ok_or(FormatError::UnexpectedEof {
expected: base,
available: file_data.len(),
})?;
return crate::gather::gather::<u8>(data, dims, elem_size, selection).map(Some);
let file_len = crate::storage::len_usize(file_data);
let eof = FormatError::UnexpectedEof {
expected: base,
available: file_len,
};
if let Some(all) = file_data.as_contiguous() {
let data = all
.get(base..)
.and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?))
.ok_or(eof)?;
return crate::gather::gather::<u8>(data, dims, elem_size, selection).map(Some);
}
// Not in memory: the same bounds check, then only the selected runs
// are read.
let len = checked_byte_len(total, elem_size)
.ok()
.filter(|&len| base <= file_len && len <= file_len - base)
.ok_or(eof)?;
return crate::gather::gather_storage(
file_data,
base as u64,
len,
dims,
elem_size,
selection,
)
.map(Some);
}
let Some((box_start, box_extent)) = bounding_box(selection, dims) else {
return Ok(None);
@@ -303,7 +347,7 @@ pub fn read_selection(
btree_address: Some(_),
..
} => {
let (chunks, chunk_dims) = list_chunks_for_read(
let (chunks, chunk_dims) = list_chunks_for_read_in(
file_data,
layout,
dataspace,
@@ -315,29 +359,37 @@ pub fn read_selection(
let rank = dims.len();
let chunk_shape: Vec<u64> = chunk_dims.iter().map(|&d| d as u64).collect();
let chunk_bytes = crate::chunked_read::checked_chunk_byte_len(&chunk_dims, elem_size)?;
// Chunks are decoded into this thread's reusable buffers.
crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> {
for chunk in &chunks {
// The chunks overlapping the box, in index order.
let wanted: Vec<&crate::chunked_read::ChunkInfo> = chunks
.iter()
.filter(|chunk| {
if chunk.offsets.len() < rank || chunk.address == u64::MAX {
continue;
return false;
}
let origin = &chunk.offsets[..rank];
let overlaps = (0..rank).all(|d| {
(0..rank).all(|d| {
origin[d] < box_start[d] + box_extent[d]
&& origin[d].saturating_add(chunk_shape[d]) > box_start[d]
});
if !overlaps {
continue;
}
let at = usize::try_from(chunk.address)
})
})
.collect();
// Their stored bytes, in one batch when the file is not in memory.
let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() {
Vec::new()
} else {
wanted
.iter()
.map(|c| (c.address, c.chunk_size as usize, true))
.collect()
};
let raw_bytes = ExtentBytes::fetch(file_data, &extents)?;
// Chunks are decoded into this thread's reusable buffers.
crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> {
for (i, chunk) in wanted.iter().enumerate() {
let origin = &chunk.offsets[..rank];
usize::try_from(chunk.address)
.map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?;
let raw = at
.checked_add(chunk.chunk_size as usize)
.and_then(|end| file_data.get(at..end))
.ok_or(FormatError::UnexpectedEof {
expected: at.saturating_add(chunk.chunk_size as usize),
available: file_data.len(),
})?;
let raw = raw_bytes.get(i, chunk.address, chunk.chunk_size as usize)?;
// Mirrors the full-read path: filter-mask bit i set means
// filter i was not applied to this chunk.
let data: &[u8] = match pipeline {