quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

fix(format): read unmapped VDS elements as the virtual dataset's fill value

Browse Source 
Elements of a virtual dataset that no mapping supplies (unmapped regions,
a missing source file, a missing source dataset) read as 0 instead of the
fill value libhdf5 returns — silent wrong data for any VDS created with a
non-zero fillvalue (read-matrix cases 0471/0472: -1 and 7 read as 0). A
missing source dataset was an error; libhdf5 reads it as fill.

Move VDS assembly into a new vds module following H5Dvirtual.c:
vds::read_virtual_dataset takes the dataset's fill value and a
VdsFileResolver that can refuse a name, and reports how many elements were
unmapped. Sources are read with their own fill value, and a source whose
datatype differs from the virtual dataset's is an error (libhdf5 converts).
File passes the dataset's fill value, resolves source names against the
virtual file's directory, and refuses names that leave it with an error
instead of reading them as fill. read_selection on a VDS goes through the
same fill-aware path.

The raw-read API (read_raw_data_full*) has no fill value, so it now errors
for a VDS with unmapped elements instead of guessing zeros.

Tests: vds_interop::vds_unmapped_regions_read_as_fill_value (external,
same-file, missing file/dataset, sparse source with its own fill, int
fill; earliest and latest format) and
vds_source_outside_directory_is_an_error_not_fill, both against h5py;
integration_test::v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 22:03:47 -05:00
co-authored by Claude Opus 5.5
parent 2c6c6c176e
commit e94a52a88b
8 changed files with 785 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_read.rs
+36 -145
View File
@@ -191,14 +191,9 @@ fn read_raw_data_full_impl(
offset_size,
length_size,
),
DataLayout::Virtual {
global_heap_address,
global_heap_index,
..
} => read_virtual_data(
DataLayout::Virtual { .. } => read_virtual_data(
file_data,
*global_heap_address,
*global_heap_index,
layout,
dataspace,
datatype,
offset_size,
@@ -465,158 +460,54 @@ pub fn read_raw_data_selection(
}
}
/// Assemble a **Virtual Dataset (VDS)** from its source mappings.
/// Assemble a **Virtual Dataset (VDS)** through the raw-read API, which has no
/// access to the dataset's fill value message.
///
/// Supports virtual datasets of any rank. Same-file sources are read directly;
/// **external-file** sources are read through the caller-supplied `resolver`,
/// which maps a stored source file name to that file's bytes. Each mapping's
/// selected source elements are scattered into the virtual buffer at the
/// positions given by the virtual selection (both enumerated in row-major
/// order, as HDF5 pairs them). Unmapped regions are left at the zero fill value.
///
/// A mapping whose external source file the resolver cannot supply (`None`) is
/// skipped, leaving its region at fill — matching HDF5's tolerance of missing
/// sources. An external source with no resolver at all is a hard error.
/// Delegates to [`crate::vds::read_virtual_dataset`]. Because the fill value
/// is unknown here, a virtual dataset with any element no mapping supplies
/// (an unmapped region, or a missing source file or dataset) is an error
/// rather than a guess at the fill value; so is one whose extent libhdf5
/// would report differently from the stored dataspace (unlimited mappings).
/// Use [`crate::vds::read_virtual_dataset`] to read those.
#[allow(clippy::too_many_arguments)]
fn read_virtual_data(
file_data: &[u8],
global_heap_address: Option<u64>,
global_heap_index: u32,
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
offset_size: u8,
length_size: u8,
resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
use crate::data_layout::parse_vds_mappings;
use crate::global_heap::GlobalHeapCollection;
use crate::selection::Selection;
let elem_size = datatype.type_size() as usize;
let mut out = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len(
dataspace.checked_num_elements()?,
elem_size,
)?)?;
let virtual_dims = &dataspace.dimensions;
let addr = global_heap_address.ok_or_else(|| {
FormatError::ChunkedReadError("virtual dataset has no mapping global heap".into())
})?;
let coll = GlobalHeapCollection::parse(file_data, addr as usize, length_size)?;
let obj =
coll.get_object(global_heap_index as u16)
.ok_or(FormatError::GlobalHeapObjectNotFound {
collection_address: addr,
index: global_heap_index as u16,
})?;
let mappings = parse_vds_mappings(&obj.data, length_size)?;
for m in &mappings {
let same_file = m.source_file.is_empty() || m.source_file == ".";
// Resolve the bytes of the file holding this source dataset.
let external;
let src_file_data: &[u8] = if same_file {
file_data
} else {
let r = resolver.ok_or_else(|| {
FormatError::ChunkedReadError(
"external-file virtual dataset sources require a file resolver".into(),
)
})?;
match r(&m.source_file) {
Some(bytes) => {
external = bytes;
&external
}
// Source file unavailable: leave this region at fill value.
None => continue,
}
};
let (vsel, _) = Selection::decode_serialized(&m.virtual_selection)?;
let (ssel, _) = Selection::decode_serialized(&m.source_selection)?;
let (src_raw, src_dims) =
read_named_dataset_raw(src_file_data, &m.source_dataset, offset_size, length_size)?;
let vidx = vsel.iter_linear(virtual_dims)?;
let sidx = ssel.iter_linear(&src_dims)?;
if vidx.len() != sidx.len() {
return Err(FormatError::ChunkedReadError(
"virtual/source selection element counts differ".into(),
));
}
for (&v, &s) in vidx.iter().zip(sidx.iter()) {
let (vo, so) = (v as usize * elem_size, s as usize * elem_size);
if vo + elem_size > out.len() || so + elem_size > src_raw.len() {
return Err(FormatError::ChunkedReadError(
"virtual dataset selection out of bounds".into(),
));
}
out[vo..vo + elem_size].copy_from_slice(&src_raw[so..so + elem_size]);
}
}
Ok(out)
}
/// Read a named dataset's raw (decoded) bytes and its dimensions, navigating
/// from the superblock. Used to pull VDS source datasets out of the same file.
fn read_named_dataset_raw(
file_data: &[u8],
path: &str,
_offset_size: u8,
_length_size: u8,
) -> Result<(Vec<u8>, Vec<u64>), FormatError> {
use crate::filter_pipeline::FilterPipeline;
use crate::group_v2::resolve_path_any;
use crate::message_type::MessageType;
use crate::object_header::ObjectHeader;
use crate::signature::find_signature;
use crate::superblock::Superblock;
let sig = find_signature(file_data)?;
let sb = Superblock::parse(file_data, sig)?;
let addr = resolve_path_any(file_data, &sb, path)?;
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?;
let find = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t);
let ds_msg = find(MessageType::Dataspace)
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no dataspace".into()))?;
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size)?;
let dt_msg = find(MessageType::Datatype)
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no datatype".into()))?;
let (datatype, _) = Datatype::parse(&dt_msg.data)?;
let dl_msg = find(MessageType::DataLayout)
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?;
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?;
// A virtual dataset whose source is itself another virtual dataset could
// form a cycle (A -> B -> A) and recurse into a stack overflow. Nested
// virtual sources are exotic and unsupported, so stop here cleanly.
if matches!(layout, DataLayout::Virtual { .. }) {
let wrapped =
resolver.map(|r| move |name: &str| -> Result<Option<Vec<u8>>, FormatError> { Ok(r(name)) });
let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver);
let v = crate::vds::read_virtual_dataset(
file_data,
layout,
dataspace,
datatype,
None,
offset_size,
length_size,
wrapped_ref,
)?;
if v.dims != dataspace.dimensions {
return Err(FormatError::ChunkedReadError(
"virtual dataset source is itself virtual (unsupported)".into(),
"virtual dataset extent differs from its stored dataspace; \
read it with vds::read_virtual_dataset"
.into(),
));
}
let pipeline = find(MessageType::FilterPipeline)
.map(|m| FilterPipeline::parse(&m.data))
.transpose()?;
let raw = read_raw_data_full(
file_data,
&layout,
&dataspace,
&datatype,
pipeline.as_ref(),
sb.offset_size,
sb.length_size,
)?;
Ok((raw, dataspace.dimensions.clone()))
if v.unmapped > 0 {
return Err(FormatError::ChunkedReadError(
"virtual dataset has elements no source supplies, which read as its \
fill value; read it with vds::read_virtual_dataset and the fill value"
.into(),
));
}
Ok(v.data)
}
/// Extract selected elements from a full dataset buffer.
pub fn extract_selection_from_buffer(
full_data: &[u8],