fix(format): read unmapped VDS elements as the virtual dataset's fill value
Elements of a virtual dataset that no mapping supplies (unmapped regions, a missing source file, a missing source dataset) read as 0 instead of the fill value libhdf5 returns — silent wrong data for any VDS created with a non-zero fillvalue (read-matrix cases 0471/0472: -1 and 7 read as 0). A missing source dataset was an error; libhdf5 reads it as fill. Move VDS assembly into a new vds module following H5Dvirtual.c: vds::read_virtual_dataset takes the dataset's fill value and a VdsFileResolver that can refuse a name, and reports how many elements were unmapped. Sources are read with their own fill value, and a source whose datatype differs from the virtual dataset's is an error (libhdf5 converts). File passes the dataset's fill value, resolves source names against the virtual file's directory, and refuses names that leave it with an error instead of reading them as fill. read_selection on a VDS goes through the same fill-aware path. The raw-read API (read_raw_data_full*) has no fill value, so it now errors for a VDS with unmapped elements instead of guessing zeros. Tests: vds_interop::vds_unmapped_regions_read_as_fill_value (external, same-file, missing file/dataset, sparse source with its own fill, int fill; earliest and latest format) and vds_source_outside_directory_is_an_error_not_fill, both against h5py; integration_test::v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,497 @@
|
||||
//! Virtual Dataset (VDS) assembly, following libhdf5's `H5Dvirtual.c`.
|
||||
//!
|
||||
//! A virtual dataset stores no data of its own: a list of mappings (kept in
|
||||
//! the global heap) pairs a selection of the virtual dataspace with a
|
||||
//! selection of a *source* dataset, in the same file (`"."`) or another one.
|
||||
//! Reading it means reading each source and scattering the selected source
|
||||
//! elements into the virtual buffer, pairing the two selections element by
|
||||
//! element in row-major order. Elements no mapping supplies — unmapped
|
||||
//! regions, and mappings whose source file or dataset does not exist — read
|
||||
//! as the virtual dataset's **fill value**, as in libhdf5.
|
||||
//!
|
||||
//! Source files other than the virtual file itself are obtained through a
|
||||
//! caller-supplied [`VdsFileResolver`], since this crate has no filesystem.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, string::String, vec, vec::Vec};
|
||||
|
||||
use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings};
|
||||
use crate::dataspace::Dataspace;
|
||||
use crate::datatype::Datatype;
|
||||
use crate::error::FormatError;
|
||||
use crate::selection::{SerializedSelection, UNLIMITED};
|
||||
|
||||
/// Resolves the name of an external VDS source file, as stored in the
|
||||
/// mapping, to that file's bytes.
|
||||
///
|
||||
/// `Ok(None)` means the file does not exist; its mappings then read as the
|
||||
/// fill value, as libhdf5 does for a missing source. `Err` refuses the name
|
||||
/// (e.g. a path the caller will not follow) and fails the read, so that a
|
||||
/// refused source is never passed off as fill.
|
||||
pub type VdsFileResolver<'a> = dyn Fn(&str) -> Result<Option<Vec<u8>>, FormatError> + 'a;
|
||||
|
||||
/// A fully assembled virtual dataset.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct VirtualData {
|
||||
/// The virtual dataset's extent.
|
||||
pub dims: Vec<u64>,
|
||||
/// Raw element bytes, row-major, in the virtual dataset's datatype.
|
||||
pub data: Vec<u8>,
|
||||
/// Number of elements no mapping supplied; they hold the fill value.
|
||||
pub unmapped: u64,
|
||||
}
|
||||
|
||||
fn vds_err(msg: impl Into<String>) -> FormatError {
|
||||
FormatError::ChunkedReadError(msg.into())
|
||||
}
|
||||
|
||||
/// One mapping with its selections decoded.
|
||||
struct Mapping {
|
||||
file: String,
|
||||
dataset: String,
|
||||
vsel: SerializedSelection,
|
||||
ssel: SerializedSelection,
|
||||
}
|
||||
|
||||
/// Load and decode the mapping list of a virtual layout.
|
||||
fn load_mappings(
|
||||
file_data: &[u8],
|
||||
layout: &DataLayout,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<Mapping>, FormatError> {
|
||||
let DataLayout::Virtual {
|
||||
global_heap_address,
|
||||
global_heap_index,
|
||||
..
|
||||
} = layout
|
||||
else {
|
||||
return Err(vds_err("not a virtual dataset layout"));
|
||||
};
|
||||
let Some(addr) = *global_heap_address else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let coll =
|
||||
crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?;
|
||||
let index = u16::try_from(*global_heap_index)
|
||||
.map_err(|_| vds_err("VDS mapping heap index out of range"))?;
|
||||
let obj = coll
|
||||
.get_object(index)
|
||||
.ok_or(FormatError::GlobalHeapObjectNotFound {
|
||||
collection_address: addr,
|
||||
index,
|
||||
})?;
|
||||
parse_vds_mappings(&obj.data, length_size)?
|
||||
.into_iter()
|
||||
.map(|m: VdsMapping| {
|
||||
let (vsel, _) = SerializedSelection::decode(&m.virtual_selection)?;
|
||||
let (ssel, _) = SerializedSelection::decode(&m.source_selection)?;
|
||||
Ok(Mapping {
|
||||
file: m.source_file,
|
||||
dataset: m.source_dataset,
|
||||
vsel,
|
||||
ssel,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The virtual dataset's extent as libhdf5 reports it (`H5Dget_space`).
|
||||
///
|
||||
/// For a virtual dataset whose mappings are all of fixed size this is the
|
||||
/// stored dataspace.
|
||||
pub fn virtual_dataset_extent(
|
||||
file_data: &[u8],
|
||||
layout: &DataLayout,
|
||||
dataspace: &Dataspace,
|
||||
_offset_size: u8,
|
||||
length_size: u8,
|
||||
_resolver: Option<&VdsFileResolver>,
|
||||
) -> Result<Vec<u64>, FormatError> {
|
||||
let mappings = load_mappings(file_data, layout, length_size)?;
|
||||
check_fixed(&mappings)?;
|
||||
Ok(dataspace.dimensions.clone())
|
||||
}
|
||||
|
||||
fn check_fixed(mappings: &[Mapping]) -> Result<(), FormatError> {
|
||||
if mappings
|
||||
.iter()
|
||||
.any(|m| m.vsel.unlimited_dim().is_some() || m.ssel.unlimited_dim().is_some())
|
||||
{
|
||||
return Err(vds_err(
|
||||
"unlimited virtual dataset mappings are not supported",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read a whole virtual dataset.
|
||||
///
|
||||
/// `fill` is the virtual dataset's fill value (from its fill value message;
|
||||
/// `None` for the default of zeros); every element no mapping supplies holds
|
||||
/// it. External source files are read through `resolver`; without one, a
|
||||
/// mapping to another file is an error.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn read_virtual_dataset(
|
||||
file_data: &[u8],
|
||||
layout: &DataLayout,
|
||||
dataspace: &Dataspace,
|
||||
datatype: &Datatype,
|
||||
fill: Option<&[u8]>,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
resolver: Option<&VdsFileResolver>,
|
||||
) -> Result<VirtualData, FormatError> {
|
||||
let mappings = load_mappings(file_data, layout, length_size)?;
|
||||
check_fixed(&mappings)?;
|
||||
let dims = virtual_dataset_extent(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
offset_size,
|
||||
length_size,
|
||||
resolver,
|
||||
)?;
|
||||
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let total = dims
|
||||
.iter()
|
||||
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
||||
.ok_or_else(|| FormatError::Overflow("virtual dataset extent".into()))?;
|
||||
let mut data = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len(
|
||||
total, elem_size,
|
||||
)?)?;
|
||||
if let Some(fill) = fill.filter(|f| f.len() == elem_size && f.iter().any(|&b| b != 0)) {
|
||||
for element in data.chunks_exact_mut(elem_size) {
|
||||
element.copy_from_slice(fill);
|
||||
}
|
||||
}
|
||||
let mut mapped = vec![false; usize::try_from(total).map_err(|_| vds_err("VDS too large"))?];
|
||||
|
||||
let mut sources = Sources::new(file_data, resolver);
|
||||
for m in &mappings {
|
||||
let Some(src) = sources.dataset(&m.file, &m.dataset, datatype)? else {
|
||||
continue; // missing source file or dataset: fill
|
||||
};
|
||||
let vidx = selection_indices(&m.vsel, &dims, None)?;
|
||||
let sidx = selection_indices(&m.ssel, &src.dims, None)?;
|
||||
scatter(&mut data, &mut mapped, &src.raw, &vidx, &sidx, elem_size)?;
|
||||
}
|
||||
|
||||
let unmapped = mapped.iter().filter(|&&m| !m).count() as u64;
|
||||
Ok(VirtualData {
|
||||
dims,
|
||||
data,
|
||||
unmapped,
|
||||
})
|
||||
}
|
||||
|
||||
/// Copy source element `sidx[i]` to virtual element `vidx[i]` for every `i`.
|
||||
fn scatter(
|
||||
out: &mut [u8],
|
||||
mapped: &mut [bool],
|
||||
src: &[u8],
|
||||
vidx: &[u64],
|
||||
sidx: &[u64],
|
||||
elem_size: usize,
|
||||
) -> Result<(), FormatError> {
|
||||
if vidx.len() != sidx.len() {
|
||||
return Err(vds_err("virtual/source selection element counts differ"));
|
||||
}
|
||||
for (&v, &s) in vidx.iter().zip(sidx) {
|
||||
let (vo, so) = (v as usize * elem_size, s as usize * elem_size);
|
||||
if vo + elem_size > out.len() || so + elem_size > src.len() {
|
||||
return Err(vds_err("virtual dataset selection out of bounds"));
|
||||
}
|
||||
out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]);
|
||||
mapped[v as usize] = true;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Row-major linear indices of the elements `sel` selects in a dataspace of
|
||||
/// shape `dims`, in the order libhdf5 iterates them (row-major).
|
||||
///
|
||||
/// `clip` = `(dim, limit)` drops every coordinate `>= limit` in `dim` — the
|
||||
/// clipping libhdf5 applies to an unlimited selection
|
||||
/// (`H5S_hyper_clip_unlim`); an unlimited selection must be clipped.
|
||||
fn selection_indices(
|
||||
sel: &SerializedSelection,
|
||||
dims: &[u64],
|
||||
clip: Option<(usize, u64)>,
|
||||
) -> Result<Vec<u64>, FormatError> {
|
||||
let overflow = || FormatError::Overflow("VDS selection index overflow".into());
|
||||
let rank = dims.len();
|
||||
let total = dims
|
||||
.iter()
|
||||
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
||||
.ok_or_else(overflow)?;
|
||||
let mut row_stride = vec![1u64; rank];
|
||||
for d in (0..rank.saturating_sub(1)).rev() {
|
||||
row_stride[d] = row_stride[d + 1]
|
||||
.checked_mul(dims[d + 1])
|
||||
.ok_or_else(overflow)?;
|
||||
}
|
||||
if sel.rank().is_some_and(|r| r != rank) {
|
||||
return Err(vds_err("VDS selection rank does not match dataspace rank"));
|
||||
}
|
||||
let limit = |d: usize| match clip {
|
||||
Some((cd, l)) if cd == d => l,
|
||||
_ => u64::MAX,
|
||||
};
|
||||
|
||||
match sel {
|
||||
SerializedSelection::All => Ok((0..total).collect()),
|
||||
SerializedSelection::None => Ok(Vec::new()),
|
||||
SerializedSelection::Regular {
|
||||
start,
|
||||
stride,
|
||||
count,
|
||||
block,
|
||||
} => {
|
||||
// Selected coordinates along each dimension, in order.
|
||||
let mut per_dim: Vec<Vec<u64>> = Vec::with_capacity(rank);
|
||||
for d in 0..rank {
|
||||
let lim = limit(d);
|
||||
if (count[d] == UNLIMITED || block[d] == UNLIMITED) && lim == u64::MAX {
|
||||
return Err(vds_err("unlimited VDS selection was not clipped"));
|
||||
}
|
||||
let mut coords = Vec::new();
|
||||
let mut ci = 0u64;
|
||||
'blocks: while ci < count[d] {
|
||||
let base = ci
|
||||
.checked_mul(stride[d])
|
||||
.and_then(|o| start[d].checked_add(o))
|
||||
.ok_or_else(overflow)?;
|
||||
if base >= lim {
|
||||
break;
|
||||
}
|
||||
let mut bi = 0u64;
|
||||
while bi < block[d] {
|
||||
let coord = base.checked_add(bi).ok_or_else(overflow)?;
|
||||
if coord >= lim {
|
||||
break 'blocks;
|
||||
}
|
||||
// Past the extent is malformed; bail before the list
|
||||
// can grow without bound.
|
||||
if coord >= dims[d] {
|
||||
return Err(vds_err("VDS selection exceeds the dataspace extent"));
|
||||
}
|
||||
coords.push(coord);
|
||||
bi += 1;
|
||||
}
|
||||
ci += 1;
|
||||
}
|
||||
per_dim.push(coords);
|
||||
}
|
||||
if per_dim.iter().any(|c| c.is_empty()) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let n = per_dim
|
||||
.iter()
|
||||
.try_fold(1usize, |acc, c| acc.checked_mul(c.len()))
|
||||
.ok_or_else(overflow)?;
|
||||
let mut out = Vec::with_capacity(n);
|
||||
let mut idx = vec![0usize; rank];
|
||||
loop {
|
||||
let lin: u64 = (0..rank).map(|d| per_dim[d][idx[d]] * row_stride[d]).sum();
|
||||
out.push(lin);
|
||||
// Mixed-radix increment, last dimension fastest.
|
||||
let mut d = rank;
|
||||
loop {
|
||||
if d == 0 {
|
||||
return Ok(out);
|
||||
}
|
||||
d -= 1;
|
||||
idx[d] += 1;
|
||||
if idx[d] < per_dim[d].len() {
|
||||
break;
|
||||
}
|
||||
idx[d] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
SerializedSelection::Blocks {
|
||||
rank: _,
|
||||
starts,
|
||||
ends,
|
||||
} => {
|
||||
// libhdf5 serializes the union as disjoint blocks, so their volumes
|
||||
// never add up to more than the dataspace.
|
||||
let mut volume = 0u64;
|
||||
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
||||
for d in 0..rank {
|
||||
if e[d] >= dims[d] {
|
||||
return Err(vds_err("VDS selection exceeds the dataspace extent"));
|
||||
}
|
||||
}
|
||||
let v = s
|
||||
.iter()
|
||||
.zip(e)
|
||||
.try_fold(1u64, |acc, (&s, &e)| acc.checked_mul(e - s + 1))
|
||||
.ok_or_else(overflow)?;
|
||||
volume = volume.checked_add(v).ok_or_else(overflow)?;
|
||||
if volume > total {
|
||||
return Err(vds_err("VDS selection blocks overlap"));
|
||||
}
|
||||
}
|
||||
let mut out = Vec::with_capacity(volume as usize);
|
||||
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
||||
let mut cur = s.to_vec();
|
||||
'block: loop {
|
||||
if (0..rank).all(|d| cur[d] < limit(d)) {
|
||||
out.push((0..rank).map(|d| cur[d] * row_stride[d]).sum());
|
||||
}
|
||||
for d in (0..rank).rev() {
|
||||
if cur[d] < e[d] {
|
||||
cur[d] += 1;
|
||||
continue 'block;
|
||||
}
|
||||
cur[d] = s[d];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.sort_unstable();
|
||||
out.dedup();
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A source dataset's decoded contents.
|
||||
struct SourceData {
|
||||
dims: Vec<u64>,
|
||||
raw: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Source files and datasets, fetched on demand. The most recently used
|
||||
/// external file is kept, since consecutive mappings usually share one.
|
||||
struct Sources<'a, 'r> {
|
||||
file_data: &'a [u8],
|
||||
resolver: Option<&'r VdsFileResolver<'r>>,
|
||||
cached_file: Option<(String, Option<Vec<u8>>)>,
|
||||
}
|
||||
|
||||
impl<'a, 'r> Sources<'a, 'r> {
|
||||
fn new(file_data: &'a [u8], resolver: Option<&'r VdsFileResolver<'r>>) -> Self {
|
||||
Sources {
|
||||
file_data,
|
||||
resolver,
|
||||
cached_file: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes of source file `name`, or `None` if it does not exist.
|
||||
fn file(&mut self, name: &str) -> Result<Option<&[u8]>, FormatError> {
|
||||
if name == "." {
|
||||
return Ok(Some(self.file_data));
|
||||
}
|
||||
if self.cached_file.as_ref().is_none_or(|(n, _)| n != name) {
|
||||
let resolver = self.resolver.ok_or_else(|| {
|
||||
vds_err("external-file virtual dataset sources require a file resolver")
|
||||
})?;
|
||||
self.cached_file = Some((String::from(name), resolver(name)?));
|
||||
}
|
||||
Ok(self.cached_file.as_ref().and_then(|(_, b)| b.as_deref()))
|
||||
}
|
||||
|
||||
/// Read source dataset `path` from file `file`, or `None` when either
|
||||
/// does not exist. Its datatype must be the virtual dataset's: libhdf5
|
||||
/// converts between types here, which is not supported.
|
||||
fn dataset(
|
||||
&mut self,
|
||||
file: &str,
|
||||
path: &str,
|
||||
datatype: &Datatype,
|
||||
) -> Result<Option<SourceData>, FormatError> {
|
||||
let Some(bytes) = self.file(file)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
read_source(bytes, path, datatype)
|
||||
}
|
||||
}
|
||||
|
||||
/// Read source dataset `path` of the file in `file_data` in full (its own
|
||||
/// fill value applied to unallocated chunks), or `None` if it does not exist.
|
||||
fn read_source(
|
||||
file_data: &[u8],
|
||||
path: &str,
|
||||
datatype: &Datatype,
|
||||
) -> Result<Option<SourceData>, FormatError> {
|
||||
use crate::filter_pipeline::FilterPipeline;
|
||||
use crate::message_type::MessageType;
|
||||
use crate::object_header::ObjectHeader;
|
||||
use crate::shared_message::message_data_with_sohm;
|
||||
|
||||
let sig = crate::signature::find_signature(file_data)?;
|
||||
let sb = crate::superblock::Superblock::parse(file_data, sig)?;
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
let addr = match crate::group_v2::resolve_path_any(file_data, &sb, path) {
|
||||
Ok(a) => a,
|
||||
Err(FormatError::PathNotFound(_)) => return Ok(None),
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let hdr = ObjectHeader::parse(file_data, addr as usize, os, ls)?;
|
||||
let msg = |t: MessageType| {
|
||||
hdr.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == t)
|
||||
.ok_or_else(|| vds_err(format!("VDS source {path} has no {t:?} message")))
|
||||
};
|
||||
let dataspace = Dataspace::parse(
|
||||
&message_data_with_sohm(file_data, msg(MessageType::Dataspace)?, os, ls)?,
|
||||
ls,
|
||||
)?;
|
||||
let (src_type, _) = Datatype::parse(&message_data_with_sohm(
|
||||
file_data,
|
||||
msg(MessageType::Datatype)?,
|
||||
os,
|
||||
ls,
|
||||
)?)?;
|
||||
if &src_type != datatype {
|
||||
return Err(vds_err(format!(
|
||||
"VDS source {path} has a different datatype from the virtual dataset \
|
||||
(type conversion is not supported)"
|
||||
)));
|
||||
}
|
||||
let layout = DataLayout::parse(&msg(MessageType::DataLayout)?.data, os, ls)?;
|
||||
// A source that is itself virtual could form a cycle (A -> B -> A) and
|
||||
// recurse without bound. Nested virtual sources are not supported.
|
||||
if matches!(layout, DataLayout::Virtual { .. }) {
|
||||
return Err(vds_err(
|
||||
"virtual dataset source is itself virtual (unsupported)",
|
||||
));
|
||||
}
|
||||
let pipeline = hdr
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::FilterPipeline)
|
||||
.map(|m| {
|
||||
message_data_with_sohm(file_data, m, os, ls).and_then(|d| FilterPipeline::parse(&d))
|
||||
})
|
||||
.transpose()?;
|
||||
let raw = crate::fill_value::read_full_with_fill(
|
||||
&hdr.messages,
|
||||
file_data,
|
||||
&layout,
|
||||
&dataspace,
|
||||
src_type.type_size() as usize,
|
||||
os,
|
||||
ls,
|
||||
|| {
|
||||
crate::data_read::read_raw_data_full(
|
||||
file_data,
|
||||
&layout,
|
||||
&dataspace,
|
||||
&src_type,
|
||||
pipeline.as_ref(),
|
||||
os,
|
||||
ls,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
Ok(Some(SourceData {
|
||||
dims: dataspace.dimensions,
|
||||
raw,
|
||||
}))
|
||||
}
|
||||
Reference in New Issue
Block a user