Merge branch 'fix/p1-vds' into fix/p1-read-gaps
# Conflicts: # CHANGELOG.md # crates/clawhdf5-format/src/data_read.rs
This commit is contained in:
osobh
@@ -74,21 +74,33 @@ pub enum DataLayout {
|
||||
},
|
||||
}
|
||||
|
||||
/// Version-1 VDS mapping flag: the source file name is stored by an earlier
|
||||
/// entry, whose index follows in place of the name.
|
||||
const VDS_SOURCE_FILE_SHARED: u8 = 0x01;
|
||||
/// Version-1 VDS mapping flag: likewise for the source dataset name.
|
||||
const VDS_SOURCE_DSET_SHARED: u8 = 0x02;
|
||||
/// Version-1 VDS mapping flag: the source is in the virtual file itself
|
||||
/// (`"."`); no file name is stored.
|
||||
const VDS_SOURCE_SAME_FILE: u8 = 0x04;
|
||||
const VDS_ALL_FLAGS: u8 = VDS_SOURCE_FILE_SHARED | VDS_SOURCE_DSET_SHARED | VDS_SOURCE_SAME_FILE;
|
||||
|
||||
/// Parse VDS mappings from global-heap object data.
|
||||
///
|
||||
/// The global-heap block holding a VDS mapping list is laid out as
|
||||
/// (reverse-engineered and validated against HDF5 2.0):
|
||||
/// (`H5D__virtual_store_layout` / `H5D__virtual_load_layout` in libhdf5):
|
||||
///
|
||||
/// ```text
|
||||
/// version(1) · nused(length_size, LE) · entry[nused] · checksum(4)
|
||||
/// ```
|
||||
///
|
||||
/// Each entry is:
|
||||
/// - source file name — a null-terminated string in **block version 0**; in
|
||||
/// **block version 1** a same-file reference is encoded as a single `0x04`
|
||||
/// marker byte (the source file is the virtual file itself) in place of the
|
||||
/// name;
|
||||
/// - source dataset name (null-terminated string);
|
||||
/// - **block version 1 only:** a flags byte. `0x04`: the source is in the
|
||||
/// virtual file itself and no file name is stored; `0x01`/`0x02`: the
|
||||
/// source file/dataset name is that of an earlier entry, whose index
|
||||
/// (`length_size` bytes) is stored instead of the name. libhdf5 2.0 writes
|
||||
/// version 1 when the file's low version bound is 2.0 and it saves space;
|
||||
/// - source file name (null-terminated string, unless flagged above);
|
||||
/// - source dataset name (null-terminated string, unless flagged above);
|
||||
/// - source selection (serialized `H5S` dataspace selection — self-describing
|
||||
/// in length);
|
||||
/// - virtual selection (serialized `H5S` dataspace selection).
|
||||
@@ -114,7 +126,7 @@ pub fn parse_vds_mappings(
|
||||
// `nused` is untrusted; don't pre-allocate from it. Each entry consumes at
|
||||
// least a few bytes, so the loop is naturally bounded by the heap data and
|
||||
// a bogus `nused` simply errors out on the first short read.
|
||||
let mut mappings = Vec::new();
|
||||
let mut mappings: Vec<VdsMapping> = Vec::new();
|
||||
// Reads one self-describing selection at `pos`, returning its raw bytes and
|
||||
// advancing past it — bounds-checked so a corrupt selection can't overrun.
|
||||
let read_selection = |heap_data: &[u8], pos: &mut usize| -> Result<Vec<u8>, FormatError> {
|
||||
@@ -134,17 +146,57 @@ pub fn parse_vds_mappings(
|
||||
Ok(bytes)
|
||||
};
|
||||
|
||||
for _ in 0..nused {
|
||||
// Source file name (with the version-1 same-file marker handled).
|
||||
let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) {
|
||||
if version > 1 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"unsupported VDS mapping block version".into(),
|
||||
));
|
||||
}
|
||||
for i in 0..nused {
|
||||
// Version 1 prefixes each entry with a flags byte; a name may then be
|
||||
// omitted (same file) or replaced by the index of an earlier entry
|
||||
// holding the same name (`H5D__virtual_load_layout`).
|
||||
let flags = if version >= 1 {
|
||||
let f = *heap_data.get(pos).ok_or(FormatError::UnexpectedEof {
|
||||
expected: pos + 1,
|
||||
available: heap_data.len(),
|
||||
})?;
|
||||
pos += 1;
|
||||
if f & !VDS_ALL_FLAGS != 0 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"unknown VDS mapping flags".into(),
|
||||
));
|
||||
}
|
||||
f
|
||||
} else {
|
||||
0
|
||||
};
|
||||
// Index of an earlier entry, for a shared name.
|
||||
let earlier = |pos: &mut usize| -> Result<usize, FormatError> {
|
||||
let idx = read_length(heap_data, *pos, length_size)?;
|
||||
*pos += ls;
|
||||
if idx >= i {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"VDS mapping shares a name with a later entry".into(),
|
||||
));
|
||||
}
|
||||
Ok(idx as usize)
|
||||
};
|
||||
|
||||
let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 {
|
||||
String::from(".")
|
||||
} else if flags & VDS_SOURCE_FILE_SHARED != 0 {
|
||||
let idx = earlier(&mut pos)?;
|
||||
mappings[idx].source_file.clone()
|
||||
} else {
|
||||
read_null_terminated_string(heap_data, &mut pos)?
|
||||
};
|
||||
|
||||
// Source dataset name.
|
||||
let source_dataset = read_null_terminated_string(heap_data, &mut pos)?;
|
||||
let source_dataset = if flags & VDS_SOURCE_DSET_SHARED != 0 {
|
||||
let idx = earlier(&mut pos)?;
|
||||
mappings[idx].source_dataset.clone()
|
||||
} else {
|
||||
read_null_terminated_string(heap_data, &mut pos)?
|
||||
};
|
||||
|
||||
// Source selection, then virtual selection (both self-describing length).
|
||||
let source_selection = read_selection(heap_data, &mut pos)?;
|
||||
@@ -1035,6 +1087,62 @@ mod tests {
|
||||
assert_eq!(v1.iter_linear_1d(8).unwrap(), vec![4, 5, 6, 7]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_vds_mappings_v1_shared_names() {
|
||||
// Written by HDF5 2.0 (h5py, libver=("v200", "v200")) for three
|
||||
// mappings from `a_rather_long_source_file.h5:a_rather_long_dataset_name`
|
||||
// and one from the same file: the entries carry flags 0x00, 0x03, 0x03
|
||||
// and 0x06, so names after the first are stored as entry indices.
|
||||
let blob: &[u8] = &[
|
||||
0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x5f, 0x72, 0x61,
|
||||
0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72,
|
||||
0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x68, 0x35, 0x00, 0x61, 0x5f, 0x72,
|
||||
0x61, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74,
|
||||
0x61, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x00, 0x02, 0x00, 0x00, 0x00,
|
||||
0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
|
||||
0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02,
|
||||
0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00,
|
||||
0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
|
||||
0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x01,
|
||||
0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02,
|
||||
0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
|
||||
0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00,
|
||||
0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x01, 0x00, 0x01, 0x00,
|
||||
0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00,
|
||||
0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
|
||||
0x01, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
|
||||
0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00,
|
||||
0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
|
||||
0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x8e, 0xa7, 0xea, 0x7a,
|
||||
];
|
||||
let mappings = parse_vds_mappings(blob, 8).unwrap();
|
||||
let names: Vec<(&str, &str)> = mappings
|
||||
.iter()
|
||||
.map(|m| (m.source_file.as_str(), m.source_dataset.as_str()))
|
||||
.collect();
|
||||
let (file, dset) = ("a_rather_long_source_file.h5", "a_rather_long_dataset_name");
|
||||
assert_eq!(
|
||||
names,
|
||||
vec![(file, dset), (file, dset), (file, dset), (".", dset)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_vds_mappings_v1_forward_reference_is_error() {
|
||||
// Entry 0 claiming to share entry 0's file name must not index past
|
||||
// the entries decoded so far.
|
||||
let mut blob = vec![0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x01];
|
||||
blob.extend_from_slice(&[0u8; 8]);
|
||||
blob.extend_from_slice(b"d\0");
|
||||
assert!(parse_vds_mappings(&blob, 8).is_err());
|
||||
// Unknown flag bits are refused.
|
||||
let blob = [0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x08, b'd', 0];
|
||||
assert!(parse_vds_mappings(&blob, 8).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_vds_mappings_external_v0() {
|
||||
// Block version 0 with an explicit (external) source file name.
|
||||
|
||||
@@ -191,14 +191,9 @@ fn read_raw_data_full_impl(
|
||||
offset_size,
|
||||
length_size,
|
||||
),
|
||||
DataLayout::Virtual {
|
||||
global_heap_address,
|
||||
global_heap_index,
|
||||
..
|
||||
} => read_virtual_data(
|
||||
DataLayout::Virtual { .. } => read_virtual_data(
|
||||
file_data,
|
||||
*global_heap_address,
|
||||
*global_heap_index,
|
||||
layout,
|
||||
dataspace,
|
||||
datatype,
|
||||
offset_size,
|
||||
@@ -465,160 +460,54 @@ pub fn read_raw_data_selection(
|
||||
}
|
||||
}
|
||||
|
||||
/// Assemble a **Virtual Dataset (VDS)** from its source mappings.
|
||||
/// Assemble a **Virtual Dataset (VDS)** through the raw-read API, which has no
|
||||
/// access to the dataset's fill value message.
|
||||
///
|
||||
/// Supports virtual datasets of any rank. Same-file sources are read directly;
|
||||
/// **external-file** sources are read through the caller-supplied `resolver`,
|
||||
/// which maps a stored source file name to that file's bytes. Each mapping's
|
||||
/// selected source elements are scattered into the virtual buffer at the
|
||||
/// positions given by the virtual selection (both enumerated in row-major
|
||||
/// order, as HDF5 pairs them). Unmapped regions are left at the zero fill value.
|
||||
///
|
||||
/// A mapping whose external source file the resolver cannot supply (`None`) is
|
||||
/// skipped, leaving its region at fill — matching HDF5's tolerance of missing
|
||||
/// sources. An external source with no resolver at all is a hard error.
|
||||
/// Delegates to [`crate::vds::read_virtual_dataset`]. Because the fill value
|
||||
/// is unknown here, a virtual dataset with any element no mapping supplies
|
||||
/// (an unmapped region, or a missing source file or dataset) is an error
|
||||
/// rather than a guess at the fill value; so is one whose extent libhdf5
|
||||
/// would report differently from the stored dataspace (unlimited mappings).
|
||||
/// Use [`crate::vds::read_virtual_dataset`] to read those.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn read_virtual_data(
|
||||
file_data: &[u8],
|
||||
global_heap_address: Option<u64>,
|
||||
global_heap_index: u32,
|
||||
layout: &DataLayout,
|
||||
dataspace: &Dataspace,
|
||||
datatype: &Datatype,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
resolver: Option<&VdsSourceResolver>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
use crate::data_layout::parse_vds_mappings;
|
||||
use crate::global_heap::GlobalHeapCollection;
|
||||
use crate::selection::Selection;
|
||||
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let mut out = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len(
|
||||
dataspace.checked_num_elements()?,
|
||||
elem_size,
|
||||
)?)?;
|
||||
|
||||
let virtual_dims = &dataspace.dimensions;
|
||||
|
||||
let addr = global_heap_address.ok_or_else(|| {
|
||||
FormatError::ChunkedReadError("virtual dataset has no mapping global heap".into())
|
||||
})?;
|
||||
let coll = GlobalHeapCollection::parse(file_data, addr as usize, length_size)?;
|
||||
let obj =
|
||||
coll.get_object(global_heap_index as u16)
|
||||
.ok_or(FormatError::GlobalHeapObjectNotFound {
|
||||
collection_address: addr,
|
||||
index: global_heap_index as u16,
|
||||
})?;
|
||||
let mappings = parse_vds_mappings(&obj.data, length_size)?;
|
||||
|
||||
for m in &mappings {
|
||||
let same_file = m.source_file.is_empty() || m.source_file == ".";
|
||||
|
||||
// Resolve the bytes of the file holding this source dataset.
|
||||
let external;
|
||||
let src_file_data: &[u8] = if same_file {
|
||||
file_data
|
||||
} else {
|
||||
let r = resolver.ok_or_else(|| {
|
||||
FormatError::ChunkedReadError(
|
||||
"external-file virtual dataset sources require a file resolver".into(),
|
||||
)
|
||||
})?;
|
||||
match r(&m.source_file) {
|
||||
Some(bytes) => {
|
||||
external = bytes;
|
||||
&external
|
||||
}
|
||||
// Source file unavailable: leave this region at fill value.
|
||||
None => continue,
|
||||
}
|
||||
};
|
||||
|
||||
let (vsel, _) = Selection::decode_serialized(&m.virtual_selection)?;
|
||||
let (ssel, _) = Selection::decode_serialized(&m.source_selection)?;
|
||||
|
||||
let (src_raw, src_dims) =
|
||||
read_named_dataset_raw(src_file_data, &m.source_dataset, offset_size, length_size)?;
|
||||
|
||||
let vidx = vsel.iter_linear(virtual_dims)?;
|
||||
let sidx = ssel.iter_linear(&src_dims)?;
|
||||
if vidx.len() != sidx.len() {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"virtual/source selection element counts differ".into(),
|
||||
));
|
||||
}
|
||||
|
||||
for (&v, &s) in vidx.iter().zip(sidx.iter()) {
|
||||
let (vo, so) = (v as usize * elem_size, s as usize * elem_size);
|
||||
if vo + elem_size > out.len() || so + elem_size > src_raw.len() {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"virtual dataset selection out of bounds".into(),
|
||||
));
|
||||
}
|
||||
out[vo..vo + elem_size].copy_from_slice(&src_raw[so..so + elem_size]);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Read a named dataset's raw (decoded) bytes and its dimensions, navigating
|
||||
/// from the superblock. Used to pull VDS source datasets out of the same file.
|
||||
fn read_named_dataset_raw(
|
||||
file_data: &[u8],
|
||||
path: &str,
|
||||
_offset_size: u8,
|
||||
_length_size: u8,
|
||||
) -> Result<(Vec<u8>, Vec<u64>), FormatError> {
|
||||
use crate::filter_pipeline::FilterPipeline;
|
||||
use crate::group_v2::resolve_path_any;
|
||||
use crate::message_type::MessageType;
|
||||
use crate::object_header::ObjectHeader;
|
||||
use crate::signature::split_user_block;
|
||||
use crate::superblock::Superblock;
|
||||
|
||||
// An external source file is handed over whole, user block included;
|
||||
// its addresses are relative to its superblock.
|
||||
let (_, file_data) = split_user_block(file_data)?;
|
||||
let sb = Superblock::parse(file_data, 0)?;
|
||||
let addr = resolve_path_any(file_data, &sb, path)?;
|
||||
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?;
|
||||
|
||||
let find = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t);
|
||||
let ds_msg = find(MessageType::Dataspace)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no dataspace".into()))?;
|
||||
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size)?;
|
||||
let dt_msg = find(MessageType::Datatype)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no datatype".into()))?;
|
||||
let (datatype, _) = Datatype::parse(&dt_msg.data)?;
|
||||
let dl_msg = find(MessageType::DataLayout)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?;
|
||||
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?;
|
||||
// A virtual dataset whose source is itself another virtual dataset could
|
||||
// form a cycle (A -> B -> A) and recurse into a stack overflow. Nested
|
||||
// virtual sources are exotic and unsupported, so stop here cleanly.
|
||||
if matches!(layout, DataLayout::Virtual { .. }) {
|
||||
let wrapped =
|
||||
resolver.map(|r| move |name: &str| -> Result<Option<Vec<u8>>, FormatError> { Ok(r(name)) });
|
||||
let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver);
|
||||
let v = crate::vds::read_virtual_dataset(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
datatype,
|
||||
None,
|
||||
offset_size,
|
||||
length_size,
|
||||
wrapped_ref,
|
||||
)?;
|
||||
if v.dims != dataspace.dimensions {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"virtual dataset source is itself virtual (unsupported)".into(),
|
||||
"virtual dataset extent differs from its stored dataspace; \
|
||||
read it with vds::read_virtual_dataset"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
let pipeline = find(MessageType::FilterPipeline)
|
||||
.map(|m| FilterPipeline::parse(&m.data))
|
||||
.transpose()?;
|
||||
|
||||
let raw = read_raw_data_full(
|
||||
file_data,
|
||||
&layout,
|
||||
&dataspace,
|
||||
&datatype,
|
||||
pipeline.as_ref(),
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
)?;
|
||||
Ok((raw, dataspace.dimensions.clone()))
|
||||
if v.unmapped > 0 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"virtual dataset has elements no source supplies, which read as its \
|
||||
fill value; read it with vds::read_virtual_dataset and the fill value"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
Ok(v.data)
|
||||
}
|
||||
|
||||
/// Extract selected elements from a full dataset buffer.
|
||||
pub fn extract_selection_from_buffer(
|
||||
full_data: &[u8],
|
||||
|
||||
@@ -101,6 +101,7 @@ pub mod signature;
|
||||
pub mod superblock;
|
||||
pub mod symbol_table;
|
||||
pub mod type_builders;
|
||||
pub mod vds;
|
||||
pub mod vl_data;
|
||||
|
||||
#[cfg(feature = "provenance")]
|
||||
|
||||
@@ -229,44 +229,47 @@ impl Selection {
|
||||
/// self-describing in length, so the count lets a caller walk a packed list
|
||||
/// of selections — as the Virtual Dataset global-heap block does).
|
||||
///
|
||||
/// Only the forms needed for VDS assembly are decoded: `ALL`, `NONE`, and
|
||||
/// **regular** hyperslabs serialized at **version 3** (the encoding HDF5
|
||||
/// 1.10+/2.0 emit). Point selections, irregular hyperslabs, and older
|
||||
/// hyperslab versions return an error rather than mis-decoding.
|
||||
/// Decodes `ALL`, `NONE`, and hyperslabs at every version libhdf5 writes
|
||||
/// (1: irregular, 4-byte coordinates — the default-format encoding; 2:
|
||||
/// regular, 8-byte; 3: either, variable width). A regular hyperslab maps
|
||||
/// to [`Selection::Hyperslab`]; an *irregular* one (a union of blocks)
|
||||
/// maps to a single-block hyperslab when it has one block, and otherwise to
|
||||
/// [`Selection::Points`] listing the union in row-major order (the order
|
||||
/// libhdf5 iterates it in). Unlimited counts/blocks decode as `u64::MAX`
|
||||
/// (see [`SerializedSelection::decode`] for the raw form). Point
|
||||
/// selections are refused: libhdf5 does not allow them in virtual datasets
|
||||
/// either.
|
||||
pub fn decode_serialized(data: &[u8]) -> Result<(Selection, usize), FormatError> {
|
||||
if data.len() < 8 {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: 8,
|
||||
available: data.len(),
|
||||
});
|
||||
}
|
||||
let sel_type = u32::from_le_bytes([data[0], data[1], data[2], data[3]]);
|
||||
let version = u32::from_le_bytes([data[4], data[5], data[6], data[7]]);
|
||||
|
||||
match sel_type {
|
||||
// ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes.
|
||||
3 | 0 => {
|
||||
if data.len() < 16 {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: 16,
|
||||
available: data.len(),
|
||||
});
|
||||
}
|
||||
let sel = if sel_type == 3 {
|
||||
Selection::All
|
||||
let (raw, len) = SerializedSelection::decode(data)?;
|
||||
let sel = match raw {
|
||||
SerializedSelection::All => Selection::All,
|
||||
SerializedSelection::None => Selection::None,
|
||||
SerializedSelection::Regular {
|
||||
start,
|
||||
stride,
|
||||
count,
|
||||
block,
|
||||
} => Selection::Hyperslab {
|
||||
start,
|
||||
stride,
|
||||
count,
|
||||
block,
|
||||
},
|
||||
SerializedSelection::Blocks { rank, starts, ends } => {
|
||||
if starts.len() == rank {
|
||||
let block = starts.iter().zip(&ends).map(|(&s, &e)| e - s + 1).collect();
|
||||
Selection::Hyperslab {
|
||||
start: starts,
|
||||
stride: vec![1; rank],
|
||||
count: vec![1; rank],
|
||||
block,
|
||||
}
|
||||
} else {
|
||||
Selection::None
|
||||
};
|
||||
Ok((sel, 16))
|
||||
Selection::Points(blocks_union_coords(rank, &starts, &ends)?)
|
||||
}
|
||||
}
|
||||
2 => decode_hyperslab_serialized(data, version),
|
||||
1 => Err(FormatError::ChunkedReadError(
|
||||
"VDS point selections are not supported".into(),
|
||||
)),
|
||||
_ => Err(FormatError::ChunkedReadError(
|
||||
"unknown dataspace selection type".into(),
|
||||
)),
|
||||
}
|
||||
};
|
||||
Ok((sel, len))
|
||||
}
|
||||
|
||||
/// Enumerate the selected element indices of a **1-D** dataspace of the
|
||||
@@ -314,6 +317,11 @@ impl Selection {
|
||||
"VDS selection rank does not match dataspace rank".into(),
|
||||
));
|
||||
}
|
||||
if count.iter().chain(block.iter()).any(|&v| v == UNLIMITED) {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"unlimited selection must be clipped before it is enumerated".into(),
|
||||
));
|
||||
}
|
||||
// Selected coordinates along each dimension, in order.
|
||||
let mut per_dim: Vec<Vec<u64>> = Vec::with_capacity(rank);
|
||||
for d in 0..rank {
|
||||
@@ -400,84 +408,279 @@ impl Selection {
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode an `H5S_SEL_HYPER` selection in its serialized form. Only version-3
|
||||
/// **regular** hyperslabs are supported.
|
||||
fn decode_hyperslab_serialized(
|
||||
data: &[u8],
|
||||
version: u32,
|
||||
) -> Result<(Selection, usize), FormatError> {
|
||||
if version != 3 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"only version-3 hyperslab selections are supported".into(),
|
||||
));
|
||||
/// Hyperslab count/block value meaning "unlimited" (`H5S_UNLIMITED`).
|
||||
pub const UNLIMITED: u64 = u64::MAX;
|
||||
|
||||
/// Largest number of elements an irregular selection is expanded to when it
|
||||
/// is converted to a point list by [`Selection::decode_serialized`].
|
||||
const MAX_EXPANDED_POINTS: u64 = 1 << 26;
|
||||
|
||||
/// A selection exactly as `H5S_select_serialize` stores it, before it is
|
||||
/// applied to any dataspace.
|
||||
///
|
||||
/// Unlike [`Selection`] this keeps an irregular hyperslab as its list of
|
||||
/// blocks, and a regular hyperslab's count/block may be [`UNLIMITED`] (the
|
||||
/// unlimited selections used by unlimited and "printf" virtual dataset
|
||||
/// mappings).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum SerializedSelection {
|
||||
/// `H5S_SEL_ALL`.
|
||||
All,
|
||||
/// `H5S_SEL_NONE`.
|
||||
None,
|
||||
/// A regular hyperslab. `count[d]` or `block[d]` may be [`UNLIMITED`].
|
||||
Regular {
|
||||
start: Vec<u64>,
|
||||
stride: Vec<u64>,
|
||||
count: Vec<u64>,
|
||||
block: Vec<u64>,
|
||||
},
|
||||
/// An irregular hyperslab: the union of `starts.len() / rank` blocks, each
|
||||
/// given by its first (`starts`) and last (`ends`, inclusive) coordinate,
|
||||
/// flattened block-major.
|
||||
Blocks {
|
||||
rank: usize,
|
||||
starts: Vec<u64>,
|
||||
ends: Vec<u64>,
|
||||
},
|
||||
}
|
||||
|
||||
fn sel_err(msg: &str) -> FormatError {
|
||||
FormatError::ChunkedReadError(msg.into())
|
||||
}
|
||||
|
||||
/// Bounds-checked little-endian reader over a serialized selection.
|
||||
struct SelReader<'a> {
|
||||
data: &'a [u8],
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl SelReader<'_> {
|
||||
fn take(&mut self, n: usize) -> Result<&[u8], FormatError> {
|
||||
let end = self.pos.checked_add(n).filter(|&e| e <= self.data.len());
|
||||
let end = end.ok_or(FormatError::UnexpectedEof {
|
||||
expected: self.pos.saturating_add(n),
|
||||
available: self.data.len(),
|
||||
})?;
|
||||
let s = &self.data[self.pos..end];
|
||||
self.pos = end;
|
||||
Ok(s)
|
||||
}
|
||||
// type(4) ver(4) flags(1) enc_size(1) rank(4) [start,stride,count,block]*rank
|
||||
if data.len() < 14 {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: 14,
|
||||
available: data.len(),
|
||||
});
|
||||
|
||||
fn uint(&mut self, size: usize) -> Result<u64, FormatError> {
|
||||
let bytes = self.take(size)?;
|
||||
Ok(bytes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.fold(0u64, |v, (i, &b)| v | (b as u64) << (i * 8)))
|
||||
}
|
||||
let flags = data[8];
|
||||
let enc_size = data[9] as usize;
|
||||
// Bit 0 set => regular hyperslab. Irregular hyperslabs list explicit blocks.
|
||||
if flags & 0x01 == 0 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"irregular VDS hyperslab selections are not supported".into(),
|
||||
));
|
||||
|
||||
fn remaining(&self) -> usize {
|
||||
self.data.len() - self.pos
|
||||
}
|
||||
if enc_size != 2 && enc_size != 4 && enc_size != 8 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"unsupported hyperslab coordinate encoding size".into(),
|
||||
));
|
||||
}
|
||||
let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize;
|
||||
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything larger so a
|
||||
// corrupt rank can't drive a huge allocation or read loop.
|
||||
if rank > 32 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"hyperslab selection rank exceeds maximum (32)".into(),
|
||||
));
|
||||
}
|
||||
let mut pos = 14;
|
||||
let read_coord = |data: &[u8], pos: usize| -> Result<u64, FormatError> {
|
||||
if pos + enc_size > data.len() {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: pos + enc_size,
|
||||
available: data.len(),
|
||||
});
|
||||
}
|
||||
|
||||
impl SerializedSelection {
|
||||
/// Decode a serialized selection, returning it and the number of bytes it
|
||||
/// occupies. Mirrors libhdf5's `H5S_select_deserialize`: `ALL`/`NONE` and
|
||||
/// hyperslab versions 1-3 are decoded; point selections (which libhdf5
|
||||
/// refuses in virtual datasets) and malformed input are errors.
|
||||
pub fn decode(data: &[u8]) -> Result<(SerializedSelection, usize), FormatError> {
|
||||
let mut r = SelReader { data, pos: 0 };
|
||||
let sel_type = r.uint(4)?;
|
||||
let version = r.uint(4)?;
|
||||
match sel_type {
|
||||
// ALL / NONE: type(4) + version(4) + reserved(4) + length(4).
|
||||
0 | 3 => {
|
||||
r.take(8)?;
|
||||
let sel = if sel_type == 3 {
|
||||
SerializedSelection::All
|
||||
} else {
|
||||
SerializedSelection::None
|
||||
};
|
||||
Ok((sel, r.pos))
|
||||
}
|
||||
2 => {
|
||||
let sel = decode_hyperslab(&mut r, version)?;
|
||||
Ok((sel, r.pos))
|
||||
}
|
||||
1 => Err(sel_err(
|
||||
"VDS point selections are not supported (libhdf5 rejects them too)",
|
||||
)),
|
||||
_ => Err(sel_err("unknown dataspace selection type")),
|
||||
}
|
||||
let mut v = 0u64;
|
||||
for (i, &b) in data[pos..pos + enc_size].iter().enumerate() {
|
||||
v |= (b as u64) << (i * 8);
|
||||
}
|
||||
|
||||
/// The single dimension in which this selection is unlimited, if any.
|
||||
pub fn unlimited_dim(&self) -> Option<usize> {
|
||||
match self {
|
||||
SerializedSelection::Regular { count, block, .. } => count
|
||||
.iter()
|
||||
.zip(block)
|
||||
.position(|(&c, &b)| c == UNLIMITED || b == UNLIMITED),
|
||||
_ => None,
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// The rank the selection was serialized with (`None` for ALL/NONE, which
|
||||
/// carry no rank).
|
||||
pub fn rank(&self) -> Option<usize> {
|
||||
match self {
|
||||
SerializedSelection::Regular { start, .. } => Some(start.len()),
|
||||
SerializedSelection::Blocks { rank, .. } => Some(*rank),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `H5S__hyper_deserialize`: after the type and version words.
|
||||
fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelection, FormatError> {
|
||||
const REGULAR: u8 = 0x01;
|
||||
let (flags, enc_size) = match version {
|
||||
// v1: reserved(4) + length(4), always irregular, 4-byte coordinates.
|
||||
1 => {
|
||||
r.take(8)?;
|
||||
(0u8, 4usize)
|
||||
}
|
||||
// v2: flags(1) + length(4), 8-byte coordinates.
|
||||
2 => {
|
||||
let flags = r.take(1)?[0];
|
||||
r.take(4)?;
|
||||
(flags, 8)
|
||||
}
|
||||
// v3: flags(1) + encoding size(1).
|
||||
3 => {
|
||||
let flags = r.take(1)?[0];
|
||||
let enc = r.take(1)?[0] as usize;
|
||||
(flags, enc)
|
||||
}
|
||||
_ => return Err(sel_err("unsupported hyperslab selection version")),
|
||||
};
|
||||
let (mut start, mut stride, mut count, mut block) = (
|
||||
Vec::with_capacity(rank),
|
||||
Vec::with_capacity(rank),
|
||||
Vec::with_capacity(rank),
|
||||
Vec::with_capacity(rank),
|
||||
);
|
||||
for _ in 0..rank {
|
||||
start.push(read_coord(data, pos)?);
|
||||
pos += enc_size;
|
||||
stride.push(read_coord(data, pos)?);
|
||||
pos += enc_size;
|
||||
count.push(read_coord(data, pos)?);
|
||||
pos += enc_size;
|
||||
block.push(read_coord(data, pos)?);
|
||||
pos += enc_size;
|
||||
if flags & !REGULAR != 0 {
|
||||
return Err(sel_err("unknown hyperslab selection flags"));
|
||||
}
|
||||
Ok((
|
||||
Selection::Hyperslab {
|
||||
if !matches!(enc_size, 2 | 4 | 8) {
|
||||
return Err(sel_err("unsupported hyperslab coordinate encoding size"));
|
||||
}
|
||||
let rank = r.uint(4)? as usize;
|
||||
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a
|
||||
// corrupt rank can't drive a huge allocation or read loop.
|
||||
if rank == 0 || rank > 32 {
|
||||
return Err(sel_err("hyperslab selection rank must be 1..=32"));
|
||||
}
|
||||
// The all-ones value of the encoding width means "unlimited".
|
||||
let unlim_raw = if enc_size == 8 {
|
||||
u64::MAX
|
||||
} else {
|
||||
(1u64 << (enc_size * 8)) - 1
|
||||
};
|
||||
|
||||
if flags & REGULAR != 0 {
|
||||
let (mut start, mut stride, mut count, mut block) = (
|
||||
Vec::with_capacity(rank),
|
||||
Vec::with_capacity(rank),
|
||||
Vec::with_capacity(rank),
|
||||
Vec::with_capacity(rank),
|
||||
);
|
||||
for _ in 0..rank {
|
||||
start.push(r.uint(enc_size)?);
|
||||
stride.push(r.uint(enc_size)?);
|
||||
let c = r.uint(enc_size)?;
|
||||
count.push(if c == unlim_raw { UNLIMITED } else { c });
|
||||
let b = r.uint(enc_size)?;
|
||||
block.push(if b == unlim_raw { UNLIMITED } else { b });
|
||||
}
|
||||
let unlimited = count
|
||||
.iter()
|
||||
.zip(&block)
|
||||
.filter(|&(&c, &b)| c == UNLIMITED || b == UNLIMITED)
|
||||
.count();
|
||||
if unlimited > 1 {
|
||||
return Err(sel_err(
|
||||
"hyperslab selection is unlimited in more than one dimension",
|
||||
));
|
||||
}
|
||||
for d in 0..rank {
|
||||
// Overlapping blocks are not a valid regular hyperslab.
|
||||
if count[d] > 1 && block[d] != UNLIMITED && block[d] > stride[d] {
|
||||
return Err(sel_err("regular hyperslab blocks overlap"));
|
||||
}
|
||||
}
|
||||
return Ok(SerializedSelection::Regular {
|
||||
start,
|
||||
stride,
|
||||
count,
|
||||
block,
|
||||
},
|
||||
pos,
|
||||
))
|
||||
});
|
||||
}
|
||||
|
||||
// Irregular: number of blocks, then each block's start and end corners.
|
||||
let nblocks = r.uint(enc_size)?;
|
||||
let per_block = (rank * 2 * enc_size) as u64;
|
||||
// Untrusted count: it must fit in what is left of the buffer.
|
||||
if nblocks
|
||||
.checked_mul(per_block)
|
||||
.is_none_or(|need| need > r.remaining() as u64)
|
||||
{
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: r
|
||||
.pos
|
||||
.saturating_add(nblocks.saturating_mul(per_block) as usize),
|
||||
available: r.data.len(),
|
||||
});
|
||||
}
|
||||
let n = nblocks as usize * rank;
|
||||
let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n));
|
||||
for _ in 0..nblocks {
|
||||
for _ in 0..rank {
|
||||
starts.push(r.uint(enc_size)?);
|
||||
}
|
||||
for _ in 0..rank {
|
||||
ends.push(r.uint(enc_size)?);
|
||||
}
|
||||
}
|
||||
if starts.iter().zip(&ends).any(|(s, e)| e < s) {
|
||||
return Err(sel_err("hyperslab block ends before it starts"));
|
||||
}
|
||||
Ok(SerializedSelection::Blocks { rank, starts, ends })
|
||||
}
|
||||
|
||||
/// The coordinates of the union of the given blocks, in row-major order.
|
||||
fn blocks_union_coords(
|
||||
rank: usize,
|
||||
starts: &[u64],
|
||||
ends: &[u64],
|
||||
) -> Result<Vec<Vec<u64>>, FormatError> {
|
||||
let mut total = 0u64;
|
||||
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
||||
let vol = s
|
||||
.iter()
|
||||
.zip(e)
|
||||
.try_fold(1u64, |acc, (&s, &e)| acc.checked_mul(e - s + 1));
|
||||
total = vol
|
||||
.and_then(|v| total.checked_add(v))
|
||||
.filter(|&t| t <= MAX_EXPANDED_POINTS)
|
||||
.ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?;
|
||||
}
|
||||
let mut out = Vec::with_capacity(total as usize);
|
||||
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
||||
let mut cur = s.to_vec();
|
||||
'block: loop {
|
||||
out.push(cur.clone());
|
||||
for d in (0..rank).rev() {
|
||||
if cur[d] < e[d] {
|
||||
cur[d] += 1;
|
||||
continue 'block;
|
||||
}
|
||||
cur[d] = s[d];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Lexicographic order of coordinates is row-major order.
|
||||
out.sort_unstable();
|
||||
out.dedup();
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -642,11 +845,100 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_irregular_hyperslab_rejected() {
|
||||
fn decode_truncated_irregular_hyperslab_is_error() {
|
||||
// Irregular, rank 1, but the block count is missing.
|
||||
let bytes = [0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x00, 0x02, 0x01, 0, 0, 0];
|
||||
assert!(Selection::decode_serialized(&bytes).is_err());
|
||||
}
|
||||
|
||||
/// Version 1 as libhdf5 writes it for the default (earliest) format bounds:
|
||||
/// type, version, reserved(4), length(4), rank(4), nblocks(4), then each
|
||||
/// block's start and inclusive end corner as 4-byte values.
|
||||
fn v1_blocks(rank: u32, blocks: &[(&[u32], &[u32])]) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
for w in [2u32, 1, 0, 0, rank, blocks.len() as u32] {
|
||||
b.extend_from_slice(&w.to_le_bytes());
|
||||
}
|
||||
for (s, e) in blocks {
|
||||
for v in s.iter().chain(e.iter()) {
|
||||
b.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_v1_irregular_single_block() {
|
||||
// Exactly what h5py/HDF5 2.0 writes for `[0:4]` with default libver.
|
||||
let bytes = v1_blocks(1, &[(&[0], &[3])]);
|
||||
let (sel, used) = Selection::decode_serialized(&bytes).unwrap();
|
||||
assert_eq!(used, bytes.len());
|
||||
assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_v1_irregular_union_is_row_major() {
|
||||
// Blocks given out of order and overlapping still enumerate once each,
|
||||
// in row-major order (libhdf5 iterates the union, not the list).
|
||||
let bytes = v1_blocks(2, &[(&[1, 0], &[1, 1]), (&[0, 2], &[1, 2])]);
|
||||
let (sel, used) = Selection::decode_serialized(&bytes).unwrap();
|
||||
assert_eq!(used, bytes.len());
|
||||
// (0,2) (1,0) (1,1) (1,2) in a 2x3 space.
|
||||
assert_eq!(sel.iter_linear(&[2, 3]).unwrap(), vec![2, 3, 4, 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_v2_regular_with_unlimited_count() {
|
||||
// v2: flags(1) + length(4), then 8-byte start/stride/count/block.
|
||||
let mut b = Vec::new();
|
||||
b.extend_from_slice(&2u32.to_le_bytes());
|
||||
b.extend_from_slice(&2u32.to_le_bytes());
|
||||
b.push(0x01);
|
||||
b.extend_from_slice(&36u32.to_le_bytes());
|
||||
b.extend_from_slice(&1u32.to_le_bytes());
|
||||
for v in [0u64, 10, u64::MAX, 10] {
|
||||
b.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
let (raw, used) = SerializedSelection::decode(&b).unwrap();
|
||||
assert_eq!(used, b.len());
|
||||
assert_eq!(raw.unlimited_dim(), Some(0));
|
||||
assert_eq!(
|
||||
raw,
|
||||
SerializedSelection::Regular {
|
||||
start: vec![0],
|
||||
stride: vec![10],
|
||||
count: vec![UNLIMITED],
|
||||
block: vec![10],
|
||||
}
|
||||
);
|
||||
// An unclipped unlimited selection cannot be enumerated.
|
||||
let (sel, _) = Selection::decode_serialized(&b).unwrap();
|
||||
assert!(sel.iter_linear_1d(100).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_v3_two_byte_all_ones_is_unlimited() {
|
||||
let bytes = [
|
||||
0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, //
|
||||
0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0xFF, 0xFF,
|
||||
];
|
||||
let (raw, _) = SerializedSelection::decode(&bytes).unwrap();
|
||||
assert_eq!(raw.unlimited_dim(), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_irregular_block_count_beyond_buffer_is_error() {
|
||||
let mut b = v1_blocks(1, &[(&[0], &[3])]);
|
||||
b[20..24].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
assert!(Selection::decode_serialized(&b).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_point_selection_is_refused() {
|
||||
let bytes = [1u8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||
assert!(Selection::decode_serialized(&bytes).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iter_linear_2d_block_row_major() {
|
||||
// A 2x2 block at the top-left of a 4x4 space => linear 0,1,4,5.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user