osobhandClaude Opus 5.5
190918a478
libhdf5 serializes a VDS hyperslab as version 1 (irregular, 4-byte block corners) for the default format bounds, and as version 2 (regular, 8-byte) for unlimited selections in the 1.10 format. Only version 3 was accepted, so every h5py VDS written with default libver failed with "only version-3 hyperslab selections are supported" (5 libhdf5 test files in the sweep). Decode all three versions following H5S__hyper_deserialize, including irregular hyperslabs (a union of blocks, enumerated in row-major order as libhdf5 iterates them) and the all-ones "unlimited" count/block marker. SerializedSelection exposes the raw form for unlimited-mapping support. Test: vds_interop::vds_version1_irregular_hyperslab_selections compares default-libver h5py VDS reads (contiguous, strided and 2-D block mappings) with libhdf5's values. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1036 lines
37 KiB
Rust
1036 lines
37 KiB
Rust
//! Hyperslab and point selection for partial dataset I/O.
|
|
//!
|
|
//! A [`Selection`] describes which elements of a dataset to read or write.
|
|
//! The most common form is a hyperslab — a regular, strided sub-region of
|
|
//! the dataspace.
|
|
//!
|
|
//! # Example
|
|
//!
|
|
//! ```ignore
|
|
//! use clawhdf5_format::selection::Selection;
|
|
//!
|
|
//! // Select rows 20..30, columns 40..60 from a 2D dataset
|
|
//! let sel = Selection::slice(&[20..30, 40..60]);
|
|
//! assert_eq!(sel.num_elements(&[100, 100]), 200); // 10 * 20
|
|
//! ```
|
|
|
|
#[cfg(not(feature = "std"))]
|
|
use alloc::{vec, vec::Vec};
|
|
|
|
use core::ops::Range;
|
|
|
|
use crate::error::FormatError;
|
|
|
|
/// A selection describing which elements of a dataset to access.
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub enum Selection {
|
|
/// Select all elements (equivalent to the entire dataspace).
|
|
All,
|
|
|
|
/// Select no elements.
|
|
None,
|
|
|
|
/// A regular hyperslab selection defined by start, stride, count, and block.
|
|
///
|
|
/// For each dimension:
|
|
/// - `start[d]` — first element index
|
|
/// - `stride[d]` — step between blocks (must be >= block[d])
|
|
/// - `count[d]` — number of blocks
|
|
/// - `block[d]` — number of consecutive elements per block
|
|
///
|
|
/// When stride == block (or stride is 1 and block is 1), this reduces
|
|
/// to a simple contiguous slice.
|
|
Hyperslab {
|
|
start: Vec<u64>,
|
|
stride: Vec<u64>,
|
|
count: Vec<u64>,
|
|
block: Vec<u64>,
|
|
},
|
|
|
|
/// Select individual points by coordinate.
|
|
Points(Vec<Vec<u64>>),
|
|
}
|
|
|
|
impl Selection {
|
|
/// Create a simple contiguous hyperslab from ranges (one per dimension).
|
|
///
|
|
/// This is equivalent to a hyperslab with stride=1 and block=1.
|
|
pub fn slice(ranges: &[Range<u64>]) -> Self {
|
|
let rank = ranges.len();
|
|
let mut start = Vec::with_capacity(rank);
|
|
let mut count = Vec::with_capacity(rank);
|
|
for r in ranges {
|
|
debug_assert!(
|
|
r.end >= r.start,
|
|
"Selection::slice: range end ({}) < start ({})",
|
|
r.end,
|
|
r.start,
|
|
);
|
|
start.push(r.start);
|
|
count.push(r.end.saturating_sub(r.start));
|
|
}
|
|
Selection::Hyperslab {
|
|
start,
|
|
stride: vec![1; rank],
|
|
count,
|
|
block: vec![1; rank],
|
|
}
|
|
}
|
|
|
|
/// Number of selected elements for a given dataspace shape.
|
|
pub fn num_elements(&self, dims: &[u64]) -> u64 {
|
|
match self {
|
|
Selection::All => dims.iter().product(),
|
|
Selection::None => 0,
|
|
Selection::Hyperslab { count, block, .. } => count
|
|
.iter()
|
|
.zip(block.iter())
|
|
.map(|(&c, &b)| c * b)
|
|
.product(),
|
|
Selection::Points(pts) => pts.len() as u64,
|
|
}
|
|
}
|
|
|
|
/// The rank (number of dimensions) of this selection.
|
|
pub fn rank(&self) -> Option<usize> {
|
|
match self {
|
|
Selection::All | Selection::None => Option::None,
|
|
Selection::Hyperslab { start, .. } => Some(start.len()),
|
|
Selection::Points(pts) => pts.first().map(|p| p.len()),
|
|
}
|
|
}
|
|
|
|
/// The shape of the selected region (output dimensions).
|
|
///
|
|
/// For hyperslabs, this is `count[d] * block[d]` per dimension.
|
|
/// For `All`, returns the dataspace shape. For `None`, returns empty.
|
|
pub fn output_shape(&self, dims: &[u64]) -> Vec<u64> {
|
|
match self {
|
|
Selection::All => dims.to_vec(),
|
|
Selection::None => vec![],
|
|
Selection::Hyperslab { count, block, .. } => count
|
|
.iter()
|
|
.zip(block.iter())
|
|
.map(|(&c, &b)| c * b)
|
|
.collect(),
|
|
Selection::Points(pts) => vec![pts.len() as u64],
|
|
}
|
|
}
|
|
|
|
/// Check whether a chunk at the given offset (with given chunk dimensions)
|
|
/// intersects this selection.
|
|
///
|
|
/// Returns `true` if any element in the chunk overlaps with the selection.
|
|
pub fn intersects_chunk(&self, chunk_offset: &[u64], chunk_dims: &[u64]) -> bool {
|
|
match self {
|
|
Selection::All => true,
|
|
Selection::None => false,
|
|
Selection::Hyperslab {
|
|
start,
|
|
stride,
|
|
count,
|
|
block,
|
|
} => {
|
|
// For each dimension, check if the chunk range overlaps the hyperslab range
|
|
for d in 0..start.len() {
|
|
let chunk_start = chunk_offset[d];
|
|
let chunk_end = chunk_start + chunk_dims[d];
|
|
|
|
// Compute the full extent of the hyperslab in this dimension
|
|
let sel_start = start[d];
|
|
let sel_end = if count[d] == 0 {
|
|
sel_start
|
|
} else {
|
|
start[d] + (count[d] - 1) * stride[d] + block[d]
|
|
};
|
|
|
|
// No overlap if chunk is entirely before or after selection
|
|
if chunk_end <= sel_start || chunk_start >= sel_end {
|
|
return false;
|
|
}
|
|
}
|
|
true
|
|
}
|
|
Selection::Points(pts) => pts.iter().any(|pt| {
|
|
pt.iter()
|
|
.zip(chunk_offset.iter().zip(chunk_dims.iter()))
|
|
.all(|(&p, (&off, &dim))| p >= off && p < off + dim)
|
|
}),
|
|
}
|
|
}
|
|
|
|
/// For a given chunk, compute the local ranges within the chunk that
|
|
/// overlap with this selection.
|
|
///
|
|
/// Returns a list of (chunk_local_start, chunk_local_end, output_offset) per
|
|
/// dimension, representing which elements from the chunk contribute to the
|
|
/// output buffer. For simple contiguous slices, this returns exactly one range
|
|
/// per dimension.
|
|
pub fn chunk_local_ranges(&self, chunk_offset: &[u64], chunk_dims: &[u64]) -> Vec<Range<u64>> {
|
|
match self {
|
|
Selection::All => chunk_dims.iter().map(|&d| 0..d).collect(),
|
|
Selection::None => vec![],
|
|
Selection::Hyperslab {
|
|
start,
|
|
stride,
|
|
count,
|
|
block,
|
|
} => {
|
|
let mut ranges = Vec::with_capacity(start.len());
|
|
for d in 0..start.len() {
|
|
let chunk_start = chunk_offset[d];
|
|
let chunk_end = chunk_start + chunk_dims[d];
|
|
|
|
// For simple contiguous selections (stride==1, block==1),
|
|
// just clamp the selection range to the chunk bounds
|
|
if stride[d] == 1 && block[d] == 1 {
|
|
let sel_start = start[d];
|
|
let sel_end = start[d] + count[d];
|
|
let local_start = sel_start.max(chunk_start) - chunk_start;
|
|
let local_end = sel_end.min(chunk_end) - chunk_start;
|
|
ranges.push(local_start..local_end);
|
|
} else {
|
|
// General strided case: find all blocks that overlap this chunk
|
|
let sel_start = start[d];
|
|
let mut min_local = chunk_dims[d];
|
|
let mut max_local = 0u64;
|
|
|
|
for bi in 0..count[d] {
|
|
let block_start = sel_start + bi * stride[d];
|
|
let block_end = block_start + block[d];
|
|
// Check overlap with chunk
|
|
if block_end > chunk_start && block_start < chunk_end {
|
|
let local_s = block_start.max(chunk_start) - chunk_start;
|
|
let local_e = block_end.min(chunk_end) - chunk_start;
|
|
min_local = min_local.min(local_s);
|
|
max_local = max_local.max(local_e);
|
|
}
|
|
}
|
|
if max_local > min_local {
|
|
ranges.push(min_local..max_local);
|
|
} else {
|
|
ranges.push(0..0);
|
|
}
|
|
}
|
|
}
|
|
ranges
|
|
}
|
|
Selection::Points(_) => {
|
|
// For point selections, return the full chunk range
|
|
// (filtering happens at the element level)
|
|
chunk_dims.iter().map(|&d| 0..d).collect()
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Decode a selection from its on-disk **`H5S_select_serialize`** form.
|
|
///
|
|
/// Returns the selection and the number of bytes consumed (selections are
|
|
/// self-describing in length, so the count lets a caller walk a packed list
|
|
/// of selections — as the Virtual Dataset global-heap block does).
|
|
///
|
|
/// Decodes `ALL`, `NONE`, and hyperslabs at every version libhdf5 writes
|
|
/// (1: irregular, 4-byte coordinates — the default-format encoding; 2:
|
|
/// regular, 8-byte; 3: either, variable width). A regular hyperslab maps
|
|
/// to [`Selection::Hyperslab`]; an *irregular* one (a union of blocks)
|
|
/// maps to a single-block hyperslab when it has one block, and otherwise to
|
|
/// [`Selection::Points`] listing the union in row-major order (the order
|
|
/// libhdf5 iterates it in). Unlimited counts/blocks decode as `u64::MAX`
|
|
/// (see [`SerializedSelection::decode`] for the raw form). Point
|
|
/// selections are refused: libhdf5 does not allow them in virtual datasets
|
|
/// either.
|
|
pub fn decode_serialized(data: &[u8]) -> Result<(Selection, usize), FormatError> {
|
|
let (raw, len) = SerializedSelection::decode(data)?;
|
|
let sel = match raw {
|
|
SerializedSelection::All => Selection::All,
|
|
SerializedSelection::None => Selection::None,
|
|
SerializedSelection::Regular {
|
|
start,
|
|
stride,
|
|
count,
|
|
block,
|
|
} => Selection::Hyperslab {
|
|
start,
|
|
stride,
|
|
count,
|
|
block,
|
|
},
|
|
SerializedSelection::Blocks { rank, starts, ends } => {
|
|
if starts.len() == rank {
|
|
let block = starts.iter().zip(&ends).map(|(&s, &e)| e - s + 1).collect();
|
|
Selection::Hyperslab {
|
|
start: starts,
|
|
stride: vec![1; rank],
|
|
count: vec![1; rank],
|
|
block,
|
|
}
|
|
} else {
|
|
Selection::Points(blocks_union_coords(rank, &starts, &ends)?)
|
|
}
|
|
}
|
|
};
|
|
Ok((sel, len))
|
|
}
|
|
|
|
/// Enumerate the selected element indices of a **1-D** dataspace of the
|
|
/// given `extent`, in row-major selection order.
|
|
///
|
|
/// Convenience wrapper over [`Selection::iter_linear`] for rank-1 spaces.
|
|
pub fn iter_linear_1d(&self, extent: u64) -> Result<Vec<u64>, FormatError> {
|
|
self.iter_linear(&[extent])
|
|
}
|
|
|
|
/// Enumerate the **row-major linear indices** of the selected elements of a
|
|
/// dataspace with shape `dims`, in row-major (C) iteration order.
|
|
///
|
|
/// This is the order HDF5 uses to pair a virtual selection with a source
|
|
/// selection in a Virtual Dataset, so the i-th index returned here for the
|
|
/// virtual selection corresponds to the i-th index for the source
|
|
/// selection. Hyperslab/point selections whose rank differs from
|
|
/// `dims.len()` are rejected.
|
|
pub fn iter_linear(&self, dims: &[u64]) -> Result<Vec<u64>, FormatError> {
|
|
let overflow = || FormatError::Overflow("VDS selection index overflow".into());
|
|
let total: u64 = dims
|
|
.iter()
|
|
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
|
.ok_or_else(overflow)?;
|
|
// Row-major strides: row_stride[d] = product(dims[d+1..]).
|
|
let rank = dims.len();
|
|
let mut row_stride = vec![1u64; rank];
|
|
for d in (0..rank.saturating_sub(1)).rev() {
|
|
row_stride[d] = row_stride[d + 1]
|
|
.checked_mul(dims[d + 1])
|
|
.ok_or_else(overflow)?;
|
|
}
|
|
|
|
match self {
|
|
Selection::All => Ok((0..total).collect()),
|
|
Selection::None => Ok(Vec::new()),
|
|
Selection::Hyperslab {
|
|
start,
|
|
stride,
|
|
count,
|
|
block,
|
|
} => {
|
|
if start.len() != rank {
|
|
return Err(FormatError::ChunkedReadError(
|
|
"VDS selection rank does not match dataspace rank".into(),
|
|
));
|
|
}
|
|
if count.iter().chain(block.iter()).any(|&v| v == UNLIMITED) {
|
|
return Err(FormatError::ChunkedReadError(
|
|
"unlimited selection must be clipped before it is enumerated".into(),
|
|
));
|
|
}
|
|
// Selected coordinates along each dimension, in order.
|
|
let mut per_dim: Vec<Vec<u64>> = Vec::with_capacity(rank);
|
|
for d in 0..rank {
|
|
let mut coords = Vec::new();
|
|
for ci in 0..count[d] {
|
|
let base = ci
|
|
.checked_mul(stride[d])
|
|
.and_then(|o| start[d].checked_add(o))
|
|
.ok_or_else(overflow)?;
|
|
for bi in 0..block[d] {
|
|
let coord = base.checked_add(bi).ok_or_else(overflow)?;
|
|
// Anything past the extent is malformed; bail before the
|
|
// coordinate list can grow without bound.
|
|
if coord >= dims[d] {
|
|
return Err(FormatError::ChunkedReadError(
|
|
"VDS hyperslab selection exceeds dataspace extent".into(),
|
|
));
|
|
}
|
|
coords.push(coord);
|
|
}
|
|
}
|
|
per_dim.push(coords);
|
|
}
|
|
if per_dim.iter().any(|c| c.is_empty()) {
|
|
return Ok(Vec::new());
|
|
}
|
|
// Cartesian product in row-major order (dim 0 slowest-varying).
|
|
let out_len: usize = per_dim
|
|
.iter()
|
|
.try_fold(1usize, |acc, c| acc.checked_mul(c.len()))
|
|
.ok_or_else(overflow)?;
|
|
let mut out = Vec::with_capacity(out_len);
|
|
let mut idx = vec![0usize; rank];
|
|
loop {
|
|
let mut lin = 0u64;
|
|
for d in 0..rank {
|
|
lin = per_dim[d][idx[d]]
|
|
.checked_mul(row_stride[d])
|
|
.and_then(|o| lin.checked_add(o))
|
|
.ok_or_else(overflow)?;
|
|
}
|
|
out.push(lin);
|
|
// Increment the mixed-radix counter, last dimension fastest.
|
|
let mut carry = true;
|
|
for d in (0..rank).rev() {
|
|
idx[d] += 1;
|
|
if idx[d] < per_dim[d].len() {
|
|
carry = false;
|
|
break;
|
|
}
|
|
idx[d] = 0;
|
|
}
|
|
if carry {
|
|
break;
|
|
}
|
|
}
|
|
Ok(out)
|
|
}
|
|
Selection::Points(pts) => {
|
|
let mut out = Vec::with_capacity(pts.len());
|
|
for p in pts {
|
|
if p.len() != rank {
|
|
return Err(FormatError::ChunkedReadError(
|
|
"VDS point selection rank does not match dataspace rank".into(),
|
|
));
|
|
}
|
|
let mut lin = 0u64;
|
|
for d in 0..rank {
|
|
if p[d] >= dims[d] {
|
|
return Err(FormatError::ChunkedReadError(
|
|
"VDS point selection exceeds dataspace extent".into(),
|
|
));
|
|
}
|
|
lin = p[d]
|
|
.checked_mul(row_stride[d])
|
|
.and_then(|o| lin.checked_add(o))
|
|
.ok_or_else(overflow)?;
|
|
}
|
|
out.push(lin);
|
|
}
|
|
Ok(out)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Hyperslab count/block value meaning "unlimited" (`H5S_UNLIMITED`).
|
|
pub const UNLIMITED: u64 = u64::MAX;
|
|
|
|
/// Largest number of elements an irregular selection is expanded to when it
|
|
/// is converted to a point list by [`Selection::decode_serialized`].
|
|
const MAX_EXPANDED_POINTS: u64 = 1 << 26;
|
|
|
|
/// A selection exactly as `H5S_select_serialize` stores it, before it is
|
|
/// applied to any dataspace.
|
|
///
|
|
/// Unlike [`Selection`] this keeps an irregular hyperslab as its list of
|
|
/// blocks, and a regular hyperslab's count/block may be [`UNLIMITED`] (the
|
|
/// unlimited selections used by unlimited and "printf" virtual dataset
|
|
/// mappings).
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub enum SerializedSelection {
|
|
/// `H5S_SEL_ALL`.
|
|
All,
|
|
/// `H5S_SEL_NONE`.
|
|
None,
|
|
/// A regular hyperslab. `count[d]` or `block[d]` may be [`UNLIMITED`].
|
|
Regular {
|
|
start: Vec<u64>,
|
|
stride: Vec<u64>,
|
|
count: Vec<u64>,
|
|
block: Vec<u64>,
|
|
},
|
|
/// An irregular hyperslab: the union of `starts.len() / rank` blocks, each
|
|
/// given by its first (`starts`) and last (`ends`, inclusive) coordinate,
|
|
/// flattened block-major.
|
|
Blocks {
|
|
rank: usize,
|
|
starts: Vec<u64>,
|
|
ends: Vec<u64>,
|
|
},
|
|
}
|
|
|
|
fn sel_err(msg: &str) -> FormatError {
|
|
FormatError::ChunkedReadError(msg.into())
|
|
}
|
|
|
|
/// Bounds-checked little-endian reader over a serialized selection.
|
|
struct SelReader<'a> {
|
|
data: &'a [u8],
|
|
pos: usize,
|
|
}
|
|
|
|
impl SelReader<'_> {
|
|
fn take(&mut self, n: usize) -> Result<&[u8], FormatError> {
|
|
let end = self.pos.checked_add(n).filter(|&e| e <= self.data.len());
|
|
let end = end.ok_or(FormatError::UnexpectedEof {
|
|
expected: self.pos.saturating_add(n),
|
|
available: self.data.len(),
|
|
})?;
|
|
let s = &self.data[self.pos..end];
|
|
self.pos = end;
|
|
Ok(s)
|
|
}
|
|
|
|
fn uint(&mut self, size: usize) -> Result<u64, FormatError> {
|
|
let bytes = self.take(size)?;
|
|
Ok(bytes
|
|
.iter()
|
|
.enumerate()
|
|
.fold(0u64, |v, (i, &b)| v | (b as u64) << (i * 8)))
|
|
}
|
|
|
|
fn remaining(&self) -> usize {
|
|
self.data.len() - self.pos
|
|
}
|
|
}
|
|
|
|
impl SerializedSelection {
|
|
/// Decode a serialized selection, returning it and the number of bytes it
|
|
/// occupies. Mirrors libhdf5's `H5S_select_deserialize`: `ALL`/`NONE` and
|
|
/// hyperslab versions 1-3 are decoded; point selections (which libhdf5
|
|
/// refuses in virtual datasets) and malformed input are errors.
|
|
pub fn decode(data: &[u8]) -> Result<(SerializedSelection, usize), FormatError> {
|
|
let mut r = SelReader { data, pos: 0 };
|
|
let sel_type = r.uint(4)?;
|
|
let version = r.uint(4)?;
|
|
match sel_type {
|
|
// ALL / NONE: type(4) + version(4) + reserved(4) + length(4).
|
|
0 | 3 => {
|
|
r.take(8)?;
|
|
let sel = if sel_type == 3 {
|
|
SerializedSelection::All
|
|
} else {
|
|
SerializedSelection::None
|
|
};
|
|
Ok((sel, r.pos))
|
|
}
|
|
2 => {
|
|
let sel = decode_hyperslab(&mut r, version)?;
|
|
Ok((sel, r.pos))
|
|
}
|
|
1 => Err(sel_err(
|
|
"VDS point selections are not supported (libhdf5 rejects them too)",
|
|
)),
|
|
_ => Err(sel_err("unknown dataspace selection type")),
|
|
}
|
|
}
|
|
|
|
/// The single dimension in which this selection is unlimited, if any.
|
|
pub fn unlimited_dim(&self) -> Option<usize> {
|
|
match self {
|
|
SerializedSelection::Regular { count, block, .. } => count
|
|
.iter()
|
|
.zip(block)
|
|
.position(|(&c, &b)| c == UNLIMITED || b == UNLIMITED),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
/// The rank the selection was serialized with (`None` for ALL/NONE, which
|
|
/// carry no rank).
|
|
pub fn rank(&self) -> Option<usize> {
|
|
match self {
|
|
SerializedSelection::Regular { start, .. } => Some(start.len()),
|
|
SerializedSelection::Blocks { rank, .. } => Some(*rank),
|
|
_ => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `H5S__hyper_deserialize`: after the type and version words.
|
|
fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelection, FormatError> {
|
|
const REGULAR: u8 = 0x01;
|
|
let (flags, enc_size) = match version {
|
|
// v1: reserved(4) + length(4), always irregular, 4-byte coordinates.
|
|
1 => {
|
|
r.take(8)?;
|
|
(0u8, 4usize)
|
|
}
|
|
// v2: flags(1) + length(4), 8-byte coordinates.
|
|
2 => {
|
|
let flags = r.take(1)?[0];
|
|
r.take(4)?;
|
|
(flags, 8)
|
|
}
|
|
// v3: flags(1) + encoding size(1).
|
|
3 => {
|
|
let flags = r.take(1)?[0];
|
|
let enc = r.take(1)?[0] as usize;
|
|
(flags, enc)
|
|
}
|
|
_ => return Err(sel_err("unsupported hyperslab selection version")),
|
|
};
|
|
if flags & !REGULAR != 0 {
|
|
return Err(sel_err("unknown hyperslab selection flags"));
|
|
}
|
|
if !matches!(enc_size, 2 | 4 | 8) {
|
|
return Err(sel_err("unsupported hyperslab coordinate encoding size"));
|
|
}
|
|
let rank = r.uint(4)? as usize;
|
|
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a
|
|
// corrupt rank can't drive a huge allocation or read loop.
|
|
if rank == 0 || rank > 32 {
|
|
return Err(sel_err("hyperslab selection rank must be 1..=32"));
|
|
}
|
|
// The all-ones value of the encoding width means "unlimited".
|
|
let unlim_raw = if enc_size == 8 {
|
|
u64::MAX
|
|
} else {
|
|
(1u64 << (enc_size * 8)) - 1
|
|
};
|
|
|
|
if flags & REGULAR != 0 {
|
|
let (mut start, mut stride, mut count, mut block) = (
|
|
Vec::with_capacity(rank),
|
|
Vec::with_capacity(rank),
|
|
Vec::with_capacity(rank),
|
|
Vec::with_capacity(rank),
|
|
);
|
|
for _ in 0..rank {
|
|
start.push(r.uint(enc_size)?);
|
|
stride.push(r.uint(enc_size)?);
|
|
let c = r.uint(enc_size)?;
|
|
count.push(if c == unlim_raw { UNLIMITED } else { c });
|
|
let b = r.uint(enc_size)?;
|
|
block.push(if b == unlim_raw { UNLIMITED } else { b });
|
|
}
|
|
let unlimited = count
|
|
.iter()
|
|
.zip(&block)
|
|
.filter(|&(&c, &b)| c == UNLIMITED || b == UNLIMITED)
|
|
.count();
|
|
if unlimited > 1 {
|
|
return Err(sel_err(
|
|
"hyperslab selection is unlimited in more than one dimension",
|
|
));
|
|
}
|
|
for d in 0..rank {
|
|
// Overlapping blocks are not a valid regular hyperslab.
|
|
if count[d] > 1 && block[d] != UNLIMITED && block[d] > stride[d] {
|
|
return Err(sel_err("regular hyperslab blocks overlap"));
|
|
}
|
|
}
|
|
return Ok(SerializedSelection::Regular {
|
|
start,
|
|
stride,
|
|
count,
|
|
block,
|
|
});
|
|
}
|
|
|
|
// Irregular: number of blocks, then each block's start and end corners.
|
|
let nblocks = r.uint(enc_size)?;
|
|
let per_block = (rank * 2 * enc_size) as u64;
|
|
// Untrusted count: it must fit in what is left of the buffer.
|
|
if nblocks
|
|
.checked_mul(per_block)
|
|
.is_none_or(|need| need > r.remaining() as u64)
|
|
{
|
|
return Err(FormatError::UnexpectedEof {
|
|
expected: r
|
|
.pos
|
|
.saturating_add(nblocks.saturating_mul(per_block) as usize),
|
|
available: r.data.len(),
|
|
});
|
|
}
|
|
let n = nblocks as usize * rank;
|
|
let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n));
|
|
for _ in 0..nblocks {
|
|
for _ in 0..rank {
|
|
starts.push(r.uint(enc_size)?);
|
|
}
|
|
for _ in 0..rank {
|
|
ends.push(r.uint(enc_size)?);
|
|
}
|
|
}
|
|
if starts.iter().zip(&ends).any(|(s, e)| e < s) {
|
|
return Err(sel_err("hyperslab block ends before it starts"));
|
|
}
|
|
Ok(SerializedSelection::Blocks { rank, starts, ends })
|
|
}
|
|
|
|
/// The coordinates of the union of the given blocks, in row-major order.
|
|
fn blocks_union_coords(
|
|
rank: usize,
|
|
starts: &[u64],
|
|
ends: &[u64],
|
|
) -> Result<Vec<Vec<u64>>, FormatError> {
|
|
let mut total = 0u64;
|
|
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
|
let vol = s
|
|
.iter()
|
|
.zip(e)
|
|
.try_fold(1u64, |acc, (&s, &e)| acc.checked_mul(e - s + 1));
|
|
total = vol
|
|
.and_then(|v| total.checked_add(v))
|
|
.filter(|&t| t <= MAX_EXPANDED_POINTS)
|
|
.ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?;
|
|
}
|
|
let mut out = Vec::with_capacity(total as usize);
|
|
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
|
let mut cur = s.to_vec();
|
|
'block: loop {
|
|
out.push(cur.clone());
|
|
for d in (0..rank).rev() {
|
|
if cur[d] < e[d] {
|
|
cur[d] += 1;
|
|
continue 'block;
|
|
}
|
|
cur[d] = s[d];
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
// Lexicographic order of coordinates is row-major order.
|
|
out.sort_unstable();
|
|
out.dedup();
|
|
Ok(out)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn selection_all_num_elements() {
|
|
let sel = Selection::All;
|
|
assert_eq!(sel.num_elements(&[100, 200]), 20000);
|
|
}
|
|
|
|
#[test]
|
|
fn selection_none_num_elements() {
|
|
let sel = Selection::None;
|
|
assert_eq!(sel.num_elements(&[100, 200]), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn selection_slice_basic() {
|
|
let sel = Selection::slice(&[20..30, 40..60]);
|
|
assert_eq!(sel.num_elements(&[100, 100]), 200); // 10 * 20
|
|
assert_eq!(sel.output_shape(&[100, 100]), vec![10, 20]);
|
|
}
|
|
|
|
#[test]
|
|
fn selection_slice_1d() {
|
|
let sel = Selection::slice(std::slice::from_ref(&(5..15)));
|
|
assert_eq!(sel.num_elements(&[100]), 10);
|
|
assert_eq!(sel.output_shape(&[100]), vec![10]);
|
|
}
|
|
|
|
#[test]
|
|
fn selection_intersects_chunk_basic() {
|
|
let sel = Selection::slice(&[20..30, 40..60]);
|
|
|
|
// Chunk [20..30, 40..50] — overlaps
|
|
assert!(sel.intersects_chunk(&[20, 40], &[10, 10]));
|
|
|
|
// Chunk [0..10, 0..10] — no overlap
|
|
assert!(!sel.intersects_chunk(&[0, 0], &[10, 10]));
|
|
|
|
// Chunk [20..30, 50..60] — overlaps
|
|
assert!(sel.intersects_chunk(&[20, 50], &[10, 10]));
|
|
|
|
// Chunk [30..40, 40..50] — no overlap (just past end in dim 0)
|
|
assert!(!sel.intersects_chunk(&[30, 40], &[10, 10]));
|
|
}
|
|
|
|
#[test]
|
|
fn selection_chunk_local_ranges_simple() {
|
|
let sel = Selection::slice(&[25..35, 40..60]);
|
|
|
|
// Chunk [20..30, 40..50]
|
|
let ranges = sel.chunk_local_ranges(&[20, 40], &[10, 10]);
|
|
assert_eq!(ranges[0], 5..10); // rows 25..30 within chunk starting at 20
|
|
assert_eq!(ranges[1], 0..10); // cols 40..50 fully selected
|
|
}
|
|
|
|
#[test]
|
|
fn selection_points() {
|
|
let sel = Selection::Points(vec![vec![1, 2], vec![3, 4], vec![5, 6]]);
|
|
assert_eq!(sel.num_elements(&[10, 10]), 3);
|
|
assert_eq!(sel.rank(), Some(2));
|
|
}
|
|
|
|
#[test]
|
|
fn selection_all_intersects_any_chunk() {
|
|
let sel = Selection::All;
|
|
assert!(sel.intersects_chunk(&[0, 0], &[10, 10]));
|
|
assert!(sel.intersects_chunk(&[100, 100], &[1, 1]));
|
|
}
|
|
|
|
#[test]
|
|
fn selection_hyperslab_strided() {
|
|
// Select every other row: start=0, stride=2, count=5, block=1 in a 10-element dim
|
|
let sel = Selection::Hyperslab {
|
|
start: vec![0],
|
|
stride: vec![2],
|
|
count: vec![5],
|
|
block: vec![1],
|
|
};
|
|
assert_eq!(sel.num_elements(&[10]), 5); // 5 blocks * 1 element each
|
|
|
|
// Chunk [0..5] should intersect (contains rows 0, 2, 4)
|
|
assert!(sel.intersects_chunk(&[0], &[5]));
|
|
// Chunk [9..10] should not intersect (only row 9, but selection ends at row 8)
|
|
assert!(!sel.intersects_chunk(&[9], &[1]));
|
|
}
|
|
|
|
#[test]
|
|
fn decode_all_selection_16_bytes() {
|
|
let bytes = [3u8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
|
let (sel, consumed) = Selection::decode_serialized(&bytes).unwrap();
|
|
assert_eq!(sel, Selection::All);
|
|
assert_eq!(consumed, 16);
|
|
assert_eq!(sel.iter_linear_1d(4).unwrap(), vec![0, 1, 2, 3]);
|
|
}
|
|
|
|
#[test]
|
|
fn decode_regular_hyperslab_matches_vds_fixture() {
|
|
// Exact virtual selection for src_a in the VDS fixture:
|
|
// start=0 stride=1 count=1 block=4, version 3, enc_size 2, rank 1.
|
|
let bytes = [
|
|
0x02, 0, 0, 0, // type = HYPER
|
|
0x03, 0, 0, 0, // version 3
|
|
0x01, // flags = regular
|
|
0x02, // enc_size = 2
|
|
0x01, 0, 0, 0, // rank = 1
|
|
0x00, 0x00, // start
|
|
0x01, 0x00, // stride
|
|
0x01, 0x00, // count
|
|
0x04, 0x00, // block
|
|
];
|
|
let (sel, consumed) = Selection::decode_serialized(&bytes).unwrap();
|
|
assert_eq!(consumed, 22);
|
|
assert_eq!(
|
|
sel,
|
|
Selection::Hyperslab {
|
|
start: vec![0],
|
|
stride: vec![1],
|
|
count: vec![1],
|
|
block: vec![4],
|
|
}
|
|
);
|
|
assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]);
|
|
}
|
|
|
|
#[test]
|
|
fn decode_hyperslab_start4() {
|
|
let bytes = [
|
|
0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, //
|
|
0x04, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00,
|
|
];
|
|
let (sel, _) = Selection::decode_serialized(&bytes).unwrap();
|
|
assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![4, 5, 6, 7]);
|
|
}
|
|
|
|
#[test]
|
|
fn decode_strided_hyperslab_iter() {
|
|
// start=1 stride=3 count=2 block=2 => 1,2, 4,5
|
|
let bytes = [
|
|
0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, //
|
|
0x01, 0x00, 0x03, 0x00, 0x02, 0x00, 0x02, 0x00,
|
|
];
|
|
let (sel, _) = Selection::decode_serialized(&bytes).unwrap();
|
|
assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![1, 2, 4, 5]);
|
|
}
|
|
|
|
#[test]
|
|
fn decode_nd_hyperslab_iter_rejected() {
|
|
let bytes = [
|
|
0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x02, 0, 0, 0, // rank 2
|
|
0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 1, 0, 1, 0, 2, 0,
|
|
];
|
|
let (sel, _) = Selection::decode_serialized(&bytes).unwrap();
|
|
assert!(sel.iter_linear_1d(16).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn decode_truncated_irregular_hyperslab_is_error() {
|
|
// Irregular, rank 1, but the block count is missing.
|
|
let bytes = [0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x00, 0x02, 0x01, 0, 0, 0];
|
|
assert!(Selection::decode_serialized(&bytes).is_err());
|
|
}
|
|
|
|
/// Version 1 as libhdf5 writes it for the default (earliest) format bounds:
|
|
/// type, version, reserved(4), length(4), rank(4), nblocks(4), then each
|
|
/// block's start and inclusive end corner as 4-byte values.
|
|
fn v1_blocks(rank: u32, blocks: &[(&[u32], &[u32])]) -> Vec<u8> {
|
|
let mut b = Vec::new();
|
|
for w in [2u32, 1, 0, 0, rank, blocks.len() as u32] {
|
|
b.extend_from_slice(&w.to_le_bytes());
|
|
}
|
|
for (s, e) in blocks {
|
|
for v in s.iter().chain(e.iter()) {
|
|
b.extend_from_slice(&v.to_le_bytes());
|
|
}
|
|
}
|
|
b
|
|
}
|
|
|
|
#[test]
|
|
fn decode_v1_irregular_single_block() {
|
|
// Exactly what h5py/HDF5 2.0 writes for `[0:4]` with default libver.
|
|
let bytes = v1_blocks(1, &[(&[0], &[3])]);
|
|
let (sel, used) = Selection::decode_serialized(&bytes).unwrap();
|
|
assert_eq!(used, bytes.len());
|
|
assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]);
|
|
}
|
|
|
|
#[test]
|
|
fn decode_v1_irregular_union_is_row_major() {
|
|
// Blocks given out of order and overlapping still enumerate once each,
|
|
// in row-major order (libhdf5 iterates the union, not the list).
|
|
let bytes = v1_blocks(2, &[(&[1, 0], &[1, 1]), (&[0, 2], &[1, 2])]);
|
|
let (sel, used) = Selection::decode_serialized(&bytes).unwrap();
|
|
assert_eq!(used, bytes.len());
|
|
// (0,2) (1,0) (1,1) (1,2) in a 2x3 space.
|
|
assert_eq!(sel.iter_linear(&[2, 3]).unwrap(), vec![2, 3, 4, 5]);
|
|
}
|
|
|
|
#[test]
|
|
fn decode_v2_regular_with_unlimited_count() {
|
|
// v2: flags(1) + length(4), then 8-byte start/stride/count/block.
|
|
let mut b = Vec::new();
|
|
b.extend_from_slice(&2u32.to_le_bytes());
|
|
b.extend_from_slice(&2u32.to_le_bytes());
|
|
b.push(0x01);
|
|
b.extend_from_slice(&36u32.to_le_bytes());
|
|
b.extend_from_slice(&1u32.to_le_bytes());
|
|
for v in [0u64, 10, u64::MAX, 10] {
|
|
b.extend_from_slice(&v.to_le_bytes());
|
|
}
|
|
let (raw, used) = SerializedSelection::decode(&b).unwrap();
|
|
assert_eq!(used, b.len());
|
|
assert_eq!(raw.unlimited_dim(), Some(0));
|
|
assert_eq!(
|
|
raw,
|
|
SerializedSelection::Regular {
|
|
start: vec![0],
|
|
stride: vec![10],
|
|
count: vec![UNLIMITED],
|
|
block: vec![10],
|
|
}
|
|
);
|
|
// An unclipped unlimited selection cannot be enumerated.
|
|
let (sel, _) = Selection::decode_serialized(&b).unwrap();
|
|
assert!(sel.iter_linear_1d(100).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn decode_v3_two_byte_all_ones_is_unlimited() {
|
|
let bytes = [
|
|
0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, //
|
|
0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0xFF, 0xFF,
|
|
];
|
|
let (raw, _) = SerializedSelection::decode(&bytes).unwrap();
|
|
assert_eq!(raw.unlimited_dim(), Some(0));
|
|
}
|
|
|
|
#[test]
|
|
fn decode_irregular_block_count_beyond_buffer_is_error() {
|
|
let mut b = v1_blocks(1, &[(&[0], &[3])]);
|
|
b[20..24].copy_from_slice(&u32::MAX.to_le_bytes());
|
|
assert!(Selection::decode_serialized(&b).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn decode_point_selection_is_refused() {
|
|
let bytes = [1u8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
|
assert!(Selection::decode_serialized(&bytes).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn iter_linear_2d_block_row_major() {
|
|
// A 2x2 block at the top-left of a 4x4 space => linear 0,1,4,5.
|
|
let sel = Selection::Hyperslab {
|
|
start: vec![0, 0],
|
|
stride: vec![1, 1],
|
|
count: vec![1, 1],
|
|
block: vec![2, 2],
|
|
};
|
|
assert_eq!(sel.iter_linear(&[4, 4]).unwrap(), vec![0, 1, 4, 5]);
|
|
|
|
// The same block shifted to the bottom-right => 10,11,14,15.
|
|
let sel2 = Selection::Hyperslab {
|
|
start: vec![2, 2],
|
|
stride: vec![1, 1],
|
|
count: vec![1, 1],
|
|
block: vec![2, 2],
|
|
};
|
|
assert_eq!(sel2.iter_linear(&[4, 4]).unwrap(), vec![10, 11, 14, 15]);
|
|
}
|
|
|
|
#[test]
|
|
fn iter_linear_2d_strided() {
|
|
// start=(0,0) stride=(2,2) count=(2,2) block=(1,1) over 4x4 =>
|
|
// coords (0,0)(0,2)(2,0)(2,2) => linear 0,2,8,10.
|
|
let sel = Selection::Hyperslab {
|
|
start: vec![0, 0],
|
|
stride: vec![2, 2],
|
|
count: vec![2, 2],
|
|
block: vec![1, 1],
|
|
};
|
|
assert_eq!(sel.iter_linear(&[4, 4]).unwrap(), vec![0, 2, 8, 10]);
|
|
}
|
|
|
|
#[test]
|
|
fn iter_linear_all_2d() {
|
|
assert_eq!(
|
|
Selection::All.iter_linear(&[2, 3]).unwrap(),
|
|
(0..6).collect::<Vec<_>>()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn iter_linear_rank_mismatch_rejected() {
|
|
let sel = Selection::Hyperslab {
|
|
start: vec![0],
|
|
stride: vec![1],
|
|
count: vec![1],
|
|
block: vec![2],
|
|
};
|
|
assert!(sel.iter_linear(&[4, 4]).is_err());
|
|
}
|
|
|
|
// ----- Adversarial / hardening: malformed input must error, never panic -----
|
|
|
|
#[test]
|
|
fn decode_all_truncated_does_not_overrun() {
|
|
// ALL claims to consume 16 bytes but only 8 are present.
|
|
let bytes = [3u8, 0, 0, 0, 1, 0, 0, 0];
|
|
assert!(Selection::decode_serialized(&bytes).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn decode_hyperslab_huge_rank_rejected() {
|
|
// rank = 0xFFFFFFFF must not drive a giant allocation.
|
|
let bytes = [
|
|
0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0xFF, 0xFF, 0xFF, 0xFF,
|
|
];
|
|
assert!(Selection::decode_serialized(&bytes).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn iter_linear_hyperslab_overflow_is_error() {
|
|
// start/stride/count near u64::MAX must not panic on multiply/add.
|
|
let sel = Selection::Hyperslab {
|
|
start: vec![u64::MAX - 1],
|
|
stride: vec![u64::MAX],
|
|
count: vec![u64::MAX],
|
|
block: vec![u64::MAX],
|
|
};
|
|
assert!(sel.iter_linear(&[100]).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn iter_linear_dims_product_overflow_is_error() {
|
|
assert!(Selection::All.iter_linear(&[u64::MAX, u64::MAX]).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn decode_empty_or_short_is_error_not_panic() {
|
|
assert!(Selection::decode_serialized(&[]).is_err());
|
|
assert!(Selection::decode_serialized(&[2, 0, 0, 0, 3, 0]).is_err());
|
|
}
|
|
}
|