From 190918a478bfc8bc412f9c58e5a70b087e560d0b Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:57:32 -0500 Subject: [PATCH] feat(format): decode hyperslab selection versions 1 and 2 in VDS mappings libhdf5 serializes a VDS hyperslab as version 1 (irregular, 4-byte block corners) for the default format bounds, and as version 2 (regular, 8-byte) for unlimited selections in the 1.10 format. Only version 3 was accepted, so every h5py VDS written with default libver failed with "only version-3 hyperslab selections are supported" (5 libhdf5 test files in the sweep). Decode all three versions following H5S__hyper_deserialize, including irregular hyperslabs (a union of blocks, enumerated in row-major order as libhdf5 iterates them) and the all-ones "unlimited" count/block marker. SerializedSelection exposes the raw form for unlimited-mapping support. Test: vds_interop::vds_version1_irregular_hyperslab_selections compares default-libver h5py VDS reads (contiguous, strided and 2-D block mappings) with libhdf5's values. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 + crates/clawhdf5-format/src/selection.rs | 498 +++++++++++++++++++----- crates/clawhdf5/tests/vds_interop.rs | 156 ++++++++ docs/known-issues.md | 3 +- 4 files changed, 562 insertions(+), 104 deletions(-) create mode 100644 crates/clawhdf5/tests/vds_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b87415..a349fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,6 +273,15 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- `clawhdf5-format` virtual datasets (VDS), checked against HDF5 2.0 through + h5py (`crates/clawhdf5/tests/vds_interop.rs`): + - Hyperslab selection versions 1 and 2 were refused ("only version-3 + hyperslab selections are supported"). Version 1 is what libhdf5 writes for + every VDS created with the default format bounds (h5py's default), so + those could not be read at all; version 2 is its encoding of an unlimited + selection. Both are decoded now, as are irregular hyperslabs (a union of + blocks, read in row-major order as libhdf5 iterates them). + `SerializedSelection` exposes the raw form, including unlimited counts. - `clawhdf5-format` reader — **values returned wrong with no error:** - Fixed Array and Extensible Array chunk indexes were laid out by the dataset's current shape instead of its max shape (23 libhdf5 test files, diff --git a/crates/clawhdf5-format/src/selection.rs b/crates/clawhdf5-format/src/selection.rs index e1f98aa..de31e46 100644 --- a/crates/clawhdf5-format/src/selection.rs +++ b/crates/clawhdf5-format/src/selection.rs @@ -229,44 +229,47 @@ impl Selection { /// self-describing in length, so the count lets a caller walk a packed list /// of selections — as the Virtual Dataset global-heap block does). /// - /// Only the forms needed for VDS assembly are decoded: `ALL`, `NONE`, and - /// **regular** hyperslabs serialized at **version 3** (the encoding HDF5 - /// 1.10+/2.0 emit). Point selections, irregular hyperslabs, and older - /// hyperslab versions return an error rather than mis-decoding. + /// Decodes `ALL`, `NONE`, and hyperslabs at every version libhdf5 writes + /// (1: irregular, 4-byte coordinates — the default-format encoding; 2: + /// regular, 8-byte; 3: either, variable width). A regular hyperslab maps + /// to [`Selection::Hyperslab`]; an *irregular* one (a union of blocks) + /// maps to a single-block hyperslab when it has one block, and otherwise to + /// [`Selection::Points`] listing the union in row-major order (the order + /// libhdf5 iterates it in). Unlimited counts/blocks decode as `u64::MAX` + /// (see [`SerializedSelection::decode`] for the raw form). Point + /// selections are refused: libhdf5 does not allow them in virtual datasets + /// either. pub fn decode_serialized(data: &[u8]) -> Result<(Selection, usize), FormatError> { - if data.len() < 8 { - return Err(FormatError::UnexpectedEof { - expected: 8, - available: data.len(), - }); - } - let sel_type = u32::from_le_bytes([data[0], data[1], data[2], data[3]]); - let version = u32::from_le_bytes([data[4], data[5], data[6], data[7]]); - - match sel_type { - // ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes. - 3 | 0 => { - if data.len() < 16 { - return Err(FormatError::UnexpectedEof { - expected: 16, - available: data.len(), - }); - } - let sel = if sel_type == 3 { - Selection::All + let (raw, len) = SerializedSelection::decode(data)?; + let sel = match raw { + SerializedSelection::All => Selection::All, + SerializedSelection::None => Selection::None, + SerializedSelection::Regular { + start, + stride, + count, + block, + } => Selection::Hyperslab { + start, + stride, + count, + block, + }, + SerializedSelection::Blocks { rank, starts, ends } => { + if starts.len() == rank { + let block = starts.iter().zip(&ends).map(|(&s, &e)| e - s + 1).collect(); + Selection::Hyperslab { + start: starts, + stride: vec![1; rank], + count: vec![1; rank], + block, + } } else { - Selection::None - }; - Ok((sel, 16)) + Selection::Points(blocks_union_coords(rank, &starts, &ends)?) + } } - 2 => decode_hyperslab_serialized(data, version), - 1 => Err(FormatError::ChunkedReadError( - "VDS point selections are not supported".into(), - )), - _ => Err(FormatError::ChunkedReadError( - "unknown dataspace selection type".into(), - )), - } + }; + Ok((sel, len)) } /// Enumerate the selected element indices of a **1-D** dataspace of the @@ -314,6 +317,11 @@ impl Selection { "VDS selection rank does not match dataspace rank".into(), )); } + if count.iter().chain(block.iter()).any(|&v| v == UNLIMITED) { + return Err(FormatError::ChunkedReadError( + "unlimited selection must be clipped before it is enumerated".into(), + )); + } // Selected coordinates along each dimension, in order. let mut per_dim: Vec> = Vec::with_capacity(rank); for d in 0..rank { @@ -400,84 +408,279 @@ impl Selection { } } -/// Decode an `H5S_SEL_HYPER` selection in its serialized form. Only version-3 -/// **regular** hyperslabs are supported. -fn decode_hyperslab_serialized( - data: &[u8], - version: u32, -) -> Result<(Selection, usize), FormatError> { - if version != 3 { - return Err(FormatError::ChunkedReadError( - "only version-3 hyperslab selections are supported".into(), - )); +/// Hyperslab count/block value meaning "unlimited" (`H5S_UNLIMITED`). +pub const UNLIMITED: u64 = u64::MAX; + +/// Largest number of elements an irregular selection is expanded to when it +/// is converted to a point list by [`Selection::decode_serialized`]. +const MAX_EXPANDED_POINTS: u64 = 1 << 26; + +/// A selection exactly as `H5S_select_serialize` stores it, before it is +/// applied to any dataspace. +/// +/// Unlike [`Selection`] this keeps an irregular hyperslab as its list of +/// blocks, and a regular hyperslab's count/block may be [`UNLIMITED`] (the +/// unlimited selections used by unlimited and "printf" virtual dataset +/// mappings). +#[derive(Debug, Clone, PartialEq)] +pub enum SerializedSelection { + /// `H5S_SEL_ALL`. + All, + /// `H5S_SEL_NONE`. + None, + /// A regular hyperslab. `count[d]` or `block[d]` may be [`UNLIMITED`]. + Regular { + start: Vec, + stride: Vec, + count: Vec, + block: Vec, + }, + /// An irregular hyperslab: the union of `starts.len() / rank` blocks, each + /// given by its first (`starts`) and last (`ends`, inclusive) coordinate, + /// flattened block-major. + Blocks { + rank: usize, + starts: Vec, + ends: Vec, + }, +} + +fn sel_err(msg: &str) -> FormatError { + FormatError::ChunkedReadError(msg.into()) +} + +/// Bounds-checked little-endian reader over a serialized selection. +struct SelReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl SelReader<'_> { + fn take(&mut self, n: usize) -> Result<&[u8], FormatError> { + let end = self.pos.checked_add(n).filter(|&e| e <= self.data.len()); + let end = end.ok_or(FormatError::UnexpectedEof { + expected: self.pos.saturating_add(n), + available: self.data.len(), + })?; + let s = &self.data[self.pos..end]; + self.pos = end; + Ok(s) } - // type(4) ver(4) flags(1) enc_size(1) rank(4) [start,stride,count,block]*rank - if data.len() < 14 { - return Err(FormatError::UnexpectedEof { - expected: 14, - available: data.len(), - }); + + fn uint(&mut self, size: usize) -> Result { + let bytes = self.take(size)?; + Ok(bytes + .iter() + .enumerate() + .fold(0u64, |v, (i, &b)| v | (b as u64) << (i * 8))) } - let flags = data[8]; - let enc_size = data[9] as usize; - // Bit 0 set => regular hyperslab. Irregular hyperslabs list explicit blocks. - if flags & 0x01 == 0 { - return Err(FormatError::ChunkedReadError( - "irregular VDS hyperslab selections are not supported".into(), - )); + + fn remaining(&self) -> usize { + self.data.len() - self.pos } - if enc_size != 2 && enc_size != 4 && enc_size != 8 { - return Err(FormatError::ChunkedReadError( - "unsupported hyperslab coordinate encoding size".into(), - )); - } - let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize; - // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything larger so a - // corrupt rank can't drive a huge allocation or read loop. - if rank > 32 { - return Err(FormatError::ChunkedReadError( - "hyperslab selection rank exceeds maximum (32)".into(), - )); - } - let mut pos = 14; - let read_coord = |data: &[u8], pos: usize| -> Result { - if pos + enc_size > data.len() { - return Err(FormatError::UnexpectedEof { - expected: pos + enc_size, - available: data.len(), - }); +} + +impl SerializedSelection { + /// Decode a serialized selection, returning it and the number of bytes it + /// occupies. Mirrors libhdf5's `H5S_select_deserialize`: `ALL`/`NONE` and + /// hyperslab versions 1-3 are decoded; point selections (which libhdf5 + /// refuses in virtual datasets) and malformed input are errors. + pub fn decode(data: &[u8]) -> Result<(SerializedSelection, usize), FormatError> { + let mut r = SelReader { data, pos: 0 }; + let sel_type = r.uint(4)?; + let version = r.uint(4)?; + match sel_type { + // ALL / NONE: type(4) + version(4) + reserved(4) + length(4). + 0 | 3 => { + r.take(8)?; + let sel = if sel_type == 3 { + SerializedSelection::All + } else { + SerializedSelection::None + }; + Ok((sel, r.pos)) + } + 2 => { + let sel = decode_hyperslab(&mut r, version)?; + Ok((sel, r.pos)) + } + 1 => Err(sel_err( + "VDS point selections are not supported (libhdf5 rejects them too)", + )), + _ => Err(sel_err("unknown dataspace selection type")), } - let mut v = 0u64; - for (i, &b) in data[pos..pos + enc_size].iter().enumerate() { - v |= (b as u64) << (i * 8); + } + + /// The single dimension in which this selection is unlimited, if any. + pub fn unlimited_dim(&self) -> Option { + match self { + SerializedSelection::Regular { count, block, .. } => count + .iter() + .zip(block) + .position(|(&c, &b)| c == UNLIMITED || b == UNLIMITED), + _ => None, } - Ok(v) + } + + /// The rank the selection was serialized with (`None` for ALL/NONE, which + /// carry no rank). + pub fn rank(&self) -> Option { + match self { + SerializedSelection::Regular { start, .. } => Some(start.len()), + SerializedSelection::Blocks { rank, .. } => Some(*rank), + _ => None, + } + } +} + +/// `H5S__hyper_deserialize`: after the type and version words. +fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result { + const REGULAR: u8 = 0x01; + let (flags, enc_size) = match version { + // v1: reserved(4) + length(4), always irregular, 4-byte coordinates. + 1 => { + r.take(8)?; + (0u8, 4usize) + } + // v2: flags(1) + length(4), 8-byte coordinates. + 2 => { + let flags = r.take(1)?[0]; + r.take(4)?; + (flags, 8) + } + // v3: flags(1) + encoding size(1). + 3 => { + let flags = r.take(1)?[0]; + let enc = r.take(1)?[0] as usize; + (flags, enc) + } + _ => return Err(sel_err("unsupported hyperslab selection version")), }; - let (mut start, mut stride, mut count, mut block) = ( - Vec::with_capacity(rank), - Vec::with_capacity(rank), - Vec::with_capacity(rank), - Vec::with_capacity(rank), - ); - for _ in 0..rank { - start.push(read_coord(data, pos)?); - pos += enc_size; - stride.push(read_coord(data, pos)?); - pos += enc_size; - count.push(read_coord(data, pos)?); - pos += enc_size; - block.push(read_coord(data, pos)?); - pos += enc_size; + if flags & !REGULAR != 0 { + return Err(sel_err("unknown hyperslab selection flags")); } - Ok(( - Selection::Hyperslab { + if !matches!(enc_size, 2 | 4 | 8) { + return Err(sel_err("unsupported hyperslab coordinate encoding size")); + } + let rank = r.uint(4)? as usize; + // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a + // corrupt rank can't drive a huge allocation or read loop. + if rank == 0 || rank > 32 { + return Err(sel_err("hyperslab selection rank must be 1..=32")); + } + // The all-ones value of the encoding width means "unlimited". + let unlim_raw = if enc_size == 8 { + u64::MAX + } else { + (1u64 << (enc_size * 8)) - 1 + }; + + if flags & REGULAR != 0 { + let (mut start, mut stride, mut count, mut block) = ( + Vec::with_capacity(rank), + Vec::with_capacity(rank), + Vec::with_capacity(rank), + Vec::with_capacity(rank), + ); + for _ in 0..rank { + start.push(r.uint(enc_size)?); + stride.push(r.uint(enc_size)?); + let c = r.uint(enc_size)?; + count.push(if c == unlim_raw { UNLIMITED } else { c }); + let b = r.uint(enc_size)?; + block.push(if b == unlim_raw { UNLIMITED } else { b }); + } + let unlimited = count + .iter() + .zip(&block) + .filter(|&(&c, &b)| c == UNLIMITED || b == UNLIMITED) + .count(); + if unlimited > 1 { + return Err(sel_err( + "hyperslab selection is unlimited in more than one dimension", + )); + } + for d in 0..rank { + // Overlapping blocks are not a valid regular hyperslab. + if count[d] > 1 && block[d] != UNLIMITED && block[d] > stride[d] { + return Err(sel_err("regular hyperslab blocks overlap")); + } + } + return Ok(SerializedSelection::Regular { start, stride, count, block, - }, - pos, - )) + }); + } + + // Irregular: number of blocks, then each block's start and end corners. + let nblocks = r.uint(enc_size)?; + let per_block = (rank * 2 * enc_size) as u64; + // Untrusted count: it must fit in what is left of the buffer. + if nblocks + .checked_mul(per_block) + .is_none_or(|need| need > r.remaining() as u64) + { + return Err(FormatError::UnexpectedEof { + expected: r + .pos + .saturating_add(nblocks.saturating_mul(per_block) as usize), + available: r.data.len(), + }); + } + let n = nblocks as usize * rank; + let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n)); + for _ in 0..nblocks { + for _ in 0..rank { + starts.push(r.uint(enc_size)?); + } + for _ in 0..rank { + ends.push(r.uint(enc_size)?); + } + } + if starts.iter().zip(&ends).any(|(s, e)| e < s) { + return Err(sel_err("hyperslab block ends before it starts")); + } + Ok(SerializedSelection::Blocks { rank, starts, ends }) +} + +/// The coordinates of the union of the given blocks, in row-major order. +fn blocks_union_coords( + rank: usize, + starts: &[u64], + ends: &[u64], +) -> Result>, FormatError> { + let mut total = 0u64; + for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { + let vol = s + .iter() + .zip(e) + .try_fold(1u64, |acc, (&s, &e)| acc.checked_mul(e - s + 1)); + total = vol + .and_then(|v| total.checked_add(v)) + .filter(|&t| t <= MAX_EXPANDED_POINTS) + .ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?; + } + let mut out = Vec::with_capacity(total as usize); + for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { + let mut cur = s.to_vec(); + 'block: loop { + out.push(cur.clone()); + for d in (0..rank).rev() { + if cur[d] < e[d] { + cur[d] += 1; + continue 'block; + } + cur[d] = s[d]; + } + break; + } + } + // Lexicographic order of coordinates is row-major order. + out.sort_unstable(); + out.dedup(); + Ok(out) } // --------------------------------------------------------------------------- @@ -642,11 +845,100 @@ mod tests { } #[test] - fn decode_irregular_hyperslab_rejected() { + fn decode_truncated_irregular_hyperslab_is_error() { + // Irregular, rank 1, but the block count is missing. let bytes = [0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x00, 0x02, 0x01, 0, 0, 0]; assert!(Selection::decode_serialized(&bytes).is_err()); } + /// Version 1 as libhdf5 writes it for the default (earliest) format bounds: + /// type, version, reserved(4), length(4), rank(4), nblocks(4), then each + /// block's start and inclusive end corner as 4-byte values. + fn v1_blocks(rank: u32, blocks: &[(&[u32], &[u32])]) -> Vec { + let mut b = Vec::new(); + for w in [2u32, 1, 0, 0, rank, blocks.len() as u32] { + b.extend_from_slice(&w.to_le_bytes()); + } + for (s, e) in blocks { + for v in s.iter().chain(e.iter()) { + b.extend_from_slice(&v.to_le_bytes()); + } + } + b + } + + #[test] + fn decode_v1_irregular_single_block() { + // Exactly what h5py/HDF5 2.0 writes for `[0:4]` with default libver. + let bytes = v1_blocks(1, &[(&[0], &[3])]); + let (sel, used) = Selection::decode_serialized(&bytes).unwrap(); + assert_eq!(used, bytes.len()); + assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]); + } + + #[test] + fn decode_v1_irregular_union_is_row_major() { + // Blocks given out of order and overlapping still enumerate once each, + // in row-major order (libhdf5 iterates the union, not the list). + let bytes = v1_blocks(2, &[(&[1, 0], &[1, 1]), (&[0, 2], &[1, 2])]); + let (sel, used) = Selection::decode_serialized(&bytes).unwrap(); + assert_eq!(used, bytes.len()); + // (0,2) (1,0) (1,1) (1,2) in a 2x3 space. + assert_eq!(sel.iter_linear(&[2, 3]).unwrap(), vec![2, 3, 4, 5]); + } + + #[test] + fn decode_v2_regular_with_unlimited_count() { + // v2: flags(1) + length(4), then 8-byte start/stride/count/block. + let mut b = Vec::new(); + b.extend_from_slice(&2u32.to_le_bytes()); + b.extend_from_slice(&2u32.to_le_bytes()); + b.push(0x01); + b.extend_from_slice(&36u32.to_le_bytes()); + b.extend_from_slice(&1u32.to_le_bytes()); + for v in [0u64, 10, u64::MAX, 10] { + b.extend_from_slice(&v.to_le_bytes()); + } + let (raw, used) = SerializedSelection::decode(&b).unwrap(); + assert_eq!(used, b.len()); + assert_eq!(raw.unlimited_dim(), Some(0)); + assert_eq!( + raw, + SerializedSelection::Regular { + start: vec![0], + stride: vec![10], + count: vec![UNLIMITED], + block: vec![10], + } + ); + // An unclipped unlimited selection cannot be enumerated. + let (sel, _) = Selection::decode_serialized(&b).unwrap(); + assert!(sel.iter_linear_1d(100).is_err()); + } + + #[test] + fn decode_v3_two_byte_all_ones_is_unlimited() { + let bytes = [ + 0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, // + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0xFF, 0xFF, + ]; + let (raw, _) = SerializedSelection::decode(&bytes).unwrap(); + assert_eq!(raw.unlimited_dim(), Some(0)); + } + + #[test] + fn decode_irregular_block_count_beyond_buffer_is_error() { + let mut b = v1_blocks(1, &[(&[0], &[3])]); + b[20..24].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(Selection::decode_serialized(&b).is_err()); + } + + #[test] + fn decode_point_selection_is_refused() { + let bytes = [1u8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + assert!(Selection::decode_serialized(&bytes).is_err()); + } + #[test] fn iter_linear_2d_block_row_major() { // A 2x2 block at the top-left of a 4x4 space => linear 0,1,4,5. diff --git a/crates/clawhdf5/tests/vds_interop.rs b/crates/clawhdf5/tests/vds_interop.rs new file mode 100644 index 0000000..43eaee4 --- /dev/null +++ b/crates/clawhdf5/tests/vds_interop.rs @@ -0,0 +1,156 @@ +//! Virtual Dataset (VDS) reads checked against libhdf5 (through h5py). +//! +//! Each test has h5py build virtual datasets and their source files in a temp +//! directory, record what libhdf5 reads back (shape and values) next to them, +//! and then compares that with what clawhdf5 reads from the same files. +//! +//! Skipped when python3 or h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::Command; + +use clawhdf5::File; + +/// The Python interpreter to drive interop checks with (`CLAWHDF5_PYTHON` +/// lets these run against a virtualenv holding h5py). +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency +/// is a test failure instead of a silent skip. +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +/// Prelude for every generator script: `expect(file, dset, tag)` records what +/// libhdf5 reads for `file:dset` as `.expect` (shape line, values line). +const PRELUDE: &str = r#" +import h5py, numpy as np +def expect(fn, dset, tag): + with h5py.File(fn, "r") as f: + d = f[dset] + a = d[...] + with open(tag + ".expect", "w") as out: + out.write(" ".join(str(n) for n in d.shape) + "\n") + out.write(" ".join(repr(float(v)) for v in a.ravel()) + "\n") +"#; + +/// Run `body` (after [`PRELUDE`]) with `dir` as the working directory, so +/// relative source file names land next to the virtual file. +fn generate(dir: &Path, body: &str) { + let script = format!("{PRELUDE}\n{body}"); + let out = Command::new(python()) + .args(["-c", &script]) + .current_dir(dir) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "generator failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); +} + +/// What libhdf5 read for `tag`: (shape, values as f64). +fn expected(dir: &Path, tag: &str) -> (Vec, Vec) { + let text = std::fs::read_to_string(dir.join(format!("{tag}.expect"))).unwrap(); + let mut lines = text.lines(); + let parse_line = |l: Option<&str>| -> Vec { + l.unwrap_or("") + .split_whitespace() + .map(str::to_string) + .collect() + }; + let shape = parse_line(lines.next()) + .iter() + .map(|s| s.parse().unwrap()) + .collect(); + let values = parse_line(lines.next()) + .iter() + .map(|s| s.parse().unwrap()) + .collect(); + (shape, values) +} + +/// Assert clawhdf5 reads `file:dset` exactly as libhdf5 did for `tag`. +fn assert_matches_libhdf5(dir: &Path, file: &str, dset: &str, tag: &str) { + let (shape, values) = expected(dir, tag); + let f = File::open(dir.join(file)).unwrap(); + let ds = f.dataset(dset).unwrap(); + assert_eq!( + ds.shape().unwrap(), + shape, + "{tag}: shape differs from libhdf5" + ); + let got = ds + .read_f64() + .unwrap_or_else(|e| panic!("{tag}: read failed: {e}")); + assert_eq!(got.len(), values.len(), "{tag}: element count differs"); + for (i, (g, e)) in got.iter().zip(&values).enumerate() { + assert!( + g == e || (g.is_nan() && e.is_nan()), + "{tag}: element {i} is {g}, libhdf5 reads {e}\n ours: {got:?}\n libhdf5: {values:?}" + ); + } +} + +// --------------------------------------------------------------------------- +// Selection encodings +// --------------------------------------------------------------------------- + +/// Files written with the default (earliest) format bounds serialize every +/// VDS hyperslab as a version-1 *irregular* selection (4-byte block corners), +/// and a strided selection as many blocks. These were refused outright. +#[test] +fn vds_version1_irregular_hyperslab_selections() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + generate( + dir.path(), + r#" +with h5py.File("src.h5", "w") as s: + s.create_dataset("a", data=np.arange(12.0)) + s.create_dataset("m", data=np.arange(20.0).reshape(4, 5)) +with h5py.File("v1.h5", "w") as f: # default libver: hyperslab version 1 + f.create_dataset("local", data=np.arange(10.0) * -1) + lay = h5py.VirtualLayout(shape=(12,), dtype="f8") + lay[0:4] = h5py.VirtualSource(".", "local", shape=(10,))[2:6] + lay[4:10] = h5py.VirtualSource("src.h5", "a", shape=(12,))[::2] + lay[10:12] = h5py.VirtualSource("src.h5", "a", shape=(12,))[10:12] + f.create_virtual_dataset("strided", lay) + lay = h5py.VirtualLayout(shape=(4, 6), dtype="f8") + lay[:, 0:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 3:5] + lay[:, 2:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 0:2] + lay[:, 3:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 2:4] + f.create_virtual_dataset("grid", lay) +expect("v1.h5", "strided", "strided") +expect("v1.h5", "grid", "grid") +"#, + ); + assert_matches_libhdf5(dir.path(), "v1.h5", "strided", "strided"); + assert_matches_libhdf5(dir.path(), "v1.h5", "grid", "grid"); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 6dfa369..c15e824 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -71,7 +71,8 @@ the VDS item, which is marked. - **Virtual datasets:** - **Wrong data:** unmapped regions read as 0 instead of the fill value. - `%b` printf-style source names are not expanded. - - Hyperslab selection versions 1 and 2 are refused. + - ~~Hyperslab selection versions 1 and 2 are refused.~~ Fixed 2026-09-25: + versions 1-3 and irregular hyperslabs are decoded. - **Files with a user block:** the base address is not applied. - **Old-style shared messages (version 1)** read the wrong address. - **Groups and links:**