//! HDF5 Data Layout message parsing (message type 0x0008). #[cfg(not(feature = "std"))] use alloc::{format, string::String, vec::Vec}; #[cfg(feature = "std")] use std::string::String; use crate::error::FormatError; /// A single VDS (Virtual Dataset) source mapping. /// /// Maps a region of the virtual dataset to a region of a source dataset /// in a (possibly external) HDF5 file. #[derive(Debug, Clone, PartialEq)] pub struct VdsMapping { /// Source file name (may be "." for the same file). pub source_file: String, /// Source dataset path within the source file. pub source_dataset: String, /// Serialized source selection bytes (dataspace selection). pub source_selection: Vec, /// Serialized virtual selection bytes (dataspace selection). pub virtual_selection: Vec, } /// Most dimensions a layout message can list (libhdf5 `H5O_LAYOUT_NDIMS`): /// 32 dataspace dimensions plus the element size. const MAX_LAYOUT_NDIMS: usize = 33; /// libhdf5's checks on a chunked layout message's dimensions /// (`H5O__layout_decode`): at most [`MAX_LAYOUT_NDIMS`], no dimension 0, and /// before version 4 at least one dataspace dimension plus the element size. /// A zero chunk dimension used to read the dataset as all fill values. fn check_chunk_dims(dims: Vec, layout_version: u8) -> Result, FormatError> { if dims.len() > MAX_LAYOUT_NDIMS { return Err(FormatError::InvalidChunkDimensions( "dimensionality is too large".into(), )); } if layout_version < 4 && dims.len() < 2 { return Err(FormatError::InvalidChunkDimensions( "bad dimensions for chunked storage".into(), )); } if let Some(u) = dims.iter().position(|&d| d == 0) { return Err(FormatError::InvalidChunkDimensions(format!( "bad chunk dimension value when parsing layout message - chunk dimension must be \ positive: mesg->u.chunk.dim[{u}] = 0" ))); } Ok(dims) } /// Parsed HDF5 data layout message. #[derive(Debug, Clone, PartialEq)] pub enum DataLayout { /// Compact: data stored inline in the message. Compact { /// The inline raw data bytes. data: Vec, }, /// Contiguous: data stored at a single address in the file. Contiguous { /// File address of the data, or `None` if undefined (all 0xFF). address: Option, /// Size of the data in bytes. size: u64, }, /// Chunked: data stored in chunks via a B-tree. Chunked { /// Chunk dimension sizes. chunk_dimensions: Vec, /// B-tree address, or `None` if undefined. btree_address: Option, /// Layout version (3 or 4). Version 1/2 messages (HDF5 1.4/1.6-era) /// use the same version-1 B-tree chunk index as version 3 and are /// reported as 3. version: u8, /// Chunk index type (v4 only). chunk_index_type: Option, /// Filtered size for v4 single chunk with filters. single_chunk_filtered_size: Option, /// Filter mask for v4 single chunk with filters. single_chunk_filter_mask: Option, /// Layout v4 flag bit 0 (`H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS`): /// partial edge chunks — those extending past the dataset's current /// extent in some dimension — are stored without the filter pipeline, /// even though their filter mask is 0. Always `false` for v3. dont_filter_partial_edge_chunks: bool, }, /// Virtual dataset layout (v4 only). Virtual { /// Layout version. version: u8, /// Global heap address where VDS mappings are stored. global_heap_address: Option, /// Index of the object in the global heap collection. global_heap_index: u32, /// Parsed VDS source mappings (populated after global heap lookup). mappings: Vec, }, } /// Version-1 VDS mapping flag: the source file name is stored by an earlier /// entry, whose index follows in place of the name. const VDS_SOURCE_FILE_SHARED: u8 = 0x01; /// Version-1 VDS mapping flag: likewise for the source dataset name. const VDS_SOURCE_DSET_SHARED: u8 = 0x02; /// Version-1 VDS mapping flag: the source is in the virtual file itself /// (`"."`); no file name is stored. const VDS_SOURCE_SAME_FILE: u8 = 0x04; const VDS_ALL_FLAGS: u8 = VDS_SOURCE_FILE_SHARED | VDS_SOURCE_DSET_SHARED | VDS_SOURCE_SAME_FILE; /// Parse VDS mappings from global-heap object data. /// /// The global-heap block holding a VDS mapping list is laid out as /// (`H5D__virtual_store_layout` / `H5D__virtual_load_layout` in libhdf5): /// /// ```text /// version(1) · nused(length_size, LE) · entry[nused] · checksum(4) /// ``` /// /// Each entry is: /// - **block version 1 only:** a flags byte. `0x04`: the source is in the /// virtual file itself and no file name is stored; `0x01`/`0x02`: the /// source file/dataset name is that of an earlier entry, whose index /// (`length_size` bytes) is stored instead of the name. libhdf5 2.0 writes /// version 1 when the file's low version bound is 2.0 and it saves space; /// - source file name (null-terminated string, unless flagged above); /// - source dataset name (null-terminated string, unless flagged above); /// - source selection (serialized `H5S` dataspace selection — self-describing /// in length); /// - virtual selection (serialized `H5S` dataspace selection). /// /// The selections are decoded with [`crate::selection::Selection`] purely to /// learn their byte length so the entry list can be walked; the raw selection /// bytes are retained on each [`VdsMapping`] for the reader to interpret. pub fn parse_vds_mappings( heap_data: &[u8], length_size: u8, ) -> Result, FormatError> { use crate::selection::Selection; let ls = length_size as usize; if heap_data.len() < 1 + ls { return Ok(Vec::new()); } let version = heap_data[0]; let mut pos = 1; let nused = read_length(heap_data, pos, length_size)?; pos += ls; // `nused` is untrusted; don't pre-allocate from it. Each entry consumes at // least a few bytes, so the loop is naturally bounded by the heap data and // a bogus `nused` simply errors out on the first short read. let mut mappings: Vec = Vec::new(); // Reads one self-describing selection at `pos`, returning its raw bytes and // advancing past it — bounds-checked so a corrupt selection can't overrun. let read_selection = |heap_data: &[u8], pos: &mut usize| -> Result, FormatError> { let rest = heap_data.get(*pos..).ok_or(FormatError::UnexpectedEof { expected: *pos, available: heap_data.len(), })?; let (_, len) = Selection::decode_serialized(rest)?; let bytes = rest .get(..len) .ok_or(FormatError::UnexpectedEof { expected: pos.saturating_add(len), available: heap_data.len(), })? .to_vec(); *pos += len; Ok(bytes) }; if version > 1 { return Err(FormatError::ChunkedReadError( "unsupported VDS mapping block version".into(), )); } for i in 0..nused { // Version 1 prefixes each entry with a flags byte; a name may then be // omitted (same file) or replaced by the index of an earlier entry // holding the same name (`H5D__virtual_load_layout`). let flags = if version >= 1 { let f = *heap_data.get(pos).ok_or(FormatError::UnexpectedEof { expected: pos + 1, available: heap_data.len(), })?; pos += 1; if f & !VDS_ALL_FLAGS != 0 { return Err(FormatError::ChunkedReadError( "unknown VDS mapping flags".into(), )); } f } else { 0 }; // Index of an earlier entry, for a shared name. let earlier = |pos: &mut usize| -> Result { let idx = read_length(heap_data, *pos, length_size)?; *pos += ls; if idx >= i { return Err(FormatError::ChunkedReadError( "VDS mapping shares a name with a later entry".into(), )); } Ok(idx as usize) }; let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 { String::from(".") } else if flags & VDS_SOURCE_FILE_SHARED != 0 { let idx = earlier(&mut pos)?; mappings[idx].source_file.clone() } else { read_null_terminated_string(heap_data, &mut pos)? }; let source_dataset = if flags & VDS_SOURCE_DSET_SHARED != 0 { let idx = earlier(&mut pos)?; mappings[idx].source_dataset.clone() } else { read_null_terminated_string(heap_data, &mut pos)? }; // Source selection, then virtual selection (both self-describing length). let source_selection = read_selection(heap_data, &mut pos)?; let virtual_selection = read_selection(heap_data, &mut pos)?; mappings.push(VdsMapping { source_file, source_dataset, source_selection, virtual_selection, }); } Ok(mappings) } /// Read a null-terminated UTF-8 string from data starting at `pos`. fn read_null_terminated_string(data: &[u8], pos: &mut usize) -> Result { let start = *pos; while *pos < data.len() && data[*pos] != 0 { *pos += 1; } if *pos >= data.len() { return Err(FormatError::UnexpectedEof { expected: start + 1, available: data.len(), }); } let s = String::from_utf8_lossy(&data[start..*pos]).into_owned(); *pos += 1; // skip null terminator Ok(s) } fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { match offset.checked_add(needed) { Some(end) if end <= data.len() => Ok(()), _ => Err(FormatError::UnexpectedEof { expected: offset.saturating_add(needed), available: data.len(), }), } } fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { let s = size as usize; ensure_len(data, pos, s)?; let slice = &data[pos..pos + s]; Ok(match size { 2 => u16::from_le_bytes([slice[0], slice[1]]) as u64, 4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64, 8 => u64::from_le_bytes([ slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7], ]), _ => { return Err(FormatError::InvalidOffsetSize(size)); } }) } fn read_length(data: &[u8], pos: usize, size: u8) -> Result { read_offset(data, pos, size) } /// Check if all bytes in a slice are 0xFF (undefined address). fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool { let s = size as usize; if pos + s > data.len() { return false; } data[pos..pos + s].iter().all(|&b| b == 0xFF) } impl DataLayout { /// For a Virtual layout, resolve VDS mappings from the global heap. /// /// Reads the global heap collection at the stored address and parses the /// VDS mapping entries from the referenced object. After calling this /// method, the `mappings` field will be populated. /// /// No-op for non-Virtual layouts. pub fn resolve_vds_mappings( &mut self, file_data: &[u8], length_size: u8, ) -> Result<(), FormatError> { if let DataLayout::Virtual { global_heap_address, global_heap_index, mappings, .. } = self && let Some(addr) = *global_heap_address { let coll = crate::global_heap::GlobalHeapCollection::parse( file_data, addr as usize, length_size, )?; let obj = coll.get_object(*global_heap_index as u16).ok_or( FormatError::GlobalHeapObjectNotFound { collection_address: addr, index: *global_heap_index as u16, }, )?; *mappings = parse_vds_mappings(&obj.data, length_size)?; } Ok(()) } /// Parse a data layout message from raw message bytes. /// /// `offset_size` and `length_size` come from the superblock. pub fn parse(data: &[u8], offset_size: u8, length_size: u8) -> Result { ensure_len(data, 0, 2)?; let version = data[0]; let layout_class = data[1]; match version { 1 | 2 => Self::parse_v1_v2(data, offset_size), 3 => Self::parse_v3(data, layout_class, offset_size, length_size), // v5 (emitted by HDF5 1.14+/2.0 with `libver=latest`) uses the same // message structure as v4 — only the version number was bumped. 4 | 5 => Self::parse_v4(data, layout_class, offset_size, length_size), _ => Err(FormatError::InvalidLayoutVersion(version)), } } /// Layout message versions 1 and 2 (HDF5 before 1.6.3): /// /// ```text /// version(1) · dimensionality(1) · layout class(1) · reserved(5) /// · address(offset_size) — contiguous and chunked only /// · dimension sizes(4 × dimensionality) /// · compact data size(4) · compact raw data — compact only /// ``` /// /// The dimension sizes are the dataset's (contiguous/compact) or the /// chunk's (chunked) extent plus a trailing element-size dimension, as in /// version 3's chunked form. libhdf5 ignores them for contiguous storage /// and sizes the data from the dataspace; the product of the stored /// dimensions is that same size, and a disagreement (a dimension that was /// truncated to 32 bits) is caught by the reader's size check rather than /// returning wrong data. fn parse_v1_v2(data: &[u8], offset_size: u8) -> Result { ensure_len(data, 0, 8)?; let dimensionality = data[1] as usize; let layout_class = data[2]; // H5O_LAYOUT_NDIMS: 32 dataspace dimensions + the element-size one. if dimensionality > 33 { return Err(FormatError::Overflow(format!( "data layout dimensionality {dimensionality} exceeds 33" ))); } let mut p = 8; let os = offset_size as usize; let address = match layout_class { 1 | 2 => { ensure_len(data, p, os)?; let a = if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) }; p += os; a } 0 => None, _ => return Err(FormatError::InvalidLayoutClass(layout_class)), }; ensure_len(data, p, dimensionality * 4)?; let dims: Vec = data[p..p + dimensionality * 4] .as_chunks::<4>() .0 .iter() .map(|c| u32::from_le_bytes(*c)) .collect(); p += dimensionality * 4; match layout_class { 0 => { ensure_len(data, p, 4)?; let size = u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]) as usize; ensure_len(data, p + 4, size)?; Ok(DataLayout::Compact { data: data[p + 4..p + 4 + size].to_vec(), }) } 1 => { let size = dims .iter() .try_fold(1u64, |acc, &d| acc.checked_mul(d as u64)) .ok_or_else(|| { FormatError::Overflow(format!("contiguous layout size {dims:?}")) })?; Ok(DataLayout::Contiguous { address, size }) } _ => Ok(DataLayout::Chunked { chunk_dimensions: check_chunk_dims(dims, 2)?, btree_address: address, version: 3, chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, dont_filter_partial_edge_chunks: false, }), } } fn parse_v3( data: &[u8], layout_class: u8, offset_size: u8, length_size: u8, ) -> Result { let pos = 2; match layout_class { 0 => { // Compact ensure_len(data, pos, 2)?; let data_size = u16::from_le_bytes([data[pos], data[pos + 1]]) as usize; ensure_len(data, pos + 2, data_size)?; let raw = data[pos + 2..pos + 2 + data_size].to_vec(); Ok(DataLayout::Compact { data: raw }) } 1 => { // Contiguous let os = offset_size as usize; let ls = length_size as usize; ensure_len(data, pos, os + ls)?; let address = if is_undefined(data, pos, offset_size) { None } else { Some(read_offset(data, pos, offset_size)?) }; let size = read_length(data, pos + os, length_size)?; Ok(DataLayout::Contiguous { address, size }) } 2 => { // Chunked ensure_len(data, pos, 1)?; let dimensionality = data[pos] as usize; let mut p = pos + 1; // btree address first let os = offset_size as usize; ensure_len(data, p, os)?; let btree_address = if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) }; p += os; // chunk dim sizes: dimensionality × 4 bytes each ensure_len(data, p, dimensionality * 4)?; let mut chunk_dimensions = Vec::with_capacity(dimensionality); for _ in 0..dimensionality { let dim = u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]); chunk_dimensions.push(dim); p += 4; } Ok(DataLayout::Chunked { chunk_dimensions: check_chunk_dims(chunk_dimensions, 3)?, btree_address, version: 3, chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, dont_filter_partial_edge_chunks: false, }) } _ => Err(FormatError::InvalidLayoutClass(layout_class)), } } fn parse_v4( data: &[u8], layout_class: u8, offset_size: u8, length_size: u8, ) -> Result { let pos = 2; match layout_class { 0 => { // Compact — same as v3 ensure_len(data, pos, 2)?; let data_size = u16::from_le_bytes([data[pos], data[pos + 1]]) as usize; ensure_len(data, pos + 2, data_size)?; let raw = data[pos + 2..pos + 2 + data_size].to_vec(); Ok(DataLayout::Compact { data: raw }) } 1 => { // Contiguous — same as v3 let os = offset_size as usize; let ls = length_size as usize; ensure_len(data, pos, os + ls)?; let address = if is_undefined(data, pos, offset_size) { None } else { Some(read_offset(data, pos, offset_size)?) }; let size = read_length(data, pos + os, length_size)?; Ok(DataLayout::Contiguous { address, size }) } 2 => { // Chunked v4 ensure_len(data, pos, 3)?; let flags = data[pos]; let dimensionality = data[pos + 1] as usize; let dim_size_encoded_length = data[pos + 2] as usize; let mut p = pos + 3; if dimensionality > MAX_LAYOUT_NDIMS { return Err(FormatError::InvalidChunkDimensions( "dimensionality is too large".into(), )); } // Each dimension takes 1 to 8 bytes (libhdf5 writes the // fewest that hold the largest one, so 3, 5, 6 and 7 occur: // a chunk dimension of 70 000 takes 3). libhdf5 refuses 0 // and more than 8. if dim_size_encoded_length == 0 || dim_size_encoded_length > 8 { return Err(FormatError::InvalidChunkDimensions( "encoded chunk dimension size is too large".into(), )); } ensure_len(data, p, dimensionality * dim_size_encoded_length)?; let mut chunk_dimensions = Vec::with_capacity(dimensionality); for _ in 0..dimensionality { let val = data[p..p + dim_size_encoded_length] .iter() .rev() .fold(0u64, |acc, &b| (acc << 8) | u64::from(b)); // Chunk dimensions are held as u32; HDF5 2.0 can write // larger ones (layout version 5), which are refused // rather than truncated. let val = u32::try_from(val).map_err(|_| { FormatError::InvalidChunkDimensions(format!( "chunk dimension {val} is larger than 2^32 - 1, which is not supported" )) })?; chunk_dimensions.push(val); p += dim_size_encoded_length; } let chunk_dimensions = check_chunk_dims(chunk_dimensions, 4)?; // chunk index type ensure_len(data, p, 1)?; let chunk_index_type = data[p]; p += 1; // Parse index-specific fields let mut single_chunk_filtered_size = None; let mut single_chunk_filter_mask = None; let btree_address = match chunk_index_type { 1 => { // Single chunk // H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER = 0x02 let filters_present = flags & 0x02 != 0; if filters_present { // filtered_size(length_size) + filter_mask(4) + address(offset_size) let ls = length_size as usize; let os = offset_size as usize; ensure_len(data, p, ls + 4 + os)?; single_chunk_filtered_size = Some(read_length(data, p, length_size)?); p += ls; single_chunk_filter_mask = Some(u32::from_le_bytes([ data[p], data[p + 1], data[p + 2], data[p + 3], ])); p += 4; if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) } } else { // just address(offset_size) ensure_len(data, p, offset_size as usize)?; if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) } } } 2 => { // Implicit: just address ensure_len(data, p, offset_size as usize)?; if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) } } 3 => { // Fixed Array: max_dblk_page_nelmts_bits(1) + address(offset_size) ensure_len(data, p, 1 + offset_size as usize)?; p += 1; // skip max_dblk_page_nelmts_bits if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) } } 4 => { // Extensible Array: 5 creation params + address(offset_size) ensure_len(data, p, 5 + offset_size as usize)?; p += 5; // skip EA creation parameters if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) } } 5 => { // B-tree v2: node_size(4) + split_percent(1) + merge_percent(1) + address ensure_len(data, p, 6 + offset_size as usize)?; p += 6; if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) } } _ => { // Unknown index type: try just address ensure_len(data, p, offset_size as usize)?; if is_undefined(data, p, offset_size) { None } else { Some(read_offset(data, p, offset_size)?) } } }; Ok(DataLayout::Chunked { chunk_dimensions, btree_address, version: 4, chunk_index_type: Some(chunk_index_type), single_chunk_filtered_size, single_chunk_filter_mask, dont_filter_partial_edge_chunks: flags & 0x01 != 0, }) } 3 => { // Virtual: global_heap_address(offset_size) + global_heap_index(4) let os = offset_size as usize; ensure_len(data, pos, os + 4)?; let global_heap_address = if is_undefined(data, pos, offset_size) { None } else { Some(read_offset(data, pos, offset_size)?) }; let idx_pos = pos + os; let global_heap_index = u32::from_le_bytes([ data[idx_pos], data[idx_pos + 1], data[idx_pos + 2], data[idx_pos + 3], ]); Ok(DataLayout::Virtual { version: 4, global_heap_address, global_heap_index, mappings: Vec::new(), }) } _ => Err(FormatError::InvalidLayoutClass(layout_class)), } } } #[cfg(test)] mod tests { use super::*; /// Version 1/2 header: version, dimensionality, class, reserved(5). fn v1v2_header(version: u8, ndims: u8, class: u8) -> Vec { vec![version, ndims, class, 0, 0, 0, 0, 0] } #[test] fn v2_compact() { let mut buf = v1v2_header(2, 2, 0); // dims (3 elements of 2 bytes) — no address for compact buf.extend_from_slice(&3u32.to_le_bytes()); buf.extend_from_slice(&2u32.to_le_bytes()); buf.extend_from_slice(&6u32.to_le_bytes()); // compact size (u32 in v1/v2) buf.extend_from_slice(&[1, 0, 2, 0, 3, 0]); assert_eq!( DataLayout::parse(&buf, 8, 8).unwrap(), DataLayout::Compact { data: vec![1, 0, 2, 0, 3, 0] } ); } #[test] fn v1_contiguous_size_from_dimensions() { let mut buf = v1v2_header(1, 3, 1); buf.extend_from_slice(&0x800u32.to_le_bytes()); // 4-byte address for d in [10u32, 20, 4] { buf.extend_from_slice(&d.to_le_bytes()); } assert_eq!( DataLayout::parse(&buf, 4, 4).unwrap(), DataLayout::Contiguous { address: Some(0x800), size: 800, } ); } #[test] fn v1_contiguous_undefined_address() { let mut buf = v1v2_header(1, 2, 1); buf.extend_from_slice(&[0xFF; 8]); buf.extend_from_slice(&5u32.to_le_bytes()); buf.extend_from_slice(&8u32.to_le_bytes()); assert_eq!( DataLayout::parse(&buf, 8, 8).unwrap(), DataLayout::Contiguous { address: None, size: 40, } ); } #[test] fn v1_chunked_maps_to_btree_v1_index() { let mut buf = v1v2_header(1, 3, 2); buf.extend_from_slice(&0x1234u64.to_le_bytes()); for d in [50u32, 50, 4] { buf.extend_from_slice(&d.to_le_bytes()); } assert_eq!( DataLayout::parse(&buf, 8, 8).unwrap(), DataLayout::Chunked { chunk_dimensions: vec![50, 50, 4], btree_address: Some(0x1234), version: 3, chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, dont_filter_partial_edge_chunks: false, } ); } /// A v3 chunked layout message with these dims (element size last). fn v3_chunked_msg(dims: &[u32]) -> Vec { let mut buf = vec![3u8, 2, dims.len() as u8]; buf.extend_from_slice(&0x1000u64.to_le_bytes()); for d in dims { buf.extend_from_slice(&d.to_le_bytes()); } buf } #[test] fn chunk_dimensions_are_checked_when_the_layout_is_parsed() { assert!(DataLayout::parse(&v3_chunked_msg(&[4, 4, 8]), 8, 8).is_ok()); // A zero chunk dimension used to read as all fill values. let err = DataLayout::parse(&v3_chunked_msg(&[4, 0, 8]), 8, 8).unwrap_err(); assert!( matches!(&err, FormatError::InvalidChunkDimensions(m) if m.contains("dim[1] = 0")), "{err:?}" ); // Only the element-size dimension: libhdf5 "bad dimensions". assert_eq!( DataLayout::parse(&v3_chunked_msg(&[8]), 8, 8).unwrap_err(), FormatError::InvalidChunkDimensions("bad dimensions for chunked storage".into()) ); assert_eq!( DataLayout::parse(&v3_chunked_msg(&[1; 34]), 8, 8).unwrap_err(), FormatError::InvalidChunkDimensions("dimensionality is too large".into()) ); // v1/v2 and v4 messages get the zero check too. let mut v1 = v1v2_header(1, 2, 2); v1.extend_from_slice(&0x1000u64.to_le_bytes()); v1.extend_from_slice(&0u32.to_le_bytes()); v1.extend_from_slice(&8u32.to_le_bytes()); assert!(matches!( DataLayout::parse(&v1, 8, 8), Err(FormatError::InvalidChunkDimensions(_)) )); let mut v4 = vec![4u8, 2, 0, 2, 4]; v4.extend_from_slice(&0u32.to_le_bytes()); v4.extend_from_slice(&8u32.to_le_bytes()); v4.push(3); // fixed array index v4.push(0); // page bits v4.extend_from_slice(&0x1000u64.to_le_bytes()); assert!(matches!( DataLayout::parse(&v4, 8, 8), Err(FormatError::InvalidChunkDimensions(_)) )); } /// A v4 chunked layout (fixed array index) whose `dims` are each /// encoded in `width` bytes. fn v4_chunked_msg(width: u8, dims: &[u64]) -> Vec { let mut m = vec![4u8, 2, 0, dims.len() as u8, width]; for &d in dims { m.extend_from_slice(&d.to_le_bytes()[..width.min(8) as usize]); } m.push(3); // fixed array index m.push(0); // page bits m.extend_from_slice(&0x1000u64.to_le_bytes()); m } #[test] fn v4_chunk_dimensions_take_1_to_8_bytes() { // libhdf5 encodes each dimension in the fewest bytes that hold the // largest: a chunk dimension of 70 000 takes 3, and 3, 5, 6 and 7 // were refused ("UnexpectedEof"). for width in 1..=8u8 { let dims = [if width >= 3 { 70_000 } else { 200 }, 8]; let layout = DataLayout::parse(&v4_chunked_msg(width, &dims), 8, 8) .unwrap_or_else(|e| panic!("width {width}: {e:?}")); assert!( matches!(&layout, DataLayout::Chunked { chunk_dimensions, .. } if chunk_dimensions.iter().map(|&d| u64::from(d)).eq(dims)), "width {width}: {layout:?}" ); } // libhdf5 refuses 0 and more than 8 bytes. for width in [0u8, 9] { assert_eq!( DataLayout::parse(&v4_chunked_msg(width, &[4, 8]), 8, 8).unwrap_err(), FormatError::InvalidChunkDimensions( "encoded chunk dimension size is too large".into() ) ); } // A dimension past u32 cannot be represented and is refused, not // truncated. assert!(matches!( DataLayout::parse(&v4_chunked_msg(5, &[1 << 32, 8]), 8, 8), Err(FormatError::InvalidChunkDimensions(m)) if m.contains("2^32") )); } #[test] fn v1v2_rejects_bad_class_dimensionality_and_truncation() { assert_eq!( DataLayout::parse(&v1v2_header(1, 1, 3), 8, 8).unwrap_err(), FormatError::InvalidLayoutClass(3) ); assert!(matches!( DataLayout::parse(&v1v2_header(2, 34, 1), 8, 8).unwrap_err(), FormatError::Overflow(_) )); // Chunked, dims cut short. let mut buf = v1v2_header(1, 2, 2); buf.extend_from_slice(&0x10u64.to_le_bytes()); buf.extend_from_slice(&7u32.to_le_bytes()); assert!(matches!( DataLayout::parse(&buf, 8, 8).unwrap_err(), FormatError::UnexpectedEof { .. } )); // Compact, raw data shorter than its declared size. let mut buf = v1v2_header(2, 1, 0); buf.extend_from_slice(&4u32.to_le_bytes()); buf.extend_from_slice(&100u32.to_le_bytes()); buf.extend_from_slice(&[0; 4]); assert!(matches!( DataLayout::parse(&buf, 8, 8).unwrap_err(), FormatError::UnexpectedEof { .. } )); } #[test] fn v3_compact() { let mut buf = vec![3u8, 0]; // version=3, class=0 (compact) buf.extend_from_slice(&5u16.to_le_bytes()); // data_size=5 buf.extend_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE]); // data let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Compact { data: vec![0xAA, 0xBB, 0xCC, 0xDD, 0xEE] } ); } #[test] fn v3_contiguous() { let mut buf = vec![3u8, 1]; // version=3, class=1 (contiguous) buf.extend_from_slice(&0x1000u64.to_le_bytes()); // address buf.extend_from_slice(&256u64.to_le_bytes()); // size let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Contiguous { address: Some(0x1000), size: 256, } ); } #[test] fn v3_contiguous_undefined_address() { let mut buf = vec![3u8, 1]; buf.extend_from_slice(&[0xFF; 8]); // undefined address buf.extend_from_slice(&0u64.to_le_bytes()); // size let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Contiguous { address: None, size: 0, } ); } #[test] fn v3_chunked() { let mut buf = vec![3u8, 2]; // version=3, class=2 (chunked) buf.push(3); // dimensionality=3 (rank+1) buf.extend_from_slice(&0x2000u64.to_le_bytes()); // btree address // 3 chunk dim sizes × 4 bytes buf.extend_from_slice(&100u32.to_le_bytes()); buf.extend_from_slice(&200u32.to_le_bytes()); buf.extend_from_slice(&8u32.to_le_bytes()); // last = element size let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Chunked { chunk_dimensions: vec![100, 200, 8], btree_address: Some(0x2000), version: 3, chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, dont_filter_partial_edge_chunks: false, } ); } #[test] fn v4_compact() { let mut buf = vec![4u8, 0]; // version=4, class=0 buf.extend_from_slice(&3u16.to_le_bytes()); buf.extend_from_slice(&[1, 2, 3]); let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Compact { data: vec![1, 2, 3] } ); } #[test] fn v4_contiguous() { let mut buf = vec![4u8, 1]; buf.extend_from_slice(&0x5000u64.to_le_bytes()); buf.extend_from_slice(&512u64.to_le_bytes()); let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Contiguous { address: Some(0x5000), size: 512, } ); } #[test] fn v5_chunked_from_hdf5_2_0() { // Real data layout message from h5py 3.16 / HDF5 2.0 (`libver=latest`) // for a gzip-compressed 1-D chunked dataset. Version 5 uses the same // structure as v4 (here: chunked, Fixed Array index). Regression guard // for reading modern-format chunked datasets. let bytes: [u8; 17] = [ 0x05, 0x02, 0x00, 0x02, 0x01, 0x0a, 0x08, 0x03, 0x0a, 0xef, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ]; let layout = DataLayout::parse(&bytes, 8, 8).unwrap(); match layout { DataLayout::Chunked { chunk_dimensions, chunk_index_type, .. } => { assert_eq!(chunk_dimensions, vec![10, 8]); assert_eq!(chunk_index_type, Some(3)); // Fixed Array } other => panic!("expected Chunked, got {other:?}"), } } #[test] fn v4_chunked_single_chunk_no_filters() { let mut buf = vec![4u8, 2]; // version=4, class=2 buf.push(0); // flags (no filters) buf.push(2); // dimensionality=2 buf.push(4); // dim_size_encoded_length=4 buf.extend_from_slice(&64u32.to_le_bytes()); // dim 0 buf.extend_from_slice(&32u32.to_le_bytes()); // dim 1 buf.push(1); // chunk_index_type=1 (single chunk) buf.extend_from_slice(&0x3000u64.to_le_bytes()); // chunk address let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Chunked { chunk_dimensions: vec![64, 32], btree_address: Some(0x3000), version: 4, chunk_index_type: Some(1), single_chunk_filtered_size: None, single_chunk_filter_mask: None, dont_filter_partial_edge_chunks: false, } ); } #[test] fn v4_chunked_dont_filter_partial_edge_chunks_flag() { let mut buf = vec![4u8, 2]; // version=4, class=2 buf.push(0x01); // flags bit 0 = don't filter partial edge chunks buf.push(2); // dimensionality=2 buf.push(4); // dim_size_encoded_length=4 buf.extend_from_slice(&5u32.to_le_bytes()); buf.extend_from_slice(&4u32.to_le_bytes()); buf.push(3); // Fixed Array buf.push(10); // max_dblk_page_nelmts_bits buf.extend_from_slice(&0x3000u64.to_le_bytes()); match DataLayout::parse(&buf, 8, 8).unwrap() { DataLayout::Chunked { dont_filter_partial_edge_chunks, btree_address, .. } => { assert!(dont_filter_partial_edge_chunks); assert_eq!(btree_address, Some(0x3000)); } other => panic!("expected Chunked, got {other:?}"), } } #[test] fn v4_chunked_single_chunk_with_filters() { let mut buf = vec![4u8, 2]; // version=4, class=2 buf.push(0x02); // flags bit 1 = single chunk with filter buf.push(1); // dimensionality=1 buf.push(4); // dim_size_encoded_length=4 buf.extend_from_slice(&128u32.to_le_bytes()); // dim 0 buf.push(1); // chunk_index_type=1 (single chunk) // filters present: filtered_size(8) + filter_mask(4) + address(8) buf.extend_from_slice(&1024u64.to_le_bytes()); // filtered size buf.extend_from_slice(&0u32.to_le_bytes()); // filter mask buf.extend_from_slice(&0x4000u64.to_le_bytes()); // address let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Chunked { chunk_dimensions: vec![128], btree_address: Some(0x4000), version: 4, chunk_index_type: Some(1), single_chunk_filtered_size: Some(1024), single_chunk_filter_mask: Some(0), dont_filter_partial_edge_chunks: false, } ); } #[test] fn invalid_version() { // v3-v5 are supported; v6 is not a real layout message version. let buf = vec![6u8, 0, 0, 0]; let err = DataLayout::parse(&buf, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLayoutVersion(6)); } #[test] fn invalid_class_v3() { let buf = vec![3u8, 5]; let err = DataLayout::parse(&buf, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLayoutClass(5)); } #[test] fn invalid_class_v4() { let buf = vec![4u8, 7]; let err = DataLayout::parse(&buf, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLayoutClass(7)); } #[test] fn v3_contiguous_4byte_offsets() { let mut buf = vec![3u8, 1]; buf.extend_from_slice(&0x800u32.to_le_bytes()); buf.extend_from_slice(&24u32.to_le_bytes()); let layout = DataLayout::parse(&buf, 4, 4).unwrap(); assert_eq!( layout, DataLayout::Contiguous { address: Some(0x800), size: 24, } ); } #[test] fn v4_virtual() { let mut buf = vec![4u8, 3]; // version=4, class=3 (virtual) buf.extend_from_slice(&0x5000u64.to_le_bytes()); // global heap address buf.extend_from_slice(&1u32.to_le_bytes()); // global heap index let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Virtual { version: 4, global_heap_address: Some(0x5000), global_heap_index: 1, mappings: Vec::new(), } ); } #[test] fn v4_virtual_undefined_address() { let mut buf = vec![4u8, 3]; buf.extend_from_slice(&[0xFF; 8]); // undefined address buf.extend_from_slice(&0u32.to_le_bytes()); let layout = DataLayout::parse(&buf, 8, 8).unwrap(); assert_eq!( layout, DataLayout::Virtual { version: 4, global_heap_address: None, global_heap_index: 0, mappings: Vec::new(), } ); } #[test] fn parse_vds_mappings_same_file_v1() { // The exact global-heap block written by HDF5 2.0 for a same-file VDS // with two sources: src_a -> virtual[0:4], src_b -> virtual[4:8]. let blob = [ 0x01u8, // block version 1 0x02, 0, 0, 0, 0, 0, 0, 0, // nused = 2 (length_size = 8) // entry 0 0x04, // same-file marker (replaces file name) 0x73, 0x72, 0x63, 0x5f, 0x61, 0x00, // "src_a\0" 0x03, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // source sel = ALL 0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, // virtual sel: HYPER v3 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, // start0 stride1 count1 block4 // entry 1 0x04, 0x73, 0x72, 0x63, 0x5f, 0x62, 0x00, // "src_b\0" 0x03, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // source sel = ALL 0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, // virtual sel: HYPER v3 0x04, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, // start4 stride1 count1 block4 0x68, 0xf0, 0x3e, 0xe4, // checksum (ignored) ]; let mappings = parse_vds_mappings(&blob, 8).unwrap(); assert_eq!(mappings.len(), 2); assert_eq!(mappings[0].source_file, "."); assert_eq!(mappings[0].source_dataset, "src_a"); assert_eq!(mappings[1].source_file, "."); assert_eq!(mappings[1].source_dataset, "src_b"); // Virtual selections decode to [0:4] and [4:8]. use crate::selection::Selection; let (v0, _) = Selection::decode_serialized(&mappings[0].virtual_selection).unwrap(); let (v1, _) = Selection::decode_serialized(&mappings[1].virtual_selection).unwrap(); assert_eq!(v0.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]); assert_eq!(v1.iter_linear_1d(8).unwrap(), vec![4, 5, 6, 7]); } #[test] fn parse_vds_mappings_v1_shared_names() { // Written by HDF5 2.0 (h5py, libver=("v200", "v200")) for three // mappings from `a_rather_long_source_file.h5:a_rather_long_dataset_name` // and one from the same file: the entries carry flags 0x00, 0x03, 0x03 // and 0x06, so names after the first are stored as entry indices. let blob: &[u8] = &[ 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x5f, 0x72, 0x61, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x68, 0x35, 0x00, 0x61, 0x5f, 0x72, 0x61, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x8e, 0xa7, 0xea, 0x7a, ]; let mappings = parse_vds_mappings(blob, 8).unwrap(); let names: Vec<(&str, &str)> = mappings .iter() .map(|m| (m.source_file.as_str(), m.source_dataset.as_str())) .collect(); let (file, dset) = ("a_rather_long_source_file.h5", "a_rather_long_dataset_name"); assert_eq!( names, vec![(file, dset), (file, dset), (file, dset), (".", dset)] ); } #[test] fn parse_vds_mappings_v1_forward_reference_is_error() { // Entry 0 claiming to share entry 0's file name must not index past // the entries decoded so far. let mut blob = vec![0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x01]; blob.extend_from_slice(&[0u8; 8]); blob.extend_from_slice(b"d\0"); assert!(parse_vds_mappings(&blob, 8).is_err()); // Unknown flag bits are refused. let blob = [0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x08, b'd', 0]; assert!(parse_vds_mappings(&blob, 8).is_err()); } #[test] fn parse_vds_mappings_external_v0() { // Block version 0 with an explicit (external) source file name. let blob = [ 0x00u8, // block version 0 0x01, 0, 0, 0, 0, 0, 0, 0, // nused = 1 0x73, 0x72, 0x63, 0x5f, 0x65, 0x78, 0x74, 0x2e, 0x68, 0x35, 0x00, // "src_ext.h5\0" 0x64, 0x61, 0x74, 0x61, 0x00, // "data\0" 0x03, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // source sel = ALL 0x03, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // virtual sel = ALL ]; let mappings = parse_vds_mappings(&blob, 8).unwrap(); assert_eq!(mappings.len(), 1); assert_eq!(mappings[0].source_file, "src_ext.h5"); assert_eq!(mappings[0].source_dataset, "data"); } #[test] fn parse_vds_mappings_huge_nused_does_not_oom_or_panic() { // nused = u64::MAX with no entry data: must error, not pre-allocate or // overrun. let mut blob = vec![0x01u8]; blob.extend_from_slice(&u64::MAX.to_le_bytes()); assert!(parse_vds_mappings(&blob, 8).is_err()); } #[test] fn parse_vds_mappings_truncated_selection_does_not_overrun() { // One entry whose source selection (ALL) is truncated to 8 of 16 bytes. let blob = [ 0x01u8, // version 1 0x01, 0, 0, 0, 0, 0, 0, 0, // nused = 1 0x04, // same-file marker 0x78, 0x00, // "x\0" 0x03, 0, 0, 0, 0x01, 0, 0, 0, // ALL header, truncated (8 of 16 bytes) ]; assert!(parse_vds_mappings(&blob, 8).is_err()); } #[test] fn parse_vds_mappings_empty_is_ok_empty() { assert!(parse_vds_mappings(&[], 8).unwrap().is_empty()); // Header present, nused = 0. let blob = [0x01u8, 0, 0, 0, 0, 0, 0, 0, 0]; assert!(parse_vds_mappings(&blob, 8).unwrap().is_empty()); } }