//! Raw data reading and typed conversion for HDF5 datasets. #[cfg(not(feature = "std"))] use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] use std::collections::BTreeMap; #[cfg(feature = "std")] use crate::chunk_cache::ChunkCache; use crate::chunked_read::read_chunked_data; #[cfg(feature = "std")] use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::datatype::{Datatype, DatatypeByteOrder}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; /// Checks that `[offset, offset + needed)` fits within `data`, guarding the /// addition against `usize` overflow from a crafted near-`usize::MAX` offset. fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { if offset .checked_add(needed) .is_none_or(|end| end > data.len()) { return Err(FormatError::UnexpectedEof { expected: offset.saturating_add(needed), available: data.len(), }); } Ok(()) } /// Zero-copy read of contiguous raw data, returning a borrowed slice. /// /// For contiguous layouts, returns a direct `&[u8]` slice into `file_data`. /// For compact or chunked layouts, returns `Ok(None)` — the caller should /// fall back to `read_raw_data` for those. pub fn read_raw_data_zerocopy<'a>( file_data: &'a [u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { let num_elements = dataspace.num_elements() as usize; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( "num_elements({num_elements}) * elem_size({elem_size})" )) })?; match layout { DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = addr as usize; let sz = *size as usize; if sz != expected_size { return Err(FormatError::DataSizeMismatch { expected: expected_size, actual: sz, }); } ensure_len(file_data, addr, sz)?; Ok(Some(&file_data[addr..addr + sz])) } _ => Ok(None), } } /// Read raw bytes for a dataset given its layout and the file data buffer. /// /// For compact layouts, returns the inline data. /// For contiguous layouts, reads from the address in the file buffer. /// For chunked layouts, traverses the B-tree and assembles chunks. pub fn read_raw_data( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8) } /// Resolves a Virtual Dataset source **file name** (as stored in the mapping, /// e.g. `"ext_src.h5"`) to that file's raw bytes. /// /// The pure-byte read API has no filesystem of its own, so external-file VDS /// sources are read through a caller-supplied resolver. The std file API wires /// one that reads relative to the virtual file's directory; callers can supply /// their own (e.g. an in-memory map) in `no_std` builds. Returning `None` means /// the source file is unavailable and the mapping is skipped. pub type VdsSourceResolver<'a> = dyn Fn(&str) -> Option> + 'a; /// Read raw bytes with full parameters including filter pipeline and sizes. pub fn read_raw_data_full( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, None, ) } /// Like [`read_raw_data_full`], but with a resolver for external-file Virtual /// Dataset sources. For non-virtual layouts the resolver is ignored. #[allow(clippy::too_many_arguments)] pub fn read_raw_data_full_with_resolver( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, resolver, ) } #[allow(clippy::too_many_arguments)] fn read_raw_data_full_impl( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { let num_elements = dataspace.num_elements() as usize; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( "num_elements({num_elements}) * elem_size({elem_size})" )) })?; match layout { DataLayout::Compact { data } => { if data.len() != expected_size { return Err(FormatError::DataSizeMismatch { expected: expected_size, actual: data.len(), }); } Ok(data.clone()) } DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = addr as usize; let sz = *size as usize; if sz != expected_size { return Err(FormatError::DataSizeMismatch { expected: expected_size, actual: sz, }); } ensure_len(file_data, addr, sz)?; Ok(file_data[addr..addr + sz].to_vec()) } DataLayout::Chunked { .. } => read_chunked_data( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ), DataLayout::Virtual { global_heap_address, global_heap_index, .. } => read_virtual_data( file_data, *global_heap_address, *global_heap_index, dataspace, datatype, offset_size, length_size, resolver, ), } } /// Read raw bytes with chunk cache support. /// /// For chunked layouts the `cache` is used to avoid repeated B-tree /// traversals and to cache decompressed chunk data. For compact and /// contiguous layouts this behaves identically to [`read_raw_data_full`]. #[cfg(feature = "std")] #[allow(clippy::too_many_arguments)] pub fn read_raw_data_cached( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, cache: &ChunkCache, ) -> Result, FormatError> { match layout { DataLayout::Chunked { .. } => read_chunked_data_cached( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, cache, ), _ => read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ), } } /// Read raw bytes with chunk B-tree index cache and pre-computed layout. /// /// For chunked layouts this uses the optimized `read_chunked_data_indexed` path /// which pre-computes contiguous row-copy operations, eliminating per-element /// N-D coordinate math on repeated reads. For compact and contiguous layouts /// this behaves identically to [`read_raw_data_full`]. #[cfg(feature = "std")] #[allow(clippy::too_many_arguments)] pub fn read_raw_data_indexed( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, cache: &ChunkCache, ) -> Result, FormatError> { match layout { DataLayout::Chunked { .. } => read_chunked_data_indexed( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, cache, ), _ => read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ), } } /// Read raw bytes for only the selected elements of a dataset. /// /// For chunked layouts, only chunks that intersect the selection are read /// and decompressed. For compact/contiguous layouts, the full data is read /// and then the selection is extracted. #[allow(clippy::too_many_arguments)] pub fn read_raw_data_selection( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, selection: &crate::selection::Selection, ) -> Result, FormatError> { use crate::selection::Selection; match selection { Selection::All => { return read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ); } Selection::None => return Ok(Vec::new()), _ => {} } let dims = &dataspace.dimensions; let elem_size = datatype.type_size() as usize; match layout { DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => { // Read all data, then extract the selection let full_data = read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, )?; extract_selection_from_buffer(&full_data, dims, elem_size, selection) } DataLayout::Chunked { chunk_dimensions, btree_address, version, chunk_index_type, .. } => { // For chunked data, only read chunks that intersect the selection let chunk_dims: Vec = chunk_dimensions.iter().map(|&d| d as u64).collect(); let rank = dims.len(); // Collect chunk info from B-tree let chunks = if *version == 4 { match chunk_index_type { Some(2) => { // Implicit index crate::chunked_read::generate_implicit_chunks( btree_address.unwrap_or(0), dims, chunk_dimensions, elem_size as u32, ) } _ => { if let Some(_addr) = btree_address { // Use extensible array or fixed array // Fall back to full read for complex v4 index types let full_data = read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, )?; return extract_selection_from_buffer( &full_data, dims, elem_size, selection, ); } else { return Ok(Vec::new()); } } } } else { // v3: B-tree v1 if let Some(addr) = btree_address { crate::chunked_read::collect_chunk_info( file_data, *addr, rank + 1, offset_size, length_size, )? } else { return Ok(Vec::new()); } }; // Filter chunks to only those that intersect the selection let intersecting: Vec<_> = chunks .iter() .filter(|ci| { let offsets: Vec = ci.offsets.iter().take(rank).copied().collect(); selection.intersects_chunk(&offsets, &chunk_dims[..rank]) }) .collect(); if intersecting.is_empty() { return Ok(Vec::new()); } // Decompress only the intersecting chunks let _chunk_total_bytes: usize = chunk_dims.iter().map(|&d| d as usize).product::() * elem_size; let _element_size_u32 = elem_size as u32; // First, assemble only the intersecting chunks into a partial buffer, // then extract the selection. For simplicity, we assemble into a full // dataset buffer and extract (same as contiguous path). let full_data = read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, )?; extract_selection_from_buffer(&full_data, dims, elem_size, selection) } DataLayout::Virtual { .. } => { // Assemble the full virtual dataset, then apply the read selection. let full_data = read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, )?; extract_selection_from_buffer(&full_data, dims, elem_size, selection) } } } /// Assemble a **Virtual Dataset (VDS)** from its source mappings. /// /// Supports virtual datasets of any rank. Same-file sources are read directly; /// **external-file** sources are read through the caller-supplied `resolver`, /// which maps a stored source file name to that file's bytes. Each mapping's /// selected source elements are scattered into the virtual buffer at the /// positions given by the virtual selection (both enumerated in row-major /// order, as HDF5 pairs them). Unmapped regions are left at the zero fill value. /// /// A mapping whose external source file the resolver cannot supply (`None`) is /// skipped, leaving its region at fill — matching HDF5's tolerance of missing /// sources. An external source with no resolver at all is a hard error. #[allow(clippy::too_many_arguments)] fn read_virtual_data( file_data: &[u8], global_heap_address: Option, global_heap_index: u32, dataspace: &Dataspace, datatype: &Datatype, offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { use crate::data_layout::parse_vds_mappings; use crate::global_heap::GlobalHeapCollection; use crate::selection::Selection; let elem_size = datatype.type_size() as usize; let total_elems = dataspace.num_elements() as usize; let mut out = vec![0u8; total_elems.saturating_mul(elem_size)]; let virtual_dims = &dataspace.dimensions; let addr = global_heap_address.ok_or_else(|| { FormatError::ChunkedReadError("virtual dataset has no mapping global heap".into()) })?; let coll = GlobalHeapCollection::parse(file_data, addr as usize, length_size)?; let obj = coll.get_object(global_heap_index as u16) .ok_or(FormatError::GlobalHeapObjectNotFound { collection_address: addr, index: global_heap_index as u16, })?; let mappings = parse_vds_mappings(&obj.data, length_size)?; for m in &mappings { let same_file = m.source_file.is_empty() || m.source_file == "."; // Resolve the bytes of the file holding this source dataset. let external; let src_file_data: &[u8] = if same_file { file_data } else { let r = resolver.ok_or_else(|| { FormatError::ChunkedReadError( "external-file virtual dataset sources require a file resolver".into(), ) })?; match r(&m.source_file) { Some(bytes) => { external = bytes; &external } // Source file unavailable: leave this region at fill value. None => continue, } }; let (vsel, _) = Selection::decode_serialized(&m.virtual_selection)?; let (ssel, _) = Selection::decode_serialized(&m.source_selection)?; let (src_raw, src_dims) = read_named_dataset_raw(src_file_data, &m.source_dataset, offset_size, length_size)?; let vidx = vsel.iter_linear(virtual_dims)?; let sidx = ssel.iter_linear(&src_dims)?; if vidx.len() != sidx.len() { return Err(FormatError::ChunkedReadError( "virtual/source selection element counts differ".into(), )); } for (&v, &s) in vidx.iter().zip(sidx.iter()) { let (vo, so) = (v as usize * elem_size, s as usize * elem_size); if vo + elem_size > out.len() || so + elem_size > src_raw.len() { return Err(FormatError::ChunkedReadError( "virtual dataset selection out of bounds".into(), )); } out[vo..vo + elem_size].copy_from_slice(&src_raw[so..so + elem_size]); } } Ok(out) } /// Read a named dataset's raw (decoded) bytes and its dimensions, navigating /// from the superblock. Used to pull VDS source datasets out of the same file. fn read_named_dataset_raw( file_data: &[u8], path: &str, _offset_size: u8, _length_size: u8, ) -> Result<(Vec, Vec), FormatError> { use crate::filter_pipeline::FilterPipeline; use crate::group_v2::resolve_path_any; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; use crate::signature::find_signature; use crate::superblock::Superblock; let sig = find_signature(file_data)?; let sb = Superblock::parse(file_data, sig)?; let addr = resolve_path_any(file_data, &sb, path)?; let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?; let find = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t); let ds_msg = find(MessageType::Dataspace) .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no dataspace".into()))?; let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size)?; let dt_msg = find(MessageType::Datatype) .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no datatype".into()))?; let (datatype, _) = Datatype::parse(&dt_msg.data)?; let dl_msg = find(MessageType::DataLayout) .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?; let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?; // A virtual dataset whose source is itself another virtual dataset could // form a cycle (A -> B -> A) and recurse into a stack overflow. Nested // virtual sources are exotic and unsupported, so stop here cleanly. if matches!(layout, DataLayout::Virtual { .. }) { return Err(FormatError::ChunkedReadError( "virtual dataset source is itself virtual (unsupported)".into(), )); } let pipeline = find(MessageType::FilterPipeline) .map(|m| FilterPipeline::parse(&m.data)) .transpose()?; let raw = read_raw_data_full( file_data, &layout, &dataspace, &datatype, pipeline.as_ref(), sb.offset_size, sb.length_size, )?; Ok((raw, dataspace.dimensions.clone())) } /// Extract selected elements from a full dataset buffer. fn extract_selection_from_buffer( full_data: &[u8], dims: &[u64], elem_size: usize, selection: &crate::selection::Selection, ) -> Result, FormatError> { use crate::selection::Selection; match selection { Selection::All => Ok(full_data.to_vec()), Selection::None => Ok(Vec::new()), Selection::Hyperslab { start, stride, count, block, } => { let rank = dims.len(); let output_elements: usize = count .iter() .zip(block.iter()) .map(|(&c, &b)| (c * b) as usize) .product(); let mut output = vec![0u8; output_elements * elem_size]; // Compute dataset strides (row-major) let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; } // Compute output shape and strides let output_dims: Vec = count .iter() .zip(block.iter()) .map(|(&c, &b)| (c * b) as usize) .collect(); let mut out_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { out_strides[i] = out_strides[i + 1] * output_dims[i + 1]; } // Iterate over all selected elements // For each block in the hyperslab, copy the elements let mut out_linear = 0usize; let _block_coords = vec![0u64; rank]; #[allow(clippy::too_many_arguments)] fn iterate_hyperslab( d: usize, rank: usize, start: &[u64], stride: &[u64], count: &[u64], block: &[u64], dims: &[u64], ds_strides: &[usize], elem_size: usize, full_data: &[u8], output: &mut [u8], out_linear: &mut usize, current_ds_offset: usize, ) { if d == rank { // Copy one element let src = current_ds_offset * elem_size; let dst = *out_linear * elem_size; if src + elem_size <= full_data.len() && dst + elem_size <= output.len() { output[dst..dst + elem_size] .copy_from_slice(&full_data[src..src + elem_size]); } *out_linear += 1; return; } for bi in 0..count[d] { let block_start = start[d] + bi * stride[d]; for bj in 0..block[d] { let coord = block_start + bj; if coord < dims[d] { iterate_hyperslab( d + 1, rank, start, stride, count, block, dims, ds_strides, elem_size, full_data, output, out_linear, current_ds_offset + coord as usize * ds_strides[d], ); } } } } iterate_hyperslab( 0, rank, start, stride, count, block, dims, &ds_strides, elem_size, full_data, &mut output, &mut out_linear, 0, ); Ok(output) } Selection::Points(pts) => { let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; } let mut output = Vec::with_capacity(pts.len() * elem_size); for pt in pts { let flat: usize = pt .iter() .zip(ds_strides.iter()) .map(|(&p, &s)| p as usize * s) .sum(); let src = flat * elem_size; if src + elem_size <= full_data.len() { output.extend_from_slice(&full_data[src..src + elem_size]); } else { output.extend_from_slice(&vec![0u8; elem_size]); } } Ok(output) } } } /// Zero-copy transmute of raw bytes to `&[f64]`. /// /// Returns `Some(&[f64])` when the datatype is native little-endian `f64` /// and the data pointer is 8-byte aligned. Returns `None` for any other /// type or alignment — the caller should fall back to [`read_as_f64`]. pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f64]> { // Only native LE f64 is eligible #[cfg(target_endian = "little")] { if !matches!( datatype, Datatype::FloatingPoint { size: 8, byte_order: DatatypeByteOrder::LittleEndian, .. } ) { return None; } if !raw.len().is_multiple_of(8) { return None; } let ptr = raw.as_ptr(); if !(ptr as usize).is_multiple_of(core::mem::align_of::()) { return None; } let count = raw.len() / 8; // SAFETY: We verified alignment (8-byte), size (multiple of 8), and // the on-disk format matches the in-memory representation (LE f64). Some(unsafe { core::slice::from_raw_parts(ptr as *const f64, count) }) } #[cfg(not(target_endian = "little"))] { let _ = (raw, datatype); None } } /// Zero-copy transmute of raw bytes to `&[f32]`. /// /// Returns `Some(&[f32])` when the datatype is native little-endian `f32` /// and the data pointer is 4-byte aligned. Returns `None` otherwise. pub fn read_as_f32_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f32]> { #[cfg(target_endian = "little")] { if !matches!( datatype, Datatype::FloatingPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, .. } ) { return None; } if !raw.len().is_multiple_of(4) { return None; } let ptr = raw.as_ptr(); if !(ptr as usize).is_multiple_of(core::mem::align_of::()) { return None; } let count = raw.len() / 4; // SAFETY: We verified alignment (4-byte), size (multiple of 4), and // the on-disk format matches the in-memory representation (LE f32). Some(unsafe { core::slice::from_raw_parts(ptr as *const f32, count) }) } #[cfg(not(target_endian = "little"))] { let _ = (raw, datatype); None } } fn datatype_name(dt: &Datatype) -> &'static str { match dt { Datatype::FixedPoint { .. } => "FixedPoint", Datatype::FloatingPoint { .. } => "FloatingPoint", Datatype::String { .. } => "String", Datatype::Time { .. } => "Time", Datatype::BitField { .. } => "BitField", Datatype::Opaque { .. } => "Opaque", Datatype::Compound { .. } => "Compound", Datatype::Reference { .. } => "Reference", Datatype::Enumeration { .. } => "Enumeration", Datatype::VariableLength { .. } => "VariableLength", Datatype::Array { .. } => "Array", } } fn ensure_numeric(dt: &Datatype, expected: &'static str) -> Result<(), FormatError> { match dt { Datatype::FixedPoint { .. } | Datatype::FloatingPoint { .. } => Ok(()), _ => Err(FormatError::TypeMismatch { expected, actual: datatype_name(dt), }), } } fn get_byte_order(dt: &Datatype) -> DatatypeByteOrder { match dt { Datatype::FixedPoint { byte_order, .. } => byte_order.clone(), Datatype::FloatingPoint { byte_order, .. } => byte_order.clone(), _ => DatatypeByteOrder::LittleEndian, } } fn get_size(dt: &Datatype) -> usize { dt.type_size() as usize } /// Convert raw bytes to `f64` values. pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes (e.g. an array-typed compound member) are read as a flat // sequence of their base elements. if let Datatype::Array { base_type, .. } = datatype { return read_as_f64(raw, base_type); } ensure_numeric(datatype, "FloatingPoint or FixedPoint")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native-endian f64 — single bulk memcpy #[cfg(target_endian = "little")] if matches!( datatype, Datatype::FloatingPoint { size: 8, byte_order: DatatypeByteOrder::LittleEndian, .. } ) { let mut result = vec![0.0f64; count]; // SAFETY: On LE platforms, f64 in-memory representation matches LE bytes. // We copy raw bytes directly into the f64 buffer. // SAFETY: The byte slice is properly aligned for this type and the length is divisible by size_of::(). unsafe { core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr() as *mut u8, raw.len()); } return Ok(result); } let order = get_byte_order(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let val = convert_to_f64(chunk, datatype, &order)?; result.push(val); } Ok(result) } fn convert_to_f64( bytes: &[u8], dt: &Datatype, order: &DatatypeByteOrder, ) -> Result { match dt { Datatype::FloatingPoint { size, .. } => match size { 4 => { let v = read_f32_bytes(bytes, order); Ok(v as f64) } 8 => Ok(read_f64_bytes(bytes, order)), 2 => Ok(read_f16_bytes(bytes, order) as f64), _ => Err(FormatError::DataSizeMismatch { expected: 8, actual: *size as usize, }), }, Datatype::FixedPoint { size, signed, bit_offset, bit_precision, .. } => { let full = read_unsigned_int(bytes, *size as usize, order); let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); let v = if *signed { extract_signed(full, off, prec) as f64 } else { extract_unsigned(full, off, prec) as f64 }; Ok(v) } _ => Err(FormatError::TypeMismatch { expected: "numeric", actual: datatype_name(dt), }), } } /// Convert raw bytes to `i64` values. pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_i64(raw, base_type); } ensure_numeric(datatype, "FixedPoint (signed)")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native LE i64 — single bulk memcpy #[cfg(target_endian = "little")] if elem_size == 8 && is_full_width(datatype) && matches!( datatype, Datatype::FixedPoint { byte_order: DatatypeByteOrder::LittleEndian, signed: true, .. } ) { let mut result = vec![0i64; count]; // SAFETY: The byte slice is properly aligned for this type and the length is divisible by size_of::(). unsafe { core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr() as *mut u8, raw.len()); } return Ok(result); } let order = get_byte_order(datatype); let (off, prec) = fixed_bits(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let full = read_unsigned_int(chunk, elem_size, &order); result.push(extract_signed(full, off, prec)); } Ok(result) } /// Convert raw bytes to `u64` values. pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_u64(raw, base_type); } ensure_numeric(datatype, "FixedPoint (unsigned)")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let order = get_byte_order(datatype); let (off, prec) = fixed_bits(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let full = read_unsigned_int(chunk, elem_size, &order); result.push(extract_unsigned(full, off, prec)); } Ok(result) } /// Convert raw bytes to `f32` values. pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_f32(raw, base_type); } ensure_numeric(datatype, "FloatingPoint")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native-endian f32 — single bulk memcpy #[cfg(target_endian = "little")] if matches!( datatype, Datatype::FloatingPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, .. } ) { let mut result = vec![0.0f32; count]; // SAFETY: The byte slice is properly aligned for this type and the length is divisible by size_of::(). unsafe { core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr() as *mut u8, raw.len()); } return Ok(result); } let order = get_byte_order(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; match datatype { Datatype::FloatingPoint { size: 4, .. } => { result.push(read_f32_bytes(chunk, &order)); } Datatype::FloatingPoint { size: 8, .. } => { result.push(read_f64_bytes(chunk, &order) as f32); } Datatype::FloatingPoint { size: 2, .. } => { result.push(read_f16_bytes(chunk, &order)); } Datatype::FixedPoint { signed: true, size, bit_offset, bit_precision, .. } => { let full = read_unsigned_int(chunk, *size as usize, &order); let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); result.push(extract_signed(full, off, prec) as f32); } Datatype::FixedPoint { signed: false, size, bit_offset, bit_precision, .. } => { let full = read_unsigned_int(chunk, *size as usize, &order); let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); result.push(extract_unsigned(full, off, prec) as f32); } _ => { return Err(FormatError::TypeMismatch { expected: "numeric", actual: datatype_name(datatype), }); } } } Ok(result) } /// Convert raw bytes to `i32` values. pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_i32(raw, base_type); } ensure_numeric(datatype, "FixedPoint")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native LE i32 — single bulk memcpy #[cfg(target_endian = "little")] if elem_size == 4 && is_full_width(datatype) && matches!( datatype, Datatype::FixedPoint { byte_order: DatatypeByteOrder::LittleEndian, .. } ) { let mut result = vec![0i32; count]; // SAFETY: The byte slice is properly aligned for this type and the length is divisible by size_of::(). unsafe { core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr() as *mut u8, raw.len()); } return Ok(result); } let order = get_byte_order(datatype); let (off, prec) = fixed_bits(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let full = read_unsigned_int(chunk, elem_size, &order); result.push(extract_signed(full, off, prec) as i32); } Ok(result) } /// Read fixed-length strings from raw bytes. pub fn read_as_strings(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { match datatype { Datatype::String { size, padding, .. } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let s = match padding { crate::datatype::StringPadding::NullTerminate => { let end = chunk.iter().position(|&b| b == 0).unwrap_or(chunk.len()); String::from_utf8_lossy(&chunk[..end]).into_owned() } crate::datatype::StringPadding::NullPad => { let end = chunk.iter().rposition(|&b| b != 0).map_or(0, |p| p + 1); String::from_utf8_lossy(&chunk[..end]).into_owned() } crate::datatype::StringPadding::SpacePad => { let end = chunk.iter().rposition(|&b| b != b' ').map_or(0, |p| p + 1); String::from_utf8_lossy(&chunk[..end]).into_owned() } }; result.push(s); } Ok(result) } _ => Err(FormatError::TypeMismatch { expected: "String", actual: datatype_name(datatype), }), } } // --- Compound type reading --- /// A single field extracted from compound data, containing the raw bytes for that field /// across all elements. #[derive(Debug, Clone)] pub struct CompoundFieldData { /// Field name. pub name: String, /// Datatype of this field. pub datatype: Datatype, /// Raw bytes for this field across all elements (len = num_elements * field_type_size). pub raw_data: Vec, } /// Read compound dataset and return all fields as separate data vectors. /// /// Each returned `CompoundFieldData` contains the raw bytes for that field /// across all elements, suitable for further typed conversion with `read_as_f64`, etc. pub fn read_compound_fields( raw: &[u8], datatype: &Datatype, ) -> Result, FormatError> { match datatype { Datatype::Compound { size, members } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut fields = Vec::with_capacity(members.len()); for m in members { let field_size = m.datatype.type_size() as usize; let offset = m.byte_offset as usize; if offset .checked_add(field_size) .is_none_or(|end| end > elem_size) { return Err(FormatError::Overflow(format!( "compound member '{}': byte_offset({offset}) + field_size({field_size}) exceeds element size({elem_size})", m.name ))); } let mut field_raw = Vec::with_capacity(count * field_size); for i in 0..count { let elem_start = i * elem_size + offset; field_raw.extend_from_slice(&raw[elem_start..elem_start + field_size]); } fields.push(CompoundFieldData { name: m.name.clone(), datatype: m.datatype.clone(), raw_data: field_raw, }); } Ok(fields) } _ => Err(FormatError::TypeMismatch { expected: "Compound", actual: datatype_name(datatype), }), } } /// Extract a single field by name from compound raw data. pub fn read_compound_field( raw: &[u8], datatype: &Datatype, field_name: &str, ) -> Result { let fields = read_compound_fields(raw, datatype)?; fields .into_iter() .find(|f| f.name == field_name) .ok_or_else(|| FormatError::PathNotFound(field_name.into())) } // --- Enum type reading --- /// A single value from an enum dataset, containing both the integer value and string name. #[derive(Debug, Clone)] pub struct EnumValue { /// The string name for this enum value. pub name: String, /// The raw integer value. pub raw_value: Vec, } /// Read enum dataset values, mapping integer values to their string names. /// /// Returns one `EnumValue` per element. Unknown values get name `UNKNOWN(hex)`. pub fn read_enum_values(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { match datatype { Datatype::Enumeration { size, members, .. } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Build lookup map: raw bytes -> name let mut lookup = BTreeMap::new(); for m in members { lookup.insert(m.value.clone(), m.name.clone()); } let mut result = Vec::with_capacity(count); for i in 0..count { let val_bytes = raw[i * elem_size..(i + 1) * elem_size].to_vec(); let name = lookup.get(&val_bytes).cloned().unwrap_or_else(|| { let hex: Vec = val_bytes .iter() .map(|b| { let mut s = String::new(); core::fmt::Write::write_fmt(&mut s, format_args!("{b:02x}")).ok(); s }) .collect(); let mut result = String::from("UNKNOWN(0x"); for h in &hex { result.push_str(h); } result.push(')'); result }); result.push(EnumValue { name, raw_value: val_bytes, }); } Ok(result) } _ => Err(FormatError::TypeMismatch { expected: "Enumeration", actual: datatype_name(datatype), }), } } /// Read enum dataset and return just the string names. pub fn read_enum_names(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { let values = read_enum_values(raw, datatype)?; Ok(values.into_iter().map(|v| v.name).collect()) } // --- Reference type reading --- /// A resolved object reference: the file address of the referenced object header. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ObjectReference { /// The file address of the referenced object header. /// A value of `u64::MAX` (all 0xFF bytes) indicates a null reference. pub address: u64, } impl ObjectReference { /// Returns `true` if this is a null (unset) reference. pub fn is_null(&self) -> bool { self.address == u64::MAX } } /// A region reference: raw bytes that encode a dataset selection. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RegionReference { /// The raw region reference bytes. A region reference is typically 12 bytes /// (object address + dataspace selection) but the exact layout depends on /// the file's offset size and the selection type. pub raw: Vec, } /// Read object references from raw bytes. /// /// Object references are stored as `offset_size`-byte file addresses pointing /// to the object header of the referenced object. A reference consisting of /// all `0xFF` bytes is a null (unset) reference. /// /// # Arguments /// * `raw` — raw bytes read from the dataset /// * `datatype` — must be `Datatype::Reference` with `ReferenceType::Object` /// * `offset_size` — the file's offset size (from superblock), typically 8 pub fn read_object_references( raw: &[u8], datatype: &Datatype, offset_size: u8, ) -> Result, FormatError> { match datatype { Datatype::Reference { ref_type: crate::datatype::ReferenceType::Object, size, } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut result = Vec::with_capacity(count); let read_size = (offset_size as usize).min(elem_size); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let address = read_ref_address(chunk, read_size); result.push(ObjectReference { address }); } Ok(result) } _ => Err(FormatError::TypeMismatch { expected: "Reference(Object)", actual: datatype_name(datatype), }), } } /// Read region references from raw bytes. /// /// Region references encode a dataset selection (hyperslab, point list, etc.) /// along with the address of the target dataset. This function returns the /// raw bytes for each reference without decoding the selection, since the /// full region reference format is complex and depends on the selection type. /// /// # Arguments /// * `raw` — raw bytes read from the dataset /// * `datatype` — must be `Datatype::Reference` with `ReferenceType::DatasetRegion` pub fn read_region_references( raw: &[u8], datatype: &Datatype, ) -> Result, FormatError> { match datatype { Datatype::Reference { ref_type: crate::datatype::ReferenceType::DatasetRegion, size, } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(RegionReference { raw: chunk.to_vec(), }); } Ok(result) } _ => Err(FormatError::TypeMismatch { expected: "Reference(DatasetRegion)", actual: datatype_name(datatype), }), } } /// Read a file address from reference bytes (little-endian). fn read_ref_address(bytes: &[u8], size: usize) -> u64 { let mut buf = [0xFFu8; 8]; let len = size.min(bytes.len()).min(8); buf[..len].copy_from_slice(&bytes[..len]); // If we read fewer than 8 bytes, check if ALL read bytes are 0xFF (null ref) if len < 8 && bytes[..len].iter().all(|&b| b == 0xFF) { return u64::MAX; } // Zero-extend upper bytes for non-null refs if len < 8 && !bytes[..len].iter().all(|&b| b == 0xFF) { for b in buf[len..].iter_mut() { *b = 0; } } u64::from_le_bytes(buf) } // --- Array type reading --- /// Read array-typed dataset elements, returning the raw base-type data. /// /// For an array type with dimensions [D1, D2, ...] and base type T, /// each dataset element contains D1*D2*... values of type T. /// This function returns the raw bytes as a flat buffer that can be /// converted with `read_as_f64`, `read_as_i32`, etc. using the base type. pub fn read_array_flat( raw: &[u8], datatype: &Datatype, ) -> Result<(Vec, Datatype, Vec), FormatError> { match datatype { Datatype::Array { base_type, dimensions, } => Ok((raw.to_vec(), *base_type.clone(), dimensions.clone())), _ => Err(FormatError::TypeMismatch { expected: "Array", actual: datatype_name(datatype), }), } } // --- Low-level byte conversion helpers --- fn reorder_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> [u8; 8] { let mut buf = [0u8; 8]; let len = bytes.len().min(8); match order { DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => { buf[..len].copy_from_slice(&bytes[..len]); } DatatypeByteOrder::BigEndian => { // Reverse bytes into LE order for i in 0..len { buf[i] = bytes[len - 1 - i]; } } } buf } fn read_f64_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f64 { let buf = reorder_bytes(bytes, order); f64::from_le_bytes(buf) } /// Decode an IEEE-754 half-precision (binary16) value to `f32`. Pure integer /// bit manipulation (no_std-safe, no `powi`/`libm`). fn read_f16_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 { let mut buf = [0u8; 2]; let len = bytes.len().min(2); match order { DatatypeByteOrder::BigEndian => { for i in 0..len { buf[i] = bytes[len - 1 - i]; } } _ => buf[..len].copy_from_slice(&bytes[..len]), } f16_bits_to_f32(u16::from_le_bytes(buf)) } /// Convert the bit pattern of an IEEE-754 half (binary16) to an `f32`. fn f16_bits_to_f32(h: u16) -> f32 { let h = h as u32; let sign = (h & 0x8000) << 16; let exp = (h >> 10) & 0x1f; let mant = h & 0x3ff; let bits = if exp == 0 { if mant == 0 { sign // signed zero } else { // Subnormal: normalize into an f32 normal. let mut e: i32 = -1; let mut m = mant; loop { e += 1; m <<= 1; if m & 0x400 != 0 { break; } } let m = m & 0x3ff; sign | (((127 - 15 - e) as u32) << 23) | (m << 13) } } else if exp == 0x1f { sign | 0x7f80_0000 | (mant << 13) // inf / NaN } else { sign | ((exp + (127 - 15)) << 23) | (mant << 13) }; f32::from_bits(bits) } fn read_f32_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 { let mut buf = [0u8; 4]; let len = bytes.len().min(4); match order { DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => { buf[..len].copy_from_slice(&bytes[..len]); } DatatypeByteOrder::BigEndian => { for i in 0..len { buf[i] = bytes[len - 1 - i]; } } } f32::from_le_bytes(buf) } /// Effective (bit offset, bit precision) for a fixed-point field, defaulting a /// zero precision to the full storage width. fn effective_bits(size: usize, bit_offset: u16, bit_precision: u16) -> (u32, u32) { let prec = if bit_precision == 0 { (size * 8) as u32 } else { bit_precision as u32 }; (bit_offset as u32, prec) } /// `(bit_offset, bit_precision)` for a fixed-point datatype, full width for /// other types. fn fixed_bits(datatype: &Datatype) -> (u32, u32) { match datatype { Datatype::FixedPoint { size, bit_offset, bit_precision, .. } => effective_bits(*size as usize, *bit_offset, *bit_precision), _ => (0, 0), } } /// Whether a datatype occupies its full storage width (bit offset 0, precision /// == size·8), in which case the bulk-copy fast read paths apply. Non /// fixed-point types are treated as full width. fn is_full_width(datatype: &Datatype) -> bool { match datatype { Datatype::FixedPoint { size, bit_offset, bit_precision, .. } => *bit_offset == 0 && *bit_precision as u32 == *size * 8, _ => true, } } /// Extract the `precision`-bit field at `offset` from a full-width integer read /// and sign-extend it. Full-width fields read as an ordinary signed integer; /// reduced-precision fields sign-extend from the field's top bit (HDF5 stores /// reduced-precision values zero-filled, so the sign lives in the precision /// field, not the storage word). fn extract_signed(full: u64, offset: u32, precision: u32) -> i64 { if precision == 0 || precision >= 64 { return full as i64; } let field = (full >> offset) & ((1u64 << precision) - 1); let shift = 64 - precision; ((field << shift) as i64) >> shift } /// Extract the `precision`-bit field at `offset` from a full-width integer read. fn extract_unsigned(full: u64, offset: u32, precision: u32) -> u64 { if precision == 0 || precision >= 64 { return full; } (full >> offset) & ((1u64 << precision) - 1) } fn read_unsigned_int(bytes: &[u8], size: usize, order: &DatatypeByteOrder) -> u64 { let buf = reorder_bytes(bytes, order); match size { 1 => buf[0] as u64, 2 => u16::from_le_bytes([buf[0], buf[1]]) as u64, 4 => u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as u64, 8 => u64::from_le_bytes(buf), _ => { // Generic: read as LE let mut val = 0u64; for (i, &byte) in buf.iter().enumerate().take(size.min(8)) { val |= (byte as u64) << (i * 8); } val } } } // --- Type conversion cost analysis --- /// Cost classification for type conversions. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ConversionCost { /// No conversion needed (same type). None, /// Widening conversion (no data loss, e.g. f32 → f64, i32 → i64). Widening, /// Narrowing conversion (potential precision loss, e.g. f64 → f32). Narrowing, /// Lossy conversion (potential data corruption, e.g. float → int). Lossy, } /// Information about a type conversion. #[derive(Debug, Clone)] pub struct ConversionInfo { /// Source datatype name. pub source_type: &'static str, /// Target datatype name. pub target_type: &'static str, /// Cost of the conversion. pub cost: ConversionCost, } /// Check the cost of converting from one datatype to another. /// /// This helps callers understand whether a read operation involves /// potentially lossy type coercion. pub fn check_conversion_cost(source: &Datatype, target: &'static str) -> ConversionInfo { let source_name = datatype_name(source); let source_size = source.type_size(); let cost = match (source_name, target) { // Same type (s, t) if s == t => ConversionCost::None, // Float widening: f32 → f64 ("f32", "f64") => ConversionCost::Widening, // Float narrowing: f64 → f32 ("f64", "f32") => ConversionCost::Narrowing, // Integer widening ("i32", "i64") | ("u8", "i32") | ("u8", "i64") | ("u8", "u64") | ("i32", "u64") => { ConversionCost::Widening } // Integer narrowing ("i64", "i32") | ("i64", "u8") | ("i32", "u8") | ("u64", "i32") | ("u64", "u8") => { ConversionCost::Narrowing } // Float ↔ Integer: lossy ("f32" | "f64", "i32" | "i64" | "u8" | "u64") => ConversionCost::Lossy, ("i32" | "i64" | "u8" | "u64", "f32" | "f64") => { // Int → Float: widening if source fits, but technically lossy for large ints if source_size <= 4 && target == "f64" { ConversionCost::Widening } else { ConversionCost::Narrowing } } // Unknown combinations _ => ConversionCost::Lossy, }; ConversionInfo { source_type: source_name, target_type: target, cost, } } #[cfg(test)] mod tests { use super::*; use crate::dataspace::{Dataspace, DataspaceType}; use crate::datatype::{CharacterSet, StringPadding}; fn f16_datatype() -> Datatype { Datatype::FloatingPoint { size: 2, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 16, exponent_location: 10, exponent_size: 5, mantissa_location: 0, mantissa_size: 10, exponent_bias: 15, } } // IEEE-754 half bit patterns for known values. fn f16_bits(v: f32) -> u16 { // Encode a few exact values used by the test. match v { x if x == 0.0 => 0x0000, x if x == 1.0 => 0x3c00, x if x == -2.0 => 0xc000, x if x == 0.5 => 0x3800, x if x == 65504.0 => 0x7bff, // f16 max _ => panic!("unsupported test value {v}"), } } #[test] fn read_f16_as_f32_and_f64() { let values = [0.0f32, 1.0, -2.0, 0.5, 65504.0]; let raw: Vec = values .iter() .flat_map(|&v| f16_bits(v).to_le_bytes()) .collect(); let dt = f16_datatype(); let got32 = read_as_f32(&raw, &dt).unwrap(); assert_eq!(got32, values); let got64 = read_as_f64(&raw, &dt).unwrap(); let expect64: Vec = values.iter().map(|&v| v as f64).collect(); assert_eq!(got64, expect64); } fn reduced_int(signed: bool, precision: u16) -> Datatype { Datatype::FixedPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, signed, bit_offset: 0, bit_precision: precision, } } #[test] fn reduced_precision_signed_sign_extends() { // 16-bit-precision signed values stored zero-filled (HDF5's canonical // layout, e.g. after N-Bit): the reader must sign-extend from bit 15. let dt = reduced_int(true, 16); // [-1, 100, -50, -32768] as 0x0000ffff / 0x00000064 / 0x0000ffce / 0x00008000 let raw: Vec = vec![ 0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xce, 0xff, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, ]; assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]); assert_eq!(read_as_i64(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]); } #[test] fn reduced_precision_unsigned_masks() { // 12-bit-precision unsigned: high bits must read as zero, not sign. let dt = reduced_int(false, 12); // [4095, 1, 2048] as 0x00000fff / 0x00000001 / 0x00000800 let raw: Vec = vec![ 0xff, 0x0f, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, ]; assert_eq!(read_as_u64(&raw, &dt).unwrap(), vec![4095, 1, 2048]); } #[test] fn full_width_signed_unchanged() { // Regression: full-width 32-bit signed must be unaffected. let dt = reduced_int(true, 32); let raw: Vec = vec![0xff, 0xff, 0xff, 0xff, 0x2a, 0x00, 0x00, 0x00]; assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 42]); } #[test] fn array_datatype_reads_flat_base_elements() { // An array-typed (e.g. compound member) datatype reads as a flat // sequence of its base elements, applying base-type precision rules. let arr = Datatype::Array { base_type: Box::new(reduced_int(true, 16)), dimensions: vec![2], }; // [-1, 100, 1000, -32768] stored zero-filled at 16-bit precision. let raw: Vec = vec![ 0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, ]; assert_eq!( read_as_i32(&raw, &arr).unwrap(), vec![-1, 100, 1000, -32768] ); // Nested array-of-array unwraps recursively. let nested = Datatype::Array { base_type: Box::new(arr), dimensions: vec![2], }; assert_eq!( read_as_i32(&raw, &nested).unwrap(), vec![-1, 100, 1000, -32768] ); } fn make_f64_le_type() -> Datatype { Datatype::FloatingPoint { size: 8, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 64, exponent_location: 52, exponent_size: 11, mantissa_location: 0, mantissa_size: 52, exponent_bias: 1023, } } fn make_f32_be_type() -> Datatype { Datatype::FloatingPoint { size: 4, byte_order: DatatypeByteOrder::BigEndian, bit_offset: 0, bit_precision: 32, exponent_location: 23, exponent_size: 8, mantissa_location: 0, mantissa_size: 23, exponent_bias: 127, } } fn make_i32_le_type() -> Datatype { Datatype::FixedPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, signed: true, bit_offset: 0, bit_precision: 32, } } fn make_i16_le_type() -> Datatype { Datatype::FixedPoint { size: 2, byte_order: DatatypeByteOrder::LittleEndian, signed: true, bit_offset: 0, bit_precision: 16, } } fn make_u8_type() -> Datatype { Datatype::FixedPoint { size: 1, byte_order: DatatypeByteOrder::LittleEndian, signed: false, bit_offset: 0, bit_precision: 8, } } fn make_simple_dataspace(dims: &[u64]) -> Dataspace { Dataspace { space_type: DataspaceType::Simple, rank: dims.len() as u8, dimensions: dims.to_vec(), max_dimensions: None, } } #[test] fn read_f64_compact() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let mut data = Vec::new(); data.extend_from_slice(&1.0f64.to_le_bytes()); data.extend_from_slice(&2.0f64.to_le_bytes()); data.extend_from_slice(&3.0f64.to_le_bytes()); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); assert_eq!(raw, data); let values = read_as_f64(&raw, &dt).unwrap(); assert_eq!(values, vec![1.0, 2.0, 3.0]); } #[test] fn read_i32_contiguous() { let dt = make_i32_le_type(); let ds = make_simple_dataspace(&[4]); let mut file_data = vec![0u8; 1024]; let offset = 256usize; let vals: Vec = vec![10, -20, 30, -40]; for (i, v) in vals.iter().enumerate() { let bytes = v.to_le_bytes(); file_data[offset + i * 4..offset + i * 4 + 4].copy_from_slice(&bytes); } let layout = DataLayout::Contiguous { address: Some(offset as u64), size: 16, }; let raw = read_raw_data(&file_data, &layout, &ds, &dt).unwrap(); let result = read_as_i32(&raw, &dt).unwrap(); assert_eq!(result, vec![10, -20, 30, -40]); } #[test] fn read_u8_data() { let dt = make_u8_type(); let ds = make_simple_dataspace(&[5]); let data = vec![10u8, 20, 30, 40, 50]; let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_u64(&raw, &dt).unwrap(); assert_eq!(result, vec![10, 20, 30, 40, 50]); } #[test] fn read_f32_be() { let dt = make_f32_be_type(); let ds = make_simple_dataspace(&[2]); let mut data = Vec::new(); // Store as big-endian data.extend_from_slice(&1.5f32.to_be_bytes()); data.extend_from_slice(&2.5f32.to_be_bytes()); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_f32(&raw, &dt).unwrap(); assert_eq!(result, vec![1.5, 2.5]); } #[test] fn read_i16_le() { let dt = make_i16_le_type(); let ds = make_simple_dataspace(&[3]); let mut data = Vec::new(); data.extend_from_slice(&(-100i16).to_le_bytes()); data.extend_from_slice(&200i16.to_le_bytes()); data.extend_from_slice(&(-300i16).to_le_bytes()); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_i64(&raw, &dt).unwrap(); assert_eq!(result, vec![-100, 200, -300]); } #[test] fn read_strings_compact() { let dt = Datatype::String { size: 5, padding: StringPadding::NullPad, charset: CharacterSet::Ascii, }; let ds = make_simple_dataspace(&[2]); let mut data = Vec::new(); data.extend_from_slice(b"hello"); data.extend_from_slice(b"hi\0\0\0"); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_strings(&raw, &dt).unwrap(); assert_eq!(result, vec!["hello", "hi"]); } #[test] fn type_mismatch_f64_on_string() { let dt = Datatype::String { size: 4, padding: StringPadding::NullTerminate, charset: CharacterSet::Ascii, }; let raw = vec![0u8; 8]; let err = read_as_f64(&raw, &dt).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn size_mismatch_compact() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let data = vec![0u8; 16]; // wrong: should be 24 let layout = DataLayout::Compact { data }; let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::DataSizeMismatch { .. })); } #[test] fn no_data_allocated() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let layout = DataLayout::Contiguous { address: None, size: 24, }; let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::NoDataAllocated)); } #[test] fn string_type_mismatch_on_read_as_strings() { let dt = make_i32_le_type(); let raw = vec![0u8; 8]; let err = read_as_strings(&raw, &dt).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn read_f64_from_i32() { // read_as_f64 should work on FixedPoint types too let dt = make_i32_le_type(); let mut raw = Vec::new(); raw.extend_from_slice(&42i32.to_le_bytes()); raw.extend_from_slice(&(-7i32).to_le_bytes()); let result = read_as_f64(&raw, &dt).unwrap(); assert_eq!(result, vec![42.0, -7.0]); } #[test] fn read_strings_space_padded() { let dt = Datatype::String { size: 8, padding: StringPadding::SpacePad, charset: CharacterSet::Ascii, }; let raw = b"hello world "; let result = read_as_strings(raw, &dt).unwrap(); assert_eq!(result, vec!["hello", "world"]); } #[test] fn read_strings_null_terminated() { let dt = Datatype::String { size: 6, padding: StringPadding::NullTerminate, charset: CharacterSet::Ascii, }; let raw = b"abc\0\0\0de\0\0\0\0"; let result = read_as_strings(raw, &dt).unwrap(); assert_eq!(result, vec!["abc", "de"]); } #[test] fn read_compound_two_fields() { use crate::datatype::CompoundMember; // Compound: { x: f64, id: i32 } => size = 12 let dt = Datatype::Compound { size: 12, members: vec![ CompoundMember { name: "x".to_string(), byte_offset: 0, datatype: make_f64_le_type(), }, CompoundMember { name: "id".to_string(), byte_offset: 8, datatype: make_i32_le_type(), }, ], }; // Two elements let mut raw = Vec::new(); raw.extend_from_slice(&1.5f64.to_le_bytes()); raw.extend_from_slice(&10i32.to_le_bytes()); raw.extend_from_slice(&2.5f64.to_le_bytes()); raw.extend_from_slice(&20i32.to_le_bytes()); let fields = read_compound_fields(&raw, &dt).unwrap(); assert_eq!(fields.len(), 2); assert_eq!(fields[0].name, "x"); let x_vals = read_as_f64(&fields[0].raw_data, &fields[0].datatype).unwrap(); assert_eq!(x_vals, vec![1.5, 2.5]); assert_eq!(fields[1].name, "id"); let id_vals = read_as_i32(&fields[1].raw_data, &fields[1].datatype).unwrap(); assert_eq!(id_vals, vec![10, 20]); } #[test] fn read_compound_rejects_byte_offset_overrun() { use crate::datatype::CompoundMember; // Compound declares size=8, but the member's byte_offset(4) + its // field_size(8, f64) = 12 > 8 — a crafted out-of-range byte_offset. let dt = Datatype::Compound { size: 8, members: vec![CompoundMember { name: "bad".to_string(), byte_offset: 4, datatype: make_f64_le_type(), }], }; let raw = vec![0u8; 8]; // one element, matches declared size let result = read_compound_fields(&raw, &dt); assert!( matches!(result, Err(FormatError::Overflow(_))), "expected a clean Overflow error, got {result:?}" ); } #[test] fn read_raw_data_zerocopy_rejects_near_usize_max_offset() { let file_data = vec![0u8; 64]; let dataspace = make_simple_dataspace(&[4]); let datatype = make_i32_le_type(); let layout = DataLayout::Contiguous { address: Some(u64::MAX - 4), size: 16, }; let result = read_raw_data_zerocopy(&file_data, &layout, &dataspace, &datatype); assert!( matches!(result, Err(FormatError::UnexpectedEof { .. })), "expected a clean UnexpectedEof, got {result:?}" ); } #[test] fn read_compound_single_field_by_name() { use crate::datatype::CompoundMember; let dt = Datatype::Compound { size: 12, members: vec![ CompoundMember { name: "x".to_string(), byte_offset: 0, datatype: make_f64_le_type(), }, CompoundMember { name: "id".to_string(), byte_offset: 8, datatype: make_i32_le_type(), }, ], }; let mut raw = Vec::new(); raw.extend_from_slice(&3.14f64.to_le_bytes()); raw.extend_from_slice(&42i32.to_le_bytes()); let field = read_compound_field(&raw, &dt, "id").unwrap(); let vals = read_as_i32(&field.raw_data, &field.datatype).unwrap(); assert_eq!(vals, vec![42]); // Non-existent field let err = read_compound_field(&raw, &dt, "missing").unwrap_err(); assert!(matches!(err, FormatError::PathNotFound(_))); } #[test] fn read_enum_values_basic() { use crate::datatype::EnumMember; let dt = Datatype::Enumeration { size: 4, base_type: Box::new(make_i32_le_type()), members: vec![ EnumMember { name: "RED".to_string(), value: 0i32.to_le_bytes().to_vec(), }, EnumMember { name: "GREEN".to_string(), value: 1i32.to_le_bytes().to_vec(), }, EnumMember { name: "BLUE".to_string(), value: 2i32.to_le_bytes().to_vec(), }, ], }; let mut raw = Vec::new(); raw.extend_from_slice(&1i32.to_le_bytes()); // GREEN raw.extend_from_slice(&0i32.to_le_bytes()); // RED raw.extend_from_slice(&2i32.to_le_bytes()); // BLUE raw.extend_from_slice(&99i32.to_le_bytes()); // unknown let names = read_enum_names(&raw, &dt).unwrap(); assert_eq!(names[0], "GREEN"); assert_eq!(names[1], "RED"); assert_eq!(names[2], "BLUE"); assert!(names[3].starts_with("UNKNOWN(")); } #[test] fn read_array_flat_basic() { // Array[3] of f64 let dt = Datatype::Array { base_type: Box::new(make_f64_le_type()), dimensions: vec![3], }; let mut raw = Vec::new(); for v in &[1.0f64, 2.0, 3.0] { raw.extend_from_slice(&v.to_le_bytes()); } let (data, base_dt, dims) = read_array_flat(&raw, &dt).unwrap(); assert_eq!(dims, vec![3]); let vals = read_as_f64(&data, &base_dt).unwrap(); assert_eq!(vals, vec![1.0, 2.0, 3.0]); } #[test] fn read_object_references_basic() { use crate::datatype::ReferenceType; let dt = Datatype::Reference { size: 8, ref_type: ReferenceType::Object, }; let mut raw = Vec::new(); raw.extend_from_slice(&1024u64.to_le_bytes()); // valid ref raw.extend_from_slice(&u64::MAX.to_le_bytes()); // null ref raw.extend_from_slice(&2048u64.to_le_bytes()); // valid ref let refs = read_object_references(&raw, &dt, 8).unwrap(); assert_eq!(refs.len(), 3); assert_eq!(refs[0].address, 1024); assert!(!refs[0].is_null()); assert!(refs[1].is_null()); assert_eq!(refs[2].address, 2048); } #[test] fn read_object_references_4byte_offset() { use crate::datatype::ReferenceType; let dt = Datatype::Reference { size: 4, ref_type: ReferenceType::Object, }; let mut raw = Vec::new(); raw.extend_from_slice(&512u32.to_le_bytes()); raw.extend_from_slice(&u32::MAX.to_le_bytes()); // null ref let refs = read_object_references(&raw, &dt, 4).unwrap(); assert_eq!(refs.len(), 2); assert_eq!(refs[0].address, 512); assert!(refs[1].is_null()); } #[test] fn read_object_references_type_mismatch() { let dt = make_f64_le_type(); let raw = vec![0u8; 8]; let err = read_object_references(&raw, &dt, 8).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn read_region_references_basic() { use crate::datatype::ReferenceType; let dt = Datatype::Reference { size: 12, ref_type: ReferenceType::DatasetRegion, }; let raw = vec![0xABu8; 24]; // two 12-byte region refs let refs = read_region_references(&raw, &dt).unwrap(); assert_eq!(refs.len(), 2); assert_eq!(refs[0].raw.len(), 12); assert_eq!(refs[1].raw.len(), 12); } #[test] fn read_region_references_type_mismatch() { let dt = make_i32_le_type(); let raw = vec![0u8; 12]; let err = read_region_references(&raw, &dt).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn zerocopy_contiguous_returns_slice_into_file_data() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let mut file_data = vec![0u8; 1024]; let offset = 256usize; let vals = [1.0f64, 2.0, 3.0]; for (i, v) in vals.iter().enumerate() { file_data[offset + i * 8..offset + i * 8 + 8].copy_from_slice(&v.to_le_bytes()); } let layout = DataLayout::Contiguous { address: Some(offset as u64), size: 24, }; let result = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap(); let slice = result.expect("contiguous should return Some"); // Pointer identity: the slice must point into file_data, not a copy let file_range = file_data.as_ptr_range(); assert!(file_range.contains(&slice.as_ptr())); assert_eq!(slice.len(), 24); // Verify the actual data let values = read_as_f64(slice, &dt).unwrap(); assert_eq!(values, vec![1.0, 2.0, 3.0]); } #[test] fn zerocopy_compact_returns_none() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[1]); let data = vec![0u8; 8]; let layout = DataLayout::Compact { data }; let result = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap(); assert!(result.is_none()); } #[test] fn zerocopy_no_data_allocated() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[1]); let layout = DataLayout::Contiguous { address: None, size: 8, }; let err = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::NoDataAllocated)); } #[test] fn read_as_f64_zerocopy_aligned() { let dt = make_f64_le_type(); // Create aligned data — Vec guarantees 8-byte alignment let values = [1.0f64, 2.0, 3.0, 4.0]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) }; let result = read_as_f64_zerocopy(raw, &dt); assert!(result.is_some(), "aligned native LE f64 should succeed"); let slice = result.unwrap(); assert_eq!(slice, &[1.0, 2.0, 3.0, 4.0]); // Verify it's the same memory (zero-copy) assert_eq!(slice.as_ptr() as *const u8, raw.as_ptr()); } #[test] fn read_as_f64_zerocopy_wrong_type() { let dt = make_i32_le_type(); let values = [1.0f64; 4]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) }; assert!(read_as_f64_zerocopy(raw, &dt).is_none()); } #[test] fn read_as_f64_zerocopy_big_endian() { let dt = Datatype::FloatingPoint { size: 8, byte_order: DatatypeByteOrder::BigEndian, bit_offset: 0, bit_precision: 64, exponent_location: 52, exponent_size: 11, mantissa_location: 0, mantissa_size: 52, exponent_bias: 1023, }; let values = [1.0f64; 4]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) }; assert!(read_as_f64_zerocopy(raw, &dt).is_none()); } #[test] fn read_as_f64_zerocopy_odd_size() { let dt = make_f64_le_type(); let raw = &[0u8; 13]; // not a multiple of 8 assert!(read_as_f64_zerocopy(raw, &dt).is_none()); } #[test] fn read_as_f32_zerocopy_aligned() { let dt = Datatype::FloatingPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 32, exponent_location: 23, exponent_size: 8, mantissa_location: 0, mantissa_size: 23, exponent_bias: 127, }; let values = [1.5f32, 2.5, 3.5]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 4) }; let result = read_as_f32_zerocopy(raw, &dt); assert!(result.is_some()); assert_eq!(result.unwrap(), &[1.5f32, 2.5, 3.5]); } #[test] fn zerocopy_size_mismatch() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let file_data = vec![0u8; 1024]; let layout = DataLayout::Contiguous { address: Some(0), size: 16, // wrong: should be 24 }; let err = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::DataSizeMismatch { .. })); } }