//! Raw data reading and typed conversion for HDF5 datasets. #[cfg(not(feature = "std"))] use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] use std::collections::BTreeMap; #[cfg(feature = "std")] use crate::chunk_cache::ChunkCache; use crate::chunked_read::read_chunked_data; #[cfg(feature = "std")] use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::datatype::{Datatype, DatatypeByteOrder}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; /// Checks that `[offset, offset + needed)` fits within `data`, guarding the /// addition against `usize` overflow from a crafted near-`usize::MAX` offset. fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { if offset .checked_add(needed) .is_none_or(|end| end > data.len()) { return Err(FormatError::UnexpectedEof { expected: offset.saturating_add(needed), available: data.len(), }); } Ok(()) } /// Zero-copy read of contiguous raw data, returning a borrowed slice. /// /// For contiguous layouts, returns a direct `&[u8]` slice into `file_data`. /// For compact or chunked layouts, returns `Ok(None)` — the caller should /// fall back to `read_raw_data` for those. pub fn read_raw_data_zerocopy<'a>( file_data: &'a [u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { let num_elements = dataspace.num_elements() as usize; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( "num_elements({num_elements}) * elem_size({elem_size})" )) })?; match layout { DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = addr as usize; let sz = *size as usize; if sz != expected_size { return Err(FormatError::DataSizeMismatch { expected: expected_size, actual: sz, }); } ensure_len(file_data, addr, sz)?; Ok(Some(&file_data[addr..addr + sz])) } _ => Ok(None), } } /// Read raw bytes for a dataset given its layout and the file data buffer. /// /// For compact layouts, returns the inline data. /// For contiguous layouts, reads from the address in the file buffer. /// For chunked layouts, traverses the B-tree and assembles chunks. pub fn read_raw_data( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8) } /// Resolves a Virtual Dataset source **file name** (as stored in the mapping, /// e.g. `"ext_src.h5"`) to that file's raw bytes. /// /// The pure-byte read API has no filesystem of its own, so external-file VDS /// sources are read through a caller-supplied resolver. The std file API wires /// one that reads relative to the virtual file's directory; callers can supply /// their own (e.g. an in-memory map) in `no_std` builds. Returning `None` means /// the source file is unavailable and the mapping is skipped. pub type VdsSourceResolver<'a> = dyn Fn(&str) -> Option> + 'a; /// Read raw bytes with full parameters including filter pipeline and sizes. pub fn read_raw_data_full( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, None, ) } /// Like [`read_raw_data_full`], but with a resolver for external-file Virtual /// Dataset sources. For non-virtual layouts the resolver is ignored. #[allow(clippy::too_many_arguments)] pub fn read_raw_data_full_with_resolver( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, resolver, ) } #[allow(clippy::too_many_arguments)] fn read_raw_data_full_impl( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { let num_elements = dataspace.num_elements() as usize; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( "num_elements({num_elements}) * elem_size({elem_size})" )) })?; match layout { DataLayout::Compact { data } => { if data.len() != expected_size { return Err(FormatError::DataSizeMismatch { expected: expected_size, actual: data.len(), }); } Ok(data.clone()) } DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = addr as usize; let sz = *size as usize; if sz != expected_size { return Err(FormatError::DataSizeMismatch { expected: expected_size, actual: sz, }); } ensure_len(file_data, addr, sz)?; let mut out = crate::bulk_alloc::vec_for_bulk(sz); out.extend_from_slice(&file_data[addr..addr + sz]); Ok(out) } DataLayout::Chunked { .. } => read_chunked_data( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ), DataLayout::Virtual { .. } => read_virtual_data( file_data, layout, dataspace, datatype, offset_size, length_size, resolver, ), } } /// Read raw bytes with chunk cache support. /// /// For chunked layouts the `cache` is used to avoid repeated B-tree /// traversals and to cache decompressed chunk data. For compact and /// contiguous layouts this behaves identically to [`read_raw_data_full`]. #[cfg(feature = "std")] #[allow(clippy::too_many_arguments)] pub fn read_raw_data_cached( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, cache: &ChunkCache, ) -> Result, FormatError> { match layout { DataLayout::Chunked { .. } => read_chunked_data_cached( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, cache, ), _ => read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ), } } /// Read raw bytes with chunk B-tree index cache and pre-computed layout. /// /// For chunked layouts this uses the optimized `read_chunked_data_indexed` path /// which pre-computes contiguous row-copy operations, eliminating per-element /// N-D coordinate math on repeated reads. For compact and contiguous layouts /// this behaves identically to [`read_raw_data_full`]. #[cfg(feature = "std")] #[allow(clippy::too_many_arguments)] pub fn read_raw_data_indexed( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, cache: &ChunkCache, ) -> Result, FormatError> { match layout { DataLayout::Chunked { .. } => read_chunked_data_indexed( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, cache, ), _ => read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ), } } /// Read raw bytes for only the selected elements of a dataset. /// /// When the selection's bounding box covers at most half the dataset, only /// that box is materialised — the overlapping rows of a contiguous dataset, /// the overlapping chunks of a chunked one, whatever its chunk index (see /// [`crate::partial_read`]). Otherwise, and for compact and virtual /// layouts, the whole dataset is decoded and the selection extracted. #[allow(clippy::too_many_arguments)] pub fn read_raw_data_selection( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, selection: &crate::selection::Selection, ) -> Result, FormatError> { use crate::selection::Selection; crate::partial_read::validate(selection, &dataspace.dimensions)?; crate::chunked_read::check_chunk_element_size(layout, datatype, offset_size)?; // Read only what the selection's bounding box touches when that is // possible; everything below is the decode-everything-then-pick path, // kept for the cases `partial_read` declines. if let Some(selected) = crate::partial_read::read_selection( file_data, layout, dataspace, datatype.type_size() as usize, pipeline, offset_size, length_size, selection, )? { return Ok(selected); } match selection { Selection::All => { return read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, ); } Selection::None => return Ok(Vec::new()), _ => {} } let dims = &dataspace.dimensions; let elem_size = datatype.type_size() as usize; match layout { DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => { // Read all data, then extract the selection let full_data = read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, )?; extract_selection_from_buffer(&full_data, dims, elem_size, selection) } DataLayout::Chunked { chunk_dimensions, version, .. } => { // `partial_read` declined (a bounding box covering most of the // dataset, or a selection it doesn't box), so decode every chunk // and pick the selection out, whatever the chunk index. This arm // used to enumerate the chunks first — passing the layout's // chunk dimensions, element-size dimension included, to the // implicit-index generator, which then indexed past the rank and // panicked — only to decode the full dataset anyway. crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?; let full_data = read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, )?; extract_selection_from_buffer(&full_data, dims, elem_size, selection) } DataLayout::Virtual { .. } => { // Assemble the full virtual dataset, then apply the read selection. let full_data = read_raw_data_full( file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, )?; extract_selection_from_buffer(&full_data, dims, elem_size, selection) } } } /// Assemble a **Virtual Dataset (VDS)** through the raw-read API, which has no /// access to the dataset's fill value message. /// /// Delegates to [`crate::vds::read_virtual_dataset`]. Because the fill value /// is unknown here, a virtual dataset with any element no mapping supplies /// (an unmapped region, or a missing source file or dataset) is an error /// rather than a guess at the fill value; so is one whose extent libhdf5 /// would report differently from the stored dataspace (unlimited mappings). /// Use [`crate::vds::read_virtual_dataset`] to read those. #[allow(clippy::too_many_arguments)] fn read_virtual_data( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { let wrapped = resolver.map(|r| move |name: &str| -> Result>, FormatError> { Ok(r(name)) }); let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver); let v = crate::vds::read_virtual_dataset( file_data, layout, dataspace, datatype, None, offset_size, length_size, wrapped_ref, )?; if v.dims != dataspace.dimensions { return Err(FormatError::ChunkedReadError( "virtual dataset extent differs from its stored dataspace; \ read it with vds::read_virtual_dataset" .into(), )); } if v.unmapped > 0 { return Err(FormatError::ChunkedReadError( "virtual dataset has elements no source supplies, which read as its \ fill value; read it with vds::read_virtual_dataset and the fill value" .into(), )); } Ok(v.data) } /// Extract selected elements from a full dataset buffer. pub fn extract_selection_from_buffer( full_data: &[u8], dims: &[u64], elem_size: usize, selection: &crate::selection::Selection, ) -> Result, FormatError> { use crate::selection::Selection; match selection { Selection::All => Ok(full_data.to_vec()), Selection::None => Ok(Vec::new()), Selection::Hyperslab { start, stride, count, block, } => { let rank = dims.len(); if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { return Err(FormatError::SelectionOutOfBounds(format!( "hyperslab rank does not match dataset rank {rank}" ))); } let output_elements = count .iter() .zip(block.iter()) .try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?)) .ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?; let mut output = crate::chunked_read::alloc_output( crate::chunked_read::checked_byte_len(output_elements, elem_size)?, )?; // One copy per run of elements contiguous in `full_data` // (`gather`'s runs). Coordinates past the extent are skipped and // runs past the end of `full_data` left as zeros, element by // element, as this extractor always did; validated selections // never hit either. let mut out_at = 0usize; crate::gather::hyperslab_runs(dims, start, stride, count, block, |first, n| { let big = |v: u64| usize::try_from(v).unwrap_or(usize::MAX); let (first, n) = (big(first), big(n)); let len = n.saturating_mul(elem_size); let src = first.saturating_mul(elem_size); let out_end = out_at.saturating_add(len); if let (Some(from), Some(to)) = ( full_data.get(src..src.saturating_add(len)), output.get_mut(out_at..out_end), ) { to.copy_from_slice(from); } else { for k in 0..n { let s = first.saturating_add(k).saturating_mul(elem_size); let o = out_at.saturating_add(k.saturating_mul(elem_size)); if o >= output.len() { break; } if let (Some(from), Some(to)) = ( full_data.get(s..s.saturating_add(elem_size)), output.get_mut(o..o.saturating_add(elem_size)), ) { to.copy_from_slice(from); } } } out_at = out_end; }); Ok(output) } Selection::Points(pts) => { let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; } let mut output = Vec::with_capacity(pts.len() * elem_size); for pt in pts { let flat: usize = pt .iter() .zip(ds_strides.iter()) .map(|(&p, &s)| p as usize * s) .sum(); let src = flat * elem_size; if src + elem_size <= full_data.len() { output.extend_from_slice(&full_data[src..src + elem_size]); } else { output.extend_from_slice(&vec![0u8; elem_size]); } } Ok(output) } } } /// Zero-copy transmute of raw bytes to `&[f64]`. /// /// Returns `Some(&[f64])` when the datatype is native little-endian `f64` /// and the data pointer is 8-byte aligned. Returns `None` for any other /// type or alignment — the caller should fall back to [`read_as_f64`]. pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f64]> { // Only native LE f64 is eligible #[cfg(target_endian = "little")] { if !is_native_le_float(datatype, FloatFormat::Double) { return None; } if !raw.len().is_multiple_of(8) { return None; } let ptr = raw.as_ptr(); if !(ptr as usize).is_multiple_of(core::mem::align_of::()) { return None; } let count = raw.len() / 8; // SAFETY: We verified alignment (8-byte), size (multiple of 8), and // the on-disk format matches the in-memory representation (LE f64). Some(unsafe { core::slice::from_raw_parts(ptr as *const f64, count) }) } #[cfg(not(target_endian = "little"))] { let _ = (raw, datatype); None } } /// Zero-copy transmute of raw bytes to `&[f32]`. /// /// Returns `Some(&[f32])` when the datatype is native little-endian `f32` /// and the data pointer is 4-byte aligned. Returns `None` otherwise. pub fn read_as_f32_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f32]> { #[cfg(target_endian = "little")] { if !is_native_le_float(datatype, FloatFormat::Single) { return None; } if !raw.len().is_multiple_of(4) { return None; } let ptr = raw.as_ptr(); if !(ptr as usize).is_multiple_of(core::mem::align_of::()) { return None; } let count = raw.len() / 4; // SAFETY: We verified alignment (4-byte), size (multiple of 4), and // the on-disk format matches the in-memory representation (LE f32). Some(unsafe { core::slice::from_raw_parts(ptr as *const f32, count) }) } #[cfg(not(target_endian = "little"))] { let _ = (raw, datatype); None } } fn datatype_name(dt: &Datatype) -> &'static str { match dt { Datatype::FixedPoint { .. } => "FixedPoint", Datatype::FloatingPoint { .. } => "FloatingPoint", Datatype::String { .. } => "String", Datatype::Time { .. } => "Time", Datatype::BitField { .. } => "BitField", Datatype::Opaque { .. } => "Opaque", Datatype::Compound { .. } => "Compound", Datatype::Reference { .. } => "Reference", Datatype::Enumeration { .. } => "Enumeration", Datatype::VariableLength { .. } => "VariableLength", Datatype::Array { .. } => "Array", } } fn ensure_numeric(dt: &Datatype, expected: &'static str) -> Result<(), FormatError> { match dt { Datatype::FixedPoint { .. } | Datatype::FloatingPoint { .. } => Ok(()), _ => Err(FormatError::TypeMismatch { expected, actual: datatype_name(dt), }), } } fn get_byte_order(dt: &Datatype) -> DatatypeByteOrder { match dt { Datatype::FixedPoint { byte_order, .. } => byte_order.clone(), Datatype::FloatingPoint { byte_order, .. } => byte_order.clone(), _ => DatatypeByteOrder::LittleEndian, } } fn get_size(dt: &Datatype) -> usize { dt.type_size() as usize } mod sealed { pub trait Sealed {} } /// A numeric type whose values can be copied straight out of a dataset's /// bytes when the dataset stores exactly that type in the target's byte /// order: `u8`, `i32`, `i64`, `u64`, `f32` and `f64`. /// /// # Safety /// /// Implementors have no padding and no invalid bit patterns, so a buffer of /// them may be filled by copying bytes. The trait is sealed. pub unsafe trait NativeElement: sealed::Sealed + Copy + 'static { /// Whether `datatype`'s stored bytes are this type's native in-memory /// representation (same size, byte order, signedness, full precision, /// IEEE layout), so reading needs a copy and no conversion. fn is_native(datatype: &Datatype) -> bool; } /// A full-width fixed-point type of `size` bytes and the given signedness in /// the target's byte order. fn is_native_int(datatype: &Datatype, size: u32, want_signed: bool) -> bool { let order = if cfg!(target_endian = "little") { DatatypeByteOrder::LittleEndian } else { DatatypeByteOrder::BigEndian }; matches!( datatype, Datatype::FixedPoint { size: s, signed, byte_order, .. } if *s == size && *signed == want_signed && (size == 1 || *byte_order == order) ) && is_full_width(datatype) } macro_rules! native_element { ($($t:ty => |$dt:ident| $check:expr;)*) => {$( impl sealed::Sealed for $t {} // SAFETY: a primitive integer or float: no padding, and every bit // pattern is a valid value. unsafe impl NativeElement for $t { fn is_native($dt: &Datatype) -> bool { $check } } )*}; } native_element! { u8 => |dt| is_native_int(dt, 1, false); i32 => |dt| is_native_int(dt, 4, true); i64 => |dt| is_native_int(dt, 8, true); u64 => |dt| is_native_int(dt, 8, false); f32 => |dt| cfg!(target_endian = "little") && is_native_le_float(dt, FloatFormat::Single); f64 => |dt| cfg!(target_endian = "little") && is_native_le_float(dt, FloatFormat::Double); } /// Copy `count` values of `T` out of `raw`, which holds them in `T`'s native /// representation (see [`NativeElement::is_native`]), in one copy. /// /// The buffer is allocated uninitialised and filled by the copy. It used to be /// `vec![0; count]` first, which for a large dataset meant writing every page /// twice (zero it, then overwrite it) — about as expensive as the copy itself. fn native_to_vec(raw: &[u8], count: usize) -> Vec { let bytes = count * core::mem::size_of::(); assert!(bytes <= raw.len(), "native_to_vec: source too short"); let mut result: Vec = crate::bulk_alloc::vec_for_bulk(count); // SAFETY: `result` has capacity for `count` values of `T`, i.e. `bytes` // bytes; `raw` holds at least `bytes` bytes (asserted); the regions // cannot overlap because `result` was just allocated. `T: NativeElement` // is valid for any bit pattern, so after the copy all `count` values are // initialised and `set_len` is sound. unsafe { core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr().cast::(), bytes); result.set_len(count); } result } /// Read `selection` of a dataset whose raw bytes (all of them, row-major, of /// shape `dims`) are `raw` — typically a contiguous dataset's bytes borrowed /// from the file — straight into a `Vec`, copying each contiguous run of /// selected elements once. /// /// Returns `Ok(None)` when `datatype` is not `T`'s native representation /// ([`NativeElement::is_native`]); the caller then converts through /// [`read_raw_data_selection`] and the `read_as_*` functions. The selection is /// validated like every selection read: out-of-range coordinates are /// [`FormatError::SelectionOutOfBounds`]. pub fn read_selection_native( raw: &[u8], dims: &[u64], datatype: &Datatype, selection: &crate::selection::Selection, ) -> Result>, FormatError> { if !T::is_native(datatype) { return Ok(None); } let elem_size = core::mem::size_of::(); let total = dims .iter() .try_fold(1u64, |acc, &d| acc.checked_mul(d)) .ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?; let expected = crate::chunked_read::checked_byte_len(total, elem_size)?; if raw.len() != expected { return Err(FormatError::DataSizeMismatch { expected, actual: raw.len(), }); } if let crate::selection::Selection::All = selection { return Ok(Some(native_to_vec(raw, expected / elem_size))); } crate::partial_read::validate(selection, dims)?; crate::gather::gather::(raw, dims, elem_size, selection).map(Some) } /// Convert raw bytes to `f64` values. pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes read as a flat sequence of their base elements, and // enumerations (h5py's bool among them) as their integer values. if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_f64(raw, base_type); } ensure_numeric(datatype, "FloatingPoint or FixedPoint")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native-endian f64 — single bulk memcpy if f64::is_native(datatype) { return Ok(native_to_vec::(raw, count)); } let order = get_byte_order(datatype); let mut result = crate::bulk_alloc::vec_for_bulk(count); if let Datatype::FloatingPoint { .. } = datatype { let format = FloatFormat::of(datatype)?; for chunk in raw.chunks_exact(elem_size) { result.push(format.decode(chunk, &order)); } return Ok(result); } for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let val = convert_to_f64(chunk, datatype, &order)?; result.push(val); } Ok(result) } fn convert_to_f64( bytes: &[u8], dt: &Datatype, order: &DatatypeByteOrder, ) -> Result { match dt { Datatype::FloatingPoint { .. } => Ok(FloatFormat::of(dt)?.decode(bytes, order)), Datatype::FixedPoint { size, signed, bit_offset, bit_precision, .. } => { let full = read_unsigned_int(bytes, *size as usize, order); let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); let v = if *signed { extract_signed(full, off, prec) as f64 } else { extract_unsigned(full, off, prec) as f64 }; Ok(v) } _ => Err(FormatError::TypeMismatch { expected: "numeric", actual: datatype_name(dt), }), } } /// One numeric element as stored, before conversion to the caller's type. #[derive(Debug, Clone, Copy, PartialEq)] enum Scalar { Signed(i64), Unsigned(u64), Float(f64), } impl Scalar { // Every conversion follows libhdf5's default (hard) conversions: a value // outside the target type's range saturates to its minimum or maximum — // including a negative value read as unsigned, which reads as 0 — rather // than being truncated to its low bits. Floats truncate toward zero; NaN // converts to 0 (libhdf5 leaves that case to the C cast, whose result is // platform-dependent). fn to_i64(self) -> i64 { match self { Scalar::Signed(v) => v, Scalar::Unsigned(v) => i64::try_from(v).unwrap_or(i64::MAX), Scalar::Float(v) => v as i64, } } fn to_u64(self) -> u64 { match self { Scalar::Signed(v) => u64::try_from(v).unwrap_or(0), Scalar::Unsigned(v) => v, Scalar::Float(v) => v as u64, } } fn to_i32(self) -> i32 { match self { Scalar::Signed(v) => v.clamp(i32::MIN.into(), i32::MAX.into()) as i32, Scalar::Unsigned(v) => i32::try_from(v).unwrap_or(i32::MAX), Scalar::Float(v) => v as i32, } } } /// Decode one element of a numeric datatype. fn decode_scalar( bytes: &[u8], dt: &Datatype, order: &DatatypeByteOrder, ) -> Result { match dt { Datatype::FixedPoint { size, signed, bit_offset, bit_precision, .. } => { let full = read_unsigned_int(bytes, *size as usize, order); let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); Ok(if *signed { Scalar::Signed(extract_signed(full, off, prec)) } else { Scalar::Unsigned(extract_unsigned(full, off, prec)) }) } _ => convert_to_f64(bytes, dt, order).map(Scalar::Float), } } /// Convert raw bytes to `i64` values. /// /// Values are converted the way libhdf5 converts them: integers outside the /// target range saturate at its minimum or maximum (a negative value read as /// unsigned is 0), and floating-point data is truncated toward zero and /// saturated, with NaN read as 0. pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes read as a flat sequence of their base elements, and // enumerations (h5py's bool among them) as their integer values. if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_i64(raw, base_type); } ensure_numeric(datatype, "FixedPoint (signed)")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native LE i64 — single bulk memcpy if i64::is_native(datatype) { return Ok(native_to_vec::(raw, count)); } let order = get_byte_order(datatype); let mut result = crate::bulk_alloc::vec_for_bulk(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(decode_scalar(chunk, datatype, &order)?.to_i64()); } Ok(result) } /// Convert raw bytes to `u64` values. /// /// Values are converted the way libhdf5 converts them: integers outside the /// target range saturate at its minimum or maximum (a negative value read as /// unsigned is 0), and floating-point data is truncated toward zero and /// saturated, with NaN read as 0. pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes read as a flat sequence of their base elements, and // enumerations (h5py's bool among them) as their integer values. if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_u64(raw, base_type); } ensure_numeric(datatype, "FixedPoint (unsigned)")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native u64 — single bulk memcpy if u64::is_native(datatype) { return Ok(native_to_vec::(raw, count)); } let order = get_byte_order(datatype); let mut result = crate::bulk_alloc::vec_for_bulk(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(decode_scalar(chunk, datatype, &order)?.to_u64()); } Ok(result) } /// Convert raw bytes to `f32` values. pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes read as a flat sequence of their base elements, and // enumerations (h5py's bool among them) as their integer values. if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_f32(raw, base_type); } ensure_numeric(datatype, "FloatingPoint")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native-endian f32 — single bulk memcpy if f32::is_native(datatype) { return Ok(native_to_vec::(raw, count)); } // Little-endian IEEE half precision (numpy float16): widen directly. if is_native_le_float(datatype, FloatFormat::Half) { let (halves, _) = raw[..count * 2].as_chunks::<2>(); let mut result = crate::bulk_alloc::vec_for_bulk(count); result.extend( halves .iter() .map(|&b| f16_bits_to_f32(u16::from_le_bytes(b))), ); return Ok(result); } let order = get_byte_order(datatype); let mut result = crate::bulk_alloc::vec_for_bulk(count); if let Datatype::FloatingPoint { .. } = datatype { let format = FloatFormat::of(datatype)?; for chunk in raw.chunks_exact(elem_size) { result.push(match format { FloatFormat::Single => read_f32_bytes(chunk, &order), FloatFormat::Half => read_f16_bytes(chunk, &order), // Double rounds; every other supported layout (bfloat16, FP8) // is exact in f32. _ => format.decode(chunk, &order) as f32, }); } return Ok(result); } for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; match datatype { Datatype::FixedPoint { signed: true, size, bit_offset, bit_precision, .. } => { let full = read_unsigned_int(chunk, *size as usize, &order); let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); result.push(extract_signed(full, off, prec) as f32); } Datatype::FixedPoint { signed: false, size, bit_offset, bit_precision, .. } => { let full = read_unsigned_int(chunk, *size as usize, &order); let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); result.push(extract_unsigned(full, off, prec) as f32); } _ => { return Err(FormatError::TypeMismatch { expected: "numeric", actual: datatype_name(datatype), }); } } } Ok(result) } /// Convert raw bytes to `i32` values. /// /// Values are converted the way libhdf5 converts them: integers outside the /// target range saturate at its minimum or maximum (a negative value read as /// unsigned is 0), and floating-point data is truncated toward zero and /// saturated, with NaN read as 0. pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes read as a flat sequence of their base elements, and // enumerations (h5py's bool among them) as their integer values. if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_i32(raw, base_type); } ensure_numeric(datatype, "FixedPoint")?; let elem_size = get_size(datatype); if elem_size == 0 || !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Fast path: native LE i32 — single bulk memcpy if i32::is_native(datatype) { return Ok(native_to_vec::(raw, count)); } let order = get_byte_order(datatype); let mut result = crate::bulk_alloc::vec_for_bulk(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(decode_scalar(chunk, datatype, &order)?.to_i32()); } Ok(result) } /// Read fixed-length strings from raw bytes. pub fn read_as_strings(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { match datatype { Datatype::String { size, padding, .. } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let s = match padding { crate::datatype::StringPadding::NullTerminate => { let end = chunk.iter().position(|&b| b == 0).unwrap_or(chunk.len()); String::from_utf8_lossy(&chunk[..end]).into_owned() } crate::datatype::StringPadding::NullPad => { let end = chunk.iter().rposition(|&b| b != 0).map_or(0, |p| p + 1); String::from_utf8_lossy(&chunk[..end]).into_owned() } crate::datatype::StringPadding::SpacePad => { let end = chunk.iter().rposition(|&b| b != b' ').map_or(0, |p| p + 1); String::from_utf8_lossy(&chunk[..end]).into_owned() } }; result.push(s); } Ok(result) } _ => Err(FormatError::TypeMismatch { expected: "String", actual: datatype_name(datatype), }), } } // --- Compound type reading --- /// A single field extracted from compound data, containing the raw bytes for that field /// across all elements. #[derive(Debug, Clone)] pub struct CompoundFieldData { /// Field name. pub name: String, /// Datatype of this field. pub datatype: Datatype, /// Raw bytes for this field across all elements (len = num_elements * field_type_size). pub raw_data: Vec, } /// Read compound dataset and return all fields as separate data vectors. /// /// Each returned `CompoundFieldData` contains the raw bytes for that field /// across all elements, suitable for further typed conversion with `read_as_f64`, etc. pub fn read_compound_fields( raw: &[u8], datatype: &Datatype, ) -> Result, FormatError> { match datatype { Datatype::Compound { size, members } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut fields = Vec::with_capacity(members.len()); for m in members { let field_size = m.datatype.type_size() as usize; let offset = m.byte_offset as usize; if offset .checked_add(field_size) .is_none_or(|end| end > elem_size) { return Err(FormatError::Overflow(format!( "compound member '{}': byte_offset({offset}) + field_size({field_size}) exceeds element size({elem_size})", m.name ))); } let mut field_raw = Vec::with_capacity(count * field_size); for i in 0..count { let elem_start = i * elem_size + offset; field_raw.extend_from_slice(&raw[elem_start..elem_start + field_size]); } fields.push(CompoundFieldData { name: m.name.clone(), datatype: m.datatype.clone(), raw_data: field_raw, }); } Ok(fields) } _ => Err(FormatError::TypeMismatch { expected: "Compound", actual: datatype_name(datatype), }), } } /// Extract a single field by name from compound raw data. pub fn read_compound_field( raw: &[u8], datatype: &Datatype, field_name: &str, ) -> Result { let fields = read_compound_fields(raw, datatype)?; fields .into_iter() .find(|f| f.name == field_name) .ok_or_else(|| FormatError::PathNotFound(field_name.into())) } // --- Enum type reading --- /// A single value from an enum dataset, containing both the integer value and string name. #[derive(Debug, Clone)] pub struct EnumValue { /// The string name for this enum value. pub name: String, /// The raw integer value. pub raw_value: Vec, } /// Read enum dataset values, mapping integer values to their string names. /// /// Returns one `EnumValue` per element. Unknown values get name `UNKNOWN(hex)`. pub fn read_enum_values(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { match datatype { Datatype::Enumeration { size, members, .. } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; // Build lookup map: raw bytes -> name let mut lookup = BTreeMap::new(); for m in members { lookup.insert(m.value.clone(), m.name.clone()); } let mut result = Vec::with_capacity(count); for i in 0..count { let val_bytes = raw[i * elem_size..(i + 1) * elem_size].to_vec(); let name = lookup.get(&val_bytes).cloned().unwrap_or_else(|| { let hex: Vec = val_bytes .iter() .map(|b| { let mut s = String::new(); core::fmt::Write::write_fmt(&mut s, format_args!("{b:02x}")).ok(); s }) .collect(); let mut result = String::from("UNKNOWN(0x"); for h in &hex { result.push_str(h); } result.push(')'); result }); result.push(EnumValue { name, raw_value: val_bytes, }); } Ok(result) } _ => Err(FormatError::TypeMismatch { expected: "Enumeration", actual: datatype_name(datatype), }), } } /// Read enum dataset and return just the string names. pub fn read_enum_names(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { let values = read_enum_values(raw, datatype)?; Ok(values.into_iter().map(|v| v.name).collect()) } // --- Reference type reading --- /// A resolved object reference: the file address of the referenced object header. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ObjectReference { /// The file address of the referenced object header. /// A value of `u64::MAX` (all 0xFF bytes) indicates a null reference. pub address: u64, } impl ObjectReference { /// Returns `true` if this is a null (unset) reference. pub fn is_null(&self) -> bool { self.address == u64::MAX } } /// A region reference: raw bytes that encode a dataset selection. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RegionReference { /// The raw region reference bytes. A region reference is typically 12 bytes /// (object address + dataspace selection) but the exact layout depends on /// the file's offset size and the selection type. pub raw: Vec, } /// Read object references from raw bytes. /// /// Object references are stored as `offset_size`-byte file addresses pointing /// to the object header of the referenced object. A reference consisting of /// all `0xFF` bytes is a null (unset) reference. /// /// # Arguments /// * `raw` — raw bytes read from the dataset /// * `datatype` — must be `Datatype::Reference` with `ReferenceType::Object` /// * `offset_size` — the file's offset size (from superblock), typically 8 pub fn read_object_references( raw: &[u8], datatype: &Datatype, offset_size: u8, ) -> Result, FormatError> { match datatype { Datatype::Reference { ref_type: crate::datatype::ReferenceType::Object, size, } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut result = Vec::with_capacity(count); let read_size = (offset_size as usize).min(elem_size); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let address = read_ref_address(chunk, read_size); result.push(ObjectReference { address }); } Ok(result) } Datatype::Reference { ref_type: crate::datatype::ReferenceType::Object2, size, } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } raw.chunks_exact(elem_size) .map(|element| { decode_std_object_ref(element).map(|address| ObjectReference { address }) }) .collect() } _ => Err(FormatError::TypeMismatch { expected: "Reference(Object)", actual: datatype_name(datatype), }), } } /// Decode one `H5T_STD_REF` object reference as stored in a dataset: /// `type(1) flags(1) token_size(1) token(token_size)`, zero-padded to the /// element size. For a reference within the same file the token is the target /// object's header address. An all-zero element is a null reference and /// decodes to the undefined address (`u64::MAX`). fn decode_std_object_ref(element: &[u8]) -> Result { const STD_REF_OBJECT: u8 = 2; const FLAG_EXTERNAL: u8 = 0x01; if element.iter().all(|&b| b == 0) { return Ok(u64::MAX); } let [ref_type, flags, token_size, token @ ..] = element else { return Err(FormatError::UnexpectedEof { expected: 3, available: element.len(), }); }; if *ref_type != STD_REF_OBJECT { return Err(FormatError::InvalidReferenceType(*ref_type)); } if flags & FLAG_EXTERNAL != 0 { // Carries a file name as well; nothing here follows those. return Err(FormatError::TypeMismatch { expected: "object reference within this file", actual: "external object reference", }); } let n = *token_size as usize; if n == 0 || n > 8 || n > token.len() { return Err(FormatError::UnexpectedEof { expected: 3 + n, available: element.len(), }); } Ok(token[..n] .iter() .rev() .fold(0u64, |addr, &byte| (addr << 8) | u64::from(byte))) } /// Read region references from raw bytes. /// /// Region references encode a dataset selection (hyperslab, point list, etc.) /// along with the address of the target dataset. This function returns the /// raw bytes for each reference without decoding the selection, since the /// full region reference format is complex and depends on the selection type. /// /// # Arguments /// * `raw` — raw bytes read from the dataset /// * `datatype` — must be `Datatype::Reference` with `ReferenceType::DatasetRegion` pub fn read_region_references( raw: &[u8], datatype: &Datatype, ) -> Result, FormatError> { match datatype { Datatype::Reference { ref_type: crate::datatype::ReferenceType::DatasetRegion, size, } => { let elem_size = *size as usize; if elem_size == 0 { return Ok(Vec::new()); } if !raw.len().is_multiple_of(elem_size) { return Err(FormatError::DataSizeMismatch { expected: 0, actual: raw.len(), }); } let count = raw.len() / elem_size; let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(RegionReference { raw: chunk.to_vec(), }); } Ok(result) } _ => Err(FormatError::TypeMismatch { expected: "Reference(DatasetRegion)", actual: datatype_name(datatype), }), } } /// Read a file address from reference bytes (little-endian). fn read_ref_address(bytes: &[u8], size: usize) -> u64 { let mut buf = [0xFFu8; 8]; let len = size.min(bytes.len()).min(8); buf[..len].copy_from_slice(&bytes[..len]); // If we read fewer than 8 bytes, check if ALL read bytes are 0xFF (null ref) if len < 8 && bytes[..len].iter().all(|&b| b == 0xFF) { return u64::MAX; } // Zero-extend upper bytes for non-null refs if len < 8 && !bytes[..len].iter().all(|&b| b == 0xFF) { for b in buf[len..].iter_mut() { *b = 0; } } u64::from_le_bytes(buf) } // --- Array type reading --- /// Read array-typed dataset elements, returning the raw base-type data. /// /// For an array type with dimensions [D1, D2, ...] and base type T, /// each dataset element contains D1*D2*... values of type T. /// This function returns the raw bytes as a flat buffer that can be /// converted with `read_as_f64`, `read_as_i32`, etc. using the base type. pub fn read_array_flat( raw: &[u8], datatype: &Datatype, ) -> Result<(Vec, Datatype, Vec), FormatError> { match datatype { Datatype::Array { base_type, dimensions, } => Ok((raw.to_vec(), *base_type.clone(), dimensions.clone())), _ => Err(FormatError::TypeMismatch { expected: "Array", actual: datatype_name(datatype), }), } } // --- Low-level byte conversion helpers --- fn reorder_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> [u8; 8] { let mut buf = [0u8; 8]; let len = bytes.len().min(8); match order { DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => { buf[..len].copy_from_slice(&bytes[..len]); } DatatypeByteOrder::BigEndian => { // Reverse bytes into LE order for i in 0..len { buf[i] = bytes[len - 1 - i]; } } } buf } /// How the bits of a floating-point datatype are laid out, read from the /// datatype message's fields rather than assumed from its size (a 2-byte /// float may be IEEE half or bfloat16). #[derive(Debug, Clone, Copy, PartialEq)] enum FloatFormat { /// IEEE-754 binary16. Half, /// IEEE-754 binary32. Single, /// IEEE-754 binary64. Double, /// Any other IEEE-style layout (implied leading mantissa bit, all-ones /// exponent for infinity/NaN) whose values are all exact in `f64`: /// bfloat16, the FP8 formats, and similar. Other(FloatLayout), } #[derive(Debug, Clone, Copy, PartialEq)] struct FloatLayout { exponent_location: u32, exponent_size: u32, mantissa_location: u32, mantissa_size: u32, exponent_bias: u32, } impl FloatFormat { fn of(dt: &Datatype) -> Result { let Datatype::FloatingPoint { size, exponent_location, exponent_size, mantissa_location, mantissa_size, exponent_bias, .. } = dt else { return Err(FormatError::TypeMismatch { expected: "FloatingPoint", actual: datatype_name(dt), }); }; let layout = FloatLayout { exponent_location: u32::from(*exponent_location), exponent_size: u32::from(*exponent_size), mantissa_location: u32::from(*mantissa_location), mantissa_size: u32::from(*mantissa_size), exponent_bias: *exponent_bias, }; let fields = ( layout.exponent_location, layout.exponent_size, layout.mantissa_location, layout.mantissa_size, layout.exponent_bias, ); let bits = size.saturating_mul(8); // The sign bit is not kept in `Datatype`; every standard layout has it // directly above the exponent, with the mantissa below. let well_formed = layout.exponent_size > 0 && layout.mantissa_size > 0 && layout.mantissa_location + layout.mantissa_size <= layout.exponent_location && layout.exponent_location + layout.exponent_size < bits; match (size, fields) { (2, (10, 5, 0, 10, 15)) => Ok(FloatFormat::Half), (4, (23, 8, 0, 23, 127)) => Ok(FloatFormat::Single), (8, (52, 11, 0, 52, 1023)) => Ok(FloatFormat::Double), _ if well_formed && *size <= 8 && layout.exponent_size <= 11 && layout.mantissa_size <= 52 => { Ok(FloatFormat::Other(layout)) } // Fields that cannot describe any float (e.g. left zeroed by a // hand-built datatype): fall back to the IEEE type of that size. (2, _) if !well_formed => Ok(FloatFormat::Half), (4, _) if !well_formed => Ok(FloatFormat::Single), (8, _) if !well_formed => Ok(FloatFormat::Double), // x87 80-bit extended, binary128, ...: not representable in f64. _ => Err(FormatError::TypeMismatch { expected: "floating point of at most 64 bits (IEEE-style layout)", actual: "FloatingPoint", }), } } fn decode(self, bytes: &[u8], order: &DatatypeByteOrder) -> f64 { match self { FloatFormat::Half => f64::from(read_f16_bytes(bytes, order)), FloatFormat::Single => f64::from(read_f32_bytes(bytes, order)), FloatFormat::Double => read_f64_bytes(bytes, order), FloatFormat::Other(layout) => { layout.decode(read_unsigned_int(bytes, bytes.len(), order)) } } } } impl FloatLayout { /// Decode the value held in the low `size * 8` bits of `bits`. fn decode(self, bits: u64) -> f64 { let field = |location: u32, size: u32| (bits >> location) & ((1u64 << size) - 1); let exponent = field(self.exponent_location, self.exponent_size); let mantissa = field(self.mantissa_location, self.mantissa_size); let negative = field(self.exponent_location + self.exponent_size, 1) == 1; let max_exponent = (1u64 << self.exponent_size) - 1; let magnitude = if exponent == max_exponent { if mantissa == 0 { f64::INFINITY } else { f64::NAN } } else { let bias = i64::from(self.exponent_bias); let msize = i64::from(self.mantissa_size); // value = significand * 2^power, with an implied leading 1 unless // the number is subnormal (exponent field 0). let (significand, power) = if exponent == 0 { (mantissa, 1 - bias - msize) } else { ( mantissa | (1u64 << self.mantissa_size), exponent as i64 - bias - msize, ) }; scale_by_pow2(significand as f64, power) }; if negative { -magnitude } else { magnitude } } } /// `x * 2^power` without `std` (no `powi`/`libm`). `x` is a non-negative /// integer below 2^53, so it is exact. fn scale_by_pow2(x: f64, power: i64) -> f64 { if x == 0.0 || power < -1200 { return 0.0; } if power > 1100 { return f64::INFINITY; } let pow2 = |p: i64| f64::from_bits(((p + 1023) as u64) << 52); let mut x = x; let mut power = power; while power > 1023 { x *= pow2(1023); power -= 1023; } while power < -1022 { x *= pow2(-1022); power += 1022; } x * pow2(power) } /// Whether `datatype` is the little-endian IEEE float `format`, whose bytes /// can be copied straight into native values on a little-endian target. fn is_native_le_float(datatype: &Datatype, format: FloatFormat) -> bool { matches!( datatype, Datatype::FloatingPoint { byte_order: DatatypeByteOrder::LittleEndian, .. } ) && FloatFormat::of(datatype).is_ok_and(|f| f == format) } fn read_f64_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f64 { let buf = reorder_bytes(bytes, order); f64::from_le_bytes(buf) } /// Decode an IEEE-754 half-precision (binary16) value to `f32`. Pure integer /// bit manipulation (no_std-safe, no `powi`/`libm`). fn read_f16_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 { let mut buf = [0u8; 2]; let len = bytes.len().min(2); match order { DatatypeByteOrder::BigEndian => { for i in 0..len { buf[i] = bytes[len - 1 - i]; } } _ => buf[..len].copy_from_slice(&bytes[..len]), } f16_bits_to_f32(u16::from_le_bytes(buf)) } use crate::float16::f16_bits_to_f32; fn read_f32_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 { let mut buf = [0u8; 4]; let len = bytes.len().min(4); match order { DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => { buf[..len].copy_from_slice(&bytes[..len]); } DatatypeByteOrder::BigEndian => { for i in 0..len { buf[i] = bytes[len - 1 - i]; } } } f32::from_le_bytes(buf) } /// Effective (bit offset, bit precision) for a fixed-point field, defaulting a /// zero precision to the full storage width. fn effective_bits(size: usize, bit_offset: u16, bit_precision: u16) -> (u32, u32) { let prec = if bit_precision == 0 { (size * 8) as u32 } else { bit_precision as u32 }; (bit_offset as u32, prec) } /// Whether a datatype occupies its full storage width (bit offset 0, precision /// == size·8), in which case the bulk-copy fast read paths apply. Non /// fixed-point types are treated as full width. fn is_full_width(datatype: &Datatype) -> bool { match datatype { Datatype::FixedPoint { size, bit_offset, bit_precision, .. } => *bit_offset == 0 && *bit_precision as u32 == *size * 8, _ => true, } } /// Extract the `precision`-bit field at `offset` from a full-width integer read /// and sign-extend it. Full-width fields read as an ordinary signed integer; /// reduced-precision fields sign-extend from the field's top bit (HDF5 stores /// reduced-precision values zero-filled, so the sign lives in the precision /// field, not the storage word). fn extract_signed(full: u64, offset: u32, precision: u32) -> i64 { if precision == 0 || precision >= 64 { return full as i64; } let field = (full >> offset) & ((1u64 << precision) - 1); let shift = 64 - precision; ((field << shift) as i64) >> shift } /// Extract the `precision`-bit field at `offset` from a full-width integer read. fn extract_unsigned(full: u64, offset: u32, precision: u32) -> u64 { if precision == 0 || precision >= 64 { return full; } (full >> offset) & ((1u64 << precision) - 1) } fn read_unsigned_int(bytes: &[u8], size: usize, order: &DatatypeByteOrder) -> u64 { let buf = reorder_bytes(bytes, order); match size { 1 => buf[0] as u64, 2 => u16::from_le_bytes([buf[0], buf[1]]) as u64, 4 => u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as u64, 8 => u64::from_le_bytes(buf), _ => { // Generic: read as LE let mut val = 0u64; for (i, &byte) in buf.iter().enumerate().take(size.min(8)) { val |= (byte as u64) << (i * 8); } val } } } // --- Type conversion cost analysis --- /// Cost classification for type conversions. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ConversionCost { /// No conversion needed (same type). None, /// Widening conversion (no data loss, e.g. f32 → f64, i32 → i64). Widening, /// Narrowing conversion (potential precision loss, e.g. f64 → f32). Narrowing, /// Lossy conversion (potential data corruption, e.g. float → int). Lossy, } /// Information about a type conversion. #[derive(Debug, Clone)] pub struct ConversionInfo { /// Source datatype name. pub source_type: &'static str, /// Target datatype name. pub target_type: &'static str, /// Cost of the conversion. pub cost: ConversionCost, } /// Check the cost of converting from one datatype to another. /// /// This helps callers understand whether a read operation involves /// potentially lossy type coercion. pub fn check_conversion_cost(source: &Datatype, target: &'static str) -> ConversionInfo { let source_name = datatype_name(source); let source_size = source.type_size(); let cost = match (source_name, target) { // Same type (s, t) if s == t => ConversionCost::None, // Float widening: f32 → f64 ("f32", "f64") => ConversionCost::Widening, // Float narrowing: f64 → f32 ("f64", "f32") => ConversionCost::Narrowing, // Integer widening ("i32", "i64") | ("u8", "i32") | ("u8", "i64") | ("u8", "u64") | ("i32", "u64") => { ConversionCost::Widening } // Integer narrowing ("i64", "i32") | ("i64", "u8") | ("i32", "u8") | ("u64", "i32") | ("u64", "u8") => { ConversionCost::Narrowing } // Float ↔ Integer: lossy ("f32" | "f64", "i32" | "i64" | "u8" | "u64") => ConversionCost::Lossy, ("i32" | "i64" | "u8" | "u64", "f32" | "f64") => { // Int → Float: widening if source fits, but technically lossy for large ints if source_size <= 4 && target == "f64" { ConversionCost::Widening } else { ConversionCost::Narrowing } } // Unknown combinations _ => ConversionCost::Lossy, }; ConversionInfo { source_type: source_name, target_type: target, cost, } } #[cfg(test)] mod tests { use super::*; use crate::dataspace::{Dataspace, DataspaceType}; use crate::datatype::{CharacterSet, StringPadding}; fn f16_datatype() -> Datatype { Datatype::FloatingPoint { size: 2, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 16, exponent_location: 10, exponent_size: 5, mantissa_location: 0, mantissa_size: 10, exponent_bias: 15, } } // IEEE-754 half bit patterns for known values. fn f16_bits(v: f32) -> u16 { // Encode a few exact values used by the test. match v { 0.0 => 0x0000, 1.0 => 0x3c00, -2.0 => 0xc000, 0.5 => 0x3800, 65504.0 => 0x7bff, // f16 max _ => panic!("unsupported test value {v}"), } } #[test] fn read_f16_as_f32_and_f64() { let values = [0.0f32, 1.0, -2.0, 0.5, 65504.0]; let raw: Vec = values .iter() .flat_map(|&v| f16_bits(v).to_le_bytes()) .collect(); let dt = f16_datatype(); let got32 = read_as_f32(&raw, &dt).unwrap(); assert_eq!(got32, values); let got64 = read_as_f64(&raw, &dt).unwrap(); let expect64: Vec = values.iter().map(|&v| v as f64).collect(); assert_eq!(got64, expect64); } fn reduced_int(signed: bool, precision: u16) -> Datatype { Datatype::FixedPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, signed, bit_offset: 0, bit_precision: precision, } } #[test] fn reduced_precision_signed_sign_extends() { // 16-bit-precision signed values stored zero-filled (HDF5's canonical // layout, e.g. after N-Bit): the reader must sign-extend from bit 15. let dt = reduced_int(true, 16); // [-1, 100, -50, -32768] as 0x0000ffff / 0x00000064 / 0x0000ffce / 0x00008000 let raw: Vec = vec![ 0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xce, 0xff, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, ]; assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]); assert_eq!(read_as_i64(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]); } #[test] fn reduced_precision_unsigned_masks() { // 12-bit-precision unsigned: high bits must read as zero, not sign. let dt = reduced_int(false, 12); // [4095, 1, 2048] as 0x00000fff / 0x00000001 / 0x00000800 let raw: Vec = vec![ 0xff, 0x0f, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, ]; assert_eq!(read_as_u64(&raw, &dt).unwrap(), vec![4095, 1, 2048]); } #[test] fn float_to_int_truncates_and_saturates() { // Values libhdf5 hands to an undefined C cast: NaN reads as 0 and // exactly 2^63 saturates instead of wrapping to i64::MIN. let dt = make_f64_le_type(); let vals = [f64::NAN, 2f64.powi(63), -2.5, 2.0f64.powi(64)]; let raw: Vec = vals.iter().flat_map(|v| v.to_le_bytes()).collect(); assert_eq!( read_as_i64(&raw, &dt).unwrap(), vec![0, i64::MAX, -2, i64::MAX] ); assert_eq!( read_as_u64(&raw, &dt).unwrap(), vec![0, 1 << 63, 0, u64::MAX] ); assert_eq!( read_as_i32(&raw, &dt).unwrap(), vec![0, i32::MAX, -2, i32::MAX] ); } #[test] fn bfloat16_and_fp8_decode_by_fields() { // bfloat16 is a 2-byte float that is not IEEE half. let bf16 = Datatype::FloatingPoint { size: 2, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 16, exponent_location: 7, exponent_size: 8, mantissa_location: 0, mantissa_size: 7, exponent_bias: 127, }; let raw: Vec = [0x3FC0u16, 0xC010, 0x7F80, 0x0001] .iter() .flat_map(|v| v.to_le_bytes()) .collect(); let got = read_as_f64(&raw, &bf16).unwrap(); assert_eq!(&got[..3], &[1.5, -2.25, f64::INFINITY]); assert_eq!(got[3], 2f64.powi(-133)); // smallest subnormal assert_eq!(read_as_f32(&raw, &bf16).unwrap()[..2], [1.5, -2.25]); // FP8 E4M3: 1, -1, 2, 0, NaN (IEEE-style, as libhdf5 treats it). let e4m3 = Datatype::FloatingPoint { size: 1, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 8, exponent_location: 3, exponent_size: 4, mantissa_location: 0, mantissa_size: 3, exponent_bias: 7, }; let got = read_as_f64(&[0x38, 0xB8, 0x40, 0x00, 0x7E], &e4m3).unwrap(); assert_eq!(&got[..4], &[1.0, -1.0, 2.0, 0.0]); assert!(got[4].is_nan()); } #[test] fn full_width_signed_unchanged() { // Regression: full-width 32-bit signed must be unaffected. let dt = reduced_int(true, 32); let raw: Vec = vec![0xff, 0xff, 0xff, 0xff, 0x2a, 0x00, 0x00, 0x00]; assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 42]); } #[test] fn array_datatype_reads_flat_base_elements() { // An array-typed (e.g. compound member) datatype reads as a flat // sequence of its base elements, applying base-type precision rules. let arr = Datatype::Array { base_type: Box::new(reduced_int(true, 16)), dimensions: vec![2], }; // [-1, 100, 1000, -32768] stored zero-filled at 16-bit precision. let raw: Vec = vec![ 0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, ]; assert_eq!( read_as_i32(&raw, &arr).unwrap(), vec![-1, 100, 1000, -32768] ); // Nested array-of-array unwraps recursively. let nested = Datatype::Array { base_type: Box::new(arr), dimensions: vec![2], }; assert_eq!( read_as_i32(&raw, &nested).unwrap(), vec![-1, 100, 1000, -32768] ); } fn make_f64_le_type() -> Datatype { Datatype::FloatingPoint { size: 8, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 64, exponent_location: 52, exponent_size: 11, mantissa_location: 0, mantissa_size: 52, exponent_bias: 1023, } } fn make_f32_be_type() -> Datatype { Datatype::FloatingPoint { size: 4, byte_order: DatatypeByteOrder::BigEndian, bit_offset: 0, bit_precision: 32, exponent_location: 23, exponent_size: 8, mantissa_location: 0, mantissa_size: 23, exponent_bias: 127, } } fn make_i32_le_type() -> Datatype { Datatype::FixedPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, signed: true, bit_offset: 0, bit_precision: 32, } } fn make_i16_le_type() -> Datatype { Datatype::FixedPoint { size: 2, byte_order: DatatypeByteOrder::LittleEndian, signed: true, bit_offset: 0, bit_precision: 16, } } fn make_u8_type() -> Datatype { Datatype::FixedPoint { size: 1, byte_order: DatatypeByteOrder::LittleEndian, signed: false, bit_offset: 0, bit_precision: 8, } } fn make_simple_dataspace(dims: &[u64]) -> Dataspace { Dataspace { space_type: DataspaceType::Simple, rank: dims.len() as u8, dimensions: dims.to_vec(), max_dimensions: None, } } #[test] fn read_f64_compact() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let mut data = Vec::new(); data.extend_from_slice(&1.0f64.to_le_bytes()); data.extend_from_slice(&2.0f64.to_le_bytes()); data.extend_from_slice(&3.0f64.to_le_bytes()); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); assert_eq!(raw, data); let values = read_as_f64(&raw, &dt).unwrap(); assert_eq!(values, vec![1.0, 2.0, 3.0]); } #[test] fn read_i32_contiguous() { let dt = make_i32_le_type(); let ds = make_simple_dataspace(&[4]); let mut file_data = vec![0u8; 1024]; let offset = 256usize; let vals: Vec = vec![10, -20, 30, -40]; for (i, v) in vals.iter().enumerate() { let bytes = v.to_le_bytes(); file_data[offset + i * 4..offset + i * 4 + 4].copy_from_slice(&bytes); } let layout = DataLayout::Contiguous { address: Some(offset as u64), size: 16, }; let raw = read_raw_data(&file_data, &layout, &ds, &dt).unwrap(); let result = read_as_i32(&raw, &dt).unwrap(); assert_eq!(result, vec![10, -20, 30, -40]); } #[test] fn read_u8_data() { let dt = make_u8_type(); let ds = make_simple_dataspace(&[5]); let data = vec![10u8, 20, 30, 40, 50]; let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_u64(&raw, &dt).unwrap(); assert_eq!(result, vec![10, 20, 30, 40, 50]); } #[test] fn read_f32_be() { let dt = make_f32_be_type(); let ds = make_simple_dataspace(&[2]); let mut data = Vec::new(); // Store as big-endian data.extend_from_slice(&1.5f32.to_be_bytes()); data.extend_from_slice(&2.5f32.to_be_bytes()); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_f32(&raw, &dt).unwrap(); assert_eq!(result, vec![1.5, 2.5]); } #[test] fn read_i16_le() { let dt = make_i16_le_type(); let ds = make_simple_dataspace(&[3]); let mut data = Vec::new(); data.extend_from_slice(&(-100i16).to_le_bytes()); data.extend_from_slice(&200i16.to_le_bytes()); data.extend_from_slice(&(-300i16).to_le_bytes()); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_i64(&raw, &dt).unwrap(); assert_eq!(result, vec![-100, 200, -300]); } #[test] fn read_strings_compact() { let dt = Datatype::String { size: 5, padding: StringPadding::NullPad, charset: CharacterSet::Ascii, }; let ds = make_simple_dataspace(&[2]); let mut data = Vec::new(); data.extend_from_slice(b"hello"); data.extend_from_slice(b"hi\0\0\0"); let layout = DataLayout::Compact { data: data.clone() }; let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap(); let result = read_as_strings(&raw, &dt).unwrap(); assert_eq!(result, vec!["hello", "hi"]); } #[test] fn type_mismatch_f64_on_string() { let dt = Datatype::String { size: 4, padding: StringPadding::NullTerminate, charset: CharacterSet::Ascii, }; let raw = vec![0u8; 8]; let err = read_as_f64(&raw, &dt).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn size_mismatch_compact() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let data = vec![0u8; 16]; // wrong: should be 24 let layout = DataLayout::Compact { data }; let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::DataSizeMismatch { .. })); } #[test] fn no_data_allocated() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let layout = DataLayout::Contiguous { address: None, size: 24, }; let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::NoDataAllocated)); } #[test] fn string_type_mismatch_on_read_as_strings() { let dt = make_i32_le_type(); let raw = vec![0u8; 8]; let err = read_as_strings(&raw, &dt).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn read_f64_from_i32() { // read_as_f64 should work on FixedPoint types too let dt = make_i32_le_type(); let mut raw = Vec::new(); raw.extend_from_slice(&42i32.to_le_bytes()); raw.extend_from_slice(&(-7i32).to_le_bytes()); let result = read_as_f64(&raw, &dt).unwrap(); assert_eq!(result, vec![42.0, -7.0]); } #[test] fn read_strings_space_padded() { let dt = Datatype::String { size: 8, padding: StringPadding::SpacePad, charset: CharacterSet::Ascii, }; let raw = b"hello world "; let result = read_as_strings(raw, &dt).unwrap(); assert_eq!(result, vec!["hello", "world"]); } #[test] fn read_strings_null_terminated() { let dt = Datatype::String { size: 6, padding: StringPadding::NullTerminate, charset: CharacterSet::Ascii, }; let raw = b"abc\0\0\0de\0\0\0\0"; let result = read_as_strings(raw, &dt).unwrap(); assert_eq!(result, vec!["abc", "de"]); } #[test] fn read_compound_two_fields() { use crate::datatype::CompoundMember; // Compound: { x: f64, id: i32 } => size = 12 let dt = Datatype::Compound { size: 12, members: vec![ CompoundMember { name: "x".to_string(), byte_offset: 0, datatype: make_f64_le_type(), }, CompoundMember { name: "id".to_string(), byte_offset: 8, datatype: make_i32_le_type(), }, ], }; // Two elements let mut raw = Vec::new(); raw.extend_from_slice(&1.5f64.to_le_bytes()); raw.extend_from_slice(&10i32.to_le_bytes()); raw.extend_from_slice(&2.5f64.to_le_bytes()); raw.extend_from_slice(&20i32.to_le_bytes()); let fields = read_compound_fields(&raw, &dt).unwrap(); assert_eq!(fields.len(), 2); assert_eq!(fields[0].name, "x"); let x_vals = read_as_f64(&fields[0].raw_data, &fields[0].datatype).unwrap(); assert_eq!(x_vals, vec![1.5, 2.5]); assert_eq!(fields[1].name, "id"); let id_vals = read_as_i32(&fields[1].raw_data, &fields[1].datatype).unwrap(); assert_eq!(id_vals, vec![10, 20]); } #[test] fn read_compound_rejects_byte_offset_overrun() { use crate::datatype::CompoundMember; // Compound declares size=8, but the member's byte_offset(4) + its // field_size(8, f64) = 12 > 8 — a crafted out-of-range byte_offset. let dt = Datatype::Compound { size: 8, members: vec![CompoundMember { name: "bad".to_string(), byte_offset: 4, datatype: make_f64_le_type(), }], }; let raw = vec![0u8; 8]; // one element, matches declared size let result = read_compound_fields(&raw, &dt); assert!( matches!(result, Err(FormatError::Overflow(_))), "expected a clean Overflow error, got {result:?}" ); } #[test] fn read_raw_data_zerocopy_rejects_near_usize_max_offset() { let file_data = vec![0u8; 64]; let dataspace = make_simple_dataspace(&[4]); let datatype = make_i32_le_type(); let layout = DataLayout::Contiguous { address: Some(u64::MAX - 4), size: 16, }; let result = read_raw_data_zerocopy(&file_data, &layout, &dataspace, &datatype); assert!( matches!(result, Err(FormatError::UnexpectedEof { .. })), "expected a clean UnexpectedEof, got {result:?}" ); } #[test] fn read_compound_single_field_by_name() { use crate::datatype::CompoundMember; let dt = Datatype::Compound { size: 12, members: vec![ CompoundMember { name: "x".to_string(), byte_offset: 0, datatype: make_f64_le_type(), }, CompoundMember { name: "id".to_string(), byte_offset: 8, datatype: make_i32_le_type(), }, ], }; let mut raw = Vec::new(); raw.extend_from_slice(&3.25f64.to_le_bytes()); raw.extend_from_slice(&42i32.to_le_bytes()); let field = read_compound_field(&raw, &dt, "id").unwrap(); let vals = read_as_i32(&field.raw_data, &field.datatype).unwrap(); assert_eq!(vals, vec![42]); // Non-existent field let err = read_compound_field(&raw, &dt, "missing").unwrap_err(); assert!(matches!(err, FormatError::PathNotFound(_))); } #[test] fn read_enum_values_basic() { use crate::datatype::EnumMember; let dt = Datatype::Enumeration { size: 4, base_type: Box::new(make_i32_le_type()), members: vec![ EnumMember { name: "RED".to_string(), value: 0i32.to_le_bytes().to_vec(), }, EnumMember { name: "GREEN".to_string(), value: 1i32.to_le_bytes().to_vec(), }, EnumMember { name: "BLUE".to_string(), value: 2i32.to_le_bytes().to_vec(), }, ], }; let mut raw = Vec::new(); raw.extend_from_slice(&1i32.to_le_bytes()); // GREEN raw.extend_from_slice(&0i32.to_le_bytes()); // RED raw.extend_from_slice(&2i32.to_le_bytes()); // BLUE raw.extend_from_slice(&99i32.to_le_bytes()); // unknown let names = read_enum_names(&raw, &dt).unwrap(); assert_eq!(names[0], "GREEN"); assert_eq!(names[1], "RED"); assert_eq!(names[2], "BLUE"); assert!(names[3].starts_with("UNKNOWN(")); } #[test] fn read_array_flat_basic() { // Array[3] of f64 let dt = Datatype::Array { base_type: Box::new(make_f64_le_type()), dimensions: vec![3], }; let mut raw = Vec::new(); for v in &[1.0f64, 2.0, 3.0] { raw.extend_from_slice(&v.to_le_bytes()); } let (data, base_dt, dims) = read_array_flat(&raw, &dt).unwrap(); assert_eq!(dims, vec![3]); let vals = read_as_f64(&data, &base_dt).unwrap(); assert_eq!(vals, vec![1.0, 2.0, 3.0]); } #[test] fn read_object_references_basic() { use crate::datatype::ReferenceType; let dt = Datatype::Reference { size: 8, ref_type: ReferenceType::Object, }; let mut raw = Vec::new(); raw.extend_from_slice(&1024u64.to_le_bytes()); // valid ref raw.extend_from_slice(&u64::MAX.to_le_bytes()); // null ref raw.extend_from_slice(&2048u64.to_le_bytes()); // valid ref let refs = read_object_references(&raw, &dt, 8).unwrap(); assert_eq!(refs.len(), 3); assert_eq!(refs[0].address, 1024); assert!(!refs[0].is_null()); assert!(refs[1].is_null()); assert_eq!(refs[2].address, 2048); } #[test] fn read_object_references_4byte_offset() { use crate::datatype::ReferenceType; let dt = Datatype::Reference { size: 4, ref_type: ReferenceType::Object, }; let mut raw = Vec::new(); raw.extend_from_slice(&512u32.to_le_bytes()); raw.extend_from_slice(&u32::MAX.to_le_bytes()); // null ref let refs = read_object_references(&raw, &dt, 4).unwrap(); assert_eq!(refs.len(), 2); assert_eq!(refs[0].address, 512); assert!(refs[1].is_null()); } #[test] fn read_object_references_type_mismatch() { let dt = make_f64_le_type(); let raw = vec![0u8; 8]; let err = read_object_references(&raw, &dt, 8).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn read_region_references_basic() { use crate::datatype::ReferenceType; let dt = Datatype::Reference { size: 12, ref_type: ReferenceType::DatasetRegion, }; let raw = vec![0xABu8; 24]; // two 12-byte region refs let refs = read_region_references(&raw, &dt).unwrap(); assert_eq!(refs.len(), 2); assert_eq!(refs[0].raw.len(), 12); assert_eq!(refs[1].raw.len(), 12); } #[test] fn read_region_references_type_mismatch() { let dt = make_i32_le_type(); let raw = vec![0u8; 12]; let err = read_region_references(&raw, &dt).unwrap_err(); assert!(matches!(err, FormatError::TypeMismatch { .. })); } #[test] fn zerocopy_contiguous_returns_slice_into_file_data() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let mut file_data = vec![0u8; 1024]; let offset = 256usize; let vals = [1.0f64, 2.0, 3.0]; for (i, v) in vals.iter().enumerate() { file_data[offset + i * 8..offset + i * 8 + 8].copy_from_slice(&v.to_le_bytes()); } let layout = DataLayout::Contiguous { address: Some(offset as u64), size: 24, }; let result = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap(); let slice = result.expect("contiguous should return Some"); // Pointer identity: the slice must point into file_data, not a copy let file_range = file_data.as_ptr_range(); assert!(file_range.contains(&slice.as_ptr())); assert_eq!(slice.len(), 24); // Verify the actual data let values = read_as_f64(slice, &dt).unwrap(); assert_eq!(values, vec![1.0, 2.0, 3.0]); } #[test] fn zerocopy_compact_returns_none() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[1]); let data = vec![0u8; 8]; let layout = DataLayout::Compact { data }; let result = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap(); assert!(result.is_none()); } #[test] fn zerocopy_no_data_allocated() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[1]); let layout = DataLayout::Contiguous { address: None, size: 8, }; let err = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::NoDataAllocated)); } #[test] fn read_as_f64_zerocopy_aligned() { let dt = make_f64_le_type(); // Create aligned data — Vec guarantees 8-byte alignment let values = [1.0f64, 2.0, 3.0, 4.0]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) }; let result = read_as_f64_zerocopy(raw, &dt); assert!(result.is_some(), "aligned native LE f64 should succeed"); let slice = result.unwrap(); assert_eq!(slice, &[1.0, 2.0, 3.0, 4.0]); // Verify it's the same memory (zero-copy) assert_eq!(slice.as_ptr() as *const u8, raw.as_ptr()); } #[test] fn read_as_f64_zerocopy_wrong_type() { let dt = make_i32_le_type(); let values = [1.0f64; 4]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) }; assert!(read_as_f64_zerocopy(raw, &dt).is_none()); } #[test] fn read_as_f64_zerocopy_big_endian() { let dt = Datatype::FloatingPoint { size: 8, byte_order: DatatypeByteOrder::BigEndian, bit_offset: 0, bit_precision: 64, exponent_location: 52, exponent_size: 11, mantissa_location: 0, mantissa_size: 52, exponent_bias: 1023, }; let values = [1.0f64; 4]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) }; assert!(read_as_f64_zerocopy(raw, &dt).is_none()); } #[test] fn read_as_f64_zerocopy_odd_size() { let dt = make_f64_le_type(); let raw = &[0u8; 13]; // not a multiple of 8 assert!(read_as_f64_zerocopy(raw, &dt).is_none()); } #[test] fn read_as_f32_zerocopy_aligned() { let dt = Datatype::FloatingPoint { size: 4, byte_order: DatatypeByteOrder::LittleEndian, bit_offset: 0, bit_precision: 32, exponent_location: 23, exponent_size: 8, mantissa_location: 0, mantissa_size: 23, exponent_bias: 127, }; let values = [1.5f32, 2.5, 3.5]; let raw: &[u8] = // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe. unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 4) }; let result = read_as_f32_zerocopy(raw, &dt); assert!(result.is_some()); assert_eq!(result.unwrap(), &[1.5f32, 2.5, 3.5]); } #[test] fn zerocopy_size_mismatch() { let dt = make_f64_le_type(); let ds = make_simple_dataspace(&[3]); let file_data = vec![0u8; 1024]; let layout = DataLayout::Contiguous { address: Some(0), size: 16, // wrong: should be 24 }; let err = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap_err(); assert!(matches!(err, FormatError::DataSizeMismatch { .. })); } }