quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity
Files
386bd1d41e47e455279777143b26bd7151ebe30f
clawhdf5/crates/clawhdf5-format/src/data_read.rs
T
osobhandClaude Opus 5.5 e94a52a88b fix(format): read unmapped VDS elements as the virtual dataset's fill value 
Elements of a virtual dataset that no mapping supplies (unmapped regions,
a missing source file, a missing source dataset) read as 0 instead of the
fill value libhdf5 returns — silent wrong data for any VDS created with a
non-zero fillvalue (read-matrix cases 0471/0472: -1 and 7 read as 0). A
missing source dataset was an error; libhdf5 reads it as fill.

Move VDS assembly into a new vds module following H5Dvirtual.c:
vds::read_virtual_dataset takes the dataset's fill value and a
VdsFileResolver that can refuse a name, and reports how many elements were
unmapped. Sources are read with their own fill value, and a source whose
datatype differs from the virtual dataset's is an error (libhdf5 converts).
File passes the dataset's fill value, resolves source names against the
virtual file's directory, and refuses names that leave it with an error
instead of reading them as fill. read_selection on a VDS goes through the
same fill-aware path.

The raw-read API (read_raw_data_full*) has no fill value, so it now errors
for a VDS with unmapped elements instead of guessing zeros.

Tests: vds_interop::vds_unmapped_regions_read_as_fill_value (external,
same-file, missing file/dataset, sparse source with its own fill, int
fill; earliest and latest format) and
vds_source_outside_directory_is_an_error_not_fill, both against h5py;
integration_test::v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-25 22:03:47 -05:00

2684 lines
92 KiB
Rust
Raw Blame History

//! Raw data reading and typed conversion for HDF5 datasets.
#[cfg(not(feature = "std"))]
use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
#[cfg(feature = "std")]
use std::collections::BTreeMap;
#[cfg(feature = "std")]
use crate::chunk_cache::ChunkCache;
use crate::chunked_read::read_chunked_data;
#[cfg(feature = "std")]
use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed};
use crate::data_layout::DataLayout;
use crate::dataspace::Dataspace;
use crate::datatype::{Datatype, DatatypeByteOrder};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
/// Checks that `[offset, offset + needed)` fits within `data`, guarding the
/// addition against `usize` overflow from a crafted near-`usize::MAX` offset.
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
if offset
.checked_add(needed)
.is_none_or(|end| end > data.len())
{
return Err(FormatError::UnexpectedEof {
expected: offset.saturating_add(needed),
available: data.len(),
});
}
Ok(())
}
/// Zero-copy read of contiguous raw data, returning a borrowed slice.
///
/// For contiguous layouts, returns a direct `&[u8]` slice into `file_data`.
/// For compact or chunked layouts, returns `Ok(None)` — the caller should
/// fall back to `read_raw_data` for those.
pub fn read_raw_data_zerocopy<'a>(
file_data: &'a [u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
) -> Result<Option<&'a [u8]>, FormatError> {
let num_elements = dataspace.num_elements() as usize;
let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
FormatError::Overflow(format!(
"num_elements({num_elements}) * elem_size({elem_size})"
))
})?;
match layout {
DataLayout::Contiguous { address, size } => {
let addr = address.ok_or(FormatError::NoDataAllocated)?;
let addr = addr as usize;
let sz = *size as usize;
if sz != expected_size {
return Err(FormatError::DataSizeMismatch {
expected: expected_size,
actual: sz,
});
}
ensure_len(file_data, addr, sz)?;
Ok(Some(&file_data[addr..addr + sz]))
}
_ => Ok(None),
}
}
/// Read raw bytes for a dataset given its layout and the file data buffer.
///
/// For compact layouts, returns the inline data.
/// For contiguous layouts, reads from the address in the file buffer.
/// For chunked layouts, traverses the B-tree and assembles chunks.
pub fn read_raw_data(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8)
}
/// Resolves a Virtual Dataset source **file name** (as stored in the mapping,
/// e.g. `"ext_src.h5"`) to that file's raw bytes.
///
/// The pure-byte read API has no filesystem of its own, so external-file VDS
/// sources are read through a caller-supplied resolver. The std file API wires
/// one that reads relative to the virtual file's directory; callers can supply
/// their own (e.g. an in-memory map) in `no_std` builds. Returning `None` means
/// the source file is unavailable and the mapping is skipped.
pub type VdsSourceResolver<'a> = dyn Fn(&str) -> Option<Vec<u8>> + 'a;
/// Read raw bytes with full parameters including filter pipeline and sizes.
pub fn read_raw_data_full(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full_impl(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
None,
)
}
/// Like [`read_raw_data_full`], but with a resolver for external-file Virtual
/// Dataset sources. For non-virtual layouts the resolver is ignored.
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_full_with_resolver(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
read_raw_data_full_impl(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
resolver,
)
}
#[allow(clippy::too_many_arguments)]
fn read_raw_data_full_impl(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
let num_elements = dataspace.num_elements() as usize;
let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
FormatError::Overflow(format!(
"num_elements({num_elements}) * elem_size({elem_size})"
))
})?;
match layout {
DataLayout::Compact { data } => {
if data.len() != expected_size {
return Err(FormatError::DataSizeMismatch {
expected: expected_size,
actual: data.len(),
});
}
Ok(data.clone())
}
DataLayout::Contiguous { address, size } => {
let addr = address.ok_or(FormatError::NoDataAllocated)?;
let addr = addr as usize;
let sz = *size as usize;
if sz != expected_size {
return Err(FormatError::DataSizeMismatch {
expected: expected_size,
actual: sz,
});
}
ensure_len(file_data, addr, sz)?;
Ok(file_data[addr..addr + sz].to_vec())
}
DataLayout::Chunked { .. } => read_chunked_data(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
),
DataLayout::Virtual { .. } => read_virtual_data(
file_data,
layout,
dataspace,
datatype,
offset_size,
length_size,
resolver,
),
}
}
/// Read raw bytes with chunk cache support.
///
/// For chunked layouts the `cache` is used to avoid repeated B-tree
/// traversals and to cache decompressed chunk data. For compact and
/// contiguous layouts this behaves identically to [`read_raw_data_full`].
#[cfg(feature = "std")]
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_cached(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
match layout {
DataLayout::Chunked { .. } => read_chunked_data_cached(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
cache,
),
_ => read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
),
}
}
/// Read raw bytes with chunk B-tree index cache and pre-computed layout.
///
/// For chunked layouts this uses the optimized `read_chunked_data_indexed` path
/// which pre-computes contiguous row-copy operations, eliminating per-element
/// N-D coordinate math on repeated reads. For compact and contiguous layouts
/// this behaves identically to [`read_raw_data_full`].
#[cfg(feature = "std")]
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_indexed(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
match layout {
DataLayout::Chunked { .. } => read_chunked_data_indexed(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
cache,
),
_ => read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
),
}
}
/// Read raw bytes for only the selected elements of a dataset.
///
/// For chunked layouts, only chunks that intersect the selection are read
/// and decompressed. For compact/contiguous layouts, the full data is read
/// and then the selection is extracted.
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_selection(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
pipeline: Option<&FilterPipeline>,
offset_size: u8,
length_size: u8,
selection: &crate::selection::Selection,
) -> Result<Vec<u8>, FormatError> {
use crate::selection::Selection;
crate::partial_read::validate(selection, &dataspace.dimensions)?;
// Read only what the selection's bounding box touches when that is
// possible; everything below is the decode-everything-then-pick path,
// kept for the cases `partial_read` declines.
if let Some(selected) = crate::partial_read::read_selection(
file_data,
layout,
dataspace,
datatype.type_size() as usize,
pipeline,
offset_size,
length_size,
selection,
)? {
return Ok(selected);
}
match selection {
Selection::All => {
return read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
);
}
Selection::None => return Ok(Vec::new()),
_ => {}
}
let dims = &dataspace.dimensions;
let elem_size = datatype.type_size() as usize;
match layout {
DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => {
// Read all data, then extract the selection
let full_data = read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
)?;
extract_selection_from_buffer(&full_data, dims, elem_size, selection)
}
DataLayout::Chunked {
chunk_dimensions,
btree_address,
version,
chunk_index_type,
..
} => {
// For chunked data, only read chunks that intersect the selection
let chunk_dims: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let rank = dims.len();
// Collect chunk info from B-tree
let chunks = if *version == 4 {
match chunk_index_type {
Some(2) => {
// Implicit index
crate::chunked_read::generate_implicit_chunks(
btree_address.unwrap_or(0),
dims,
chunk_dimensions,
elem_size as u32,
)
}
_ => {
if let Some(_addr) = btree_address {
// Use extensible array or fixed array
// Fall back to full read for complex v4 index types
let full_data = read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
)?;
return extract_selection_from_buffer(
&full_data, dims, elem_size, selection,
);
} else {
return Ok(Vec::new());
}
}
}
} else {
// v3: B-tree v1
if let Some(addr) = btree_address {
crate::chunked_read::collect_chunk_info(
file_data,
*addr,
rank + 1,
offset_size,
length_size,
)?
} else {
return Ok(Vec::new());
}
};
// Filter chunks to only those that intersect the selection
let intersecting: Vec<_> = chunks
.iter()
.filter(|ci| {
let offsets: Vec<u64> = ci.offsets.iter().take(rank).copied().collect();
selection.intersects_chunk(&offsets, &chunk_dims[..rank])
})
.collect();
if intersecting.is_empty() {
return Ok(Vec::new());
}
// Decompress only the intersecting chunks
let _chunk_total_bytes: usize =
chunk_dims.iter().map(|&d| d as usize).product::<usize>() * elem_size;
let _element_size_u32 = elem_size as u32;
// First, assemble only the intersecting chunks into a partial buffer,
// then extract the selection. For simplicity, we assemble into a full
// dataset buffer and extract (same as contiguous path).
let full_data = read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
)?;
extract_selection_from_buffer(&full_data, dims, elem_size, selection)
}
DataLayout::Virtual { .. } => {
// Assemble the full virtual dataset, then apply the read selection.
let full_data = read_raw_data_full(
file_data,
layout,
dataspace,
datatype,
pipeline,
offset_size,
length_size,
)?;
extract_selection_from_buffer(&full_data, dims, elem_size, selection)
}
}
}
/// Assemble a **Virtual Dataset (VDS)** through the raw-read API, which has no
/// access to the dataset's fill value message.
///
/// Delegates to [`crate::vds::read_virtual_dataset`]. Because the fill value
/// is unknown here, a virtual dataset with any element no mapping supplies
/// (an unmapped region, or a missing source file or dataset) is an error
/// rather than a guess at the fill value; so is one whose extent libhdf5
/// would report differently from the stored dataspace (unlimited mappings).
/// Use [`crate::vds::read_virtual_dataset`] to read those.
#[allow(clippy::too_many_arguments)]
fn read_virtual_data(
file_data: &[u8],
layout: &DataLayout,
dataspace: &Dataspace,
datatype: &Datatype,
offset_size: u8,
length_size: u8,
resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
let wrapped =
resolver.map(|r| move |name: &str| -> Result<Option<Vec<u8>>, FormatError> { Ok(r(name)) });
let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver);
let v = crate::vds::read_virtual_dataset(
file_data,
layout,
dataspace,
datatype,
None,
offset_size,
length_size,
wrapped_ref,
)?;
if v.dims != dataspace.dimensions {
return Err(FormatError::ChunkedReadError(
"virtual dataset extent differs from its stored dataspace; \
read it with vds::read_virtual_dataset"
.into(),
));
}
if v.unmapped > 0 {
return Err(FormatError::ChunkedReadError(
"virtual dataset has elements no source supplies, which read as its \
fill value; read it with vds::read_virtual_dataset and the fill value"
.into(),
));
}
Ok(v.data)
}
/// Extract selected elements from a full dataset buffer.
pub fn extract_selection_from_buffer(
full_data: &[u8],
dims: &[u64],
elem_size: usize,
selection: &crate::selection::Selection,
) -> Result<Vec<u8>, FormatError> {
use crate::selection::Selection;
match selection {
Selection::All => Ok(full_data.to_vec()),
Selection::None => Ok(Vec::new()),
Selection::Hyperslab {
start,
stride,
count,
block,
} => {
let rank = dims.len();
let output_elements = count
.iter()
.zip(block.iter())
.try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?))
.ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?;
let mut output = crate::chunked_read::alloc_output(
crate::chunked_read::checked_byte_len(output_elements, elem_size)?,
)?;
// Compute dataset strides (row-major)
let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
}
// Compute output shape and strides
let output_dims: Vec<usize> = count
.iter()
.zip(block.iter())
.map(|(&c, &b)| (c * b) as usize)
.collect();
let mut out_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
out_strides[i] = out_strides[i + 1] * output_dims[i + 1];
}
// Iterate over all selected elements
// For each block in the hyperslab, copy the elements
let mut out_linear = 0usize;
let _block_coords = vec![0u64; rank];
#[allow(clippy::too_many_arguments)]
fn iterate_hyperslab(
d: usize,
rank: usize,
start: &[u64],
stride: &[u64],
count: &[u64],
block: &[u64],
dims: &[u64],
ds_strides: &[usize],
elem_size: usize,
full_data: &[u8],
output: &mut [u8],
out_linear: &mut usize,
current_ds_offset: usize,
) {
if d == rank {
// Copy one element
let src = current_ds_offset * elem_size;
let dst = *out_linear * elem_size;
if src + elem_size <= full_data.len() && dst + elem_size <= output.len() {
output[dst..dst + elem_size]
.copy_from_slice(&full_data[src..src + elem_size]);
}
*out_linear += 1;
return;
}
for bi in 0..count[d] {
let block_start = start[d] + bi * stride[d];
for bj in 0..block[d] {
let coord = block_start + bj;
if coord < dims[d] {
iterate_hyperslab(
d + 1,
rank,
start,
stride,
count,
block,
dims,
ds_strides,
elem_size,
full_data,
output,
out_linear,
current_ds_offset + coord as usize * ds_strides[d],
);
}
}
}
}
iterate_hyperslab(
0,
rank,
start,
stride,
count,
block,
dims,
&ds_strides,
elem_size,
full_data,
&mut output,
&mut out_linear,
0,
);
Ok(output)
}
Selection::Points(pts) => {
let rank = dims.len();
let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
}
let mut output = Vec::with_capacity(pts.len() * elem_size);
for pt in pts {
let flat: usize = pt
.iter()
.zip(ds_strides.iter())
.map(|(&p, &s)| p as usize * s)
.sum();
let src = flat * elem_size;
if src + elem_size <= full_data.len() {
output.extend_from_slice(&full_data[src..src + elem_size]);
} else {
output.extend_from_slice(&vec![0u8; elem_size]);
}
}
Ok(output)
}
}
}
/// Zero-copy transmute of raw bytes to `&[f64]`.
///
/// Returns `Some(&[f64])` when the datatype is native little-endian `f64`
/// and the data pointer is 8-byte aligned. Returns `None` for any other
/// type or alignment — the caller should fall back to [`read_as_f64`].
pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f64]> {
// Only native LE f64 is eligible
#[cfg(target_endian = "little")]
{
if !is_native_le_float(datatype, FloatFormat::Double) {
return None;
}
if !raw.len().is_multiple_of(8) {
return None;
}
let ptr = raw.as_ptr();
if !(ptr as usize).is_multiple_of(core::mem::align_of::<f64>()) {
return None;
}
let count = raw.len() / 8;
// SAFETY: We verified alignment (8-byte), size (multiple of 8), and
// the on-disk format matches the in-memory representation (LE f64).
Some(unsafe { core::slice::from_raw_parts(ptr as *const f64, count) })
}
#[cfg(not(target_endian = "little"))]
{
let _ = (raw, datatype);
None
}
}
/// Zero-copy transmute of raw bytes to `&[f32]`.
///
/// Returns `Some(&[f32])` when the datatype is native little-endian `f32`
/// and the data pointer is 4-byte aligned. Returns `None` otherwise.
pub fn read_as_f32_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f32]> {
#[cfg(target_endian = "little")]
{
if !is_native_le_float(datatype, FloatFormat::Single) {
return None;
}
if !raw.len().is_multiple_of(4) {
return None;
}
let ptr = raw.as_ptr();
if !(ptr as usize).is_multiple_of(core::mem::align_of::<f32>()) {
return None;
}
let count = raw.len() / 4;
// SAFETY: We verified alignment (4-byte), size (multiple of 4), and
// the on-disk format matches the in-memory representation (LE f32).
Some(unsafe { core::slice::from_raw_parts(ptr as *const f32, count) })
}
#[cfg(not(target_endian = "little"))]
{
let _ = (raw, datatype);
None
}
}
fn datatype_name(dt: &Datatype) -> &'static str {
match dt {
Datatype::FixedPoint { .. } => "FixedPoint",
Datatype::FloatingPoint { .. } => "FloatingPoint",
Datatype::String { .. } => "String",
Datatype::Time { .. } => "Time",
Datatype::BitField { .. } => "BitField",
Datatype::Opaque { .. } => "Opaque",
Datatype::Compound { .. } => "Compound",
Datatype::Reference { .. } => "Reference",
Datatype::Enumeration { .. } => "Enumeration",
Datatype::VariableLength { .. } => "VariableLength",
Datatype::Array { .. } => "Array",
}
}
fn ensure_numeric(dt: &Datatype, expected: &'static str) -> Result<(), FormatError> {
match dt {
Datatype::FixedPoint { .. } | Datatype::FloatingPoint { .. } => Ok(()),
_ => Err(FormatError::TypeMismatch {
expected,
actual: datatype_name(dt),
}),
}
}
fn get_byte_order(dt: &Datatype) -> DatatypeByteOrder {
match dt {
Datatype::FixedPoint { byte_order, .. } => byte_order.clone(),
Datatype::FloatingPoint { byte_order, .. } => byte_order.clone(),
_ => DatatypeByteOrder::LittleEndian,
}
}
fn get_size(dt: &Datatype) -> usize {
dt.type_size() as usize
}
/// Reinterpret little-endian bytes as `count` native values of `T` on a
/// little-endian target, in one copy.
///
/// The buffer is allocated uninitialised and filled by the copy. It used to be
/// `vec![0; count]` first, which for a large dataset meant writing every page
/// twice (zero it, then overwrite it) — about as expensive as the copy itself.
#[cfg(target_endian = "little")]
fn native_le_to_vec<T: Copy>(raw: &[u8], count: usize) -> Vec<T> {
let bytes = count * core::mem::size_of::<T>();
debug_assert!(bytes <= raw.len());
let mut result: Vec<T> = Vec::with_capacity(count);
// SAFETY: `result` has capacity for `count` values of `T`, i.e. `bytes`
// bytes; `raw` holds at least `bytes` bytes (callers derive `count` from
// `raw.len() / size_of::<T>()`); the regions cannot overlap because
// `result` was just allocated. Every `T` used here (f32/f64/i32/i64) is
// valid for any bit pattern, so after the copy all `count` values are
// initialised and `set_len` is sound.
unsafe {
core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr().cast::<u8>(), bytes);
result.set_len(count);
}
result
}
/// Convert raw bytes to `f64` values.
pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatError> {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_f64(raw, base_type);
}
ensure_numeric(datatype, "FloatingPoint or FixedPoint")?;
let elem_size = get_size(datatype);
if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
// Fast path: native-endian f64 — single bulk memcpy
#[cfg(target_endian = "little")]
if is_native_le_float(datatype, FloatFormat::Double) {
return Ok(native_le_to_vec::<f64>(raw, count));
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
if let Datatype::FloatingPoint { .. } = datatype {
let format = FloatFormat::of(datatype)?;
for chunk in raw.chunks_exact(elem_size) {
result.push(format.decode(chunk, &order));
}
return Ok(result);
}
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
let val = convert_to_f64(chunk, datatype, &order)?;
result.push(val);
}
Ok(result)
}
fn convert_to_f64(
bytes: &[u8],
dt: &Datatype,
order: &DatatypeByteOrder,
) -> Result<f64, FormatError> {
match dt {
Datatype::FloatingPoint { .. } => Ok(FloatFormat::of(dt)?.decode(bytes, order)),
Datatype::FixedPoint {
size,
signed,
bit_offset,
bit_precision,
..
} => {
let full = read_unsigned_int(bytes, *size as usize, order);
let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
let v = if *signed {
extract_signed(full, off, prec) as f64
} else {
extract_unsigned(full, off, prec) as f64
};
Ok(v)
}
_ => Err(FormatError::TypeMismatch {
expected: "numeric",
actual: datatype_name(dt),
}),
}
}
/// One numeric element as stored, before conversion to the caller's type.
#[derive(Debug, Clone, Copy, PartialEq)]
enum Scalar {
Signed(i64),
Unsigned(u64),
Float(f64),
}
impl Scalar {
// Every conversion follows libhdf5's default (hard) conversions: a value
// outside the target type's range saturates to its minimum or maximum —
// including a negative value read as unsigned, which reads as 0 — rather
// than being truncated to its low bits. Floats truncate toward zero; NaN
// converts to 0 (libhdf5 leaves that case to the C cast, whose result is
// platform-dependent).
fn to_i64(self) -> i64 {
match self {
Scalar::Signed(v) => v,
Scalar::Unsigned(v) => i64::try_from(v).unwrap_or(i64::MAX),
Scalar::Float(v) => v as i64,
}
}
fn to_u64(self) -> u64 {
match self {
Scalar::Signed(v) => u64::try_from(v).unwrap_or(0),
Scalar::Unsigned(v) => v,
Scalar::Float(v) => v as u64,
}
}
fn to_i32(self) -> i32 {
match self {
Scalar::Signed(v) => v.clamp(i32::MIN.into(), i32::MAX.into()) as i32,
Scalar::Unsigned(v) => i32::try_from(v).unwrap_or(i32::MAX),
Scalar::Float(v) => v as i32,
}
}
}
/// Decode one element of a numeric datatype.
fn decode_scalar(
bytes: &[u8],
dt: &Datatype,
order: &DatatypeByteOrder,
) -> Result<Scalar, FormatError> {
match dt {
Datatype::FixedPoint {
size,
signed,
bit_offset,
bit_precision,
..
} => {
let full = read_unsigned_int(bytes, *size as usize, order);
let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
Ok(if *signed {
Scalar::Signed(extract_signed(full, off, prec))
} else {
Scalar::Unsigned(extract_unsigned(full, off, prec))
})
}
_ => convert_to_f64(bytes, dt, order).map(Scalar::Float),
}
}
/// Convert raw bytes to `i64` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result<Vec<i64>, FormatError> {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_i64(raw, base_type);
}
ensure_numeric(datatype, "FixedPoint (signed)")?;
let elem_size = get_size(datatype);
if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
// Fast path: native LE i64 — single bulk memcpy
#[cfg(target_endian = "little")]
if elem_size == 8
&& is_full_width(datatype)
&& matches!(
datatype,
Datatype::FixedPoint {
byte_order: DatatypeByteOrder::LittleEndian,
signed: true,
..
}
)
{
return Ok(native_le_to_vec::<i64>(raw, count));
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
result.push(decode_scalar(chunk, datatype, &order)?.to_i64());
}
Ok(result)
}
/// Convert raw bytes to `u64` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result<Vec<u64>, FormatError> {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_u64(raw, base_type);
}
ensure_numeric(datatype, "FixedPoint (unsigned)")?;
let elem_size = get_size(datatype);
if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
result.push(decode_scalar(chunk, datatype, &order)?.to_u64());
}
Ok(result)
}
/// Convert raw bytes to `f32` values.
pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatError> {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_f32(raw, base_type);
}
ensure_numeric(datatype, "FloatingPoint")?;
let elem_size = get_size(datatype);
if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
// Fast path: native-endian f32 — single bulk memcpy
#[cfg(target_endian = "little")]
if is_native_le_float(datatype, FloatFormat::Single) {
return Ok(native_le_to_vec::<f32>(raw, count));
}
// Little-endian IEEE half precision (numpy float16): widen directly.
if is_native_le_float(datatype, FloatFormat::Half) {
let (halves, _) = raw[..count * 2].as_chunks::<2>();
return Ok(halves
.iter()
.map(|&b| f16_bits_to_f32(u16::from_le_bytes(b)))
.collect());
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
if let Datatype::FloatingPoint { .. } = datatype {
let format = FloatFormat::of(datatype)?;
for chunk in raw.chunks_exact(elem_size) {
result.push(match format {
FloatFormat::Single => read_f32_bytes(chunk, &order),
FloatFormat::Half => read_f16_bytes(chunk, &order),
// Double rounds; every other supported layout (bfloat16, FP8)
// is exact in f32.
_ => format.decode(chunk, &order) as f32,
});
}
return Ok(result);
}
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
match datatype {
Datatype::FixedPoint {
signed: true,
size,
bit_offset,
bit_precision,
..
} => {
let full = read_unsigned_int(chunk, *size as usize, &order);
let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
result.push(extract_signed(full, off, prec) as f32);
}
Datatype::FixedPoint {
signed: false,
size,
bit_offset,
bit_precision,
..
} => {
let full = read_unsigned_int(chunk, *size as usize, &order);
let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
result.push(extract_unsigned(full, off, prec) as f32);
}
_ => {
return Err(FormatError::TypeMismatch {
expected: "numeric",
actual: datatype_name(datatype),
});
}
}
}
Ok(result)
}
/// Convert raw bytes to `i32` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatError> {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_i32(raw, base_type);
}
ensure_numeric(datatype, "FixedPoint")?;
let elem_size = get_size(datatype);
if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
// Fast path: native LE i32 — single bulk memcpy
#[cfg(target_endian = "little")]
if elem_size == 4
&& is_full_width(datatype)
&& matches!(
datatype,
Datatype::FixedPoint {
byte_order: DatatypeByteOrder::LittleEndian,
signed: true,
..
}
)
{
return Ok(native_le_to_vec::<i32>(raw, count));
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
result.push(decode_scalar(chunk, datatype, &order)?.to_i32());
}
Ok(result)
}
/// Read fixed-length strings from raw bytes.
pub fn read_as_strings(raw: &[u8], datatype: &Datatype) -> Result<Vec<String>, FormatError> {
match datatype {
Datatype::String { size, padding, .. } => {
let elem_size = *size as usize;
if elem_size == 0 {
return Ok(Vec::new());
}
if !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
let s = match padding {
crate::datatype::StringPadding::NullTerminate => {
let end = chunk.iter().position(|&b| b == 0).unwrap_or(chunk.len());
String::from_utf8_lossy(&chunk[..end]).into_owned()
}
crate::datatype::StringPadding::NullPad => {
let end = chunk.iter().rposition(|&b| b != 0).map_or(0, |p| p + 1);
String::from_utf8_lossy(&chunk[..end]).into_owned()
}
crate::datatype::StringPadding::SpacePad => {
let end = chunk.iter().rposition(|&b| b != b' ').map_or(0, |p| p + 1);
String::from_utf8_lossy(&chunk[..end]).into_owned()
}
};
result.push(s);
}
Ok(result)
}
_ => Err(FormatError::TypeMismatch {
expected: "String",
actual: datatype_name(datatype),
}),
}
}
// --- Compound type reading ---
/// A single field extracted from compound data, containing the raw bytes for that field
/// across all elements.
#[derive(Debug, Clone)]
pub struct CompoundFieldData {
/// Field name.
pub name: String,
/// Datatype of this field.
pub datatype: Datatype,
/// Raw bytes for this field across all elements (len = num_elements * field_type_size).
pub raw_data: Vec<u8>,
}
/// Read compound dataset and return all fields as separate data vectors.
///
/// Each returned `CompoundFieldData` contains the raw bytes for that field
/// across all elements, suitable for further typed conversion with `read_as_f64`, etc.
pub fn read_compound_fields(
raw: &[u8],
datatype: &Datatype,
) -> Result<Vec<CompoundFieldData>, FormatError> {
match datatype {
Datatype::Compound { size, members } => {
let elem_size = *size as usize;
if elem_size == 0 {
return Ok(Vec::new());
}
if !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
let mut fields = Vec::with_capacity(members.len());
for m in members {
let field_size = m.datatype.type_size() as usize;
let offset = m.byte_offset as usize;
if offset
.checked_add(field_size)
.is_none_or(|end| end > elem_size)
{
return Err(FormatError::Overflow(format!(
"compound member '{}': byte_offset({offset}) + field_size({field_size}) exceeds element size({elem_size})",
m.name
)));
}
let mut field_raw = Vec::with_capacity(count * field_size);
for i in 0..count {
let elem_start = i * elem_size + offset;
field_raw.extend_from_slice(&raw[elem_start..elem_start + field_size]);
}
fields.push(CompoundFieldData {
name: m.name.clone(),
datatype: m.datatype.clone(),
raw_data: field_raw,
});
}
Ok(fields)
}
_ => Err(FormatError::TypeMismatch {
expected: "Compound",
actual: datatype_name(datatype),
}),
}
}
/// Extract a single field by name from compound raw data.
pub fn read_compound_field(
raw: &[u8],
datatype: &Datatype,
field_name: &str,
) -> Result<CompoundFieldData, FormatError> {
let fields = read_compound_fields(raw, datatype)?;
fields
.into_iter()
.find(|f| f.name == field_name)
.ok_or_else(|| FormatError::PathNotFound(field_name.into()))
}
// --- Enum type reading ---
/// A single value from an enum dataset, containing both the integer value and string name.
#[derive(Debug, Clone)]
pub struct EnumValue {
/// The string name for this enum value.
pub name: String,
/// The raw integer value.
pub raw_value: Vec<u8>,
}
/// Read enum dataset values, mapping integer values to their string names.
///
/// Returns one `EnumValue` per element. Unknown values get name `UNKNOWN(hex)`.
pub fn read_enum_values(raw: &[u8], datatype: &Datatype) -> Result<Vec<EnumValue>, FormatError> {
match datatype {
Datatype::Enumeration { size, members, .. } => {
let elem_size = *size as usize;
if elem_size == 0 {
return Ok(Vec::new());
}
if !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
// Build lookup map: raw bytes -> name
let mut lookup = BTreeMap::new();
for m in members {
lookup.insert(m.value.clone(), m.name.clone());
}
let mut result = Vec::with_capacity(count);
for i in 0..count {
let val_bytes = raw[i * elem_size..(i + 1) * elem_size].to_vec();
let name = lookup.get(&val_bytes).cloned().unwrap_or_else(|| {
let hex: Vec<String> = val_bytes
.iter()
.map(|b| {
let mut s = String::new();
core::fmt::Write::write_fmt(&mut s, format_args!("{b:02x}")).ok();
s
})
.collect();
let mut result = String::from("UNKNOWN(0x");
for h in &hex {
result.push_str(h);
}
result.push(')');
result
});
result.push(EnumValue {
name,
raw_value: val_bytes,
});
}
Ok(result)
}
_ => Err(FormatError::TypeMismatch {
expected: "Enumeration",
actual: datatype_name(datatype),
}),
}
}
/// Read enum dataset and return just the string names.
pub fn read_enum_names(raw: &[u8], datatype: &Datatype) -> Result<Vec<String>, FormatError> {
let values = read_enum_values(raw, datatype)?;
Ok(values.into_iter().map(|v| v.name).collect())
}
// --- Reference type reading ---
/// A resolved object reference: the file address of the referenced object header.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ObjectReference {
/// The file address of the referenced object header.
/// A value of `u64::MAX` (all 0xFF bytes) indicates a null reference.
pub address: u64,
}
impl ObjectReference {
/// Returns `true` if this is a null (unset) reference.
pub fn is_null(&self) -> bool {
self.address == u64::MAX
}
}
/// A region reference: raw bytes that encode a dataset selection.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RegionReference {
/// The raw region reference bytes. A region reference is typically 12 bytes
/// (object address + dataspace selection) but the exact layout depends on
/// the file's offset size and the selection type.
pub raw: Vec<u8>,
}
/// Read object references from raw bytes.
///
/// Object references are stored as `offset_size`-byte file addresses pointing
/// to the object header of the referenced object. A reference consisting of
/// all `0xFF` bytes is a null (unset) reference.
///
/// # Arguments
/// * `raw` — raw bytes read from the dataset
/// * `datatype` — must be `Datatype::Reference` with `ReferenceType::Object`
/// * `offset_size` — the file's offset size (from superblock), typically 8
pub fn read_object_references(
raw: &[u8],
datatype: &Datatype,
offset_size: u8,
) -> Result<Vec<ObjectReference>, FormatError> {
match datatype {
Datatype::Reference {
ref_type: crate::datatype::ReferenceType::Object,
size,
} => {
let elem_size = *size as usize;
if elem_size == 0 {
return Ok(Vec::new());
}
if !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
let mut result = Vec::with_capacity(count);
let read_size = (offset_size as usize).min(elem_size);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
let address = read_ref_address(chunk, read_size);
result.push(ObjectReference { address });
}
Ok(result)
}
Datatype::Reference {
ref_type: crate::datatype::ReferenceType::Object2,
size,
} => {
let elem_size = *size as usize;
if elem_size == 0 {
return Ok(Vec::new());
}
if !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
raw.chunks_exact(elem_size)
.map(|element| {
decode_std_object_ref(element).map(|address| ObjectReference { address })
})
.collect()
}
_ => Err(FormatError::TypeMismatch {
expected: "Reference(Object)",
actual: datatype_name(datatype),
}),
}
}
/// Decode one `H5T_STD_REF` object reference as stored in a dataset:
/// `type(1) flags(1) token_size(1) token(token_size)`, zero-padded to the
/// element size. For a reference within the same file the token is the target
/// object's header address. An all-zero element is a null reference and
/// decodes to the undefined address (`u64::MAX`).
fn decode_std_object_ref(element: &[u8]) -> Result<u64, FormatError> {
const STD_REF_OBJECT: u8 = 2;
const FLAG_EXTERNAL: u8 = 0x01;
if element.iter().all(|&b| b == 0) {
return Ok(u64::MAX);
}
let [ref_type, flags, token_size, token @ ..] = element else {
return Err(FormatError::UnexpectedEof {
expected: 3,
available: element.len(),
});
};
if *ref_type != STD_REF_OBJECT {
return Err(FormatError::InvalidReferenceType(*ref_type));
}
if flags & FLAG_EXTERNAL != 0 {
// Carries a file name as well; nothing here follows those.
return Err(FormatError::TypeMismatch {
expected: "object reference within this file",
actual: "external object reference",
});
}
let n = *token_size as usize;
if n == 0 || n > 8 || n > token.len() {
return Err(FormatError::UnexpectedEof {
expected: 3 + n,
available: element.len(),
});
}
Ok(token[..n]
.iter()
.rev()
.fold(0u64, |addr, &byte| (addr << 8) | u64::from(byte)))
}
/// Read region references from raw bytes.
///
/// Region references encode a dataset selection (hyperslab, point list, etc.)
/// along with the address of the target dataset. This function returns the
/// raw bytes for each reference without decoding the selection, since the
/// full region reference format is complex and depends on the selection type.
///
/// # Arguments
/// * `raw` — raw bytes read from the dataset
/// * `datatype` — must be `Datatype::Reference` with `ReferenceType::DatasetRegion`
pub fn read_region_references(
raw: &[u8],
datatype: &Datatype,
) -> Result<Vec<RegionReference>, FormatError> {
match datatype {
Datatype::Reference {
ref_type: crate::datatype::ReferenceType::DatasetRegion,
size,
} => {
let elem_size = *size as usize;
if elem_size == 0 {
return Ok(Vec::new());
}
if !raw.len().is_multiple_of(elem_size) {
return Err(FormatError::DataSizeMismatch {
expected: 0,
actual: raw.len(),
});
}
let count = raw.len() / elem_size;
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
result.push(RegionReference {
raw: chunk.to_vec(),
});
}
Ok(result)
}
_ => Err(FormatError::TypeMismatch {
expected: "Reference(DatasetRegion)",
actual: datatype_name(datatype),
}),
}
}
/// Read a file address from reference bytes (little-endian).
fn read_ref_address(bytes: &[u8], size: usize) -> u64 {
let mut buf = [0xFFu8; 8];
let len = size.min(bytes.len()).min(8);
buf[..len].copy_from_slice(&bytes[..len]);
// If we read fewer than 8 bytes, check if ALL read bytes are 0xFF (null ref)
if len < 8 && bytes[..len].iter().all(|&b| b == 0xFF) {
return u64::MAX;
}
// Zero-extend upper bytes for non-null refs
if len < 8 && !bytes[..len].iter().all(|&b| b == 0xFF) {
for b in buf[len..].iter_mut() {
*b = 0;
}
}
u64::from_le_bytes(buf)
}
// --- Array type reading ---
/// Read array-typed dataset elements, returning the raw base-type data.
///
/// For an array type with dimensions [D1, D2, ...] and base type T,
/// each dataset element contains D1*D2*... values of type T.
/// This function returns the raw bytes as a flat buffer that can be
/// converted with `read_as_f64`, `read_as_i32`, etc. using the base type.
pub fn read_array_flat(
raw: &[u8],
datatype: &Datatype,
) -> Result<(Vec<u8>, Datatype, Vec<u32>), FormatError> {
match datatype {
Datatype::Array {
base_type,
dimensions,
} => Ok((raw.to_vec(), *base_type.clone(), dimensions.clone())),
_ => Err(FormatError::TypeMismatch {
expected: "Array",
actual: datatype_name(datatype),
}),
}
}
// --- Low-level byte conversion helpers ---
fn reorder_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> [u8; 8] {
let mut buf = [0u8; 8];
let len = bytes.len().min(8);
match order {
DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => {
buf[..len].copy_from_slice(&bytes[..len]);
}
DatatypeByteOrder::BigEndian => {
// Reverse bytes into LE order
for i in 0..len {
buf[i] = bytes[len - 1 - i];
}
}
}
buf
}
/// How the bits of a floating-point datatype are laid out, read from the
/// datatype message's fields rather than assumed from its size (a 2-byte
/// float may be IEEE half or bfloat16).
#[derive(Debug, Clone, Copy, PartialEq)]
enum FloatFormat {
/// IEEE-754 binary16.
Half,
/// IEEE-754 binary32.
Single,
/// IEEE-754 binary64.
Double,
/// Any other IEEE-style layout (implied leading mantissa bit, all-ones
/// exponent for infinity/NaN) whose values are all exact in `f64`:
/// bfloat16, the FP8 formats, and similar.
Other(FloatLayout),
}
#[derive(Debug, Clone, Copy, PartialEq)]
struct FloatLayout {
exponent_location: u32,
exponent_size: u32,
mantissa_location: u32,
mantissa_size: u32,
exponent_bias: u32,
}
impl FloatFormat {
fn of(dt: &Datatype) -> Result<FloatFormat, FormatError> {
let Datatype::FloatingPoint {
size,
exponent_location,
exponent_size,
mantissa_location,
mantissa_size,
exponent_bias,
..
} = dt
else {
return Err(FormatError::TypeMismatch {
expected: "FloatingPoint",
actual: datatype_name(dt),
});
};
let layout = FloatLayout {
exponent_location: u32::from(*exponent_location),
exponent_size: u32::from(*exponent_size),
mantissa_location: u32::from(*mantissa_location),
mantissa_size: u32::from(*mantissa_size),
exponent_bias: *exponent_bias,
};
let fields = (
layout.exponent_location,
layout.exponent_size,
layout.mantissa_location,
layout.mantissa_size,
layout.exponent_bias,
);
let bits = size.saturating_mul(8);
// The sign bit is not kept in `Datatype`; every standard layout has it
// directly above the exponent, with the mantissa below.
let well_formed = layout.exponent_size > 0
&& layout.mantissa_size > 0
&& layout.mantissa_location + layout.mantissa_size <= layout.exponent_location
&& layout.exponent_location + layout.exponent_size < bits;
match (size, fields) {
(2, (10, 5, 0, 10, 15)) => Ok(FloatFormat::Half),
(4, (23, 8, 0, 23, 127)) => Ok(FloatFormat::Single),
(8, (52, 11, 0, 52, 1023)) => Ok(FloatFormat::Double),
_ if well_formed
&& *size <= 8
&& layout.exponent_size <= 11
&& layout.mantissa_size <= 52 =>
{
Ok(FloatFormat::Other(layout))
}
// Fields that cannot describe any float (e.g. left zeroed by a
// hand-built datatype): fall back to the IEEE type of that size.
(2, _) if !well_formed => Ok(FloatFormat::Half),
(4, _) if !well_formed => Ok(FloatFormat::Single),
(8, _) if !well_formed => Ok(FloatFormat::Double),
// x87 80-bit extended, binary128, ...: not representable in f64.
_ => Err(FormatError::TypeMismatch {
expected: "floating point of at most 64 bits (IEEE-style layout)",
actual: "FloatingPoint",
}),
}
}
fn decode(self, bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
match self {
FloatFormat::Half => f64::from(read_f16_bytes(bytes, order)),
FloatFormat::Single => f64::from(read_f32_bytes(bytes, order)),
FloatFormat::Double => read_f64_bytes(bytes, order),
FloatFormat::Other(layout) => {
layout.decode(read_unsigned_int(bytes, bytes.len(), order))
}
}
}
}
impl FloatLayout {
/// Decode the value held in the low `size * 8` bits of `bits`.
fn decode(self, bits: u64) -> f64 {
let field = |location: u32, size: u32| (bits >> location) & ((1u64 << size) - 1);
let exponent = field(self.exponent_location, self.exponent_size);
let mantissa = field(self.mantissa_location, self.mantissa_size);
let negative = field(self.exponent_location + self.exponent_size, 1) == 1;
let max_exponent = (1u64 << self.exponent_size) - 1;
let magnitude = if exponent == max_exponent {
if mantissa == 0 {
f64::INFINITY
} else {
f64::NAN
}
} else {
let bias = i64::from(self.exponent_bias);
let msize = i64::from(self.mantissa_size);
// value = significand * 2^power, with an implied leading 1 unless
// the number is subnormal (exponent field 0).
let (significand, power) = if exponent == 0 {
(mantissa, 1 - bias - msize)
} else {
(
mantissa | (1u64 << self.mantissa_size),
exponent as i64 - bias - msize,
)
};
scale_by_pow2(significand as f64, power)
};
if negative { -magnitude } else { magnitude }
}
}
/// `x * 2^power` without `std` (no `powi`/`libm`). `x` is a non-negative
/// integer below 2^53, so it is exact.
fn scale_by_pow2(x: f64, power: i64) -> f64 {
if x == 0.0 || power < -1200 {
return 0.0;
}
if power > 1100 {
return f64::INFINITY;
}
let pow2 = |p: i64| f64::from_bits(((p + 1023) as u64) << 52);
let mut x = x;
let mut power = power;
while power > 1023 {
x *= pow2(1023);
power -= 1023;
}
while power < -1022 {
x *= pow2(-1022);
power += 1022;
}
x * pow2(power)
}
/// Whether `datatype` is the little-endian IEEE float `format`, whose bytes
/// can be copied straight into native values on a little-endian target.
fn is_native_le_float(datatype: &Datatype, format: FloatFormat) -> bool {
matches!(
datatype,
Datatype::FloatingPoint {
byte_order: DatatypeByteOrder::LittleEndian,
..
}
) && FloatFormat::of(datatype).is_ok_and(|f| f == format)
}
fn read_f64_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
let buf = reorder_bytes(bytes, order);
f64::from_le_bytes(buf)
}
/// Decode an IEEE-754 half-precision (binary16) value to `f32`. Pure integer
/// bit manipulation (no_std-safe, no `powi`/`libm`).
fn read_f16_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 {
let mut buf = [0u8; 2];
let len = bytes.len().min(2);
match order {
DatatypeByteOrder::BigEndian => {
for i in 0..len {
buf[i] = bytes[len - 1 - i];
}
}
_ => buf[..len].copy_from_slice(&bytes[..len]),
}
f16_bits_to_f32(u16::from_le_bytes(buf))
}
use crate::float16::f16_bits_to_f32;
fn read_f32_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 {
let mut buf = [0u8; 4];
let len = bytes.len().min(4);
match order {
DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => {
buf[..len].copy_from_slice(&bytes[..len]);
}
DatatypeByteOrder::BigEndian => {
for i in 0..len {
buf[i] = bytes[len - 1 - i];
}
}
}
f32::from_le_bytes(buf)
}
/// Effective (bit offset, bit precision) for a fixed-point field, defaulting a
/// zero precision to the full storage width.
fn effective_bits(size: usize, bit_offset: u16, bit_precision: u16) -> (u32, u32) {
let prec = if bit_precision == 0 {
(size * 8) as u32
} else {
bit_precision as u32
};
(bit_offset as u32, prec)
}
/// Whether a datatype occupies its full storage width (bit offset 0, precision
/// == size·8), in which case the bulk-copy fast read paths apply. Non
/// fixed-point types are treated as full width.
fn is_full_width(datatype: &Datatype) -> bool {
match datatype {
Datatype::FixedPoint {
size,
bit_offset,
bit_precision,
..
} => *bit_offset == 0 && *bit_precision as u32 == *size * 8,
_ => true,
}
}
/// Extract the `precision`-bit field at `offset` from a full-width integer read
/// and sign-extend it. Full-width fields read as an ordinary signed integer;
/// reduced-precision fields sign-extend from the field's top bit (HDF5 stores
/// reduced-precision values zero-filled, so the sign lives in the precision
/// field, not the storage word).
fn extract_signed(full: u64, offset: u32, precision: u32) -> i64 {
if precision == 0 || precision >= 64 {
return full as i64;
}
let field = (full >> offset) & ((1u64 << precision) - 1);
let shift = 64 - precision;
((field << shift) as i64) >> shift
}
/// Extract the `precision`-bit field at `offset` from a full-width integer read.
fn extract_unsigned(full: u64, offset: u32, precision: u32) -> u64 {
if precision == 0 || precision >= 64 {
return full;
}
(full >> offset) & ((1u64 << precision) - 1)
}
fn read_unsigned_int(bytes: &[u8], size: usize, order: &DatatypeByteOrder) -> u64 {
let buf = reorder_bytes(bytes, order);
match size {
1 => buf[0] as u64,
2 => u16::from_le_bytes([buf[0], buf[1]]) as u64,
4 => u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as u64,
8 => u64::from_le_bytes(buf),
_ => {
// Generic: read as LE
let mut val = 0u64;
for (i, &byte) in buf.iter().enumerate().take(size.min(8)) {
val |= (byte as u64) << (i * 8);
}
val
}
}
}
// --- Type conversion cost analysis ---
/// Cost classification for type conversions.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConversionCost {
/// No conversion needed (same type).
None,
/// Widening conversion (no data loss, e.g. f32 → f64, i32 → i64).
Widening,
/// Narrowing conversion (potential precision loss, e.g. f64 → f32).
Narrowing,
/// Lossy conversion (potential data corruption, e.g. float → int).
Lossy,
}
/// Information about a type conversion.
#[derive(Debug, Clone)]
pub struct ConversionInfo {
/// Source datatype name.
pub source_type: &'static str,
/// Target datatype name.
pub target_type: &'static str,
/// Cost of the conversion.
pub cost: ConversionCost,
}
/// Check the cost of converting from one datatype to another.
///
/// This helps callers understand whether a read operation involves
/// potentially lossy type coercion.
pub fn check_conversion_cost(source: &Datatype, target: &'static str) -> ConversionInfo {
let source_name = datatype_name(source);
let source_size = source.type_size();
let cost = match (source_name, target) {
// Same type
(s, t) if s == t => ConversionCost::None,
// Float widening: f32 → f64
("f32", "f64") => ConversionCost::Widening,
// Float narrowing: f64 → f32
("f64", "f32") => ConversionCost::Narrowing,
// Integer widening
("i32", "i64") | ("u8", "i32") | ("u8", "i64") | ("u8", "u64") | ("i32", "u64") => {
ConversionCost::Widening
}
// Integer narrowing
("i64", "i32") | ("i64", "u8") | ("i32", "u8") | ("u64", "i32") | ("u64", "u8") => {
ConversionCost::Narrowing
}
// Float ↔ Integer: lossy
("f32" | "f64", "i32" | "i64" | "u8" | "u64") => ConversionCost::Lossy,
("i32" | "i64" | "u8" | "u64", "f32" | "f64") => {
// Int → Float: widening if source fits, but technically lossy for large ints
if source_size <= 4 && target == "f64" {
ConversionCost::Widening
} else {
ConversionCost::Narrowing
}
}
// Unknown combinations
_ => ConversionCost::Lossy,
};
ConversionInfo {
source_type: source_name,
target_type: target,
cost,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::dataspace::{Dataspace, DataspaceType};
use crate::datatype::{CharacterSet, StringPadding};
fn f16_datatype() -> Datatype {
Datatype::FloatingPoint {
size: 2,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 16,
exponent_location: 10,
exponent_size: 5,
mantissa_location: 0,
mantissa_size: 10,
exponent_bias: 15,
}
}
// IEEE-754 half bit patterns for known values.
fn f16_bits(v: f32) -> u16 {
// Encode a few exact values used by the test.
match v {
0.0 => 0x0000,
1.0 => 0x3c00,
-2.0 => 0xc000,
0.5 => 0x3800,
65504.0 => 0x7bff, // f16 max
_ => panic!("unsupported test value {v}"),
}
}
#[test]
fn read_f16_as_f32_and_f64() {
let values = [0.0f32, 1.0, -2.0, 0.5, 65504.0];
let raw: Vec<u8> = values
.iter()
.flat_map(|&v| f16_bits(v).to_le_bytes())
.collect();
let dt = f16_datatype();
let got32 = read_as_f32(&raw, &dt).unwrap();
assert_eq!(got32, values);
let got64 = read_as_f64(&raw, &dt).unwrap();
let expect64: Vec<f64> = values.iter().map(|&v| v as f64).collect();
assert_eq!(got64, expect64);
}
fn reduced_int(signed: bool, precision: u16) -> Datatype {
Datatype::FixedPoint {
size: 4,
byte_order: DatatypeByteOrder::LittleEndian,
signed,
bit_offset: 0,
bit_precision: precision,
}
}
#[test]
fn reduced_precision_signed_sign_extends() {
// 16-bit-precision signed values stored zero-filled (HDF5's canonical
// layout, e.g. after N-Bit): the reader must sign-extend from bit 15.
let dt = reduced_int(true, 16);
// [-1, 100, -50, -32768] as 0x0000ffff / 0x00000064 / 0x0000ffce / 0x00008000
let raw: Vec<u8> = vec![
0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xce, 0xff, 0x00, 0x00, 0x00, 0x80,
0x00, 0x00,
];
assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]);
assert_eq!(read_as_i64(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]);
}
#[test]
fn reduced_precision_unsigned_masks() {
// 12-bit-precision unsigned: high bits must read as zero, not sign.
let dt = reduced_int(false, 12);
// [4095, 1, 2048] as 0x00000fff / 0x00000001 / 0x00000800
let raw: Vec<u8> = vec![
0xff, 0x0f, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00,
];
assert_eq!(read_as_u64(&raw, &dt).unwrap(), vec![4095, 1, 2048]);
}
#[test]
fn float_to_int_truncates_and_saturates() {
// Values libhdf5 hands to an undefined C cast: NaN reads as 0 and
// exactly 2^63 saturates instead of wrapping to i64::MIN.
let dt = make_f64_le_type();
let vals = [f64::NAN, 2f64.powi(63), -2.5, 2.0f64.powi(64)];
let raw: Vec<u8> = vals.iter().flat_map(|v| v.to_le_bytes()).collect();
assert_eq!(
read_as_i64(&raw, &dt).unwrap(),
vec![0, i64::MAX, -2, i64::MAX]
);
assert_eq!(
read_as_u64(&raw, &dt).unwrap(),
vec![0, 1 << 63, 0, u64::MAX]
);
assert_eq!(
read_as_i32(&raw, &dt).unwrap(),
vec![0, i32::MAX, -2, i32::MAX]
);
}
#[test]
fn bfloat16_and_fp8_decode_by_fields() {
// bfloat16 is a 2-byte float that is not IEEE half.
let bf16 = Datatype::FloatingPoint {
size: 2,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 16,
exponent_location: 7,
exponent_size: 8,
mantissa_location: 0,
mantissa_size: 7,
exponent_bias: 127,
};
let raw: Vec<u8> = [0x3FC0u16, 0xC010, 0x7F80, 0x0001]
.iter()
.flat_map(|v| v.to_le_bytes())
.collect();
let got = read_as_f64(&raw, &bf16).unwrap();
assert_eq!(&got[..3], &[1.5, -2.25, f64::INFINITY]);
assert_eq!(got[3], 2f64.powi(-133)); // smallest subnormal
assert_eq!(read_as_f32(&raw, &bf16).unwrap()[..2], [1.5, -2.25]);
// FP8 E4M3: 1, -1, 2, 0, NaN (IEEE-style, as libhdf5 treats it).
let e4m3 = Datatype::FloatingPoint {
size: 1,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 8,
exponent_location: 3,
exponent_size: 4,
mantissa_location: 0,
mantissa_size: 3,
exponent_bias: 7,
};
let got = read_as_f64(&[0x38, 0xB8, 0x40, 0x00, 0x7E], &e4m3).unwrap();
assert_eq!(&got[..4], &[1.0, -1.0, 2.0, 0.0]);
assert!(got[4].is_nan());
}
#[test]
fn full_width_signed_unchanged() {
// Regression: full-width 32-bit signed must be unaffected.
let dt = reduced_int(true, 32);
let raw: Vec<u8> = vec![0xff, 0xff, 0xff, 0xff, 0x2a, 0x00, 0x00, 0x00];
assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 42]);
}
#[test]
fn array_datatype_reads_flat_base_elements() {
// An array-typed (e.g. compound member) datatype reads as a flat
// sequence of its base elements, applying base-type precision rules.
let arr = Datatype::Array {
base_type: Box::new(reduced_int(true, 16)),
dimensions: vec![2],
};
// [-1, 100, 1000, -32768] stored zero-filled at 16-bit precision.
let raw: Vec<u8> = vec![
0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x80,
0x00, 0x00,
];
assert_eq!(
read_as_i32(&raw, &arr).unwrap(),
vec![-1, 100, 1000, -32768]
);
// Nested array-of-array unwraps recursively.
let nested = Datatype::Array {
base_type: Box::new(arr),
dimensions: vec![2],
};
assert_eq!(
read_as_i32(&raw, &nested).unwrap(),
vec![-1, 100, 1000, -32768]
);
}
fn make_f64_le_type() -> Datatype {
Datatype::FloatingPoint {
size: 8,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 64,
exponent_location: 52,
exponent_size: 11,
mantissa_location: 0,
mantissa_size: 52,
exponent_bias: 1023,
}
}
fn make_f32_be_type() -> Datatype {
Datatype::FloatingPoint {
size: 4,
byte_order: DatatypeByteOrder::BigEndian,
bit_offset: 0,
bit_precision: 32,
exponent_location: 23,
exponent_size: 8,
mantissa_location: 0,
mantissa_size: 23,
exponent_bias: 127,
}
}
fn make_i32_le_type() -> Datatype {
Datatype::FixedPoint {
size: 4,
byte_order: DatatypeByteOrder::LittleEndian,
signed: true,
bit_offset: 0,
bit_precision: 32,
}
}
fn make_i16_le_type() -> Datatype {
Datatype::FixedPoint {
size: 2,
byte_order: DatatypeByteOrder::LittleEndian,
signed: true,
bit_offset: 0,
bit_precision: 16,
}
}
fn make_u8_type() -> Datatype {
Datatype::FixedPoint {
size: 1,
byte_order: DatatypeByteOrder::LittleEndian,
signed: false,
bit_offset: 0,
bit_precision: 8,
}
}
fn make_simple_dataspace(dims: &[u64]) -> Dataspace {
Dataspace {
space_type: DataspaceType::Simple,
rank: dims.len() as u8,
dimensions: dims.to_vec(),
max_dimensions: None,
}
}
#[test]
fn read_f64_compact() {
let dt = make_f64_le_type();
let ds = make_simple_dataspace(&[3]);
let mut data = Vec::new();
data.extend_from_slice(&1.0f64.to_le_bytes());
data.extend_from_slice(&2.0f64.to_le_bytes());
data.extend_from_slice(&3.0f64.to_le_bytes());
let layout = DataLayout::Compact { data: data.clone() };
let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
assert_eq!(raw, data);
let values = read_as_f64(&raw, &dt).unwrap();
assert_eq!(values, vec![1.0, 2.0, 3.0]);
}
#[test]
fn read_i32_contiguous() {
let dt = make_i32_le_type();
let ds = make_simple_dataspace(&[4]);
let mut file_data = vec![0u8; 1024];
let offset = 256usize;
let vals: Vec<i32> = vec![10, -20, 30, -40];
for (i, v) in vals.iter().enumerate() {
let bytes = v.to_le_bytes();
file_data[offset + i * 4..offset + i * 4 + 4].copy_from_slice(&bytes);
}
let layout = DataLayout::Contiguous {
address: Some(offset as u64),
size: 16,
};
let raw = read_raw_data(&file_data, &layout, &ds, &dt).unwrap();
let result = read_as_i32(&raw, &dt).unwrap();
assert_eq!(result, vec![10, -20, 30, -40]);
}
#[test]
fn read_u8_data() {
let dt = make_u8_type();
let ds = make_simple_dataspace(&[5]);
let data = vec![10u8, 20, 30, 40, 50];
let layout = DataLayout::Compact { data: data.clone() };
let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
let result = read_as_u64(&raw, &dt).unwrap();
assert_eq!(result, vec![10, 20, 30, 40, 50]);
}
#[test]
fn read_f32_be() {
let dt = make_f32_be_type();
let ds = make_simple_dataspace(&[2]);
let mut data = Vec::new();
// Store as big-endian
data.extend_from_slice(&1.5f32.to_be_bytes());
data.extend_from_slice(&2.5f32.to_be_bytes());
let layout = DataLayout::Compact { data: data.clone() };
let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
let result = read_as_f32(&raw, &dt).unwrap();
assert_eq!(result, vec![1.5, 2.5]);
}
#[test]
fn read_i16_le() {
let dt = make_i16_le_type();
let ds = make_simple_dataspace(&[3]);
let mut data = Vec::new();
data.extend_from_slice(&(-100i16).to_le_bytes());
data.extend_from_slice(&200i16.to_le_bytes());
data.extend_from_slice(&(-300i16).to_le_bytes());
let layout = DataLayout::Compact { data: data.clone() };
let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
let result = read_as_i64(&raw, &dt).unwrap();
assert_eq!(result, vec![-100, 200, -300]);
}
#[test]
fn read_strings_compact() {
let dt = Datatype::String {
size: 5,
padding: StringPadding::NullPad,
charset: CharacterSet::Ascii,
};
let ds = make_simple_dataspace(&[2]);
let mut data = Vec::new();
data.extend_from_slice(b"hello");
data.extend_from_slice(b"hi\0\0\0");
let layout = DataLayout::Compact { data: data.clone() };
let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
let result = read_as_strings(&raw, &dt).unwrap();
assert_eq!(result, vec!["hello", "hi"]);
}
#[test]
fn type_mismatch_f64_on_string() {
let dt = Datatype::String {
size: 4,
padding: StringPadding::NullTerminate,
charset: CharacterSet::Ascii,
};
let raw = vec![0u8; 8];
let err = read_as_f64(&raw, &dt).unwrap_err();
assert!(matches!(err, FormatError::TypeMismatch { .. }));
}
#[test]
fn size_mismatch_compact() {
let dt = make_f64_le_type();
let ds = make_simple_dataspace(&[3]);
let data = vec![0u8; 16]; // wrong: should be 24
let layout = DataLayout::Compact { data };
let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err();
assert!(matches!(err, FormatError::DataSizeMismatch { .. }));
}
#[test]
fn no_data_allocated() {
let dt = make_f64_le_type();
let ds = make_simple_dataspace(&[3]);
let layout = DataLayout::Contiguous {
address: None,
size: 24,
};
let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err();
assert!(matches!(err, FormatError::NoDataAllocated));
}
#[test]
fn string_type_mismatch_on_read_as_strings() {
let dt = make_i32_le_type();
let raw = vec![0u8; 8];
let err = read_as_strings(&raw, &dt).unwrap_err();
assert!(matches!(err, FormatError::TypeMismatch { .. }));
}
#[test]
fn read_f64_from_i32() {
// read_as_f64 should work on FixedPoint types too
let dt = make_i32_le_type();
let mut raw = Vec::new();
raw.extend_from_slice(&42i32.to_le_bytes());
raw.extend_from_slice(&(-7i32).to_le_bytes());
let result = read_as_f64(&raw, &dt).unwrap();
assert_eq!(result, vec![42.0, -7.0]);
}
#[test]
fn read_strings_space_padded() {
let dt = Datatype::String {
size: 8,
padding: StringPadding::SpacePad,
charset: CharacterSet::Ascii,
};
let raw = b"hello world ";
let result = read_as_strings(raw, &dt).unwrap();
assert_eq!(result, vec!["hello", "world"]);
}
#[test]
fn read_strings_null_terminated() {
let dt = Datatype::String {
size: 6,
padding: StringPadding::NullTerminate,
charset: CharacterSet::Ascii,
};
let raw = b"abc\0\0\0de\0\0\0\0";
let result = read_as_strings(raw, &dt).unwrap();
assert_eq!(result, vec!["abc", "de"]);
}
#[test]
fn read_compound_two_fields() {
use crate::datatype::CompoundMember;
// Compound: { x: f64, id: i32 } => size = 12
let dt = Datatype::Compound {
size: 12,
members: vec![
CompoundMember {
name: "x".to_string(),
byte_offset: 0,
datatype: make_f64_le_type(),
},
CompoundMember {
name: "id".to_string(),
byte_offset: 8,
datatype: make_i32_le_type(),
},
],
};
// Two elements
let mut raw = Vec::new();
raw.extend_from_slice(&1.5f64.to_le_bytes());
raw.extend_from_slice(&10i32.to_le_bytes());
raw.extend_from_slice(&2.5f64.to_le_bytes());
raw.extend_from_slice(&20i32.to_le_bytes());
let fields = read_compound_fields(&raw, &dt).unwrap();
assert_eq!(fields.len(), 2);
assert_eq!(fields[0].name, "x");
let x_vals = read_as_f64(&fields[0].raw_data, &fields[0].datatype).unwrap();
assert_eq!(x_vals, vec![1.5, 2.5]);
assert_eq!(fields[1].name, "id");
let id_vals = read_as_i32(&fields[1].raw_data, &fields[1].datatype).unwrap();
assert_eq!(id_vals, vec![10, 20]);
}
#[test]
fn read_compound_rejects_byte_offset_overrun() {
use crate::datatype::CompoundMember;
// Compound declares size=8, but the member's byte_offset(4) + its
// field_size(8, f64) = 12 > 8 — a crafted out-of-range byte_offset.
let dt = Datatype::Compound {
size: 8,
members: vec![CompoundMember {
name: "bad".to_string(),
byte_offset: 4,
datatype: make_f64_le_type(),
}],
};
let raw = vec![0u8; 8]; // one element, matches declared size
let result = read_compound_fields(&raw, &dt);
assert!(
matches!(result, Err(FormatError::Overflow(_))),
"expected a clean Overflow error, got {result:?}"
);
}
#[test]
fn read_raw_data_zerocopy_rejects_near_usize_max_offset() {
let file_data = vec![0u8; 64];
let dataspace = make_simple_dataspace(&[4]);
let datatype = make_i32_le_type();
let layout = DataLayout::Contiguous {
address: Some(u64::MAX - 4),
size: 16,
};
let result = read_raw_data_zerocopy(&file_data, &layout, &dataspace, &datatype);
assert!(
matches!(result, Err(FormatError::UnexpectedEof { .. })),
"expected a clean UnexpectedEof, got {result:?}"
);
}
#[test]
fn read_compound_single_field_by_name() {
use crate::datatype::CompoundMember;
let dt = Datatype::Compound {
size: 12,
members: vec![
CompoundMember {
name: "x".to_string(),
byte_offset: 0,
datatype: make_f64_le_type(),
},
CompoundMember {
name: "id".to_string(),
byte_offset: 8,
datatype: make_i32_le_type(),
},
],
};
let mut raw = Vec::new();
raw.extend_from_slice(&3.25f64.to_le_bytes());
raw.extend_from_slice(&42i32.to_le_bytes());
let field = read_compound_field(&raw, &dt, "id").unwrap();
let vals = read_as_i32(&field.raw_data, &field.datatype).unwrap();
assert_eq!(vals, vec![42]);
// Non-existent field
let err = read_compound_field(&raw, &dt, "missing").unwrap_err();
assert!(matches!(err, FormatError::PathNotFound(_)));
}
#[test]
fn read_enum_values_basic() {
use crate::datatype::EnumMember;
let dt = Datatype::Enumeration {
size: 4,
base_type: Box::new(make_i32_le_type()),
members: vec![
EnumMember {
name: "RED".to_string(),
value: 0i32.to_le_bytes().to_vec(),
},
EnumMember {
name: "GREEN".to_string(),
value: 1i32.to_le_bytes().to_vec(),
},
EnumMember {
name: "BLUE".to_string(),
value: 2i32.to_le_bytes().to_vec(),
},
],
};
let mut raw = Vec::new();
raw.extend_from_slice(&1i32.to_le_bytes()); // GREEN
raw.extend_from_slice(&0i32.to_le_bytes()); // RED
raw.extend_from_slice(&2i32.to_le_bytes()); // BLUE
raw.extend_from_slice(&99i32.to_le_bytes()); // unknown
let names = read_enum_names(&raw, &dt).unwrap();
assert_eq!(names[0], "GREEN");
assert_eq!(names[1], "RED");
assert_eq!(names[2], "BLUE");
assert!(names[3].starts_with("UNKNOWN("));
}
#[test]
fn read_array_flat_basic() {
// Array[3] of f64
let dt = Datatype::Array {
base_type: Box::new(make_f64_le_type()),
dimensions: vec![3],
};
let mut raw = Vec::new();
for v in &[1.0f64, 2.0, 3.0] {
raw.extend_from_slice(&v.to_le_bytes());
}
let (data, base_dt, dims) = read_array_flat(&raw, &dt).unwrap();
assert_eq!(dims, vec![3]);
let vals = read_as_f64(&data, &base_dt).unwrap();
assert_eq!(vals, vec![1.0, 2.0, 3.0]);
}
#[test]
fn read_object_references_basic() {
use crate::datatype::ReferenceType;
let dt = Datatype::Reference {
size: 8,
ref_type: ReferenceType::Object,
};
let mut raw = Vec::new();
raw.extend_from_slice(&1024u64.to_le_bytes()); // valid ref
raw.extend_from_slice(&u64::MAX.to_le_bytes()); // null ref
raw.extend_from_slice(&2048u64.to_le_bytes()); // valid ref
let refs = read_object_references(&raw, &dt, 8).unwrap();
assert_eq!(refs.len(), 3);
assert_eq!(refs[0].address, 1024);
assert!(!refs[0].is_null());
assert!(refs[1].is_null());
assert_eq!(refs[2].address, 2048);
}
#[test]
fn read_object_references_4byte_offset() {
use crate::datatype::ReferenceType;
let dt = Datatype::Reference {
size: 4,
ref_type: ReferenceType::Object,
};
let mut raw = Vec::new();
raw.extend_from_slice(&512u32.to_le_bytes());
raw.extend_from_slice(&u32::MAX.to_le_bytes()); // null ref
let refs = read_object_references(&raw, &dt, 4).unwrap();
assert_eq!(refs.len(), 2);
assert_eq!(refs[0].address, 512);
assert!(refs[1].is_null());
}
#[test]
fn read_object_references_type_mismatch() {
let dt = make_f64_le_type();
let raw = vec![0u8; 8];
let err = read_object_references(&raw, &dt, 8).unwrap_err();
assert!(matches!(err, FormatError::TypeMismatch { .. }));
}
#[test]
fn read_region_references_basic() {
use crate::datatype::ReferenceType;
let dt = Datatype::Reference {
size: 12,
ref_type: ReferenceType::DatasetRegion,
};
let raw = vec![0xABu8; 24]; // two 12-byte region refs
let refs = read_region_references(&raw, &dt).unwrap();
assert_eq!(refs.len(), 2);
assert_eq!(refs[0].raw.len(), 12);
assert_eq!(refs[1].raw.len(), 12);
}
#[test]
fn read_region_references_type_mismatch() {
let dt = make_i32_le_type();
let raw = vec![0u8; 12];
let err = read_region_references(&raw, &dt).unwrap_err();
assert!(matches!(err, FormatError::TypeMismatch { .. }));
}
#[test]
fn zerocopy_contiguous_returns_slice_into_file_data() {
let dt = make_f64_le_type();
let ds = make_simple_dataspace(&[3]);
let mut file_data = vec![0u8; 1024];
let offset = 256usize;
let vals = [1.0f64, 2.0, 3.0];
for (i, v) in vals.iter().enumerate() {
file_data[offset + i * 8..offset + i * 8 + 8].copy_from_slice(&v.to_le_bytes());
}
let layout = DataLayout::Contiguous {
address: Some(offset as u64),
size: 24,
};
let result = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap();
let slice = result.expect("contiguous should return Some");
// Pointer identity: the slice must point into file_data, not a copy
let file_range = file_data.as_ptr_range();
assert!(file_range.contains(&slice.as_ptr()));
assert_eq!(slice.len(), 24);
// Verify the actual data
let values = read_as_f64(slice, &dt).unwrap();
assert_eq!(values, vec![1.0, 2.0, 3.0]);
}
#[test]
fn zerocopy_compact_returns_none() {
let dt = make_f64_le_type();
let ds = make_simple_dataspace(&[1]);
let data = vec![0u8; 8];
let layout = DataLayout::Compact { data };
let result = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap();
assert!(result.is_none());
}
#[test]
fn zerocopy_no_data_allocated() {
let dt = make_f64_le_type();
let ds = make_simple_dataspace(&[1]);
let layout = DataLayout::Contiguous {
address: None,
size: 8,
};
let err = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap_err();
assert!(matches!(err, FormatError::NoDataAllocated));
}
#[test]
fn read_as_f64_zerocopy_aligned() {
let dt = make_f64_le_type();
// Create aligned data — Vec<f64> guarantees 8-byte alignment
let values = [1.0f64, 2.0, 3.0, 4.0];
let raw: &[u8] =
// SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.
unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) };
let result = read_as_f64_zerocopy(raw, &dt);
assert!(result.is_some(), "aligned native LE f64 should succeed");
let slice = result.unwrap();
assert_eq!(slice, &[1.0, 2.0, 3.0, 4.0]);
// Verify it's the same memory (zero-copy)
assert_eq!(slice.as_ptr() as *const u8, raw.as_ptr());
}
#[test]
fn read_as_f64_zerocopy_wrong_type() {
let dt = make_i32_le_type();
let values = [1.0f64; 4];
let raw: &[u8] =
// SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.
unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) };
assert!(read_as_f64_zerocopy(raw, &dt).is_none());
}
#[test]
fn read_as_f64_zerocopy_big_endian() {
let dt = Datatype::FloatingPoint {
size: 8,
byte_order: DatatypeByteOrder::BigEndian,
bit_offset: 0,
bit_precision: 64,
exponent_location: 52,
exponent_size: 11,
mantissa_location: 0,
mantissa_size: 52,
exponent_bias: 1023,
};
let values = [1.0f64; 4];
let raw: &[u8] =
// SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.
unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) };
assert!(read_as_f64_zerocopy(raw, &dt).is_none());
}
#[test]
fn read_as_f64_zerocopy_odd_size() {
let dt = make_f64_le_type();
let raw = &[0u8; 13]; // not a multiple of 8
assert!(read_as_f64_zerocopy(raw, &dt).is_none());
}
#[test]
fn read_as_f32_zerocopy_aligned() {
let dt = Datatype::FloatingPoint {
size: 4,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 32,
exponent_location: 23,
exponent_size: 8,
mantissa_location: 0,
mantissa_size: 23,
exponent_bias: 127,
};
let values = [1.5f32, 2.5, 3.5];
let raw: &[u8] =
// SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.
unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 4) };
let result = read_as_f32_zerocopy(raw, &dt);
assert!(result.is_some());
assert_eq!(result.unwrap(), &[1.5f32, 2.5, 3.5]);
}
#[test]
fn zerocopy_size_mismatch() {
let dt = make_f64_le_type();
let ds = make_simple_dataspace(&[3]);
let file_data = vec![0u8; 1024];
let layout = DataLayout::Contiguous {
address: Some(0),
size: 16, // wrong: should be 24
};
let err = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap_err();
assert!(matches!(err, FormatError::DataSizeMismatch { .. }));
}
}
Reference in New Issue View Git Blame Copy Permalink