clawhdf5/crates/clawhdf5-format/src/data_read.rs

//! Raw data reading and typed conversion for HDF5 datasets.

#[cfg(not(feature = "std"))]
use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};

#[cfg(feature = "std")]
use std::collections::BTreeMap;

#[cfg(feature = "std")]
use crate::chunk_cache::ChunkCache;
use crate::chunked_read::read_chunked_data;
#[cfg(feature = "std")]
use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed};
use crate::data_layout::DataLayout;
use crate::dataspace::Dataspace;
use crate::datatype::{Datatype, DatatypeByteOrder};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;

/// Checks that `[offset, offset + needed)` fits within `data`, guarding the
/// addition against `usize` overflow from a crafted near-`usize::MAX` offset.
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
    if offset
        .checked_add(needed)
        .is_none_or(|end| end > data.len())
    {
        return Err(FormatError::UnexpectedEof {
            expected: offset.saturating_add(needed),
            available: data.len(),
        });
    }
    Ok(())
}

/// Zero-copy read of contiguous raw data, returning a borrowed slice.
///
/// For contiguous layouts, returns a direct `&[u8]` slice into `file_data`.
/// For compact or chunked layouts, returns `Ok(None)` — the caller should
/// fall back to `read_raw_data` for those.
pub fn read_raw_data_zerocopy<'a>(
    file_data: &'a [u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
) -> Result<Option<&'a [u8]>, FormatError> {
    let num_elements = dataspace.num_elements() as usize;
    let elem_size = datatype.type_size() as usize;
    let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
        FormatError::Overflow(format!(
            "num_elements({num_elements}) * elem_size({elem_size})"
        ))
    })?;

    match layout {
        DataLayout::Contiguous { address, size } => {
            let addr = address.ok_or(FormatError::NoDataAllocated)?;
            let addr = addr as usize;
            let sz = *size as usize;
            if sz != expected_size {
                return Err(FormatError::DataSizeMismatch {
                    expected: expected_size,
                    actual: sz,
                });
            }
            ensure_len(file_data, addr, sz)?;
            Ok(Some(&file_data[addr..addr + sz]))
        }
        _ => Ok(None),
    }
}

/// Read raw bytes for a dataset given its layout and the file data buffer.
///
/// For compact layouts, returns the inline data.
/// For contiguous layouts, reads from the address in the file buffer.
/// For chunked layouts, traverses the B-tree and assembles chunks.
pub fn read_raw_data(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
) -> Result<Vec<u8>, FormatError> {
    read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8)
}

/// Resolves a Virtual Dataset source **file name** (as stored in the mapping,
/// e.g. `"ext_src.h5"`) to that file's raw bytes.
///
/// The pure-byte read API has no filesystem of its own, so external-file VDS
/// sources are read through a caller-supplied resolver. The std file API wires
/// one that reads relative to the virtual file's directory; callers can supply
/// their own (e.g. an in-memory map) in `no_std` builds. Returning `None` means
/// the source file is unavailable and the mapping is skipped.
pub type VdsSourceResolver<'a> = dyn Fn(&str) -> Option<Vec<u8>> + 'a;

/// Read raw bytes with full parameters including filter pipeline and sizes.
pub fn read_raw_data_full(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
    pipeline: Option<&FilterPipeline>,
    offset_size: u8,
    length_size: u8,
) -> Result<Vec<u8>, FormatError> {
    read_raw_data_full_impl(
        file_data,
        layout,
        dataspace,
        datatype,
        pipeline,
        offset_size,
        length_size,
        None,
    )
}

/// Like [`read_raw_data_full`], but with a resolver for external-file Virtual
/// Dataset sources. For non-virtual layouts the resolver is ignored.
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_full_with_resolver(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
    pipeline: Option<&FilterPipeline>,
    offset_size: u8,
    length_size: u8,
    resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
    read_raw_data_full_impl(
        file_data,
        layout,
        dataspace,
        datatype,
        pipeline,
        offset_size,
        length_size,
        resolver,
    )
}

#[allow(clippy::too_many_arguments)]
fn read_raw_data_full_impl(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
    pipeline: Option<&FilterPipeline>,
    offset_size: u8,
    length_size: u8,
    resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
    let num_elements = dataspace.num_elements() as usize;
    let elem_size = datatype.type_size() as usize;
    let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
        FormatError::Overflow(format!(
            "num_elements({num_elements}) * elem_size({elem_size})"
        ))
    })?;

    match layout {
        DataLayout::Compact { data } => {
            if data.len() != expected_size {
                return Err(FormatError::DataSizeMismatch {
                    expected: expected_size,
                    actual: data.len(),
                });
            }
            Ok(data.clone())
        }
        DataLayout::Contiguous { address, size } => {
            let addr = address.ok_or(FormatError::NoDataAllocated)?;
            let addr = addr as usize;
            let sz = *size as usize;
            if sz != expected_size {
                return Err(FormatError::DataSizeMismatch {
                    expected: expected_size,
                    actual: sz,
                });
            }
            ensure_len(file_data, addr, sz)?;
            let mut out = crate::bulk_alloc::vec_for_bulk(sz);
            out.extend_from_slice(&file_data[addr..addr + sz]);
            Ok(out)
        }
        DataLayout::Chunked { .. } => read_chunked_data(
            file_data,
            layout,
            dataspace,
            datatype,
            pipeline,
            offset_size,
            length_size,
        ),
        DataLayout::Virtual { .. } => read_virtual_data(
            file_data,
            layout,
            dataspace,
            datatype,
            offset_size,
            length_size,
            resolver,
        ),
    }
}

/// Read raw bytes with chunk cache support.
///
/// For chunked layouts the `cache` is used to avoid repeated B-tree
/// traversals and to cache decompressed chunk data.  For compact and
/// contiguous layouts this behaves identically to [`read_raw_data_full`].
#[cfg(feature = "std")]
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_cached(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
    pipeline: Option<&FilterPipeline>,
    offset_size: u8,
    length_size: u8,
    cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
    match layout {
        DataLayout::Chunked { .. } => read_chunked_data_cached(
            file_data,
            layout,
            dataspace,
            datatype,
            pipeline,
            offset_size,
            length_size,
            cache,
        ),
        _ => read_raw_data_full(
            file_data,
            layout,
            dataspace,
            datatype,
            pipeline,
            offset_size,
            length_size,
        ),
    }
}

/// Read raw bytes with chunk B-tree index cache and pre-computed layout.
///
/// For chunked layouts this uses the optimized `read_chunked_data_indexed` path
/// which pre-computes contiguous row-copy operations, eliminating per-element
/// N-D coordinate math on repeated reads.  For compact and contiguous layouts
/// this behaves identically to [`read_raw_data_full`].
#[cfg(feature = "std")]
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_indexed(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
    pipeline: Option<&FilterPipeline>,
    offset_size: u8,
    length_size: u8,
    cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
    match layout {
        DataLayout::Chunked { .. } => read_chunked_data_indexed(
            file_data,
            layout,
            dataspace,
            datatype,
            pipeline,
            offset_size,
            length_size,
            cache,
        ),
        _ => read_raw_data_full(
            file_data,
            layout,
            dataspace,
            datatype,
            pipeline,
            offset_size,
            length_size,
        ),
    }
}

/// Read raw bytes for only the selected elements of a dataset.
///
/// When the selection's bounding box covers at most half the dataset, only
/// that box is materialised — the overlapping rows of a contiguous dataset,
/// the overlapping chunks of a chunked one, whatever its chunk index (see
/// [`crate::partial_read`]). Otherwise, and for compact and virtual
/// layouts, the whole dataset is decoded and the selection extracted.
#[allow(clippy::too_many_arguments)]
pub fn read_raw_data_selection(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
    pipeline: Option<&FilterPipeline>,
    offset_size: u8,
    length_size: u8,
    selection: &crate::selection::Selection,
) -> Result<Vec<u8>, FormatError> {
    use crate::selection::Selection;

    crate::partial_read::validate(selection, &dataspace.dimensions)?;
    crate::chunked_read::check_chunk_element_size(layout, datatype, offset_size)?;

    // Read only what the selection's bounding box touches when that is
    // possible; everything below is the decode-everything-then-pick path,
    // kept for the cases `partial_read` declines.
    if let Some(selected) = crate::partial_read::read_selection(
        file_data,
        layout,
        dataspace,
        datatype.type_size() as usize,
        pipeline,
        offset_size,
        length_size,
        selection,
    )? {
        return Ok(selected);
    }

    match selection {
        Selection::All => {
            return read_raw_data_full(
                file_data,
                layout,
                dataspace,
                datatype,
                pipeline,
                offset_size,
                length_size,
            );
        }
        Selection::None => return Ok(Vec::new()),
        _ => {}
    }

    let dims = &dataspace.dimensions;
    let elem_size = datatype.type_size() as usize;

    match layout {
        DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => {
            // Read all data, then extract the selection
            let full_data = read_raw_data_full(
                file_data,
                layout,
                dataspace,
                datatype,
                pipeline,
                offset_size,
                length_size,
            )?;
            extract_selection_from_buffer(&full_data, dims, elem_size, selection)
        }
        DataLayout::Chunked {
            chunk_dimensions,
            version,
            ..
        } => {
            // `partial_read` declined (a bounding box covering most of the
            // dataset, or a selection it doesn't box), so decode every chunk
            // and pick the selection out, whatever the chunk index. This arm
            // used to enumerate the chunks first — passing the layout's
            // chunk dimensions, element-size dimension included, to the
            // implicit-index generator, which then indexed past the rank and
            // panicked — only to decode the full dataset anyway.
            crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?;
            let full_data = read_raw_data_full(
                file_data,
                layout,
                dataspace,
                datatype,
                pipeline,
                offset_size,
                length_size,
            )?;
            extract_selection_from_buffer(&full_data, dims, elem_size, selection)
        }
        DataLayout::Virtual { .. } => {
            // Assemble the full virtual dataset, then apply the read selection.
            let full_data = read_raw_data_full(
                file_data,
                layout,
                dataspace,
                datatype,
                pipeline,
                offset_size,
                length_size,
            )?;
            extract_selection_from_buffer(&full_data, dims, elem_size, selection)
        }
    }
}

/// Assemble a **Virtual Dataset (VDS)** through the raw-read API, which has no
/// access to the dataset's fill value message.
///
/// Delegates to [`crate::vds::read_virtual_dataset`]. Because the fill value
/// is unknown here, a virtual dataset with any element no mapping supplies
/// (an unmapped region, or a missing source file or dataset) is an error
/// rather than a guess at the fill value; so is one whose extent libhdf5
/// would report differently from the stored dataspace (unlimited mappings).
/// Use [`crate::vds::read_virtual_dataset`] to read those.
#[allow(clippy::too_many_arguments)]
fn read_virtual_data(
    file_data: &[u8],
    layout: &DataLayout,
    dataspace: &Dataspace,
    datatype: &Datatype,
    offset_size: u8,
    length_size: u8,
    resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
    let wrapped =
        resolver.map(|r| move |name: &str| -> Result<Option<Vec<u8>>, FormatError> { Ok(r(name)) });
    let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver);
    let v = crate::vds::read_virtual_dataset(
        file_data,
        layout,
        dataspace,
        datatype,
        None,
        offset_size,
        length_size,
        wrapped_ref,
    )?;
    if v.dims != dataspace.dimensions {
        return Err(FormatError::ChunkedReadError(
            "virtual dataset extent differs from its stored dataspace; \
             read it with vds::read_virtual_dataset"
                .into(),
        ));
    }
    if v.unmapped > 0 {
        return Err(FormatError::ChunkedReadError(
            "virtual dataset has elements no source supplies, which read as its \
             fill value; read it with vds::read_virtual_dataset and the fill value"
                .into(),
        ));
    }
    Ok(v.data)
}
/// Extract selected elements from a full dataset buffer.
pub fn extract_selection_from_buffer(
    full_data: &[u8],
    dims: &[u64],
    elem_size: usize,
    selection: &crate::selection::Selection,
) -> Result<Vec<u8>, FormatError> {
    use crate::selection::Selection;

    match selection {
        Selection::All => Ok(full_data.to_vec()),
        Selection::None => Ok(Vec::new()),
        Selection::Hyperslab {
            start,
            stride,
            count,
            block,
        } => {
            let rank = dims.len();
            if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] {
                return Err(FormatError::SelectionOutOfBounds(format!(
                    "hyperslab rank does not match dataset rank {rank}"
                )));
            }
            let output_elements = count
                .iter()
                .zip(block.iter())
                .try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?))
                .ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?;
            let mut output = crate::chunked_read::alloc_output(
                crate::chunked_read::checked_byte_len(output_elements, elem_size)?,
            )?;

            // One copy per run of elements contiguous in `full_data`
            // (`gather`'s runs). Coordinates past the extent are skipped and
            // runs past the end of `full_data` left as zeros, element by
            // element, as this extractor always did; validated selections
            // never hit either.
            let mut out_at = 0usize;
            crate::gather::hyperslab_runs(dims, start, stride, count, block, |first, n| {
                let big = |v: u64| usize::try_from(v).unwrap_or(usize::MAX);
                let (first, n) = (big(first), big(n));
                let len = n.saturating_mul(elem_size);
                let src = first.saturating_mul(elem_size);
                let out_end = out_at.saturating_add(len);
                if let (Some(from), Some(to)) = (
                    full_data.get(src..src.saturating_add(len)),
                    output.get_mut(out_at..out_end),
                ) {
                    to.copy_from_slice(from);
                } else {
                    for k in 0..n {
                        let s = first.saturating_add(k).saturating_mul(elem_size);
                        let o = out_at.saturating_add(k.saturating_mul(elem_size));
                        if o >= output.len() {
                            break;
                        }
                        if let (Some(from), Some(to)) = (
                            full_data.get(s..s.saturating_add(elem_size)),
                            output.get_mut(o..o.saturating_add(elem_size)),
                        ) {
                            to.copy_from_slice(from);
                        }
                    }
                }
                out_at = out_end;
            });

            Ok(output)
        }
        Selection::Points(pts) => {
            let rank = dims.len();
            let mut ds_strides = vec![1usize; rank];
            for i in (0..rank.saturating_sub(1)).rev() {
                ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
            }

            let mut output = Vec::with_capacity(pts.len() * elem_size);
            for pt in pts {
                let flat: usize = pt
                    .iter()
                    .zip(ds_strides.iter())
                    .map(|(&p, &s)| p as usize * s)
                    .sum();
                let src = flat * elem_size;
                if src + elem_size <= full_data.len() {
                    output.extend_from_slice(&full_data[src..src + elem_size]);
                } else {
                    output.extend_from_slice(&vec![0u8; elem_size]);
                }
            }
            Ok(output)
        }
    }
}

/// Zero-copy transmute of raw bytes to `&[f64]`.
///
/// Returns `Some(&[f64])` when the datatype is native little-endian `f64`
/// and the data pointer is 8-byte aligned.  Returns `None` for any other
/// type or alignment — the caller should fall back to [`read_as_f64`].
pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f64]> {
    // Only native LE f64 is eligible
    #[cfg(target_endian = "little")]
    {
        if !is_native_le_float(datatype, FloatFormat::Double) {
            return None;
        }
        if !raw.len().is_multiple_of(8) {
            return None;
        }
        let ptr = raw.as_ptr();
        if !(ptr as usize).is_multiple_of(core::mem::align_of::<f64>()) {
            return None;
        }
        let count = raw.len() / 8;
        // SAFETY: We verified alignment (8-byte), size (multiple of 8), and
        // the on-disk format matches the in-memory representation (LE f64).
        Some(unsafe { core::slice::from_raw_parts(ptr as *const f64, count) })
    }
    #[cfg(not(target_endian = "little"))]
    {
        let _ = (raw, datatype);
        None
    }
}

/// Zero-copy transmute of raw bytes to `&[f32]`.
///
/// Returns `Some(&[f32])` when the datatype is native little-endian `f32`
/// and the data pointer is 4-byte aligned.  Returns `None` otherwise.
pub fn read_as_f32_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f32]> {
    #[cfg(target_endian = "little")]
    {
        if !is_native_le_float(datatype, FloatFormat::Single) {
            return None;
        }
        if !raw.len().is_multiple_of(4) {
            return None;
        }
        let ptr = raw.as_ptr();
        if !(ptr as usize).is_multiple_of(core::mem::align_of::<f32>()) {
            return None;
        }
        let count = raw.len() / 4;
        // SAFETY: We verified alignment (4-byte), size (multiple of 4), and
        // the on-disk format matches the in-memory representation (LE f32).
        Some(unsafe { core::slice::from_raw_parts(ptr as *const f32, count) })
    }
    #[cfg(not(target_endian = "little"))]
    {
        let _ = (raw, datatype);
        None
    }
}

fn datatype_name(dt: &Datatype) -> &'static str {
    match dt {
        Datatype::FixedPoint { .. } => "FixedPoint",
        Datatype::FloatingPoint { .. } => "FloatingPoint",
        Datatype::String { .. } => "String",
        Datatype::Time { .. } => "Time",
        Datatype::BitField { .. } => "BitField",
        Datatype::Opaque { .. } => "Opaque",
        Datatype::Compound { .. } => "Compound",
        Datatype::Reference { .. } => "Reference",
        Datatype::Enumeration { .. } => "Enumeration",
        Datatype::VariableLength { .. } => "VariableLength",
        Datatype::Array { .. } => "Array",
    }
}

fn ensure_numeric(dt: &Datatype, expected: &'static str) -> Result<(), FormatError> {
    match dt {
        Datatype::FixedPoint { .. } | Datatype::FloatingPoint { .. } => Ok(()),
        _ => Err(FormatError::TypeMismatch {
            expected,
            actual: datatype_name(dt),
        }),
    }
}

fn get_byte_order(dt: &Datatype) -> DatatypeByteOrder {
    match dt {
        Datatype::FixedPoint { byte_order, .. } => byte_order.clone(),
        Datatype::FloatingPoint { byte_order, .. } => byte_order.clone(),
        _ => DatatypeByteOrder::LittleEndian,
    }
}

fn get_size(dt: &Datatype) -> usize {
    dt.type_size() as usize
}

mod sealed {
    pub trait Sealed {}
}

/// A numeric type whose values can be copied straight out of a dataset's
/// bytes when the dataset stores exactly that type in the target's byte
/// order: `u8`, `i32`, `i64`, `u64`, `f32` and `f64`.
///
/// # Safety
///
/// Implementors have no padding and no invalid bit patterns, so a buffer of
/// them may be filled by copying bytes. The trait is sealed.
pub unsafe trait NativeElement: sealed::Sealed + Copy + 'static {
    /// Whether `datatype`'s stored bytes are this type's native in-memory
    /// representation (same size, byte order, signedness, full precision,
    /// IEEE layout), so reading needs a copy and no conversion.
    fn is_native(datatype: &Datatype) -> bool;
}

/// A full-width fixed-point type of `size` bytes and the given signedness in
/// the target's byte order.
fn is_native_int(datatype: &Datatype, size: u32, want_signed: bool) -> bool {
    let order = if cfg!(target_endian = "little") {
        DatatypeByteOrder::LittleEndian
    } else {
        DatatypeByteOrder::BigEndian
    };
    matches!(
        datatype,
        Datatype::FixedPoint { size: s, signed, byte_order, .. }
            if *s == size && *signed == want_signed && (size == 1 || *byte_order == order)
    ) && is_full_width(datatype)
}

macro_rules! native_element {
    ($($t:ty => |$dt:ident| $check:expr;)*) => {$(
        impl sealed::Sealed for $t {}
        // SAFETY: a primitive integer or float: no padding, and every bit
        // pattern is a valid value.
        unsafe impl NativeElement for $t {
            fn is_native($dt: &Datatype) -> bool {
                $check
            }
        }
    )*};
}

native_element! {
    u8 => |dt| is_native_int(dt, 1, false);
    i32 => |dt| is_native_int(dt, 4, true);
    i64 => |dt| is_native_int(dt, 8, true);
    u64 => |dt| is_native_int(dt, 8, false);
    f32 => |dt| cfg!(target_endian = "little") && is_native_le_float(dt, FloatFormat::Single);
    f64 => |dt| cfg!(target_endian = "little") && is_native_le_float(dt, FloatFormat::Double);
}

/// Copy `count` values of `T` out of `raw`, which holds them in `T`'s native
/// representation (see [`NativeElement::is_native`]), in one copy.
///
/// The buffer is allocated uninitialised and filled by the copy. It used to be
/// `vec![0; count]` first, which for a large dataset meant writing every page
/// twice (zero it, then overwrite it) — about as expensive as the copy itself.
fn native_to_vec<T: NativeElement>(raw: &[u8], count: usize) -> Vec<T> {
    let bytes = count * core::mem::size_of::<T>();
    assert!(bytes <= raw.len(), "native_to_vec: source too short");
    let mut result: Vec<T> = crate::bulk_alloc::vec_for_bulk(count);
    // SAFETY: `result` has capacity for `count` values of `T`, i.e. `bytes`
    // bytes; `raw` holds at least `bytes` bytes (asserted); the regions
    // cannot overlap because `result` was just allocated. `T: NativeElement`
    // is valid for any bit pattern, so after the copy all `count` values are
    // initialised and `set_len` is sound.
    unsafe {
        core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr().cast::<u8>(), bytes);
        result.set_len(count);
    }
    result
}

/// Read `selection` of a dataset whose raw bytes (all of them, row-major, of
/// shape `dims`) are `raw` — typically a contiguous dataset's bytes borrowed
/// from the file — straight into a `Vec<T>`, copying each contiguous run of
/// selected elements once.
///
/// Returns `Ok(None)` when `datatype` is not `T`'s native representation
/// ([`NativeElement::is_native`]); the caller then converts through
/// [`read_raw_data_selection`] and the `read_as_*` functions. The selection is
/// validated like every selection read: out-of-range coordinates are
/// [`FormatError::SelectionOutOfBounds`].
pub fn read_selection_native<T: NativeElement>(
    raw: &[u8],
    dims: &[u64],
    datatype: &Datatype,
    selection: &crate::selection::Selection,
) -> Result<Option<Vec<T>>, FormatError> {
    if !T::is_native(datatype) {
        return Ok(None);
    }
    let elem_size = core::mem::size_of::<T>();
    let total = dims
        .iter()
        .try_fold(1u64, |acc, &d| acc.checked_mul(d))
        .ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?;
    let expected = crate::chunked_read::checked_byte_len(total, elem_size)?;
    if raw.len() != expected {
        return Err(FormatError::DataSizeMismatch {
            expected,
            actual: raw.len(),
        });
    }
    if let crate::selection::Selection::All = selection {
        return Ok(Some(native_to_vec(raw, expected / elem_size)));
    }
    crate::partial_read::validate(selection, dims)?;
    crate::gather::gather::<T>(raw, dims, elem_size, selection).map(Some)
}

/// Convert raw bytes to `f64` values.
pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatError> {
    // Array datatypes read as a flat sequence of their base elements, and
    // enumerations (h5py's bool among them) as their integer values.
    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
        return read_as_f64(raw, base_type);
    }
    ensure_numeric(datatype, "FloatingPoint or FixedPoint")?;
    let elem_size = get_size(datatype);
    if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
        return Err(FormatError::DataSizeMismatch {
            expected: 0,
            actual: raw.len(),
        });
    }
    let count = raw.len() / elem_size;

    // Fast path: native-endian f64 — single bulk memcpy
    if f64::is_native(datatype) {
        return Ok(native_to_vec::<f64>(raw, count));
    }

    let order = get_byte_order(datatype);
    let mut result = crate::bulk_alloc::vec_for_bulk(count);
    if let Datatype::FloatingPoint { .. } = datatype {
        let format = FloatFormat::of(datatype)?;
        for chunk in raw.chunks_exact(elem_size) {
            result.push(format.decode(chunk, &order));
        }
        return Ok(result);
    }
    for i in 0..count {
        let chunk = &raw[i * elem_size..(i + 1) * elem_size];
        let val = convert_to_f64(chunk, datatype, &order)?;
        result.push(val);
    }
    Ok(result)
}

fn convert_to_f64(
    bytes: &[u8],
    dt: &Datatype,
    order: &DatatypeByteOrder,
) -> Result<f64, FormatError> {
    match dt {
        Datatype::FloatingPoint { .. } => Ok(FloatFormat::of(dt)?.decode(bytes, order)),
        Datatype::FixedPoint {
            size,
            signed,
            bit_offset,
            bit_precision,
            ..
        } => {
            let full = read_unsigned_int(bytes, *size as usize, order);
            let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
            let v = if *signed {
                extract_signed(full, off, prec) as f64
            } else {
                extract_unsigned(full, off, prec) as f64
            };
            Ok(v)
        }
        _ => Err(FormatError::TypeMismatch {
            expected: "numeric",
            actual: datatype_name(dt),
        }),
    }
}

/// One numeric element as stored, before conversion to the caller's type.
#[derive(Debug, Clone, Copy, PartialEq)]
enum Scalar {
    Signed(i64),
    Unsigned(u64),
    Float(f64),
}

impl Scalar {
    // Every conversion follows libhdf5's default (hard) conversions: a value
    // outside the target type's range saturates to its minimum or maximum —
    // including a negative value read as unsigned, which reads as 0 — rather
    // than being truncated to its low bits. Floats truncate toward zero; NaN
    // converts to 0 (libhdf5 leaves that case to the C cast, whose result is
    // platform-dependent).

    fn to_i64(self) -> i64 {
        match self {
            Scalar::Signed(v) => v,
            Scalar::Unsigned(v) => i64::try_from(v).unwrap_or(i64::MAX),
            Scalar::Float(v) => v as i64,
        }
    }

    fn to_u64(self) -> u64 {
        match self {
            Scalar::Signed(v) => u64::try_from(v).unwrap_or(0),
            Scalar::Unsigned(v) => v,
            Scalar::Float(v) => v as u64,
        }
    }

    fn to_i32(self) -> i32 {
        match self {
            Scalar::Signed(v) => v.clamp(i32::MIN.into(), i32::MAX.into()) as i32,
            Scalar::Unsigned(v) => i32::try_from(v).unwrap_or(i32::MAX),
            Scalar::Float(v) => v as i32,
        }
    }
}

/// Decode one element of a numeric datatype.
fn decode_scalar(
    bytes: &[u8],
    dt: &Datatype,
    order: &DatatypeByteOrder,
) -> Result<Scalar, FormatError> {
    match dt {
        Datatype::FixedPoint {
            size,
            signed,
            bit_offset,
            bit_precision,
            ..
        } => {
            let full = read_unsigned_int(bytes, *size as usize, order);
            let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
            Ok(if *signed {
                Scalar::Signed(extract_signed(full, off, prec))
            } else {
                Scalar::Unsigned(extract_unsigned(full, off, prec))
            })
        }
        _ => convert_to_f64(bytes, dt, order).map(Scalar::Float),
    }
}

/// Convert raw bytes to `i64` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result<Vec<i64>, FormatError> {
    // Array datatypes read as a flat sequence of their base elements, and
    // enumerations (h5py's bool among them) as their integer values.
    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
        return read_as_i64(raw, base_type);
    }
    ensure_numeric(datatype, "FixedPoint (signed)")?;
    let elem_size = get_size(datatype);
    if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
        return Err(FormatError::DataSizeMismatch {
            expected: 0,
            actual: raw.len(),
        });
    }
    let count = raw.len() / elem_size;

    // Fast path: native LE i64 — single bulk memcpy
    if i64::is_native(datatype) {
        return Ok(native_to_vec::<i64>(raw, count));
    }

    let order = get_byte_order(datatype);
    let mut result = crate::bulk_alloc::vec_for_bulk(count);
    for i in 0..count {
        let chunk = &raw[i * elem_size..(i + 1) * elem_size];
        result.push(decode_scalar(chunk, datatype, &order)?.to_i64());
    }
    Ok(result)
}

/// Convert raw bytes to `u64` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result<Vec<u64>, FormatError> {
    // Array datatypes read as a flat sequence of their base elements, and
    // enumerations (h5py's bool among them) as their integer values.
    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
        return read_as_u64(raw, base_type);
    }
    ensure_numeric(datatype, "FixedPoint (unsigned)")?;
    let elem_size = get_size(datatype);
    if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
        return Err(FormatError::DataSizeMismatch {
            expected: 0,
            actual: raw.len(),
        });
    }
    let count = raw.len() / elem_size;

    // Fast path: native u64 — single bulk memcpy
    if u64::is_native(datatype) {
        return Ok(native_to_vec::<u64>(raw, count));
    }

    let order = get_byte_order(datatype);
    let mut result = crate::bulk_alloc::vec_for_bulk(count);
    for i in 0..count {
        let chunk = &raw[i * elem_size..(i + 1) * elem_size];
        result.push(decode_scalar(chunk, datatype, &order)?.to_u64());
    }
    Ok(result)
}

/// Convert raw bytes to `f32` values.
pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatError> {
    // Array datatypes read as a flat sequence of their base elements, and
    // enumerations (h5py's bool among them) as their integer values.
    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
        return read_as_f32(raw, base_type);
    }
    ensure_numeric(datatype, "FloatingPoint")?;
    let elem_size = get_size(datatype);
    if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
        return Err(FormatError::DataSizeMismatch {
            expected: 0,
            actual: raw.len(),
        });
    }
    let count = raw.len() / elem_size;

    // Fast path: native-endian f32 — single bulk memcpy
    if f32::is_native(datatype) {
        return Ok(native_to_vec::<f32>(raw, count));
    }
    // Little-endian IEEE half precision (numpy float16): widen directly.
    if is_native_le_float(datatype, FloatFormat::Half) {
        let (halves, _) = raw[..count * 2].as_chunks::<2>();
        let mut result = crate::bulk_alloc::vec_for_bulk(count);
        result.extend(
            halves
                .iter()
                .map(|&b| f16_bits_to_f32(u16::from_le_bytes(b))),
        );
        return Ok(result);
    }

    let order = get_byte_order(datatype);
    let mut result = crate::bulk_alloc::vec_for_bulk(count);
    if let Datatype::FloatingPoint { .. } = datatype {
        let format = FloatFormat::of(datatype)?;
        for chunk in raw.chunks_exact(elem_size) {
            result.push(match format {
                FloatFormat::Single => read_f32_bytes(chunk, &order),
                FloatFormat::Half => read_f16_bytes(chunk, &order),
                // Double rounds; every other supported layout (bfloat16, FP8)
                // is exact in f32.
                _ => format.decode(chunk, &order) as f32,
            });
        }
        return Ok(result);
    }
    for i in 0..count {
        let chunk = &raw[i * elem_size..(i + 1) * elem_size];
        match datatype {
            Datatype::FixedPoint {
                signed: true,
                size,
                bit_offset,
                bit_precision,
                ..
            } => {
                let full = read_unsigned_int(chunk, *size as usize, &order);
                let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
                result.push(extract_signed(full, off, prec) as f32);
            }
            Datatype::FixedPoint {
                signed: false,
                size,
                bit_offset,
                bit_precision,
                ..
            } => {
                let full = read_unsigned_int(chunk, *size as usize, &order);
                let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
                result.push(extract_unsigned(full, off, prec) as f32);
            }
            _ => {
                return Err(FormatError::TypeMismatch {
                    expected: "numeric",
                    actual: datatype_name(datatype),
                });
            }
        }
    }
    Ok(result)
}

/// Convert raw bytes to `i32` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatError> {
    // Array datatypes read as a flat sequence of their base elements, and
    // enumerations (h5py's bool among them) as their integer values.
    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
        return read_as_i32(raw, base_type);
    }
    ensure_numeric(datatype, "FixedPoint")?;
    let elem_size = get_size(datatype);
    if elem_size == 0 || !raw.len().is_multiple_of(elem_size) {
        return Err(FormatError::DataSizeMismatch {
            expected: 0,
            actual: raw.len(),
        });
    }
    let count = raw.len() / elem_size;

    // Fast path: native LE i32 — single bulk memcpy
    if i32::is_native(datatype) {
        return Ok(native_to_vec::<i32>(raw, count));
    }

    let order = get_byte_order(datatype);
    let mut result = crate::bulk_alloc::vec_for_bulk(count);
    for i in 0..count {
        let chunk = &raw[i * elem_size..(i + 1) * elem_size];
        result.push(decode_scalar(chunk, datatype, &order)?.to_i32());
    }
    Ok(result)
}

/// Read fixed-length strings from raw bytes.
pub fn read_as_strings(raw: &[u8], datatype: &Datatype) -> Result<Vec<String>, FormatError> {
    match datatype {
        Datatype::String { size, padding, .. } => {
            let elem_size = *size as usize;
            if elem_size == 0 {
                return Ok(Vec::new());
            }
            if !raw.len().is_multiple_of(elem_size) {
                return Err(FormatError::DataSizeMismatch {
                    expected: 0,
                    actual: raw.len(),
                });
            }
            let count = raw.len() / elem_size;
            let mut result = Vec::with_capacity(count);
            for i in 0..count {
                let chunk = &raw[i * elem_size..(i + 1) * elem_size];
                let s = match padding {
                    crate::datatype::StringPadding::NullTerminate => {
                        let end = chunk.iter().position(|&b| b == 0).unwrap_or(chunk.len());
                        String::from_utf8_lossy(&chunk[..end]).into_owned()
                    }
                    crate::datatype::StringPadding::NullPad => {
                        let end = chunk.iter().rposition(|&b| b != 0).map_or(0, |p| p + 1);
                        String::from_utf8_lossy(&chunk[..end]).into_owned()
                    }
                    crate::datatype::StringPadding::SpacePad => {
                        let end = chunk.iter().rposition(|&b| b != b' ').map_or(0, |p| p + 1);
                        String::from_utf8_lossy(&chunk[..end]).into_owned()
                    }
                };
                result.push(s);
            }
            Ok(result)
        }
        _ => Err(FormatError::TypeMismatch {
            expected: "String",
            actual: datatype_name(datatype),
        }),
    }
}

// --- Compound type reading ---

/// A single field extracted from compound data, containing the raw bytes for that field
/// across all elements.
#[derive(Debug, Clone)]
pub struct CompoundFieldData {
    /// Field name.
    pub name: String,
    /// Datatype of this field.
    pub datatype: Datatype,
    /// Raw bytes for this field across all elements (len = num_elements * field_type_size).
    pub raw_data: Vec<u8>,
}

/// Read compound dataset and return all fields as separate data vectors.
///
/// Each returned `CompoundFieldData` contains the raw bytes for that field
/// across all elements, suitable for further typed conversion with `read_as_f64`, etc.
pub fn read_compound_fields(
    raw: &[u8],
    datatype: &Datatype,
) -> Result<Vec<CompoundFieldData>, FormatError> {
    match datatype {
        Datatype::Compound { size, members } => {
            let elem_size = *size as usize;
            if elem_size == 0 {
                return Ok(Vec::new());
            }
            if !raw.len().is_multiple_of(elem_size) {
                return Err(FormatError::DataSizeMismatch {
                    expected: 0,
                    actual: raw.len(),
                });
            }
            let count = raw.len() / elem_size;
            let mut fields = Vec::with_capacity(members.len());
            for m in members {
                let field_size = m.datatype.type_size() as usize;
                let offset = m.byte_offset as usize;
                if offset
                    .checked_add(field_size)
                    .is_none_or(|end| end > elem_size)
                {
                    return Err(FormatError::Overflow(format!(
                        "compound member '{}': byte_offset({offset}) + field_size({field_size}) exceeds element size({elem_size})",
                        m.name
                    )));
                }
                let mut field_raw = Vec::with_capacity(count * field_size);
                for i in 0..count {
                    let elem_start = i * elem_size + offset;
                    field_raw.extend_from_slice(&raw[elem_start..elem_start + field_size]);
                }
                fields.push(CompoundFieldData {
                    name: m.name.clone(),
                    datatype: m.datatype.clone(),
                    raw_data: field_raw,
                });
            }
            Ok(fields)
        }
        _ => Err(FormatError::TypeMismatch {
            expected: "Compound",
            actual: datatype_name(datatype),
        }),
    }
}

/// Extract a single field by name from compound raw data.
pub fn read_compound_field(
    raw: &[u8],
    datatype: &Datatype,
    field_name: &str,
) -> Result<CompoundFieldData, FormatError> {
    let fields = read_compound_fields(raw, datatype)?;
    fields
        .into_iter()
        .find(|f| f.name == field_name)
        .ok_or_else(|| FormatError::PathNotFound(field_name.into()))
}

// --- Enum type reading ---

/// A single value from an enum dataset, containing both the integer value and string name.
#[derive(Debug, Clone)]
pub struct EnumValue {
    /// The string name for this enum value.
    pub name: String,
    /// The raw integer value.
    pub raw_value: Vec<u8>,
}

/// Read enum dataset values, mapping integer values to their string names.
///
/// Returns one `EnumValue` per element. Unknown values get name `UNKNOWN(hex)`.
pub fn read_enum_values(raw: &[u8], datatype: &Datatype) -> Result<Vec<EnumValue>, FormatError> {
    match datatype {
        Datatype::Enumeration { size, members, .. } => {
            let elem_size = *size as usize;
            if elem_size == 0 {
                return Ok(Vec::new());
            }
            if !raw.len().is_multiple_of(elem_size) {
                return Err(FormatError::DataSizeMismatch {
                    expected: 0,
                    actual: raw.len(),
                });
            }
            let count = raw.len() / elem_size;
            // Build lookup map: raw bytes -> name
            let mut lookup = BTreeMap::new();
            for m in members {
                lookup.insert(m.value.clone(), m.name.clone());
            }
            let mut result = Vec::with_capacity(count);
            for i in 0..count {
                let val_bytes = raw[i * elem_size..(i + 1) * elem_size].to_vec();
                let name = lookup.get(&val_bytes).cloned().unwrap_or_else(|| {
                    let hex: Vec<String> = val_bytes
                        .iter()
                        .map(|b| {
                            let mut s = String::new();
                            core::fmt::Write::write_fmt(&mut s, format_args!("{b:02x}")).ok();
                            s
                        })
                        .collect();
                    let mut result = String::from("UNKNOWN(0x");
                    for h in &hex {
                        result.push_str(h);
                    }
                    result.push(')');
                    result
                });
                result.push(EnumValue {
                    name,
                    raw_value: val_bytes,
                });
            }
            Ok(result)
        }
        _ => Err(FormatError::TypeMismatch {
            expected: "Enumeration",
            actual: datatype_name(datatype),
        }),
    }
}

/// Read enum dataset and return just the string names.
pub fn read_enum_names(raw: &[u8], datatype: &Datatype) -> Result<Vec<String>, FormatError> {
    let values = read_enum_values(raw, datatype)?;
    Ok(values.into_iter().map(|v| v.name).collect())
}

// --- Reference type reading ---

/// A resolved object reference: the file address of the referenced object header.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ObjectReference {
    /// The file address of the referenced object header.
    /// A value of `u64::MAX` (all 0xFF bytes) indicates a null reference.
    pub address: u64,
}

impl ObjectReference {
    /// Returns `true` if this is a null (unset) reference.
    pub fn is_null(&self) -> bool {
        self.address == u64::MAX
    }
}

/// A region reference: raw bytes that encode a dataset selection.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RegionReference {
    /// The raw region reference bytes. A region reference is typically 12 bytes
    /// (object address + dataspace selection) but the exact layout depends on
    /// the file's offset size and the selection type.
    pub raw: Vec<u8>,
}

/// Read object references from raw bytes.
///
/// Object references are stored as `offset_size`-byte file addresses pointing
/// to the object header of the referenced object. A reference consisting of
/// all `0xFF` bytes is a null (unset) reference.
///
/// # Arguments
/// * `raw` — raw bytes read from the dataset
/// * `datatype` — must be `Datatype::Reference` with `ReferenceType::Object`
/// * `offset_size` — the file's offset size (from superblock), typically 8
pub fn read_object_references(
    raw: &[u8],
    datatype: &Datatype,
    offset_size: u8,
) -> Result<Vec<ObjectReference>, FormatError> {
    match datatype {
        Datatype::Reference {
            ref_type: crate::datatype::ReferenceType::Object,
            size,
        } => {
            let elem_size = *size as usize;
            if elem_size == 0 {
                return Ok(Vec::new());
            }
            if !raw.len().is_multiple_of(elem_size) {
                return Err(FormatError::DataSizeMismatch {
                    expected: 0,
                    actual: raw.len(),
                });
            }
            let count = raw.len() / elem_size;
            let mut result = Vec::with_capacity(count);
            let read_size = (offset_size as usize).min(elem_size);
            for i in 0..count {
                let chunk = &raw[i * elem_size..(i + 1) * elem_size];
                let address = read_ref_address(chunk, read_size);
                result.push(ObjectReference { address });
            }
            Ok(result)
        }
        Datatype::Reference {
            ref_type: crate::datatype::ReferenceType::Object2,
            size,
        } => {
            let elem_size = *size as usize;
            if elem_size == 0 {
                return Ok(Vec::new());
            }
            if !raw.len().is_multiple_of(elem_size) {
                return Err(FormatError::DataSizeMismatch {
                    expected: 0,
                    actual: raw.len(),
                });
            }
            raw.chunks_exact(elem_size)
                .map(|element| {
                    decode_std_object_ref(element).map(|address| ObjectReference { address })
                })
                .collect()
        }
        _ => Err(FormatError::TypeMismatch {
            expected: "Reference(Object)",
            actual: datatype_name(datatype),
        }),
    }
}

/// Decode one `H5T_STD_REF` object reference as stored in a dataset:
/// `type(1) flags(1) token_size(1) token(token_size)`, zero-padded to the
/// element size. For a reference within the same file the token is the target
/// object's header address. An all-zero element is a null reference and
/// decodes to the undefined address (`u64::MAX`).
fn decode_std_object_ref(element: &[u8]) -> Result<u64, FormatError> {
    const STD_REF_OBJECT: u8 = 2;
    const FLAG_EXTERNAL: u8 = 0x01;
    if element.iter().all(|&b| b == 0) {
        return Ok(u64::MAX);
    }
    let [ref_type, flags, token_size, token @ ..] = element else {
        return Err(FormatError::UnexpectedEof {
            expected: 3,
            available: element.len(),
        });
    };
    if *ref_type != STD_REF_OBJECT {
        return Err(FormatError::InvalidReferenceType(*ref_type));
    }
    if flags & FLAG_EXTERNAL != 0 {
        // Carries a file name as well; nothing here follows those.
        return Err(FormatError::TypeMismatch {
            expected: "object reference within this file",
            actual: "external object reference",
        });
    }
    let n = *token_size as usize;
    if n == 0 || n > 8 || n > token.len() {
        return Err(FormatError::UnexpectedEof {
            expected: 3 + n,
            available: element.len(),
        });
    }
    Ok(token[..n]
        .iter()
        .rev()
        .fold(0u64, |addr, &byte| (addr << 8) | u64::from(byte)))
}

/// Read region references from raw bytes.
///
/// Region references encode a dataset selection (hyperslab, point list, etc.)
/// along with the address of the target dataset. This function returns the
/// raw bytes for each reference without decoding the selection, since the
/// full region reference format is complex and depends on the selection type.
///
/// # Arguments
/// * `raw` — raw bytes read from the dataset
/// * `datatype` — must be `Datatype::Reference` with `ReferenceType::DatasetRegion`
pub fn read_region_references(
    raw: &[u8],
    datatype: &Datatype,
) -> Result<Vec<RegionReference>, FormatError> {
    match datatype {
        Datatype::Reference {
            ref_type: crate::datatype::ReferenceType::DatasetRegion,
            size,
        } => {
            let elem_size = *size as usize;
            if elem_size == 0 {
                return Ok(Vec::new());
            }
            if !raw.len().is_multiple_of(elem_size) {
                return Err(FormatError::DataSizeMismatch {
                    expected: 0,
                    actual: raw.len(),
                });
            }
            let count = raw.len() / elem_size;
            let mut result = Vec::with_capacity(count);
            for i in 0..count {
                let chunk = &raw[i * elem_size..(i + 1) * elem_size];
                result.push(RegionReference {
                    raw: chunk.to_vec(),
                });
            }
            Ok(result)
        }
        _ => Err(FormatError::TypeMismatch {
            expected: "Reference(DatasetRegion)",
            actual: datatype_name(datatype),
        }),
    }
}

/// Read a file address from reference bytes (little-endian).
fn read_ref_address(bytes: &[u8], size: usize) -> u64 {
    let mut buf = [0xFFu8; 8];
    let len = size.min(bytes.len()).min(8);
    buf[..len].copy_from_slice(&bytes[..len]);
    // If we read fewer than 8 bytes, check if ALL read bytes are 0xFF (null ref)
    if len < 8 && bytes[..len].iter().all(|&b| b == 0xFF) {
        return u64::MAX;
    }
    // Zero-extend upper bytes for non-null refs
    if len < 8 && !bytes[..len].iter().all(|&b| b == 0xFF) {
        for b in buf[len..].iter_mut() {
            *b = 0;
        }
    }
    u64::from_le_bytes(buf)
}

// --- Array type reading ---

/// Read array-typed dataset elements, returning the raw base-type data.
///
/// For an array type with dimensions [D1, D2, ...] and base type T,
/// each dataset element contains D1*D2*... values of type T.
/// This function returns the raw bytes as a flat buffer that can be
/// converted with `read_as_f64`, `read_as_i32`, etc. using the base type.
pub fn read_array_flat(
    raw: &[u8],
    datatype: &Datatype,
) -> Result<(Vec<u8>, Datatype, Vec<u32>), FormatError> {
    match datatype {
        Datatype::Array {
            base_type,
            dimensions,
        } => Ok((raw.to_vec(), *base_type.clone(), dimensions.clone())),
        _ => Err(FormatError::TypeMismatch {
            expected: "Array",
            actual: datatype_name(datatype),
        }),
    }
}

// --- Low-level byte conversion helpers ---

fn reorder_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> [u8; 8] {
    let mut buf = [0u8; 8];
    let len = bytes.len().min(8);
    match order {
        DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => {
            buf[..len].copy_from_slice(&bytes[..len]);
        }
        DatatypeByteOrder::BigEndian => {
            // Reverse bytes into LE order
            for i in 0..len {
                buf[i] = bytes[len - 1 - i];
            }
        }
    }
    buf
}

/// How the bits of a floating-point datatype are laid out, read from the
/// datatype message's fields rather than assumed from its size (a 2-byte
/// float may be IEEE half or bfloat16).
#[derive(Debug, Clone, Copy, PartialEq)]
enum FloatFormat {
    /// IEEE-754 binary16.
    Half,
    /// IEEE-754 binary32.
    Single,
    /// IEEE-754 binary64.
    Double,
    /// Any other IEEE-style layout (implied leading mantissa bit, all-ones
    /// exponent for infinity/NaN) whose values are all exact in `f64`:
    /// bfloat16, the FP8 formats, and similar.
    Other(FloatLayout),
}

#[derive(Debug, Clone, Copy, PartialEq)]
struct FloatLayout {
    exponent_location: u32,
    exponent_size: u32,
    mantissa_location: u32,
    mantissa_size: u32,
    exponent_bias: u32,
}

impl FloatFormat {
    fn of(dt: &Datatype) -> Result<FloatFormat, FormatError> {
        let Datatype::FloatingPoint {
            size,
            exponent_location,
            exponent_size,
            mantissa_location,
            mantissa_size,
            exponent_bias,
            ..
        } = dt
        else {
            return Err(FormatError::TypeMismatch {
                expected: "FloatingPoint",
                actual: datatype_name(dt),
            });
        };
        let layout = FloatLayout {
            exponent_location: u32::from(*exponent_location),
            exponent_size: u32::from(*exponent_size),
            mantissa_location: u32::from(*mantissa_location),
            mantissa_size: u32::from(*mantissa_size),
            exponent_bias: *exponent_bias,
        };
        let fields = (
            layout.exponent_location,
            layout.exponent_size,
            layout.mantissa_location,
            layout.mantissa_size,
            layout.exponent_bias,
        );
        let bits = size.saturating_mul(8);
        // The sign bit is not kept in `Datatype`; every standard layout has it
        // directly above the exponent, with the mantissa below.
        let well_formed = layout.exponent_size > 0
            && layout.mantissa_size > 0
            && layout.mantissa_location + layout.mantissa_size <= layout.exponent_location
            && layout.exponent_location + layout.exponent_size < bits;
        match (size, fields) {
            (2, (10, 5, 0, 10, 15)) => Ok(FloatFormat::Half),
            (4, (23, 8, 0, 23, 127)) => Ok(FloatFormat::Single),
            (8, (52, 11, 0, 52, 1023)) => Ok(FloatFormat::Double),
            _ if well_formed
                && *size <= 8
                && layout.exponent_size <= 11
                && layout.mantissa_size <= 52 =>
            {
                Ok(FloatFormat::Other(layout))
            }
            // Fields that cannot describe any float (e.g. left zeroed by a
            // hand-built datatype): fall back to the IEEE type of that size.
            (2, _) if !well_formed => Ok(FloatFormat::Half),
            (4, _) if !well_formed => Ok(FloatFormat::Single),
            (8, _) if !well_formed => Ok(FloatFormat::Double),
            // x87 80-bit extended, binary128, ...: not representable in f64.
            _ => Err(FormatError::TypeMismatch {
                expected: "floating point of at most 64 bits (IEEE-style layout)",
                actual: "FloatingPoint",
            }),
        }
    }

    fn decode(self, bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
        match self {
            FloatFormat::Half => f64::from(read_f16_bytes(bytes, order)),
            FloatFormat::Single => f64::from(read_f32_bytes(bytes, order)),
            FloatFormat::Double => read_f64_bytes(bytes, order),
            FloatFormat::Other(layout) => {
                layout.decode(read_unsigned_int(bytes, bytes.len(), order))
            }
        }
    }
}

impl FloatLayout {
    /// Decode the value held in the low `size * 8` bits of `bits`.
    fn decode(self, bits: u64) -> f64 {
        let field = |location: u32, size: u32| (bits >> location) & ((1u64 << size) - 1);
        let exponent = field(self.exponent_location, self.exponent_size);
        let mantissa = field(self.mantissa_location, self.mantissa_size);
        let negative = field(self.exponent_location + self.exponent_size, 1) == 1;
        let max_exponent = (1u64 << self.exponent_size) - 1;
        let magnitude = if exponent == max_exponent {
            if mantissa == 0 {
                f64::INFINITY
            } else {
                f64::NAN
            }
        } else {
            let bias = i64::from(self.exponent_bias);
            let msize = i64::from(self.mantissa_size);
            // value = significand * 2^power, with an implied leading 1 unless
            // the number is subnormal (exponent field 0).
            let (significand, power) = if exponent == 0 {
                (mantissa, 1 - bias - msize)
            } else {
                (
                    mantissa | (1u64 << self.mantissa_size),
                    exponent as i64 - bias - msize,
                )
            };
            scale_by_pow2(significand as f64, power)
        };
        if negative { -magnitude } else { magnitude }
    }
}

/// `x * 2^power` without `std` (no `powi`/`libm`). `x` is a non-negative
/// integer below 2^53, so it is exact.
fn scale_by_pow2(x: f64, power: i64) -> f64 {
    if x == 0.0 || power < -1200 {
        return 0.0;
    }
    if power > 1100 {
        return f64::INFINITY;
    }
    let pow2 = |p: i64| f64::from_bits(((p + 1023) as u64) << 52);
    let mut x = x;
    let mut power = power;
    while power > 1023 {
        x *= pow2(1023);
        power -= 1023;
    }
    while power < -1022 {
        x *= pow2(-1022);
        power += 1022;
    }
    x * pow2(power)
}

/// Whether `datatype` is the little-endian IEEE float `format`, whose bytes
/// can be copied straight into native values on a little-endian target.
fn is_native_le_float(datatype: &Datatype, format: FloatFormat) -> bool {
    matches!(
        datatype,
        Datatype::FloatingPoint {
            byte_order: DatatypeByteOrder::LittleEndian,
            ..
        }
    ) && FloatFormat::of(datatype).is_ok_and(|f| f == format)
}

fn read_f64_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
    let buf = reorder_bytes(bytes, order);
    f64::from_le_bytes(buf)
}

/// Decode an IEEE-754 half-precision (binary16) value to `f32`. Pure integer
/// bit manipulation (no_std-safe, no `powi`/`libm`).
fn read_f16_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 {
    let mut buf = [0u8; 2];
    let len = bytes.len().min(2);
    match order {
        DatatypeByteOrder::BigEndian => {
            for i in 0..len {
                buf[i] = bytes[len - 1 - i];
            }
        }
        _ => buf[..len].copy_from_slice(&bytes[..len]),
    }
    f16_bits_to_f32(u16::from_le_bytes(buf))
}

use crate::float16::f16_bits_to_f32;

fn read_f32_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 {
    let mut buf = [0u8; 4];
    let len = bytes.len().min(4);
    match order {
        DatatypeByteOrder::LittleEndian | DatatypeByteOrder::Vax => {
            buf[..len].copy_from_slice(&bytes[..len]);
        }
        DatatypeByteOrder::BigEndian => {
            for i in 0..len {
                buf[i] = bytes[len - 1 - i];
            }
        }
    }
    f32::from_le_bytes(buf)
}

/// Effective (bit offset, bit precision) for a fixed-point field, defaulting a
/// zero precision to the full storage width.
fn effective_bits(size: usize, bit_offset: u16, bit_precision: u16) -> (u32, u32) {
    let prec = if bit_precision == 0 {
        (size * 8) as u32
    } else {
        bit_precision as u32
    };
    (bit_offset as u32, prec)
}

/// Whether a datatype occupies its full storage width (bit offset 0, precision
/// == size·8), in which case the bulk-copy fast read paths apply. Non
/// fixed-point types are treated as full width.
fn is_full_width(datatype: &Datatype) -> bool {
    match datatype {
        Datatype::FixedPoint {
            size,
            bit_offset,
            bit_precision,
            ..
        } => *bit_offset == 0 && *bit_precision as u32 == *size * 8,
        _ => true,
    }
}

/// Extract the `precision`-bit field at `offset` from a full-width integer read
/// and sign-extend it. Full-width fields read as an ordinary signed integer;
/// reduced-precision fields sign-extend from the field's top bit (HDF5 stores
/// reduced-precision values zero-filled, so the sign lives in the precision
/// field, not the storage word).
fn extract_signed(full: u64, offset: u32, precision: u32) -> i64 {
    if precision == 0 || precision >= 64 {
        return full as i64;
    }
    let field = (full >> offset) & ((1u64 << precision) - 1);
    let shift = 64 - precision;
    ((field << shift) as i64) >> shift
}

/// Extract the `precision`-bit field at `offset` from a full-width integer read.
fn extract_unsigned(full: u64, offset: u32, precision: u32) -> u64 {
    if precision == 0 || precision >= 64 {
        return full;
    }
    (full >> offset) & ((1u64 << precision) - 1)
}

fn read_unsigned_int(bytes: &[u8], size: usize, order: &DatatypeByteOrder) -> u64 {
    let buf = reorder_bytes(bytes, order);
    match size {
        1 => buf[0] as u64,
        2 => u16::from_le_bytes([buf[0], buf[1]]) as u64,
        4 => u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as u64,
        8 => u64::from_le_bytes(buf),
        _ => {
            // Generic: read as LE
            let mut val = 0u64;
            for (i, &byte) in buf.iter().enumerate().take(size.min(8)) {
                val |= (byte as u64) << (i * 8);
            }
            val
        }
    }
}

// --- Type conversion cost analysis ---

/// Cost classification for type conversions.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConversionCost {
    /// No conversion needed (same type).
    None,
    /// Widening conversion (no data loss, e.g. f32 → f64, i32 → i64).
    Widening,
    /// Narrowing conversion (potential precision loss, e.g. f64 → f32).
    Narrowing,
    /// Lossy conversion (potential data corruption, e.g. float → int).
    Lossy,
}

/// Information about a type conversion.
#[derive(Debug, Clone)]
pub struct ConversionInfo {
    /// Source datatype name.
    pub source_type: &'static str,
    /// Target datatype name.
    pub target_type: &'static str,
    /// Cost of the conversion.
    pub cost: ConversionCost,
}

/// Check the cost of converting from one datatype to another.
///
/// This helps callers understand whether a read operation involves
/// potentially lossy type coercion.
pub fn check_conversion_cost(source: &Datatype, target: &'static str) -> ConversionInfo {
    let source_name = datatype_name(source);
    let source_size = source.type_size();

    let cost = match (source_name, target) {
        // Same type
        (s, t) if s == t => ConversionCost::None,

        // Float widening: f32 → f64
        ("f32", "f64") => ConversionCost::Widening,
        // Float narrowing: f64 → f32
        ("f64", "f32") => ConversionCost::Narrowing,

        // Integer widening
        ("i32", "i64") | ("u8", "i32") | ("u8", "i64") | ("u8", "u64") | ("i32", "u64") => {
            ConversionCost::Widening
        }
        // Integer narrowing
        ("i64", "i32") | ("i64", "u8") | ("i32", "u8") | ("u64", "i32") | ("u64", "u8") => {
            ConversionCost::Narrowing
        }

        // Float ↔ Integer: lossy
        ("f32" | "f64", "i32" | "i64" | "u8" | "u64") => ConversionCost::Lossy,
        ("i32" | "i64" | "u8" | "u64", "f32" | "f64") => {
            // Int → Float: widening if source fits, but technically lossy for large ints
            if source_size <= 4 && target == "f64" {
                ConversionCost::Widening
            } else {
                ConversionCost::Narrowing
            }
        }

        // Unknown combinations
        _ => ConversionCost::Lossy,
    };

    ConversionInfo {
        source_type: source_name,
        target_type: target,
        cost,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::dataspace::{Dataspace, DataspaceType};
    use crate::datatype::{CharacterSet, StringPadding};

    fn f16_datatype() -> Datatype {
        Datatype::FloatingPoint {
            size: 2,
            byte_order: DatatypeByteOrder::LittleEndian,
            bit_offset: 0,
            bit_precision: 16,
            exponent_location: 10,
            exponent_size: 5,
            mantissa_location: 0,
            mantissa_size: 10,
            exponent_bias: 15,
        }
    }

    // IEEE-754 half bit patterns for known values.
    fn f16_bits(v: f32) -> u16 {
        // Encode a few exact values used by the test.
        match v {
            0.0 => 0x0000,
            1.0 => 0x3c00,
            -2.0 => 0xc000,
            0.5 => 0x3800,
            65504.0 => 0x7bff, // f16 max
            _ => panic!("unsupported test value {v}"),
        }
    }

    #[test]
    fn read_f16_as_f32_and_f64() {
        let values = [0.0f32, 1.0, -2.0, 0.5, 65504.0];
        let raw: Vec<u8> = values
            .iter()
            .flat_map(|&v| f16_bits(v).to_le_bytes())
            .collect();
        let dt = f16_datatype();
        let got32 = read_as_f32(&raw, &dt).unwrap();
        assert_eq!(got32, values);
        let got64 = read_as_f64(&raw, &dt).unwrap();
        let expect64: Vec<f64> = values.iter().map(|&v| v as f64).collect();
        assert_eq!(got64, expect64);
    }

    fn reduced_int(signed: bool, precision: u16) -> Datatype {
        Datatype::FixedPoint {
            size: 4,
            byte_order: DatatypeByteOrder::LittleEndian,
            signed,
            bit_offset: 0,
            bit_precision: precision,
        }
    }

    #[test]
    fn reduced_precision_signed_sign_extends() {
        // 16-bit-precision signed values stored zero-filled (HDF5's canonical
        // layout, e.g. after N-Bit): the reader must sign-extend from bit 15.
        let dt = reduced_int(true, 16);
        // [-1, 100, -50, -32768] as 0x0000ffff / 0x00000064 / 0x0000ffce / 0x00008000
        let raw: Vec<u8> = vec![
            0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xce, 0xff, 0x00, 0x00, 0x00, 0x80,
            0x00, 0x00,
        ];
        assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]);
        assert_eq!(read_as_i64(&raw, &dt).unwrap(), vec![-1, 100, -50, -32768]);
    }

    #[test]
    fn reduced_precision_unsigned_masks() {
        // 12-bit-precision unsigned: high bits must read as zero, not sign.
        let dt = reduced_int(false, 12);
        // [4095, 1, 2048] as 0x00000fff / 0x00000001 / 0x00000800
        let raw: Vec<u8> = vec![
            0xff, 0x0f, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00,
        ];
        assert_eq!(read_as_u64(&raw, &dt).unwrap(), vec![4095, 1, 2048]);
    }

    #[test]
    fn float_to_int_truncates_and_saturates() {
        // Values libhdf5 hands to an undefined C cast: NaN reads as 0 and
        // exactly 2^63 saturates instead of wrapping to i64::MIN.
        let dt = make_f64_le_type();
        let vals = [f64::NAN, 2f64.powi(63), -2.5, 2.0f64.powi(64)];
        let raw: Vec<u8> = vals.iter().flat_map(|v| v.to_le_bytes()).collect();
        assert_eq!(
            read_as_i64(&raw, &dt).unwrap(),
            vec![0, i64::MAX, -2, i64::MAX]
        );
        assert_eq!(
            read_as_u64(&raw, &dt).unwrap(),
            vec![0, 1 << 63, 0, u64::MAX]
        );
        assert_eq!(
            read_as_i32(&raw, &dt).unwrap(),
            vec![0, i32::MAX, -2, i32::MAX]
        );
    }

    #[test]
    fn bfloat16_and_fp8_decode_by_fields() {
        // bfloat16 is a 2-byte float that is not IEEE half.
        let bf16 = Datatype::FloatingPoint {
            size: 2,
            byte_order: DatatypeByteOrder::LittleEndian,
            bit_offset: 0,
            bit_precision: 16,
            exponent_location: 7,
            exponent_size: 8,
            mantissa_location: 0,
            mantissa_size: 7,
            exponent_bias: 127,
        };
        let raw: Vec<u8> = [0x3FC0u16, 0xC010, 0x7F80, 0x0001]
            .iter()
            .flat_map(|v| v.to_le_bytes())
            .collect();
        let got = read_as_f64(&raw, &bf16).unwrap();
        assert_eq!(&got[..3], &[1.5, -2.25, f64::INFINITY]);
        assert_eq!(got[3], 2f64.powi(-133)); // smallest subnormal
        assert_eq!(read_as_f32(&raw, &bf16).unwrap()[..2], [1.5, -2.25]);

        // FP8 E4M3: 1, -1, 2, 0, NaN (IEEE-style, as libhdf5 treats it).
        let e4m3 = Datatype::FloatingPoint {
            size: 1,
            byte_order: DatatypeByteOrder::LittleEndian,
            bit_offset: 0,
            bit_precision: 8,
            exponent_location: 3,
            exponent_size: 4,
            mantissa_location: 0,
            mantissa_size: 3,
            exponent_bias: 7,
        };
        let got = read_as_f64(&[0x38, 0xB8, 0x40, 0x00, 0x7E], &e4m3).unwrap();
        assert_eq!(&got[..4], &[1.0, -1.0, 2.0, 0.0]);
        assert!(got[4].is_nan());
    }

    #[test]
    fn full_width_signed_unchanged() {
        // Regression: full-width 32-bit signed must be unaffected.
        let dt = reduced_int(true, 32);
        let raw: Vec<u8> = vec![0xff, 0xff, 0xff, 0xff, 0x2a, 0x00, 0x00, 0x00];
        assert_eq!(read_as_i32(&raw, &dt).unwrap(), vec![-1, 42]);
    }

    #[test]
    fn array_datatype_reads_flat_base_elements() {
        // An array-typed (e.g. compound member) datatype reads as a flat
        // sequence of its base elements, applying base-type precision rules.
        let arr = Datatype::Array {
            base_type: Box::new(reduced_int(true, 16)),
            dimensions: vec![2],
        };
        // [-1, 100, 1000, -32768] stored zero-filled at 16-bit precision.
        let raw: Vec<u8> = vec![
            0xff, 0xff, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x80,
            0x00, 0x00,
        ];
        assert_eq!(
            read_as_i32(&raw, &arr).unwrap(),
            vec![-1, 100, 1000, -32768]
        );
        // Nested array-of-array unwraps recursively.
        let nested = Datatype::Array {
            base_type: Box::new(arr),
            dimensions: vec![2],
        };
        assert_eq!(
            read_as_i32(&raw, &nested).unwrap(),
            vec![-1, 100, 1000, -32768]
        );
    }

    fn make_f64_le_type() -> Datatype {
        Datatype::FloatingPoint {
            size: 8,
            byte_order: DatatypeByteOrder::LittleEndian,
            bit_offset: 0,
            bit_precision: 64,
            exponent_location: 52,
            exponent_size: 11,
            mantissa_location: 0,
            mantissa_size: 52,
            exponent_bias: 1023,
        }
    }

    fn make_f32_be_type() -> Datatype {
        Datatype::FloatingPoint {
            size: 4,
            byte_order: DatatypeByteOrder::BigEndian,
            bit_offset: 0,
            bit_precision: 32,
            exponent_location: 23,
            exponent_size: 8,
            mantissa_location: 0,
            mantissa_size: 23,
            exponent_bias: 127,
        }
    }

    fn make_i32_le_type() -> Datatype {
        Datatype::FixedPoint {
            size: 4,
            byte_order: DatatypeByteOrder::LittleEndian,
            signed: true,
            bit_offset: 0,
            bit_precision: 32,
        }
    }

    fn make_i16_le_type() -> Datatype {
        Datatype::FixedPoint {
            size: 2,
            byte_order: DatatypeByteOrder::LittleEndian,
            signed: true,
            bit_offset: 0,
            bit_precision: 16,
        }
    }

    fn make_u8_type() -> Datatype {
        Datatype::FixedPoint {
            size: 1,
            byte_order: DatatypeByteOrder::LittleEndian,
            signed: false,
            bit_offset: 0,
            bit_precision: 8,
        }
    }

    fn make_simple_dataspace(dims: &[u64]) -> Dataspace {
        Dataspace {
            space_type: DataspaceType::Simple,
            rank: dims.len() as u8,
            dimensions: dims.to_vec(),
            max_dimensions: None,
        }
    }

    #[test]
    fn read_f64_compact() {
        let dt = make_f64_le_type();
        let ds = make_simple_dataspace(&[3]);
        let mut data = Vec::new();
        data.extend_from_slice(&1.0f64.to_le_bytes());
        data.extend_from_slice(&2.0f64.to_le_bytes());
        data.extend_from_slice(&3.0f64.to_le_bytes());
        let layout = DataLayout::Compact { data: data.clone() };
        let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
        assert_eq!(raw, data);
        let values = read_as_f64(&raw, &dt).unwrap();
        assert_eq!(values, vec![1.0, 2.0, 3.0]);
    }

    #[test]
    fn read_i32_contiguous() {
        let dt = make_i32_le_type();
        let ds = make_simple_dataspace(&[4]);
        let mut file_data = vec![0u8; 1024];
        let offset = 256usize;
        let vals: Vec<i32> = vec![10, -20, 30, -40];
        for (i, v) in vals.iter().enumerate() {
            let bytes = v.to_le_bytes();
            file_data[offset + i * 4..offset + i * 4 + 4].copy_from_slice(&bytes);
        }
        let layout = DataLayout::Contiguous {
            address: Some(offset as u64),
            size: 16,
        };
        let raw = read_raw_data(&file_data, &layout, &ds, &dt).unwrap();
        let result = read_as_i32(&raw, &dt).unwrap();
        assert_eq!(result, vec![10, -20, 30, -40]);
    }

    #[test]
    fn read_u8_data() {
        let dt = make_u8_type();
        let ds = make_simple_dataspace(&[5]);
        let data = vec![10u8, 20, 30, 40, 50];
        let layout = DataLayout::Compact { data: data.clone() };
        let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
        let result = read_as_u64(&raw, &dt).unwrap();
        assert_eq!(result, vec![10, 20, 30, 40, 50]);
    }

    #[test]
    fn read_f32_be() {
        let dt = make_f32_be_type();
        let ds = make_simple_dataspace(&[2]);
        let mut data = Vec::new();
        // Store as big-endian
        data.extend_from_slice(&1.5f32.to_be_bytes());
        data.extend_from_slice(&2.5f32.to_be_bytes());
        let layout = DataLayout::Compact { data: data.clone() };
        let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
        let result = read_as_f32(&raw, &dt).unwrap();
        assert_eq!(result, vec![1.5, 2.5]);
    }

    #[test]
    fn read_i16_le() {
        let dt = make_i16_le_type();
        let ds = make_simple_dataspace(&[3]);
        let mut data = Vec::new();
        data.extend_from_slice(&(-100i16).to_le_bytes());
        data.extend_from_slice(&200i16.to_le_bytes());
        data.extend_from_slice(&(-300i16).to_le_bytes());
        let layout = DataLayout::Compact { data: data.clone() };
        let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
        let result = read_as_i64(&raw, &dt).unwrap();
        assert_eq!(result, vec![-100, 200, -300]);
    }

    #[test]
    fn read_strings_compact() {
        let dt = Datatype::String {
            size: 5,
            padding: StringPadding::NullPad,
            charset: CharacterSet::Ascii,
        };
        let ds = make_simple_dataspace(&[2]);
        let mut data = Vec::new();
        data.extend_from_slice(b"hello");
        data.extend_from_slice(b"hi\0\0\0");
        let layout = DataLayout::Compact { data: data.clone() };
        let raw = read_raw_data(&[], &layout, &ds, &dt).unwrap();
        let result = read_as_strings(&raw, &dt).unwrap();
        assert_eq!(result, vec!["hello", "hi"]);
    }

    #[test]
    fn type_mismatch_f64_on_string() {
        let dt = Datatype::String {
            size: 4,
            padding: StringPadding::NullTerminate,
            charset: CharacterSet::Ascii,
        };
        let raw = vec![0u8; 8];
        let err = read_as_f64(&raw, &dt).unwrap_err();
        assert!(matches!(err, FormatError::TypeMismatch { .. }));
    }

    #[test]
    fn size_mismatch_compact() {
        let dt = make_f64_le_type();
        let ds = make_simple_dataspace(&[3]);
        let data = vec![0u8; 16]; // wrong: should be 24
        let layout = DataLayout::Compact { data };
        let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err();
        assert!(matches!(err, FormatError::DataSizeMismatch { .. }));
    }

    #[test]
    fn no_data_allocated() {
        let dt = make_f64_le_type();
        let ds = make_simple_dataspace(&[3]);
        let layout = DataLayout::Contiguous {
            address: None,
            size: 24,
        };
        let err = read_raw_data(&[], &layout, &ds, &dt).unwrap_err();
        assert!(matches!(err, FormatError::NoDataAllocated));
    }

    #[test]
    fn string_type_mismatch_on_read_as_strings() {
        let dt = make_i32_le_type();
        let raw = vec![0u8; 8];
        let err = read_as_strings(&raw, &dt).unwrap_err();
        assert!(matches!(err, FormatError::TypeMismatch { .. }));
    }

    #[test]
    fn read_f64_from_i32() {
        // read_as_f64 should work on FixedPoint types too
        let dt = make_i32_le_type();
        let mut raw = Vec::new();
        raw.extend_from_slice(&42i32.to_le_bytes());
        raw.extend_from_slice(&(-7i32).to_le_bytes());
        let result = read_as_f64(&raw, &dt).unwrap();
        assert_eq!(result, vec![42.0, -7.0]);
    }

    #[test]
    fn read_strings_space_padded() {
        let dt = Datatype::String {
            size: 8,
            padding: StringPadding::SpacePad,
            charset: CharacterSet::Ascii,
        };
        let raw = b"hello   world   ";
        let result = read_as_strings(raw, &dt).unwrap();
        assert_eq!(result, vec!["hello", "world"]);
    }

    #[test]
    fn read_strings_null_terminated() {
        let dt = Datatype::String {
            size: 6,
            padding: StringPadding::NullTerminate,
            charset: CharacterSet::Ascii,
        };
        let raw = b"abc\0\0\0de\0\0\0\0";
        let result = read_as_strings(raw, &dt).unwrap();
        assert_eq!(result, vec!["abc", "de"]);
    }

    #[test]
    fn read_compound_two_fields() {
        use crate::datatype::CompoundMember;
        // Compound: { x: f64, id: i32 } => size = 12
        let dt = Datatype::Compound {
            size: 12,
            members: vec![
                CompoundMember {
                    name: "x".to_string(),
                    byte_offset: 0,
                    datatype: make_f64_le_type(),
                },
                CompoundMember {
                    name: "id".to_string(),
                    byte_offset: 8,
                    datatype: make_i32_le_type(),
                },
            ],
        };
        // Two elements
        let mut raw = Vec::new();
        raw.extend_from_slice(&1.5f64.to_le_bytes());
        raw.extend_from_slice(&10i32.to_le_bytes());
        raw.extend_from_slice(&2.5f64.to_le_bytes());
        raw.extend_from_slice(&20i32.to_le_bytes());

        let fields = read_compound_fields(&raw, &dt).unwrap();
        assert_eq!(fields.len(), 2);
        assert_eq!(fields[0].name, "x");
        let x_vals = read_as_f64(&fields[0].raw_data, &fields[0].datatype).unwrap();
        assert_eq!(x_vals, vec![1.5, 2.5]);

        assert_eq!(fields[1].name, "id");
        let id_vals = read_as_i32(&fields[1].raw_data, &fields[1].datatype).unwrap();
        assert_eq!(id_vals, vec![10, 20]);
    }

    #[test]
    fn read_compound_rejects_byte_offset_overrun() {
        use crate::datatype::CompoundMember;
        // Compound declares size=8, but the member's byte_offset(4) + its
        // field_size(8, f64) = 12 > 8 — a crafted out-of-range byte_offset.
        let dt = Datatype::Compound {
            size: 8,
            members: vec![CompoundMember {
                name: "bad".to_string(),
                byte_offset: 4,
                datatype: make_f64_le_type(),
            }],
        };
        let raw = vec![0u8; 8]; // one element, matches declared size
        let result = read_compound_fields(&raw, &dt);
        assert!(
            matches!(result, Err(FormatError::Overflow(_))),
            "expected a clean Overflow error, got {result:?}"
        );
    }

    #[test]
    fn read_raw_data_zerocopy_rejects_near_usize_max_offset() {
        let file_data = vec![0u8; 64];
        let dataspace = make_simple_dataspace(&[4]);
        let datatype = make_i32_le_type();
        let layout = DataLayout::Contiguous {
            address: Some(u64::MAX - 4),
            size: 16,
        };
        let result = read_raw_data_zerocopy(&file_data, &layout, &dataspace, &datatype);
        assert!(
            matches!(result, Err(FormatError::UnexpectedEof { .. })),
            "expected a clean UnexpectedEof, got {result:?}"
        );
    }

    #[test]
    fn read_compound_single_field_by_name() {
        use crate::datatype::CompoundMember;
        let dt = Datatype::Compound {
            size: 12,
            members: vec![
                CompoundMember {
                    name: "x".to_string(),
                    byte_offset: 0,
                    datatype: make_f64_le_type(),
                },
                CompoundMember {
                    name: "id".to_string(),
                    byte_offset: 8,
                    datatype: make_i32_le_type(),
                },
            ],
        };
        let mut raw = Vec::new();
        raw.extend_from_slice(&3.25f64.to_le_bytes());
        raw.extend_from_slice(&42i32.to_le_bytes());

        let field = read_compound_field(&raw, &dt, "id").unwrap();
        let vals = read_as_i32(&field.raw_data, &field.datatype).unwrap();
        assert_eq!(vals, vec![42]);

        // Non-existent field
        let err = read_compound_field(&raw, &dt, "missing").unwrap_err();
        assert!(matches!(err, FormatError::PathNotFound(_)));
    }

    #[test]
    fn read_enum_values_basic() {
        use crate::datatype::EnumMember;
        let dt = Datatype::Enumeration {
            size: 4,
            base_type: Box::new(make_i32_le_type()),
            members: vec![
                EnumMember {
                    name: "RED".to_string(),
                    value: 0i32.to_le_bytes().to_vec(),
                },
                EnumMember {
                    name: "GREEN".to_string(),
                    value: 1i32.to_le_bytes().to_vec(),
                },
                EnumMember {
                    name: "BLUE".to_string(),
                    value: 2i32.to_le_bytes().to_vec(),
                },
            ],
        };
        let mut raw = Vec::new();
        raw.extend_from_slice(&1i32.to_le_bytes()); // GREEN
        raw.extend_from_slice(&0i32.to_le_bytes()); // RED
        raw.extend_from_slice(&2i32.to_le_bytes()); // BLUE
        raw.extend_from_slice(&99i32.to_le_bytes()); // unknown

        let names = read_enum_names(&raw, &dt).unwrap();
        assert_eq!(names[0], "GREEN");
        assert_eq!(names[1], "RED");
        assert_eq!(names[2], "BLUE");
        assert!(names[3].starts_with("UNKNOWN("));
    }

    #[test]
    fn read_array_flat_basic() {
        // Array[3] of f64
        let dt = Datatype::Array {
            base_type: Box::new(make_f64_le_type()),
            dimensions: vec![3],
        };
        let mut raw = Vec::new();
        for v in &[1.0f64, 2.0, 3.0] {
            raw.extend_from_slice(&v.to_le_bytes());
        }
        let (data, base_dt, dims) = read_array_flat(&raw, &dt).unwrap();
        assert_eq!(dims, vec![3]);
        let vals = read_as_f64(&data, &base_dt).unwrap();
        assert_eq!(vals, vec![1.0, 2.0, 3.0]);
    }

    #[test]
    fn read_object_references_basic() {
        use crate::datatype::ReferenceType;
        let dt = Datatype::Reference {
            size: 8,
            ref_type: ReferenceType::Object,
        };
        let mut raw = Vec::new();
        raw.extend_from_slice(&1024u64.to_le_bytes()); // valid ref
        raw.extend_from_slice(&u64::MAX.to_le_bytes()); // null ref
        raw.extend_from_slice(&2048u64.to_le_bytes()); // valid ref

        let refs = read_object_references(&raw, &dt, 8).unwrap();
        assert_eq!(refs.len(), 3);
        assert_eq!(refs[0].address, 1024);
        assert!(!refs[0].is_null());
        assert!(refs[1].is_null());
        assert_eq!(refs[2].address, 2048);
    }

    #[test]
    fn read_object_references_4byte_offset() {
        use crate::datatype::ReferenceType;
        let dt = Datatype::Reference {
            size: 4,
            ref_type: ReferenceType::Object,
        };
        let mut raw = Vec::new();
        raw.extend_from_slice(&512u32.to_le_bytes());
        raw.extend_from_slice(&u32::MAX.to_le_bytes()); // null ref

        let refs = read_object_references(&raw, &dt, 4).unwrap();
        assert_eq!(refs.len(), 2);
        assert_eq!(refs[0].address, 512);
        assert!(refs[1].is_null());
    }

    #[test]
    fn read_object_references_type_mismatch() {
        let dt = make_f64_le_type();
        let raw = vec![0u8; 8];
        let err = read_object_references(&raw, &dt, 8).unwrap_err();
        assert!(matches!(err, FormatError::TypeMismatch { .. }));
    }

    #[test]
    fn read_region_references_basic() {
        use crate::datatype::ReferenceType;
        let dt = Datatype::Reference {
            size: 12,
            ref_type: ReferenceType::DatasetRegion,
        };
        let raw = vec![0xABu8; 24]; // two 12-byte region refs

        let refs = read_region_references(&raw, &dt).unwrap();
        assert_eq!(refs.len(), 2);
        assert_eq!(refs[0].raw.len(), 12);
        assert_eq!(refs[1].raw.len(), 12);
    }

    #[test]
    fn read_region_references_type_mismatch() {
        let dt = make_i32_le_type();
        let raw = vec![0u8; 12];
        let err = read_region_references(&raw, &dt).unwrap_err();
        assert!(matches!(err, FormatError::TypeMismatch { .. }));
    }

    #[test]
    fn zerocopy_contiguous_returns_slice_into_file_data() {
        let dt = make_f64_le_type();
        let ds = make_simple_dataspace(&[3]);
        let mut file_data = vec![0u8; 1024];
        let offset = 256usize;
        let vals = [1.0f64, 2.0, 3.0];
        for (i, v) in vals.iter().enumerate() {
            file_data[offset + i * 8..offset + i * 8 + 8].copy_from_slice(&v.to_le_bytes());
        }
        let layout = DataLayout::Contiguous {
            address: Some(offset as u64),
            size: 24,
        };
        let result = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap();
        let slice = result.expect("contiguous should return Some");
        // Pointer identity: the slice must point into file_data, not a copy
        let file_range = file_data.as_ptr_range();
        assert!(file_range.contains(&slice.as_ptr()));
        assert_eq!(slice.len(), 24);
        // Verify the actual data
        let values = read_as_f64(slice, &dt).unwrap();
        assert_eq!(values, vec![1.0, 2.0, 3.0]);
    }

    #[test]
    fn zerocopy_compact_returns_none() {
        let dt = make_f64_le_type();
        let ds = make_simple_dataspace(&[1]);
        let data = vec![0u8; 8];
        let layout = DataLayout::Compact { data };
        let result = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap();
        assert!(result.is_none());
    }

    #[test]
    fn zerocopy_no_data_allocated() {
        let dt = make_f64_le_type();
        let ds = make_simple_dataspace(&[1]);
        let layout = DataLayout::Contiguous {
            address: None,
            size: 8,
        };
        let err = read_raw_data_zerocopy(&[], &layout, &ds, &dt).unwrap_err();
        assert!(matches!(err, FormatError::NoDataAllocated));
    }

    #[test]
    fn read_as_f64_zerocopy_aligned() {
        let dt = make_f64_le_type();
        // Create aligned data — Vec<f64> guarantees 8-byte alignment
        let values = [1.0f64, 2.0, 3.0, 4.0];
        let raw: &[u8] =
            // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.

            unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) };
        let result = read_as_f64_zerocopy(raw, &dt);
        assert!(result.is_some(), "aligned native LE f64 should succeed");
        let slice = result.unwrap();
        assert_eq!(slice, &[1.0, 2.0, 3.0, 4.0]);
        // Verify it's the same memory (zero-copy)
        assert_eq!(slice.as_ptr() as *const u8, raw.as_ptr());
    }

    #[test]
    fn read_as_f64_zerocopy_wrong_type() {
        let dt = make_i32_le_type();
        let values = [1.0f64; 4];
        let raw: &[u8] =
            // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.

            unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) };
        assert!(read_as_f64_zerocopy(raw, &dt).is_none());
    }

    #[test]
    fn read_as_f64_zerocopy_big_endian() {
        let dt = Datatype::FloatingPoint {
            size: 8,
            byte_order: DatatypeByteOrder::BigEndian,
            bit_offset: 0,
            bit_precision: 64,
            exponent_location: 52,
            exponent_size: 11,
            mantissa_location: 0,
            mantissa_size: 52,
            exponent_bias: 1023,
        };
        let values = [1.0f64; 4];
        let raw: &[u8] =
            // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.

            unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 8) };
        assert!(read_as_f64_zerocopy(raw, &dt).is_none());
    }

    #[test]
    fn read_as_f64_zerocopy_odd_size() {
        let dt = make_f64_le_type();
        let raw = &[0u8; 13]; // not a multiple of 8
        assert!(read_as_f64_zerocopy(raw, &dt).is_none());
    }

    #[test]
    fn read_as_f32_zerocopy_aligned() {
        let dt = Datatype::FloatingPoint {
            size: 4,
            byte_order: DatatypeByteOrder::LittleEndian,
            bit_offset: 0,
            bit_precision: 32,
            exponent_location: 23,
            exponent_size: 8,
            mantissa_location: 0,
            mantissa_size: 23,
            exponent_bias: 127,
        };
        let values = [1.5f32, 2.5, 3.5];
        let raw: &[u8] =
            // SAFETY: values is a valid slice; reinterpreting as u8 bytes is always safe.

            unsafe { core::slice::from_raw_parts(values.as_ptr() as *const u8, values.len() * 4) };
        let result = read_as_f32_zerocopy(raw, &dt);
        assert!(result.is_some());
        assert_eq!(result.unwrap(), &[1.5f32, 2.5, 3.5]);
    }

    #[test]
    fn zerocopy_size_mismatch() {
        let dt = make_f64_le_type();
        let ds = make_simple_dataspace(&[3]);
        let file_data = vec![0u8; 1024];
        let layout = DataLayout::Contiguous {
            address: Some(0),
            size: 16, // wrong: should be 24
        };
        let err = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap_err();
        assert!(matches!(err, FormatError::DataSizeMismatch { .. }));
    }
}