clawhdf5/crates/clawhdf5-format/src/extensible_array.rs

//! HDF5 Extensible Array index parsing for chunked datasets (v4 index type 4).
//!
//! Extensible Arrays are used for datasets with exactly one unlimited dimension.
//! Structures: AEHD (header), AEIB (index block), AEDB (data block), AESB (super block).

#[cfg(not(feature = "std"))]
extern crate alloc;

#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};

use crate::addr::to_usize;
use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
use crate::storage::{PAGED_BLOCK_ONE_READ_MAX, Storage, Window, read_exact_at};

/// Verify the Jenkins lookup3 checksum stored immediately after
/// `data[start..end]`, as every Extensible Array structure carries one. `w`
/// is a window of the file and `start`/`end` are relative to it.
///
/// A corrupt chunk index yields addresses pointing at the wrong bytes, so a
/// mismatch is an error: otherwise the damage surfaces as plausible data read
/// from the wrong chunk.
#[cfg(feature = "checksum")]
fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> {
    w.ensure(end, 4)?;
    let data: &[u8] = &w.bytes;
    let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
    let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
    if computed != stored {
        return Err(FormatError::ChecksumMismatch {
            expected: stored,
            computed,
        });
    }
    Ok(())
}

#[cfg(not(feature = "checksum"))]
fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> {
    Ok(())
}

/// Parsed Extensible Array header (AEHD).
#[derive(Debug, Clone)]
pub struct ExtensibleArrayHeader {
    /// Client ID: 0 = non-filtered chunks, 1 = filtered chunks.
    pub client_id: u8,
    /// Size of each array element in bytes.
    pub element_size: u8,
    /// Max number of elements bits (log2 of the max number of data block elements per page).
    pub max_nelmts_bits: u8,
    /// Number of elements in the index block.
    pub idx_blk_elmts: u8,
    /// Minimum number of data block elements.
    pub min_dblk_nelmts: u8,
    /// Minimum number of elements in a super block.
    pub super_blk_min_nelmts: u8,
    /// Max number of data block elements bits.
    pub max_dblk_nelmts_bits: u8,
    /// Total number of elements stored.
    pub num_elements: u64,
    /// Address of the index block.
    pub index_block_address: u64,
}

fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
    let s = size as usize;
    if pos.checked_add(s).is_none_or(|end| end > data.len()) {
        return Err(FormatError::UnexpectedEof {
            expected: pos.saturating_add(s),
            available: data.len(),
        });
    }
    let slice = &data[pos..pos + s];
    Ok(match size {
        2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
        4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
        8 => u64::from_le_bytes([
            slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
        ]),
        _ => return Err(FormatError::InvalidOffsetSize(size)),
    })
}

fn is_undefined_addr(addr: u64, offset_size: u8) -> bool {
    match offset_size {
        2 => addr == 0xFFFF,
        4 => addr == 0xFFFF_FFFF,
        8 => addr == 0xFFFF_FFFF_FFFF_FFFF,
        _ => false,
    }
}

fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
    let s = size as usize;
    if pos + s > data.len() {
        return false;
    }
    data[pos..pos + s].iter().all(|&b| b == 0xFF)
}

fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
    if size > 8 || data.len() < size {
        return Err(FormatError::ChunkedReadError(
            "invalid variable-length size".into(),
        ));
    }
    let mut val = 0u64;
    for (i, &byte) in data.iter().enumerate().take(size) {
        val |= (byte as u64) << (i * 8);
    }
    Ok(val)
}

impl ExtensibleArrayHeader {
    /// Parse an Extensible Array header from file data at the given offset.
    pub fn parse(
        file_data: &[u8],
        offset: usize,
        offset_size: u8,
        length_size: u8,
    ) -> Result<Self, FormatError> {
        Self::parse_in(file_data, offset as u64, offset_size, length_size)
    }

    /// [`Self::parse`] over any [`Storage`]: one read of the header.
    pub fn parse_in<S: Storage + ?Sized>(
        file: &S,
        offset: u64,
        offset_size: u8,
        length_size: u8,
    ) -> Result<Self, FormatError> {
        // EAHD: signature(4) + version(1) + client_id(1) + element_size(1) +
        //   max_nelmts_bits(1) + idx_blk_elmts(1) + min_dblk_nelmts(1) +
        //   super_blk_min_nelmts(1) + max_dblk_nelmts_bits(1) +
        //   6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4)
        let min_size =
            4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4;
        let w = Window::read(file, offset, min_size)?;
        w.ensure(0, min_size)?;

        let d: &[u8] = &w.bytes;
        if &d[0..4] != b"EAHD" {
            return Err(FormatError::ChunkedReadError(
                "invalid Extensible Array header signature".into(),
            ));
        }

        let version = d[4];
        if version != 0 {
            return Err(FormatError::ChunkedReadError(format!(
                "unsupported Extensible Array header version: {version}"
            )));
        }

        let client_id = d[5];
        let element_size = d[6];
        let max_nelmts_bits = d[7];
        let idx_blk_elmts = d[8];
        let min_dblk_nelmts = d[9];
        let super_blk_min_nelmts = d[10];
        let max_dblk_nelmts_bits = d[11];

        let mut pos = 12;
        // 6 stats fields: [0] unknown, [1] unknown, [2] nsuper_blks_created,
        // [3] super_blk_size, [4] nelmts, [5] max_idx_set
        // We only need nelmts (field[4]) and skip the rest.
        let ls = length_size as usize;
        pos += 4 * ls; // skip first 4 stats fields
        let num_elements = read_offset(d, pos, length_size)?;
        pos += ls; // skip nelmts
        pos += ls; // skip max_idx_set (6th stats field)
        let index_block_address = read_offset(d, pos, offset_size)?;
        pos += offset_size as usize;
        verify_checksum(&w, 0, pos)?;

        Ok(ExtensibleArrayHeader {
            client_id,
            element_size,
            max_nelmts_bits,
            idx_blk_elmts,
            min_dblk_nelmts,
            super_blk_min_nelmts,
            max_dblk_nelmts_bits,
            num_elements,
            index_block_address,
        })
    }

    /// Compute the size of this header in bytes (for write support).
    pub fn serialized_size(offset_size: u8, length_size: u8) -> usize {
        4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4
    }
}

/// Read a single element at offset `pos` of the window `w`.
/// Returns (chunk_info, bytes_consumed) or None if unallocated.
#[allow(clippy::too_many_arguments)]
fn read_element(
    w: &Window<'_>,
    pos: usize,
    client_id: u8,
    element_size: u8,
    offset_size: u8,
    chunk_byte_size: u64,
    linear_index: usize,
    grid: &ChunkGrid,
) -> Result<(Option<ChunkInfo>, usize), FormatError> {
    let os = offset_size as usize;
    let data: &[u8] = &w.bytes;

    if client_id == 0 {
        // Non-filtered: just address
        w.ensure(pos, os)?;
        if is_undefined(data, pos, offset_size) {
            return Ok((None, os));
        }
        let address = read_offset(data, pos, offset_size)?;
        // A slot beyond the current extent is ignored, as the library does.
        let Some(offsets) = grid.offsets(linear_index as u64) else {
            return Ok((None, os));
        };
        Ok((
            Some(ChunkInfo {
                chunk_size: chunk_byte_size as u32,
                filter_mask: 0,
                offsets,
                address,
            }),
            os,
        ))
    } else {
        // Filtered: address + compressed_size + filter_mask
        let es = element_size as usize;
        if es < os + 4 {
            return Err(FormatError::ChunkedReadError(
                "element_size too small for filtered element".into(),
            ));
        }
        let chunk_size_bytes = es - os - 4;
        let elem_total = os + chunk_size_bytes + 4;
        w.ensure(pos, elem_total)?;
        if is_undefined(data, pos, offset_size) {
            return Ok((None, elem_total));
        }
        let address = read_offset(data, pos, offset_size)?;
        let chunk_size = read_variable_length(&data[pos + os..], chunk_size_bytes)?;
        let fm_off = pos + os + chunk_size_bytes;
        let filter_mask = u32::from_le_bytes([
            data[fm_off],
            data[fm_off + 1],
            data[fm_off + 2],
            data[fm_off + 3],
        ]);
        let Some(offsets) = grid.offsets(linear_index as u64) else {
            return Ok((None, elem_total));
        };
        Ok((
            Some(ChunkInfo {
                chunk_size: chunk_size as u32,
                filter_mask,
                offsets,
                address,
            }),
            elem_total,
        ))
    }
}

/// Collect elements from a data block at the given offset.
#[allow(clippy::too_many_arguments)]
/// Layout of super block `u`, per the HDF5 spec: the number of data blocks it
/// owns and how many elements each of them holds.
///
/// `ndblks` and `dblk_nelmts` each double every *other* level, a half-step
/// apart, so the blocks grow as 1x16, 1x32, 2x32, 2x64, 4x64 ... for a
/// 16-element minimum. Treating either as doubling every level (the previous
/// implementation) puts every element after the first data block at the wrong
/// index.
fn sblk_info(u: usize, data_blk_min_elmts: usize) -> Option<(usize, usize)> {
    let ndblks = 1usize.checked_shl((u / 2) as u32)?;
    let dblk_nelmts = 1usize
        .checked_shl(u.div_ceil(2) as u32)?
        .checked_mul(data_blk_min_elmts)?;
    Some((ndblks, dblk_nelmts))
}

/// Width of the "offset of the block in the array" field carried by super and
/// data blocks (`hdr->arr_off_size`).
fn arr_off_size(header: &ExtensibleArrayHeader) -> usize {
    (header.max_nelmts_bits as usize).div_ceil(8)
}

/// Elements per data block page, once a data block is large enough to be paged.
fn page_nelmts(header: &ExtensibleArrayHeader) -> Option<usize> {
    1usize.checked_shl(u32::from(header.max_dblk_nelmts_bits))
}

/// Read the elements of one data block (EADB).
///
/// `page_init` is the owning super block's page-init bitmap and `first_page`
/// this block's first bit in it; both are only consulted when the block is
/// paged. The bitmap lives in the super block, not here — a paged data block
/// stores only its prefix, then one slot per page.
#[allow(clippy::too_many_arguments)]
fn read_data_block_elements<S: Storage + ?Sized>(
    file: &S,
    db_offset: u64,
    nelmts: usize,
    header: &ExtensibleArrayHeader,
    offset_size: u8,
    chunk_byte_size: u64,
    start_index: usize,
    grid: &ChunkGrid,
    page_init: &[u8],
    first_page: usize,
) -> Result<Vec<ChunkInfo>, FormatError> {
    // EADB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
    //       + block offset(arr_off_size)
    let db_header_size = 4 + 1 + 1 + offset_size as usize + arr_off_size(header);
    let prefix = read_exact_at(file, db_offset, db_header_size)?;

    if &prefix[0..4] != b"EADB" {
        return Err(FormatError::ChunkedReadError(
            "invalid Extensible Array data block signature".into(),
        ));
    }

    // Positions below are relative to the data block.
    let mut pos = db_header_size;
    let page = page_nelmts(header).ok_or_else(|| {
        FormatError::Overflow("Extensible Array page element count overflows usize".into())
    })?;
    let elem_bytes = if header.client_id == 0 {
        offset_size as usize
    } else {
        header.element_size as usize
    };

    let mut chunks = Vec::new();
    let read_run = |w: &Window<'_>,
                    from: usize,
                    count: usize,
                    first_index: usize,
                    chunks: &mut Vec<ChunkInfo>|
     -> Result<usize, FormatError> {
        let mut p = from;
        for i in 0..count {
            let (info, consumed) = read_element(
                w,
                p,
                header.client_id,
                header.element_size,
                offset_size,
                chunk_byte_size,
                first_index + i,
                grid,
            )?;
            if let Some(ci) = info {
                chunks.push(ci);
            }
            p += consumed;
        }
        Ok(p)
    };

    if nelmts <= page {
        // Prefix and elements are covered by one checksum. One window holds
        // all of it (or ends at the end of the file), so its bounds checks
        // are the whole-file ones.
        let end = nelmts
            .checked_mul(elem_bytes)
            .and_then(|b| pos.checked_add(b))
            .ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?;
        // The checksum's bounds check comes first: make it before reading.
        #[cfg(feature = "checksum")]
        Window::check_extent(file, db_offset, end, 4)?;
        let w = Window::read(file, db_offset, end.saturating_add(4))?;
        verify_checksum(&w, 0, end)?;
        read_run(&w, pos, nelmts, start_index, &mut chunks)?;
        return Ok(chunks);
    }

    // Paged: the prefix ends with its own checksum, then one slot per page,
    // each holding `page` elements followed by a checksum. Pages whose bit is
    // clear were never written; their slot still occupies the file, so stride
    // over it rather than reading zeros as addresses.
    let npages = nelmts.div_ceil(page);
    // The whole data block in one window when it is small: every position
    // checked below lies inside it (or past the end of the file). A larger
    // block is read as its prefix, then each page in use on its own.
    let block_len = pos
        .saturating_add(4)
        .saturating_add(npages.saturating_mul(page.saturating_mul(elem_bytes).saturating_add(4)));
    let whole = if block_len <= PAGED_BLOCK_ONE_READ_MAX {
        Some(Window::read(file, db_offset, block_len)?)
    } else {
        None
    };
    let head_w;
    let head = match &whole {
        Some(w) => w,
        None => {
            head_w = Window::read(file, db_offset, pos + 4)?;
            &head_w
        }
    };
    verify_checksum(head, 0, pos)?;
    pos += 4;
    let page_stride = page
        .checked_mul(elem_bytes)
        .and_then(|b| b.checked_add(4))
        .ok_or_else(|| FormatError::Overflow("Extensible Array page stride".into()))?;
    for p in 0..npages {
        // One bit per page across the whole super block, packed contiguously
        // and MSB-first within each byte, as H5VM_bit_get reads it.
        let bit = first_page + p;
        let initialised = page_init
            .get(bit / 8)
            .is_some_and(|byte| byte & (0x80 >> (bit % 8)) != 0);
        if initialised {
            let count = core::cmp::min(page, nelmts - p * page);
            // `w` holds the page from `base` on (positions below are
            // relative to it, and `pos` to the data block).
            let page_w;
            let (w, base) = match &whole {
                Some(w) => (w, 0),
                None => {
                    page_w = Window::read(file, db_offset.saturating_add(pos as u64), page_stride)?;
                    (&page_w, pos)
                }
            };
            // Each page carries its own checksum, over a full page's worth of
            // slots even when the last one holds fewer live elements.
            verify_checksum(w, pos - base, pos - base + page * elem_bytes)?;
            read_run(w, pos - base, count, start_index + p * page, &mut chunks)?;
        }
        pos = pos
            .checked_add(page_stride)
            .ok_or_else(|| FormatError::Overflow("Extensible Array page offset".into()))?;
    }

    Ok(chunks)
}

/// Read chunk records from an Extensible Array.
///
/// Traverses AEHD -> AEIB -> AEDB/AESB to collect all allocated chunks.
#[allow(clippy::too_many_arguments)]
pub fn read_extensible_array_chunks(
    file_data: &[u8],
    header: &ExtensibleArrayHeader,
    dataset_dims: &[u64],
    max_dims: Option<&[u64]>,
    chunk_dimensions: &[u32],
    element_size: u32,
    offset_size: u8,
    length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
    read_extensible_array_chunks_in(
        &file_data,
        header,
        dataset_dims,
        max_dims,
        chunk_dimensions,
        element_size,
        offset_size,
        length_size,
    )
}

/// [`read_extensible_array_chunks`] over any [`Storage`]: one read of the
/// index block's prefix, one of the whole index block, and the same for
/// every super block and data block it references.
#[allow(clippy::too_many_arguments)]
pub fn read_extensible_array_chunks_in<S: Storage + ?Sized>(
    file: &S,
    header: &ExtensibleArrayHeader,
    dataset_dims: &[u64],
    max_dims: Option<&[u64]>,
    chunk_dimensions: &[u32],
    element_size: u32,
    offset_size: u8,
    _length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
    let os = offset_size as usize;

    // Linear indexes follow the maximum dimensions, with the unlimited
    // dimension swizzled to the slowest position (see `chunk_grid`).
    let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
    let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, &dims_u64)?;
    let grid = &grid;

    let chunk_byte_size: u64 =
        chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;

    // Parse index block (EAIB): signature(4) + version(1) + client_id(1)
    // + header address(offset_size), then the inline elements, then the
    // direct data block addresses, then the super block addresses.
    // Positions below are relative to the index block.
    let ib_offset = header.index_block_address;
    let ib_header_size = 4 + 1 + 1 + os;
    let prefix = read_exact_at(file, ib_offset, ib_header_size)?;

    if &prefix[0..4] != b"EAIB" {
        return Err(FormatError::ChunkedReadError(
            "invalid Extensible Array index block signature".into(),
        ));
    }
    let mut pos = ib_header_size;

    let mut chunks = Vec::new();
    let total_elements = to_usize(header.num_elements)?;

    let dmin = header.min_dblk_nelmts as usize;
    if dmin == 0 || !dmin.is_power_of_two() {
        return Err(FormatError::ChunkedReadError(
            "Extensible Array data block minimum is not a power of two".into(),
        ));
    }
    // nsblks = 1 + (max_nelmts_bits - log2(data_blk_min_elmts)), and the index
    // block holds 2 * (sup_blk_min_data_ptrs - 1) data block addresses.
    let log2_dmin = dmin.trailing_zeros() as usize;
    let nsblks = 1 + (header.max_nelmts_bits as usize).saturating_sub(log2_dmin);
    let ndblk_addrs = 2 * (header.super_blk_min_nelmts as usize).saturating_sub(1);

    // The data blocks listed directly in the index block are the first
    // `ndblk_addrs` in super-block order, each sized by the level it belongs
    // to; the super block addresses that follow resume at the next level.
    let mut direct: Vec<usize> = Vec::with_capacity(ndblk_addrs);
    let mut level = 0usize;
    while direct.len() < ndblk_addrs {
        if level >= nsblks {
            return Err(FormatError::ChunkedReadError(
                "Extensible Array index block claims more data blocks than the array has".into(),
            ));
        }
        let (ndblks, dblk_nelmts) = sblk_info(level, dmin).ok_or_else(|| {
            FormatError::Overflow("Extensible Array super block layout overflows usize".into())
        })?;
        for _ in 0..ndblks {
            direct.push(dblk_nelmts);
        }
        level += 1;
    }
    if direct.len() != ndblk_addrs {
        // A partial level in the index block is not a layout HDF5 produces,
        // and guessing where the super blocks resume would misplace elements.
        return Err(FormatError::ChunkedReadError(
            "Extensible Array index block ends mid super block".into(),
        ));
    }

    // One checksum covers the prefix, every inline element slot, and every
    // data block and super block address.
    let elem_bytes = if header.client_id == 0 {
        os
    } else {
        header.element_size as usize
    };
    let ib_end = (header.idx_blk_elmts as usize)
        .checked_mul(elem_bytes)
        .and_then(|b| pos.checked_add(b))
        .and_then(|p| {
            ndblk_addrs
                .checked_add(nsblks - level)
                .and_then(|n| n.checked_mul(os).and_then(|b| p.checked_add(b)))
        })
        .ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?;
    // The whole index block in one window: every position read below is
    // before `ib_end`.
    // The checksum's bounds check comes first: make it before reading.
    #[cfg(feature = "checksum")]
    Window::check_extent(file, ib_offset, ib_end, 4)?;
    let w = Window::read(file, ib_offset, ib_end.saturating_add(4))?;
    verify_checksum(&w, 0, ib_end)?;

    // 1. Elements stored inline in the index block.
    let n_inline = (header.idx_blk_elmts as usize).min(total_elements);
    for i in 0..n_inline {
        let (info, consumed) = read_element(
            &w,
            pos,
            header.client_id,
            header.element_size,
            offset_size,
            chunk_byte_size,
            i,
            grid,
        )?;
        if let Some(ci) = info {
            chunks.push(ci);
        }
        pos += consumed;
    }
    let mut global_index = n_inline;
    if global_index >= total_elements {
        return Ok(chunks);
    }

    // 2. Data blocks listed directly in the index block.
    for &dblk_nelmts in &direct {
        if global_index >= total_elements {
            return Ok(chunks);
        }
        w.ensure(pos, os)?;
        let addr = read_offset(&w.bytes, pos, offset_size)?;
        pos += os;
        if !is_undefined_addr(addr, offset_size) {
            if dblk_nelmts > page_nelmts(header).unwrap_or(usize::MAX) {
                // Would need a page-init bitmap, which only a super block
                // carries. HDF5 never pages these small early blocks.
                return Err(FormatError::ChunkedReadError(
                    "Extensible Array index block references a paged data block".into(),
                ));
            }
            chunks.extend(read_data_block_elements(
                file,
                addr,
                dblk_nelmts,
                header,
                offset_size,
                chunk_byte_size,
                global_index,
                grid,
                &[],
                0,
            )?);
        }
        global_index += dblk_nelmts;
    }

    // 3. Everything else lives in super blocks, one address per remaining
    //    level, starting at the level after the direct data blocks.
    for u in level..nsblks {
        if global_index >= total_elements {
            break;
        }
        w.ensure(pos, os)?;
        let sb_addr = read_offset(&w.bytes, pos, offset_size)?;
        pos += os;
        let (ndblks, dblk_nelmts) = sblk_info(u, dmin).ok_or_else(|| {
            FormatError::Overflow("Extensible Array super block layout overflows usize".into())
        })?;
        if !is_undefined_addr(sb_addr, offset_size) {
            chunks.extend(read_super_block(
                file,
                sb_addr,
                ndblks,
                dblk_nelmts,
                header,
                offset_size,
                chunk_byte_size,
                global_index,
                grid,
            )?);
        }
        global_index =
            global_index.saturating_add(ndblks.checked_mul(dblk_nelmts).ok_or_else(|| {
                FormatError::Overflow("Extensible Array super block span".into())
            })?);
    }

    Ok(chunks)
}

/// Read a super block (EASB) and the data blocks it owns.
///
/// On disk: signature(4) + version(1) + client_id(1) + header address
/// + block offset + the page-init bitmap for every data block it owns
/// + one address per data block + checksum.
#[allow(clippy::too_many_arguments)]
fn read_super_block<S: Storage + ?Sized>(
    file: &S,
    sb_offset: u64,
    ndblks: usize,
    dblk_nelmts: usize,
    header: &ExtensibleArrayHeader,
    offset_size: u8,
    chunk_byte_size: u64,
    start_index: usize,
    grid: &ChunkGrid,
) -> Result<Vec<ChunkInfo>, FormatError> {
    let os = offset_size as usize;
    let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header);
    let prefix = read_exact_at(file, sb_offset, sb_header_size)?;

    if &prefix[0..4] != b"EASB" {
        return Err(FormatError::ChunkedReadError(
            "invalid Extensible Array super block signature".into(),
        ));
    }

    // Page-init bitmap: one bit per page, `npages` bits per data block, packed
    // contiguously. HDF5 sizes the buffer `ndblks * ceil(npages / 8)`, which
    // is bigger than the bits need when `npages` is not a multiple of eight.
    // Zero-sized unless this level's data blocks are paged.
    let page = page_nelmts(header).ok_or_else(|| {
        FormatError::Overflow("Extensible Array page element count overflows usize".into())
    })?;
    let npages = if dblk_nelmts > page {
        dblk_nelmts / page
    } else {
        0
    };
    let per_dblk_bitmap = npages.div_ceil(8);
    let bitmap_bytes = per_dblk_bitmap
        .checked_mul(ndblks)
        .ok_or_else(|| FormatError::Overflow("Extensible Array page bitmap size".into()))?;
    // Positions below are relative to the super block, whose bytes (up to
    // its checksum) are all in one window.
    let bitmap_start = sb_header_size;
    // The bitmap's bounds check, then (with checksums) the checksum's, come
    // before anything else is read from the block: make them before reading
    // it, so size fields stretching it past the end of the file cost no read.
    Window::check_extent(file, sb_offset, bitmap_start, bitmap_bytes)?;
    let mut pos = bitmap_start + bitmap_bytes;

    // One checksum covers the prefix, the bitmap and every data block address.
    let sb_end = ndblks
        .checked_mul(os)
        .and_then(|b| pos.checked_add(b))
        .ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?;
    #[cfg(feature = "checksum")]
    Window::check_extent(file, sb_offset, sb_end, 4)?;
    let w = Window::read(file, sb_offset, sb_end.saturating_add(4))?;
    w.ensure(bitmap_start, bitmap_bytes)?;
    let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes];

    let mut chunks = Vec::new();
    let mut global_idx = start_index;
    verify_checksum(&w, 0, sb_end)?;

    for i in 0..ndblks {
        w.ensure(pos, os)?;
        let addr = read_offset(&w.bytes, pos, offset_size)?;
        pos += os;
        if !is_undefined_addr(addr, offset_size) {
            chunks.extend(read_data_block_elements(
                file,
                addr,
                dblk_nelmts,
                header,
                offset_size,
                chunk_byte_size,
                global_idx,
                grid,
                bitmap,
                i * npages,
            )?);
        }
        global_idx += dblk_nelmts;
    }

    Ok(chunks)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Stamp the Jenkins checksum a real file would carry over
    /// `data[start..end]`, writing it at `end`. Hand-built fixtures need this
    /// now that the reader validates it, exactly as HDF5 writes it.
    fn stamp_checksum(data: &mut [u8], start: usize, end: usize) {
        let sum = crate::checksum::jenkins_lookup3(&data[start..end]);
        data[end..end + 4].copy_from_slice(&sum.to_le_bytes());
    }
    #[test]
    fn index_to_offsets_1d() {
        let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
        assert_eq!(g.offsets(0).unwrap(), vec![0]);
        assert_eq!(g.offsets(1).unwrap(), vec![20]);
        assert_eq!(g.offsets(4).unwrap(), vec![80]);
    }

    #[test]
    fn index_to_offsets_2d() {
        let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
        assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
        assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
        assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
    }

    #[test]
    fn parse_header_valid() {
        let os: u8 = 8;
        let ls: u8 = 8;
        let mut buf = vec![0u8; 256];
        buf[0..4].copy_from_slice(b"EAHD");
        buf[4] = 0; // version
        buf[5] = 0; // client_id = non-filtered
        buf[6] = 8; // element_size
        buf[7] = 10; // max_nelmts_bits
        buf[8] = 2; // idx_blk_elmts
        buf[9] = 4; // min_dblk_nelmts
        buf[10] = 2; // super_blk_min_nelmts
        buf[11] = 8; // max_dblk_nelmts_bits
        // 6 stats fields (each 8 bytes)
        buf[12..20].copy_from_slice(&0u64.to_le_bytes()); // stat[0]
        buf[20..28].copy_from_slice(&0u64.to_le_bytes()); // stat[1]
        buf[28..36].copy_from_slice(&0u64.to_le_bytes()); // stat[2]
        buf[36..44].copy_from_slice(&0u64.to_le_bytes()); // stat[3]
        buf[44..52].copy_from_slice(&5u64.to_le_bytes()); // stat[4] = num_elements
        buf[52..60].copy_from_slice(&0u64.to_le_bytes()); // stat[5]
        buf[60..68].copy_from_slice(&0x1000u64.to_le_bytes()); // index_block_address
        stamp_checksum(&mut buf, 0, 68);

        let hdr = ExtensibleArrayHeader::parse(&buf, 0, os, ls).unwrap();
        assert_eq!(hdr.client_id, 0);
        assert_eq!(hdr.element_size, 8);
        assert_eq!(hdr.idx_blk_elmts, 2);
        assert_eq!(hdr.min_dblk_nelmts, 4);
        assert_eq!(hdr.num_elements, 5);
        assert_eq!(hdr.index_block_address, 0x1000);
    }

    #[test]
    fn parse_header_invalid_signature() {
        let mut buf = vec![0u8; 256];
        buf[0..4].copy_from_slice(b"XXXX");
        let result = ExtensibleArrayHeader::parse(&buf, 0, 8, 8);
        assert!(result.is_err());
    }

    /// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
    #[test]
    fn parse_rejects_offset_overflow() {
        let buf = vec![0u8; 64];
        let result = ExtensibleArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
        assert!(result.is_err());
    }

    /// A near-`usize::MAX` index block address must error cleanly, not overflow/panic.
    #[test]
    fn read_rejects_index_block_offset_overflow() {
        let header = ExtensibleArrayHeader {
            client_id: 0,
            element_size: 8,
            max_nelmts_bits: 10,
            idx_blk_elmts: 2,
            min_dblk_nelmts: 4,
            super_blk_min_nelmts: 2,
            max_dblk_nelmts_bits: 8,
            num_elements: 5,
            index_block_address: (usize::MAX - 4) as u64,
        };
        let buf = vec![0u8; 64];
        let r = read_extensible_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
        assert!(r.is_err());
    }

    #[test]
    fn parse_header_invalid_version() {
        let mut buf = vec![0u8; 256];
        buf[0..4].copy_from_slice(b"EAHD");
        buf[4] = 1;
        let result = ExtensibleArrayHeader::parse(&buf, 0, 8, 8);
        assert!(result.is_err());
    }

    /// Build a synthetic Extensible Array with only inline elements (simplest case).
    /// All chunks fit in the index block.
    #[test]
    fn read_inline_only() {
        let os: u8 = 8;
        let ls: u8 = 8;
        let osv = os as usize;
        let num_chunks = 2usize;
        let chunk_byte_size = 20u64 * 8; // 20 elements × 8 bytes

        let mut file_data = vec![0u8; 0x3000];

        // AEHD at offset 0x100
        let aehd_offset = 0x100usize;
        let aeib_offset = 0x200usize;

        // Build AEHD
        file_data[aehd_offset..aehd_offset + 4].copy_from_slice(b"EAHD");
        file_data[aehd_offset + 4] = 0; // version
        file_data[aehd_offset + 5] = 0; // client_id = non-filtered
        file_data[aehd_offset + 6] = osv as u8; // element_size
        file_data[aehd_offset + 7] = 10; // max_nelmts_bits
        file_data[aehd_offset + 8] = num_chunks as u8; // idx_blk_elmts (all inline)
        file_data[aehd_offset + 9] = 4; // min_dblk_nelmts
        file_data[aehd_offset + 10] = 2; // super_blk_min_nelmts
        file_data[aehd_offset + 11] = 8; // max_dblk_nelmts_bits
        // 6 stats fields (each 8 bytes), nelmts at stat[4]
        file_data[aehd_offset + 44..aehd_offset + 52]
            .copy_from_slice(&(num_chunks as u64).to_le_bytes());
        file_data[aehd_offset + 60..aehd_offset + 68]
            .copy_from_slice(&(aeib_offset as u64).to_le_bytes());
        stamp_checksum(&mut file_data, aehd_offset, aehd_offset + 68);
        // checksum (4 bytes at +68) — not validated

        // Build AEIB at aeib_offset
        file_data[aeib_offset..aeib_offset + 4].copy_from_slice(b"EAIB");
        file_data[aeib_offset + 4] = 0; // version
        file_data[aeib_offset + 5] = 0; // client_id
        file_data[aeib_offset + 6..aeib_offset + 14]
            .copy_from_slice(&(aehd_offset as u64).to_le_bytes());

        // Inline elements
        let elem_start = aeib_offset + 6 + osv;
        let base_addr = 0x1000u64;
        for i in 0..num_chunks {
            let addr = base_addr + i as u64 * chunk_byte_size;
            let p = elem_start + i * osv;
            file_data[p..p + osv].copy_from_slice(&addr.to_le_bytes());
        }
        // The index block's checksum covers its prefix, every inline element
        // slot, and every data block and super block address slot:
        // ndblk_addrs = 2 * (sup_blk_min_data_ptrs - 1), and the super block
        // pointers make up the rest of nsblks levels.
        let sup_ptrs = file_data[aehd_offset + 10] as usize;
        let dmin = file_data[aehd_offset + 9] as usize;
        let nsblks = 1 + 10 - dmin.trailing_zeros() as usize;
        let ndblk_addrs = 2 * (sup_ptrs - 1);
        // Levels consumed by those direct data blocks (1, 1, 2, 2, ... per level).
        let mut consumed = 0usize;
        let mut levels = 0usize;
        while consumed < ndblk_addrs {
            consumed += 1 << (levels / 2);
            levels += 1;
        }
        let ib_end = elem_start + num_chunks * osv + (ndblk_addrs + nsblks - levels) * osv;
        stamp_checksum(&mut file_data, aeib_offset, ib_end);

        let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
        let ds_dims = vec![40u64]; // 2 chunks × 20 elements
        let chunk_dims = vec![20u32];
        let chunks = read_extensible_array_chunks(
            &file_data,
            &header,
            &ds_dims,
            None,
            &chunk_dims,
            8,
            os,
            ls,
        )
        .unwrap();

        assert_eq!(chunks.len(), 2);
        assert_eq!(chunks[0].address, base_addr);
        assert_eq!(chunks[0].offsets, vec![0]);
        assert_eq!(chunks[0].chunk_size, chunk_byte_size as u32);
        assert_eq!(chunks[1].address, base_addr + chunk_byte_size);
        assert_eq!(chunks[1].offsets, vec![20]);
    }

    /// A synthetic EA with inline elements + one direct data block: the
    /// file, with the header at 0x100 (8-byte offsets and lengths, 4 chunks
    /// of 10 elements from 0x1000 on).
    fn build_inline_plus_data_blocks() -> Vec<u8> {
        let os: u8 = 8;
        let osv = os as usize;
        let chunk_byte_size = 10u64 * 8; // 10 elements × 8 bytes
        let idx_blk_elmts = 2u8;
        let min_dblk_nelmts = 2u8;
        let sblk_min = 2u8;
        let total_chunks = 4usize; // 2 inline + 2 in data block (1 dblk from sb_level 0)

        let mut file_data = vec![0u8; 0x5000];
        let aehd_offset = 0x100usize;
        let aeib_offset = 0x200usize;
        let aedb_offset = 0x300usize;

        // EAHD
        file_data[aehd_offset..aehd_offset + 4].copy_from_slice(b"EAHD");
        file_data[aehd_offset + 4] = 0;
        file_data[aehd_offset + 5] = 0; // client_id
        file_data[aehd_offset + 6] = osv as u8; // element_size
        file_data[aehd_offset + 7] = 10;
        file_data[aehd_offset + 8] = idx_blk_elmts;
        file_data[aehd_offset + 9] = min_dblk_nelmts;
        file_data[aehd_offset + 10] = sblk_min;
        file_data[aehd_offset + 11] = 8;
        // 6 stats fields (each 8 bytes), nelmts at stat[4] (offset 12 + 4*8 = 44)
        file_data[aehd_offset + 44..aehd_offset + 52]
            .copy_from_slice(&(total_chunks as u64).to_le_bytes());
        // idx_blk_addr at offset 12 + 6*8 = 60
        file_data[aehd_offset + 60..aehd_offset + 68]
            .copy_from_slice(&(aeib_offset as u64).to_le_bytes());
        stamp_checksum(&mut file_data, aehd_offset, aehd_offset + 68);

        // AEIB
        file_data[aeib_offset..aeib_offset + 4].copy_from_slice(b"EAIB");
        file_data[aeib_offset + 4] = 0;
        file_data[aeib_offset + 5] = 0;
        file_data[aeib_offset + 6..aeib_offset + 14]
            .copy_from_slice(&(aehd_offset as u64).to_le_bytes());

        let mut pos = aeib_offset + 6 + osv;

        // Inline elements (2 chunks)
        let base_addr = 0x1000u64;
        for i in 0..idx_blk_elmts as usize {
            let addr = base_addr + i as u64 * chunk_byte_size;
            file_data[pos..pos + osv].copy_from_slice(&addr.to_le_bytes());
            pos += osv;
        }

        // Direct data block addresses. With sup_blk_min_data_ptrs = 2 the index
        // block holds 2 * (2 - 1) = 2 of them, which are the data blocks of
        // super block levels 0 and 1: one of `min_dblk_nelmts` elements, then
        // one of twice that (ndblks = 2^(u/2), dblk_nelmts = 2^((u+1)/2) * min).
        // Only the first is allocated here; the rest of the array is empty.
        let ndblk_addrs = 2 * (sblk_min as usize - 1);
        file_data[pos..pos + osv].copy_from_slice(&(aedb_offset as u64).to_le_bytes());
        pos += osv;
        for _ in 1..ndblk_addrs {
            file_data[pos..pos + osv].copy_from_slice(&u64::MAX.to_le_bytes());
            pos += osv;
        }
        // Super block addresses fill the remaining levels; all unallocated.
        let nsblks = 1 + 10 - (min_dblk_nelmts as usize).trailing_zeros() as usize;
        let mut consumed = 0usize;
        let mut levels = 0usize;
        while consumed < ndblk_addrs {
            consumed += 1 << (levels / 2);
            levels += 1;
        }
        for _ in 0..(nsblks - levels) {
            file_data[pos..pos + osv].copy_from_slice(&u64::MAX.to_le_bytes());
            pos += osv;
        }
        stamp_checksum(&mut file_data, aeib_offset, pos);

        // EADB holding the first data block's `min_dblk_nelmts` elements.
        file_data[aedb_offset..aedb_offset + 4].copy_from_slice(b"EADB");
        file_data[aedb_offset + 4] = 0;
        file_data[aedb_offset + 5] = 0;
        file_data[aedb_offset + 6..aedb_offset + 14]
            .copy_from_slice(&(aehd_offset as u64).to_le_bytes());
        // Block offset field: ceil(max_nelmts_bits / 8) bytes, zero here.
        let blk_off_size = (10usize).div_ceil(8);
        let db_elems = aedb_offset + 6 + osv + blk_off_size;
        let mut dbpos = db_elems;
        for i in 0..min_dblk_nelmts as usize {
            let addr = base_addr + (idx_blk_elmts as u64 + i as u64) * chunk_byte_size;
            file_data[dbpos..dbpos + osv].copy_from_slice(&addr.to_le_bytes());
            dbpos += osv;
        }
        stamp_checksum(&mut file_data, aedb_offset, dbpos);
        file_data
    }

    /// Build a synthetic EA with inline elements + one direct data block.
    #[test]
    fn read_inline_plus_data_blocks() {
        let (os, ls) = (8u8, 8u8);
        let chunk_byte_size = 10u64 * 8;
        let base_addr = 0x1000u64;
        let file_data = build_inline_plus_data_blocks();
        let header = ExtensibleArrayHeader::parse(&file_data, 0x100, os, ls).unwrap();
        let ds_dims = vec![40u64];
        let chunk_dims = vec![10u32];
        let chunks = read_extensible_array_chunks(
            &file_data,
            &header,
            &ds_dims,
            None,
            &chunk_dims,
            8,
            os,
            ls,
        )
        .unwrap();

        assert_eq!(chunks.len(), 4);
        for (i, c) in chunks.iter().enumerate() {
            assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size);
            assert_eq!(c.offsets, vec![i as u64 * 10]);
        }
    }

    /// Test serialized_size computation.
    #[test]
    fn header_serialized_size() {
        // 12 fixed + 6*8 stats + 8 addr + 4 checksum = 72
        assert_eq!(ExtensibleArrayHeader::serialized_size(8, 8), 72);
        // 12 fixed + 6*4 stats + 4 addr + 4 checksum = 44
        assert_eq!(ExtensibleArrayHeader::serialized_size(4, 4), 44);
    }

    /// Verify read_element for unallocated slots.
    #[test]
    fn read_element_unallocated() {
        let data = vec![0xFFu8; 16];
        let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
        let (info, consumed) =
            read_element(&Window::whole(&data), 0, 0, 8, 8, 80, 0, &grid).unwrap();
        assert!(info.is_none());
        assert_eq!(consumed, 8);
    }

    /// Verify filtered element reading.
    #[test]
    fn read_element_filtered() {
        let os: u8 = 8;
        let chunk_size_bytes = 4usize;
        let elem_size = os as usize + chunk_size_bytes + 4;
        let mut data = vec![0u8; elem_size + 16];
        // Address
        data[0..8].copy_from_slice(&0x2000u64.to_le_bytes());
        // Compressed size (4 bytes LE)
        data[8..12].copy_from_slice(&120u32.to_le_bytes());
        // Filter mask
        data[12..16].copy_from_slice(&0u32.to_le_bytes());

        let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
        let (info, consumed) = read_element(
            &Window::whole(&data),
            0,
            1,
            elem_size as u8,
            os,
            80,
            2,
            &grid,
        )
        .unwrap();
        let ci = info.unwrap();
        assert_eq!(ci.address, 0x2000);
        assert_eq!(ci.chunk_size, 120);
        assert_eq!(ci.filter_mask, 0);
        assert_eq!(ci.offsets, vec![20]);
        assert_eq!(consumed, elem_size);
    }

    /// The Storage path reads exactly what the slice path reads: the array
    /// whole, cut at every length through its structures, and with a byte
    /// damaged in each of them, through a read_at-only CountingStorage.
    #[test]
    fn storage_reads_match_slice_reads() {
        use crate::storage::CountingStorage;
        let full = build_inline_plus_data_blocks();
        let mut files = Vec::new();
        for cut in 0x100..0x340 {
            files.push(full[..cut].to_vec());
        }
        for at in [0x104, 0x150, 0x204, 0x216, 0x230, 0x304, 0x318] {
            let mut damaged = full.clone();
            damaged[at] ^= 1;
            files.push(damaged);
        }
        files.push(full);
        let mut compared = 0;
        for f in files {
            let storage = CountingStorage::new(f.clone());
            let want = ExtensibleArrayHeader::parse(&f, 0x100, 8, 8);
            let got = ExtensibleArrayHeader::parse_in(&storage, 0x100, 8, 8);
            assert_eq!(format!("{got:?}"), format!("{want:?}"));
            let Ok(h) = want else { continue };
            for dims in [&[40u64][..], &[25]] {
                let want = read_extensible_array_chunks(&f, &h, dims, None, &[10], 8, 8, 8);
                let got = read_extensible_array_chunks_in(&storage, &h, dims, None, &[10], 8, 8, 8);
                assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len());
                compared += 1;
            }
        }
        assert!(compared > 100);
    }
}