osobhandClaude Opus 5
b41272487a
Every structure in both chunk indexes — header, index block, super block, data block and each data block page — carries a Jenkins lookup3 checksum, and all of them were parsed past and ignored. What that costs is not a warning but correct data. Flip one low bit of a chunk address and the index still has the right shape, the address still lands inside the file, and the reader returns whatever bytes now sit there as that chunk's contents. Nothing else in the parse can tell. Verified in both directions. The checksums accept files written by HDF5 2.0 from 100 to 200 000 chunks — dense, sparse, gzip-filtered and paged — which also confirms the block layouts byte for byte, since a wrong offset would fail every file. And an interop test corrupts an address to check the read fails instead of returning data: removing the verification makes that test fail with "corruption produced data instead of an error", which is what it is there to prove. The first version of that test passed with verification disabled — it corrupted a byte a structural check already rejected, so it proved nothing. Worth recording, since a test that passes for the wrong reason looks exactly like coverage. Hand-built fixtures now stamp real checksums, as HDF5 writers do, and the Extensible Array ones no longer describe the superseded layout. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1094 lines
40 KiB
Rust
1094 lines
40 KiB
Rust
//! HDF5 Extensible Array index parsing for chunked datasets (v4 index type 4).
|
||
//!
|
||
//! Extensible Arrays are used for datasets with exactly one unlimited dimension.
|
||
//! Structures: AEHD (header), AEIB (index block), AEDB (data block), AESB (super block).
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
extern crate alloc;
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{format, vec, vec::Vec};
|
||
|
||
use crate::chunked_read::ChunkInfo;
|
||
use crate::error::FormatError;
|
||
|
||
/// Verify the Jenkins lookup3 checksum stored immediately after
|
||
/// `data[start..end]`, as every Extensible Array structure carries one.
|
||
///
|
||
/// A corrupt chunk index yields addresses pointing at the wrong bytes, so a
|
||
/// mismatch is an error: otherwise the damage surfaces as plausible data read
|
||
/// from the wrong chunk.
|
||
#[cfg(feature = "checksum")]
|
||
fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> {
|
||
ensure_len(data, end, 4)?;
|
||
let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
|
||
let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
if computed != stored {
|
||
return Err(FormatError::ChecksumMismatch {
|
||
expected: stored,
|
||
computed,
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(not(feature = "checksum"))]
|
||
fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> {
|
||
Ok(())
|
||
}
|
||
|
||
/// Parsed Extensible Array header (AEHD).
|
||
#[derive(Debug, Clone)]
|
||
pub struct ExtensibleArrayHeader {
|
||
/// Client ID: 0 = non-filtered chunks, 1 = filtered chunks.
|
||
pub client_id: u8,
|
||
/// Size of each array element in bytes.
|
||
pub element_size: u8,
|
||
/// Max number of elements bits (log2 of the max number of data block elements per page).
|
||
pub max_nelmts_bits: u8,
|
||
/// Number of elements in the index block.
|
||
pub idx_blk_elmts: u8,
|
||
/// Minimum number of data block elements.
|
||
pub min_dblk_nelmts: u8,
|
||
/// Minimum number of elements in a super block.
|
||
pub super_blk_min_nelmts: u8,
|
||
/// Max number of data block elements bits.
|
||
pub max_dblk_nelmts_bits: u8,
|
||
/// Total number of elements stored.
|
||
pub num_elements: u64,
|
||
/// Address of the index block.
|
||
pub index_block_address: u64,
|
||
}
|
||
|
||
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
let s = size as usize;
|
||
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: pos.saturating_add(s),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match size {
|
||
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
||
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
||
8 => u64::from_le_bytes([
|
||
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
||
]),
|
||
_ => return Err(FormatError::InvalidOffsetSize(size)),
|
||
})
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
||
if offset
|
||
.checked_add(needed)
|
||
.is_none_or(|end| end > data.len())
|
||
{
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: offset.saturating_add(needed),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn is_undefined_addr(addr: u64, offset_size: u8) -> bool {
|
||
match offset_size {
|
||
2 => addr == 0xFFFF,
|
||
4 => addr == 0xFFFF_FFFF,
|
||
8 => addr == 0xFFFF_FFFF_FFFF_FFFF,
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
|
||
let s = size as usize;
|
||
if pos + s > data.len() {
|
||
return false;
|
||
}
|
||
data[pos..pos + s].iter().all(|&b| b == 0xFF)
|
||
}
|
||
|
||
fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
|
||
if size > 8 || data.len() < size {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid variable-length size".into(),
|
||
));
|
||
}
|
||
let mut val = 0u64;
|
||
for (i, &byte) in data.iter().enumerate().take(size) {
|
||
val |= (byte as u64) << (i * 8);
|
||
}
|
||
Ok(val)
|
||
}
|
||
|
||
impl ExtensibleArrayHeader {
|
||
/// Parse an Extensible Array header from file data at the given offset.
|
||
pub fn parse(
|
||
file_data: &[u8],
|
||
offset: usize,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Self, FormatError> {
|
||
// EAHD: signature(4) + version(1) + client_id(1) + element_size(1) +
|
||
// max_nelmts_bits(1) + idx_blk_elmts(1) + min_dblk_nelmts(1) +
|
||
// super_blk_min_nelmts(1) + max_dblk_nelmts_bits(1) +
|
||
// 6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4)
|
||
let min_size =
|
||
4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4;
|
||
ensure_len(file_data, offset, min_size)?;
|
||
|
||
let d = &file_data[offset..];
|
||
if &d[0..4] != b"EAHD" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Extensible Array header signature".into(),
|
||
));
|
||
}
|
||
|
||
let version = d[4];
|
||
if version != 0 {
|
||
return Err(FormatError::ChunkedReadError(format!(
|
||
"unsupported Extensible Array header version: {version}"
|
||
)));
|
||
}
|
||
|
||
let client_id = d[5];
|
||
let element_size = d[6];
|
||
let max_nelmts_bits = d[7];
|
||
let idx_blk_elmts = d[8];
|
||
let min_dblk_nelmts = d[9];
|
||
let super_blk_min_nelmts = d[10];
|
||
let max_dblk_nelmts_bits = d[11];
|
||
|
||
let mut pos = 12;
|
||
// 6 stats fields: [0] unknown, [1] unknown, [2] nsuper_blks_created,
|
||
// [3] super_blk_size, [4] nelmts, [5] max_idx_set
|
||
// We only need nelmts (field[4]) and skip the rest.
|
||
let ls = length_size as usize;
|
||
pos += 4 * ls; // skip first 4 stats fields
|
||
let num_elements = read_offset(d, pos, length_size)?;
|
||
pos += ls; // skip nelmts
|
||
pos += ls; // skip max_idx_set (6th stats field)
|
||
let index_block_address = read_offset(d, pos, offset_size)?;
|
||
pos += offset_size as usize;
|
||
verify_checksum(file_data, offset, offset + pos)?;
|
||
|
||
Ok(ExtensibleArrayHeader {
|
||
client_id,
|
||
element_size,
|
||
max_nelmts_bits,
|
||
idx_blk_elmts,
|
||
min_dblk_nelmts,
|
||
super_blk_min_nelmts,
|
||
max_dblk_nelmts_bits,
|
||
num_elements,
|
||
index_block_address,
|
||
})
|
||
}
|
||
|
||
/// Compute the size of this header in bytes (for write support).
|
||
pub fn serialized_size(offset_size: u8, length_size: u8) -> usize {
|
||
4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4
|
||
}
|
||
}
|
||
|
||
/// Read a single element from the extensible array element data.
|
||
/// Returns (chunk_info, bytes_consumed) or None if unallocated.
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn read_element(
|
||
data: &[u8],
|
||
pos: usize,
|
||
client_id: u8,
|
||
element_size: u8,
|
||
offset_size: u8,
|
||
chunk_byte_size: u64,
|
||
linear_index: usize,
|
||
num_chunks_per_dim: &[u64],
|
||
chunk_dimensions: &[u32],
|
||
) -> Result<(Option<ChunkInfo>, usize), FormatError> {
|
||
let os = offset_size as usize;
|
||
|
||
if client_id == 0 {
|
||
// Non-filtered: just address
|
||
if pos + os > data.len() {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: pos + os,
|
||
available: data.len(),
|
||
});
|
||
}
|
||
if is_undefined(data, pos, offset_size) {
|
||
return Ok((None, os));
|
||
}
|
||
let address = read_offset(data, pos, offset_size)?;
|
||
let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
|
||
Ok((
|
||
Some(ChunkInfo {
|
||
chunk_size: chunk_byte_size as u32,
|
||
filter_mask: 0,
|
||
offsets,
|
||
address,
|
||
}),
|
||
os,
|
||
))
|
||
} else {
|
||
// Filtered: address + compressed_size + filter_mask
|
||
let es = element_size as usize;
|
||
if es < os + 4 {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"element_size too small for filtered element".into(),
|
||
));
|
||
}
|
||
let chunk_size_bytes = es - os - 4;
|
||
let elem_total = os + chunk_size_bytes + 4;
|
||
if pos
|
||
.checked_add(elem_total)
|
||
.is_none_or(|end| end > data.len())
|
||
{
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: pos.saturating_add(elem_total),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
if is_undefined(data, pos, offset_size) {
|
||
return Ok((None, elem_total));
|
||
}
|
||
let address = read_offset(data, pos, offset_size)?;
|
||
let chunk_size = read_variable_length(&data[pos + os..], chunk_size_bytes)?;
|
||
let fm_off = pos + os + chunk_size_bytes;
|
||
let filter_mask = u32::from_le_bytes([
|
||
data[fm_off],
|
||
data[fm_off + 1],
|
||
data[fm_off + 2],
|
||
data[fm_off + 3],
|
||
]);
|
||
let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
|
||
Ok((
|
||
Some(ChunkInfo {
|
||
chunk_size: chunk_size as u32,
|
||
filter_mask,
|
||
offsets,
|
||
address,
|
||
}),
|
||
elem_total,
|
||
))
|
||
}
|
||
}
|
||
|
||
/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
|
||
fn index_to_chunk_offsets(
|
||
index: usize,
|
||
num_chunks_per_dim: &[u64],
|
||
chunk_dimensions: &[u32],
|
||
) -> Vec<u64> {
|
||
let rank = num_chunks_per_dim.len();
|
||
let mut offsets = vec![0u64; rank];
|
||
let mut remaining = index as u64;
|
||
for d in (0..rank).rev() {
|
||
let nchunks = num_chunks_per_dim[d];
|
||
if nchunks == 0 {
|
||
continue;
|
||
}
|
||
let chunk_idx = remaining % nchunks;
|
||
remaining /= nchunks;
|
||
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
|
||
}
|
||
offsets
|
||
}
|
||
|
||
/// Collect elements from a data block at the given offset.
|
||
#[allow(clippy::too_many_arguments)]
|
||
/// Layout of super block `u`, per the HDF5 spec: the number of data blocks it
|
||
/// owns and how many elements each of them holds.
|
||
///
|
||
/// `ndblks` and `dblk_nelmts` each double every *other* level, a half-step
|
||
/// apart, so the blocks grow as 1x16, 1x32, 2x32, 2x64, 4x64 ... for a
|
||
/// 16-element minimum. Treating either as doubling every level (the previous
|
||
/// implementation) puts every element after the first data block at the wrong
|
||
/// index.
|
||
fn sblk_info(u: usize, data_blk_min_elmts: usize) -> Option<(usize, usize)> {
|
||
let ndblks = 1usize.checked_shl((u / 2) as u32)?;
|
||
let dblk_nelmts = 1usize
|
||
.checked_shl(u.div_ceil(2) as u32)?
|
||
.checked_mul(data_blk_min_elmts)?;
|
||
Some((ndblks, dblk_nelmts))
|
||
}
|
||
|
||
/// Width of the "offset of the block in the array" field carried by super and
|
||
/// data blocks (`hdr->arr_off_size`).
|
||
fn arr_off_size(header: &ExtensibleArrayHeader) -> usize {
|
||
(header.max_nelmts_bits as usize).div_ceil(8)
|
||
}
|
||
|
||
/// Elements per data block page, once a data block is large enough to be paged.
|
||
fn page_nelmts(header: &ExtensibleArrayHeader) -> Option<usize> {
|
||
1usize.checked_shl(u32::from(header.max_dblk_nelmts_bits))
|
||
}
|
||
|
||
/// Read the elements of one data block (EADB).
|
||
///
|
||
/// `page_init` is the owning super block's page-init bitmap and `first_page`
|
||
/// this block's first bit in it; both are only consulted when the block is
|
||
/// paged. The bitmap lives in the super block, not here — a paged data block
|
||
/// stores only its prefix, then one slot per page.
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn read_data_block_elements(
|
||
file_data: &[u8],
|
||
db_offset: usize,
|
||
nelmts: usize,
|
||
header: &ExtensibleArrayHeader,
|
||
offset_size: u8,
|
||
chunk_byte_size: u64,
|
||
start_index: usize,
|
||
num_chunks_per_dim: &[u64],
|
||
chunk_dimensions: &[u32],
|
||
page_init: &[u8],
|
||
first_page: usize,
|
||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||
// EADB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
|
||
// + block offset(arr_off_size)
|
||
let db_header_size = 4 + 1 + 1 + offset_size as usize + arr_off_size(header);
|
||
ensure_len(file_data, db_offset, db_header_size)?;
|
||
|
||
if &file_data[db_offset..db_offset + 4] != b"EADB" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Extensible Array data block signature".into(),
|
||
));
|
||
}
|
||
|
||
let mut pos = db_offset + db_header_size;
|
||
let page = page_nelmts(header).ok_or_else(|| {
|
||
FormatError::Overflow("Extensible Array page element count overflows usize".into())
|
||
})?;
|
||
|
||
let mut chunks = Vec::new();
|
||
let read_run = |from: usize,
|
||
count: usize,
|
||
first_index: usize,
|
||
chunks: &mut Vec<ChunkInfo>|
|
||
-> Result<usize, FormatError> {
|
||
let mut p = from;
|
||
for i in 0..count {
|
||
let (info, consumed) = read_element(
|
||
file_data,
|
||
p,
|
||
header.client_id,
|
||
header.element_size,
|
||
offset_size,
|
||
chunk_byte_size,
|
||
first_index + i,
|
||
num_chunks_per_dim,
|
||
chunk_dimensions,
|
||
)?;
|
||
if let Some(ci) = info {
|
||
chunks.push(ci);
|
||
}
|
||
p += consumed;
|
||
}
|
||
Ok(p)
|
||
};
|
||
|
||
if nelmts <= page {
|
||
// Prefix and elements are covered by one checksum.
|
||
let elem_bytes = if header.client_id == 0 {
|
||
offset_size as usize
|
||
} else {
|
||
header.element_size as usize
|
||
};
|
||
let end = nelmts
|
||
.checked_mul(elem_bytes)
|
||
.and_then(|b| pos.checked_add(b))
|
||
.ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?;
|
||
verify_checksum(file_data, db_offset, end)?;
|
||
read_run(pos, nelmts, start_index, &mut chunks)?;
|
||
return Ok(chunks);
|
||
}
|
||
|
||
// Paged: the prefix ends with its own checksum, then one slot per page,
|
||
// each holding `page` elements followed by a checksum. Pages whose bit is
|
||
// clear were never written; their slot still occupies the file, so stride
|
||
// over it rather than reading zeros as addresses.
|
||
verify_checksum(file_data, db_offset, pos)?;
|
||
pos += 4;
|
||
let elem_bytes = if header.client_id == 0 {
|
||
offset_size as usize
|
||
} else {
|
||
header.element_size as usize
|
||
};
|
||
let page_stride = page
|
||
.checked_mul(elem_bytes)
|
||
.and_then(|b| b.checked_add(4))
|
||
.ok_or_else(|| FormatError::Overflow("Extensible Array page stride".into()))?;
|
||
let npages = nelmts.div_ceil(page);
|
||
for p in 0..npages {
|
||
// One bit per page across the whole super block, packed contiguously
|
||
// and MSB-first within each byte, as H5VM_bit_get reads it.
|
||
let bit = first_page + p;
|
||
let initialised = page_init
|
||
.get(bit / 8)
|
||
.is_some_and(|byte| byte & (0x80 >> (bit % 8)) != 0);
|
||
if initialised {
|
||
let count = core::cmp::min(page, nelmts - p * page);
|
||
// Each page carries its own checksum, over a full page's worth of
|
||
// slots even when the last one holds fewer live elements.
|
||
verify_checksum(file_data, pos, pos + page * elem_bytes)?;
|
||
read_run(pos, count, start_index + p * page, &mut chunks)?;
|
||
}
|
||
pos = pos
|
||
.checked_add(page_stride)
|
||
.ok_or_else(|| FormatError::Overflow("Extensible Array page offset".into()))?;
|
||
}
|
||
|
||
Ok(chunks)
|
||
}
|
||
|
||
/// Read chunk records from an Extensible Array.
|
||
///
|
||
/// Traverses AEHD -> AEIB -> AEDB/AESB to collect all allocated chunks.
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn read_extensible_array_chunks(
|
||
file_data: &[u8],
|
||
header: &ExtensibleArrayHeader,
|
||
dataset_dims: &[u64],
|
||
chunk_dimensions: &[u32],
|
||
element_size: u32,
|
||
offset_size: u8,
|
||
_length_size: u8,
|
||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let rank = chunk_dimensions.len();
|
||
let os = offset_size as usize;
|
||
|
||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||
for d in 0..rank {
|
||
let ch_dim = chunk_dimensions[d] as u64;
|
||
if ch_dim == 0 {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"chunk dimension is zero".into(),
|
||
));
|
||
}
|
||
let ds_dim = dataset_dims[d];
|
||
num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
|
||
}
|
||
|
||
let chunk_byte_size: u64 =
|
||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||
|
||
// Parse index block (EAIB): signature(4) + version(1) + client_id(1)
|
||
// + header address(offset_size), then the inline elements, then the
|
||
// direct data block addresses, then the super block addresses.
|
||
let ib_offset = header.index_block_address as usize;
|
||
let ib_header_size = 4 + 1 + 1 + os;
|
||
ensure_len(file_data, ib_offset, ib_header_size)?;
|
||
|
||
if &file_data[ib_offset..ib_offset + 4] != b"EAIB" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Extensible Array index block signature".into(),
|
||
));
|
||
}
|
||
let mut pos = ib_offset + ib_header_size;
|
||
|
||
let mut chunks = Vec::new();
|
||
let total_elements = header.num_elements as usize;
|
||
|
||
let dmin = header.min_dblk_nelmts as usize;
|
||
if dmin == 0 || !dmin.is_power_of_two() {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Extensible Array data block minimum is not a power of two".into(),
|
||
));
|
||
}
|
||
// nsblks = 1 + (max_nelmts_bits - log2(data_blk_min_elmts)), and the index
|
||
// block holds 2 * (sup_blk_min_data_ptrs - 1) data block addresses.
|
||
let log2_dmin = dmin.trailing_zeros() as usize;
|
||
let nsblks = 1 + (header.max_nelmts_bits as usize).saturating_sub(log2_dmin);
|
||
let ndblk_addrs = 2 * (header.super_blk_min_nelmts as usize).saturating_sub(1);
|
||
|
||
// The data blocks listed directly in the index block are the first
|
||
// `ndblk_addrs` in super-block order, each sized by the level it belongs
|
||
// to; the super block addresses that follow resume at the next level.
|
||
let mut direct: Vec<usize> = Vec::with_capacity(ndblk_addrs);
|
||
let mut level = 0usize;
|
||
while direct.len() < ndblk_addrs {
|
||
if level >= nsblks {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Extensible Array index block claims more data blocks than the array has".into(),
|
||
));
|
||
}
|
||
let (ndblks, dblk_nelmts) = sblk_info(level, dmin).ok_or_else(|| {
|
||
FormatError::Overflow("Extensible Array super block layout overflows usize".into())
|
||
})?;
|
||
for _ in 0..ndblks {
|
||
direct.push(dblk_nelmts);
|
||
}
|
||
level += 1;
|
||
}
|
||
if direct.len() != ndblk_addrs {
|
||
// A partial level in the index block is not a layout HDF5 produces,
|
||
// and guessing where the super blocks resume would misplace elements.
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Extensible Array index block ends mid super block".into(),
|
||
));
|
||
}
|
||
|
||
// One checksum covers the prefix, every inline element slot, and every
|
||
// data block and super block address.
|
||
let elem_bytes = if header.client_id == 0 {
|
||
os
|
||
} else {
|
||
header.element_size as usize
|
||
};
|
||
let ib_end = (header.idx_blk_elmts as usize)
|
||
.checked_mul(elem_bytes)
|
||
.and_then(|b| pos.checked_add(b))
|
||
.and_then(|p| {
|
||
ndblk_addrs
|
||
.checked_add(nsblks - level)
|
||
.and_then(|n| n.checked_mul(os).and_then(|b| p.checked_add(b)))
|
||
})
|
||
.ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?;
|
||
verify_checksum(file_data, ib_offset, ib_end)?;
|
||
|
||
// 1. Elements stored inline in the index block.
|
||
let n_inline = (header.idx_blk_elmts as usize).min(total_elements);
|
||
for i in 0..n_inline {
|
||
let (info, consumed) = read_element(
|
||
file_data,
|
||
pos,
|
||
header.client_id,
|
||
header.element_size,
|
||
offset_size,
|
||
chunk_byte_size,
|
||
i,
|
||
&num_chunks_per_dim,
|
||
chunk_dimensions,
|
||
)?;
|
||
if let Some(ci) = info {
|
||
chunks.push(ci);
|
||
}
|
||
pos += consumed;
|
||
}
|
||
let mut global_index = n_inline;
|
||
if global_index >= total_elements {
|
||
return Ok(chunks);
|
||
}
|
||
|
||
// 2. Data blocks listed directly in the index block.
|
||
for &dblk_nelmts in &direct {
|
||
if global_index >= total_elements {
|
||
return Ok(chunks);
|
||
}
|
||
ensure_len(file_data, pos, os)?;
|
||
let addr = read_offset(file_data, pos, offset_size)?;
|
||
pos += os;
|
||
if !is_undefined_addr(addr, offset_size) {
|
||
if dblk_nelmts > page_nelmts(header).unwrap_or(usize::MAX) {
|
||
// Would need a page-init bitmap, which only a super block
|
||
// carries. HDF5 never pages these small early blocks.
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Extensible Array index block references a paged data block".into(),
|
||
));
|
||
}
|
||
chunks.extend(read_data_block_elements(
|
||
file_data,
|
||
addr as usize,
|
||
dblk_nelmts,
|
||
header,
|
||
offset_size,
|
||
chunk_byte_size,
|
||
global_index,
|
||
&num_chunks_per_dim,
|
||
chunk_dimensions,
|
||
&[],
|
||
0,
|
||
)?);
|
||
}
|
||
global_index += dblk_nelmts;
|
||
}
|
||
|
||
// 3. Everything else lives in super blocks, one address per remaining
|
||
// level, starting at the level after the direct data blocks.
|
||
for u in level..nsblks {
|
||
if global_index >= total_elements {
|
||
break;
|
||
}
|
||
ensure_len(file_data, pos, os)?;
|
||
let sb_addr = read_offset(file_data, pos, offset_size)?;
|
||
pos += os;
|
||
let (ndblks, dblk_nelmts) = sblk_info(u, dmin).ok_or_else(|| {
|
||
FormatError::Overflow("Extensible Array super block layout overflows usize".into())
|
||
})?;
|
||
if !is_undefined_addr(sb_addr, offset_size) {
|
||
chunks.extend(read_super_block(
|
||
file_data,
|
||
sb_addr as usize,
|
||
ndblks,
|
||
dblk_nelmts,
|
||
header,
|
||
offset_size,
|
||
chunk_byte_size,
|
||
global_index,
|
||
&num_chunks_per_dim,
|
||
chunk_dimensions,
|
||
)?);
|
||
}
|
||
global_index =
|
||
global_index.saturating_add(ndblks.checked_mul(dblk_nelmts).ok_or_else(|| {
|
||
FormatError::Overflow("Extensible Array super block span".into())
|
||
})?);
|
||
}
|
||
|
||
Ok(chunks)
|
||
}
|
||
|
||
/// Read a super block (EASB) and the data blocks it owns.
|
||
///
|
||
/// On disk: signature(4) + version(1) + client_id(1) + header address
|
||
/// + block offset + the page-init bitmap for every data block it owns
|
||
/// + one address per data block + checksum.
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn read_super_block(
|
||
file_data: &[u8],
|
||
sb_offset: usize,
|
||
ndblks: usize,
|
||
dblk_nelmts: usize,
|
||
header: &ExtensibleArrayHeader,
|
||
offset_size: u8,
|
||
chunk_byte_size: u64,
|
||
start_index: usize,
|
||
num_chunks_per_dim: &[u64],
|
||
chunk_dimensions: &[u32],
|
||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let os = offset_size as usize;
|
||
let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header);
|
||
ensure_len(file_data, sb_offset, sb_header_size)?;
|
||
|
||
if &file_data[sb_offset..sb_offset + 4] != b"EASB" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Extensible Array super block signature".into(),
|
||
));
|
||
}
|
||
|
||
// Page-init bitmap: one bit per page, `npages` bits per data block, packed
|
||
// contiguously. HDF5 sizes the buffer `ndblks * ceil(npages / 8)`, which
|
||
// is bigger than the bits need when `npages` is not a multiple of eight.
|
||
// Zero-sized unless this level's data blocks are paged.
|
||
let page = page_nelmts(header).ok_or_else(|| {
|
||
FormatError::Overflow("Extensible Array page element count overflows usize".into())
|
||
})?;
|
||
let npages = if dblk_nelmts > page {
|
||
dblk_nelmts / page
|
||
} else {
|
||
0
|
||
};
|
||
let per_dblk_bitmap = npages.div_ceil(8);
|
||
let bitmap_bytes = per_dblk_bitmap
|
||
.checked_mul(ndblks)
|
||
.ok_or_else(|| FormatError::Overflow("Extensible Array page bitmap size".into()))?;
|
||
let bitmap_start = sb_offset + sb_header_size;
|
||
ensure_len(file_data, bitmap_start, bitmap_bytes)?;
|
||
let bitmap = &file_data[bitmap_start..bitmap_start + bitmap_bytes];
|
||
|
||
let mut pos = bitmap_start + bitmap_bytes;
|
||
let mut chunks = Vec::new();
|
||
let mut global_idx = start_index;
|
||
|
||
// One checksum covers the prefix, the bitmap and every data block address.
|
||
let sb_end = ndblks
|
||
.checked_mul(os)
|
||
.and_then(|b| pos.checked_add(b))
|
||
.ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?;
|
||
verify_checksum(file_data, sb_offset, sb_end)?;
|
||
|
||
for i in 0..ndblks {
|
||
ensure_len(file_data, pos, os)?;
|
||
let addr = read_offset(file_data, pos, offset_size)?;
|
||
pos += os;
|
||
if !is_undefined_addr(addr, offset_size) {
|
||
chunks.extend(read_data_block_elements(
|
||
file_data,
|
||
addr as usize,
|
||
dblk_nelmts,
|
||
header,
|
||
offset_size,
|
||
chunk_byte_size,
|
||
global_idx,
|
||
num_chunks_per_dim,
|
||
chunk_dimensions,
|
||
bitmap,
|
||
i * npages,
|
||
)?);
|
||
}
|
||
global_idx += dblk_nelmts;
|
||
}
|
||
|
||
Ok(chunks)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Stamp the Jenkins checksum a real file would carry over
|
||
/// `data[start..end]`, writing it at `end`. Hand-built fixtures need this
|
||
/// now that the reader validates it, exactly as HDF5 writes it.
|
||
fn stamp_checksum(data: &mut [u8], start: usize, end: usize) {
|
||
let sum = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
data[end..end + 4].copy_from_slice(&sum.to_le_bytes());
|
||
}
|
||
#[test]
|
||
fn index_to_offsets_1d() {
|
||
let num_chunks = vec![5u64];
|
||
let chunk_dims = vec![20u32];
|
||
assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||
vec![20]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
|
||
vec![80]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn index_to_offsets_2d() {
|
||
let num_chunks = vec![3u64, 2];
|
||
let chunk_dims = vec![4u32, 3];
|
||
assert_eq!(
|
||
index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
|
||
vec![0, 0]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||
vec![0, 3]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
|
||
vec![4, 0]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_header_valid() {
|
||
let os: u8 = 8;
|
||
let ls: u8 = 8;
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"EAHD");
|
||
buf[4] = 0; // version
|
||
buf[5] = 0; // client_id = non-filtered
|
||
buf[6] = 8; // element_size
|
||
buf[7] = 10; // max_nelmts_bits
|
||
buf[8] = 2; // idx_blk_elmts
|
||
buf[9] = 4; // min_dblk_nelmts
|
||
buf[10] = 2; // super_blk_min_nelmts
|
||
buf[11] = 8; // max_dblk_nelmts_bits
|
||
// 6 stats fields (each 8 bytes)
|
||
buf[12..20].copy_from_slice(&0u64.to_le_bytes()); // stat[0]
|
||
buf[20..28].copy_from_slice(&0u64.to_le_bytes()); // stat[1]
|
||
buf[28..36].copy_from_slice(&0u64.to_le_bytes()); // stat[2]
|
||
buf[36..44].copy_from_slice(&0u64.to_le_bytes()); // stat[3]
|
||
buf[44..52].copy_from_slice(&5u64.to_le_bytes()); // stat[4] = num_elements
|
||
buf[52..60].copy_from_slice(&0u64.to_le_bytes()); // stat[5]
|
||
buf[60..68].copy_from_slice(&0x1000u64.to_le_bytes()); // index_block_address
|
||
stamp_checksum(&mut buf, 0, 68);
|
||
|
||
let hdr = ExtensibleArrayHeader::parse(&buf, 0, os, ls).unwrap();
|
||
assert_eq!(hdr.client_id, 0);
|
||
assert_eq!(hdr.element_size, 8);
|
||
assert_eq!(hdr.idx_blk_elmts, 2);
|
||
assert_eq!(hdr.min_dblk_nelmts, 4);
|
||
assert_eq!(hdr.num_elements, 5);
|
||
assert_eq!(hdr.index_block_address, 0x1000);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_header_invalid_signature() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"XXXX");
|
||
let result = ExtensibleArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn parse_rejects_offset_overflow() {
|
||
let buf = vec![0u8; 64];
|
||
let result = ExtensibleArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` index block address must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn read_rejects_index_block_offset_overflow() {
|
||
let header = ExtensibleArrayHeader {
|
||
client_id: 0,
|
||
element_size: 8,
|
||
max_nelmts_bits: 10,
|
||
idx_blk_elmts: 2,
|
||
min_dblk_nelmts: 4,
|
||
super_blk_min_nelmts: 2,
|
||
max_dblk_nelmts_bits: 8,
|
||
num_elements: 5,
|
||
index_block_address: (usize::MAX - 4) as u64,
|
||
};
|
||
let buf = vec![0u8; 64];
|
||
let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn parse_header_invalid_version() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"EAHD");
|
||
buf[4] = 1;
|
||
let result = ExtensibleArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// Build a synthetic Extensible Array with only inline elements (simplest case).
|
||
/// All chunks fit in the index block.
|
||
#[test]
|
||
fn read_inline_only() {
|
||
let os: u8 = 8;
|
||
let ls: u8 = 8;
|
||
let osv = os as usize;
|
||
let num_chunks = 2usize;
|
||
let chunk_byte_size = 20u64 * 8; // 20 elements × 8 bytes
|
||
|
||
let mut file_data = vec![0u8; 0x3000];
|
||
|
||
// AEHD at offset 0x100
|
||
let aehd_offset = 0x100usize;
|
||
let aeib_offset = 0x200usize;
|
||
|
||
// Build AEHD
|
||
file_data[aehd_offset..aehd_offset + 4].copy_from_slice(b"EAHD");
|
||
file_data[aehd_offset + 4] = 0; // version
|
||
file_data[aehd_offset + 5] = 0; // client_id = non-filtered
|
||
file_data[aehd_offset + 6] = osv as u8; // element_size
|
||
file_data[aehd_offset + 7] = 10; // max_nelmts_bits
|
||
file_data[aehd_offset + 8] = num_chunks as u8; // idx_blk_elmts (all inline)
|
||
file_data[aehd_offset + 9] = 4; // min_dblk_nelmts
|
||
file_data[aehd_offset + 10] = 2; // super_blk_min_nelmts
|
||
file_data[aehd_offset + 11] = 8; // max_dblk_nelmts_bits
|
||
// 6 stats fields (each 8 bytes), nelmts at stat[4]
|
||
file_data[aehd_offset + 44..aehd_offset + 52]
|
||
.copy_from_slice(&(num_chunks as u64).to_le_bytes());
|
||
file_data[aehd_offset + 60..aehd_offset + 68]
|
||
.copy_from_slice(&(aeib_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, aehd_offset, aehd_offset + 68);
|
||
// checksum (4 bytes at +68) — not validated
|
||
|
||
// Build AEIB at aeib_offset
|
||
file_data[aeib_offset..aeib_offset + 4].copy_from_slice(b"EAIB");
|
||
file_data[aeib_offset + 4] = 0; // version
|
||
file_data[aeib_offset + 5] = 0; // client_id
|
||
file_data[aeib_offset + 6..aeib_offset + 14]
|
||
.copy_from_slice(&(aehd_offset as u64).to_le_bytes());
|
||
|
||
// Inline elements
|
||
let elem_start = aeib_offset + 6 + osv;
|
||
let base_addr = 0x1000u64;
|
||
for i in 0..num_chunks {
|
||
let addr = base_addr + i as u64 * chunk_byte_size;
|
||
let p = elem_start + i * osv;
|
||
file_data[p..p + osv].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
// The index block's checksum covers its prefix, every inline element
|
||
// slot, and every data block and super block address slot:
|
||
// ndblk_addrs = 2 * (sup_blk_min_data_ptrs - 1), and the super block
|
||
// pointers make up the rest of nsblks levels.
|
||
let sup_ptrs = file_data[aehd_offset + 10] as usize;
|
||
let dmin = file_data[aehd_offset + 9] as usize;
|
||
let nsblks = 1 + 10 - dmin.trailing_zeros() as usize;
|
||
let ndblk_addrs = 2 * (sup_ptrs - 1);
|
||
// Levels consumed by those direct data blocks (1, 1, 2, 2, ... per level).
|
||
let mut consumed = 0usize;
|
||
let mut levels = 0usize;
|
||
while consumed < ndblk_addrs {
|
||
consumed += 1 << (levels / 2);
|
||
levels += 1;
|
||
}
|
||
let ib_end = elem_start + num_chunks * osv + (ndblk_addrs + nsblks - levels) * osv;
|
||
stamp_checksum(&mut file_data, aeib_offset, ib_end);
|
||
|
||
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
|
||
let ds_dims = vec![40u64]; // 2 chunks × 20 elements
|
||
let chunk_dims = vec![20u32];
|
||
let chunks =
|
||
read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 2);
|
||
assert_eq!(chunks[0].address, base_addr);
|
||
assert_eq!(chunks[0].offsets, vec![0]);
|
||
assert_eq!(chunks[0].chunk_size, chunk_byte_size as u32);
|
||
assert_eq!(chunks[1].address, base_addr + chunk_byte_size);
|
||
assert_eq!(chunks[1].offsets, vec![20]);
|
||
}
|
||
|
||
/// Build a synthetic EA with inline elements + one direct data block.
|
||
#[test]
|
||
fn read_inline_plus_data_blocks() {
|
||
let os: u8 = 8;
|
||
let ls: u8 = 8;
|
||
let osv = os as usize;
|
||
let chunk_byte_size = 10u64 * 8; // 10 elements × 8 bytes
|
||
let idx_blk_elmts = 2u8;
|
||
let min_dblk_nelmts = 2u8;
|
||
let sblk_min = 2u8;
|
||
let total_chunks = 4usize; // 2 inline + 2 in data block (1 dblk from sb_level 0)
|
||
|
||
let mut file_data = vec![0u8; 0x5000];
|
||
let aehd_offset = 0x100usize;
|
||
let aeib_offset = 0x200usize;
|
||
let aedb_offset = 0x300usize;
|
||
|
||
// EAHD
|
||
file_data[aehd_offset..aehd_offset + 4].copy_from_slice(b"EAHD");
|
||
file_data[aehd_offset + 4] = 0;
|
||
file_data[aehd_offset + 5] = 0; // client_id
|
||
file_data[aehd_offset + 6] = osv as u8; // element_size
|
||
file_data[aehd_offset + 7] = 10;
|
||
file_data[aehd_offset + 8] = idx_blk_elmts;
|
||
file_data[aehd_offset + 9] = min_dblk_nelmts;
|
||
file_data[aehd_offset + 10] = sblk_min;
|
||
file_data[aehd_offset + 11] = 8;
|
||
// 6 stats fields (each 8 bytes), nelmts at stat[4] (offset 12 + 4*8 = 44)
|
||
file_data[aehd_offset + 44..aehd_offset + 52]
|
||
.copy_from_slice(&(total_chunks as u64).to_le_bytes());
|
||
// idx_blk_addr at offset 12 + 6*8 = 60
|
||
file_data[aehd_offset + 60..aehd_offset + 68]
|
||
.copy_from_slice(&(aeib_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, aehd_offset, aehd_offset + 68);
|
||
|
||
// AEIB
|
||
file_data[aeib_offset..aeib_offset + 4].copy_from_slice(b"EAIB");
|
||
file_data[aeib_offset + 4] = 0;
|
||
file_data[aeib_offset + 5] = 0;
|
||
file_data[aeib_offset + 6..aeib_offset + 14]
|
||
.copy_from_slice(&(aehd_offset as u64).to_le_bytes());
|
||
|
||
let mut pos = aeib_offset + 6 + osv;
|
||
|
||
// Inline elements (2 chunks)
|
||
let base_addr = 0x1000u64;
|
||
for i in 0..idx_blk_elmts as usize {
|
||
let addr = base_addr + i as u64 * chunk_byte_size;
|
||
file_data[pos..pos + osv].copy_from_slice(&addr.to_le_bytes());
|
||
pos += osv;
|
||
}
|
||
|
||
// Direct data block addresses. With sup_blk_min_data_ptrs = 2 the index
|
||
// block holds 2 * (2 - 1) = 2 of them, which are the data blocks of
|
||
// super block levels 0 and 1: one of `min_dblk_nelmts` elements, then
|
||
// one of twice that (ndblks = 2^(u/2), dblk_nelmts = 2^((u+1)/2) * min).
|
||
// Only the first is allocated here; the rest of the array is empty.
|
||
let ndblk_addrs = 2 * (sblk_min as usize - 1);
|
||
file_data[pos..pos + osv].copy_from_slice(&(aedb_offset as u64).to_le_bytes());
|
||
pos += osv;
|
||
for _ in 1..ndblk_addrs {
|
||
file_data[pos..pos + osv].copy_from_slice(&u64::MAX.to_le_bytes());
|
||
pos += osv;
|
||
}
|
||
// Super block addresses fill the remaining levels; all unallocated.
|
||
let nsblks = 1 + 10 - (min_dblk_nelmts as usize).trailing_zeros() as usize;
|
||
let mut consumed = 0usize;
|
||
let mut levels = 0usize;
|
||
while consumed < ndblk_addrs {
|
||
consumed += 1 << (levels / 2);
|
||
levels += 1;
|
||
}
|
||
for _ in 0..(nsblks - levels) {
|
||
file_data[pos..pos + osv].copy_from_slice(&u64::MAX.to_le_bytes());
|
||
pos += osv;
|
||
}
|
||
stamp_checksum(&mut file_data, aeib_offset, pos);
|
||
|
||
// EADB holding the first data block's `min_dblk_nelmts` elements.
|
||
file_data[aedb_offset..aedb_offset + 4].copy_from_slice(b"EADB");
|
||
file_data[aedb_offset + 4] = 0;
|
||
file_data[aedb_offset + 5] = 0;
|
||
file_data[aedb_offset + 6..aedb_offset + 14]
|
||
.copy_from_slice(&(aehd_offset as u64).to_le_bytes());
|
||
// Block offset field: ceil(max_nelmts_bits / 8) bytes, zero here.
|
||
let blk_off_size = (10usize).div_ceil(8);
|
||
let db_elems = aedb_offset + 6 + osv + blk_off_size;
|
||
let mut dbpos = db_elems;
|
||
for i in 0..min_dblk_nelmts as usize {
|
||
let addr = base_addr + (idx_blk_elmts as u64 + i as u64) * chunk_byte_size;
|
||
file_data[dbpos..dbpos + osv].copy_from_slice(&addr.to_le_bytes());
|
||
dbpos += osv;
|
||
}
|
||
stamp_checksum(&mut file_data, aedb_offset, dbpos);
|
||
|
||
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
|
||
let ds_dims = vec![40u64];
|
||
let chunk_dims = vec![10u32];
|
||
let chunks =
|
||
read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 4);
|
||
for (i, c) in chunks.iter().enumerate() {
|
||
assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size);
|
||
assert_eq!(c.offsets, vec![i as u64 * 10]);
|
||
}
|
||
}
|
||
|
||
/// Test serialized_size computation.
|
||
#[test]
|
||
fn header_serialized_size() {
|
||
// 12 fixed + 6*8 stats + 8 addr + 4 checksum = 72
|
||
assert_eq!(ExtensibleArrayHeader::serialized_size(8, 8), 72);
|
||
// 12 fixed + 6*4 stats + 4 addr + 4 checksum = 44
|
||
assert_eq!(ExtensibleArrayHeader::serialized_size(4, 4), 44);
|
||
}
|
||
|
||
/// Verify read_element for unallocated slots.
|
||
#[test]
|
||
fn read_element_unallocated() {
|
||
let data = vec![0xFFu8; 16];
|
||
let num_chunks = vec![5u64];
|
||
let chunk_dims = vec![10u32];
|
||
let (info, consumed) =
|
||
read_element(&data, 0, 0, 8, 8, 80, 0, &num_chunks, &chunk_dims).unwrap();
|
||
assert!(info.is_none());
|
||
assert_eq!(consumed, 8);
|
||
}
|
||
|
||
/// Verify filtered element reading.
|
||
#[test]
|
||
fn read_element_filtered() {
|
||
let os: u8 = 8;
|
||
let chunk_size_bytes = 4usize;
|
||
let elem_size = os as usize + chunk_size_bytes + 4;
|
||
let mut data = vec![0u8; elem_size + 16];
|
||
// Address
|
||
data[0..8].copy_from_slice(&0x2000u64.to_le_bytes());
|
||
// Compressed size (4 bytes LE)
|
||
data[8..12].copy_from_slice(&120u32.to_le_bytes());
|
||
// Filter mask
|
||
data[12..16].copy_from_slice(&0u32.to_le_bytes());
|
||
|
||
let num_chunks = vec![5u64];
|
||
let chunk_dims = vec![10u32];
|
||
let (info, consumed) = read_element(
|
||
&data,
|
||
0,
|
||
1,
|
||
elem_size as u8,
|
||
os,
|
||
80,
|
||
2,
|
||
&num_chunks,
|
||
&chunk_dims,
|
||
)
|
||
.unwrap();
|
||
let ci = info.unwrap();
|
||
assert_eq!(ci.address, 0x2000);
|
||
assert_eq!(ci.chunk_size, 120);
|
||
assert_eq!(ci.filter_mask, 0);
|
||
assert_eq!(ci.offsets, vec![20]);
|
||
assert_eq!(consumed, elem_size);
|
||
}
|
||
}
|