osobhandClaude Opus 5
b41272487a
Every structure in both chunk indexes — header, index block, super block, data block and each data block page — carries a Jenkins lookup3 checksum, and all of them were parsed past and ignored. What that costs is not a warning but correct data. Flip one low bit of a chunk address and the index still has the right shape, the address still lands inside the file, and the reader returns whatever bytes now sit there as that chunk's contents. Nothing else in the parse can tell. Verified in both directions. The checksums accept files written by HDF5 2.0 from 100 to 200 000 chunks — dense, sparse, gzip-filtered and paged — which also confirms the block layouts byte for byte, since a wrong offset would fail every file. And an interop test corrupts an address to check the read fails instead of returning data: removing the verification makes that test fail with "corruption produced data instead of an error", which is what it is there to prove. The first version of that test passed with verification disabled — it corrupted a byte a structural check already rejected, so it proved nothing. Worth recording, since a test that passes for the wrong reason looks exactly like coverage. Hand-built fixtures now stamp real checksums, as HDF5 writers do, and the Extensible Array ones no longer describe the superseded layout. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
862 lines
32 KiB
Rust
862 lines
32 KiB
Rust
//! HDF5 Fixed Array index parsing for chunked datasets (v4 index type 3).
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
extern crate alloc;
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{format, vec, vec::Vec};
|
||
|
||
use crate::chunked_read::ChunkInfo;
|
||
use crate::error::FormatError;
|
||
|
||
/// Verify the Jenkins lookup3 checksum stored immediately after
|
||
/// `data[start..end]`, as every Fixed Array structure carries one.
|
||
///
|
||
/// A corrupt chunk index silently yields addresses pointing at the wrong
|
||
/// bytes, so a mismatch has to be an error rather than a shrug: without this
|
||
/// the damage surfaces as plausible-looking data from the wrong chunk.
|
||
#[cfg(feature = "checksum")]
|
||
fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> {
|
||
ensure_len(data, end, 4)?;
|
||
let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
|
||
let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
if computed != stored {
|
||
return Err(FormatError::ChecksumMismatch {
|
||
expected: stored,
|
||
computed,
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(not(feature = "checksum"))]
|
||
fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> {
|
||
Ok(())
|
||
}
|
||
|
||
/// Parsed Fixed Array header (FAHD).
|
||
#[derive(Debug, Clone)]
|
||
pub struct FixedArrayHeader {
|
||
/// Client ID: 0 = non-filtered chunks, 1 = filtered chunks.
|
||
pub client_id: u8,
|
||
/// Size of each array element in bytes.
|
||
pub element_size: u8,
|
||
/// Log2 of max number of elements in a data block page.
|
||
pub max_nelmts_bits: u8,
|
||
/// Total number of elements (chunks) in the array.
|
||
pub num_elements: u64,
|
||
/// Address of the data block.
|
||
pub data_block_address: u64,
|
||
}
|
||
|
||
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
let s = size as usize;
|
||
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: pos.saturating_add(s),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match size {
|
||
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
||
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
||
8 => u64::from_le_bytes([
|
||
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
||
]),
|
||
_ => return Err(FormatError::InvalidOffsetSize(size)),
|
||
})
|
||
}
|
||
|
||
fn read_length(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
read_offset(data, pos, size)
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
||
if offset
|
||
.checked_add(needed)
|
||
.is_none_or(|end| end > data.len())
|
||
{
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: offset.saturating_add(needed),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
|
||
let s = size as usize;
|
||
if pos + s > data.len() {
|
||
return false;
|
||
}
|
||
data[pos..pos + s].iter().all(|&b| b == 0xFF)
|
||
}
|
||
|
||
impl FixedArrayHeader {
|
||
/// Parse a Fixed Array header from file data at the given offset.
|
||
pub fn parse(
|
||
file_data: &[u8],
|
||
offset: usize,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Self, FormatError> {
|
||
// FAHD signature(4) + version(1) + client_id(1) + element_size(1) +
|
||
// max_nelmts_bits(1) + num_elements(length_size) + data_block_addr(offset_size) + checksum(4)
|
||
let min_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + offset_size as usize + 4;
|
||
ensure_len(file_data, offset, min_size)?;
|
||
|
||
let d = &file_data[offset..];
|
||
if &d[0..4] != b"FAHD" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Fixed Array header signature".into(),
|
||
));
|
||
}
|
||
|
||
let version = d[4];
|
||
if version != 0 {
|
||
return Err(FormatError::ChunkedReadError(format!(
|
||
"unsupported Fixed Array header version: {version}"
|
||
)));
|
||
}
|
||
|
||
let client_id = d[5];
|
||
let element_size = d[6];
|
||
let max_nelmts_bits = d[7];
|
||
|
||
let mut pos = 8;
|
||
let num_elements = read_length(d, pos, length_size)?;
|
||
pos += length_size as usize;
|
||
let data_block_address = read_offset(d, pos, offset_size)?;
|
||
pos += offset_size as usize;
|
||
verify_checksum(file_data, offset, offset + pos)?;
|
||
|
||
Ok(FixedArrayHeader {
|
||
client_id,
|
||
element_size,
|
||
max_nelmts_bits,
|
||
num_elements,
|
||
data_block_address,
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Read chunk records from a Fixed Array data block.
|
||
///
|
||
/// Returns a `Vec<ChunkInfo>` with one entry per allocated chunk.
|
||
/// `chunk_dimensions` should be the spatial chunk dims only (not including the element-size dim).
|
||
/// `element_size` is the datatype size in bytes.
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn read_fixed_array_chunks(
|
||
file_data: &[u8],
|
||
header: &FixedArrayHeader,
|
||
dataset_dims: &[u64],
|
||
chunk_dimensions: &[u32],
|
||
element_size: u32,
|
||
offset_size: u8,
|
||
_length_size: u8,
|
||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let db_offset = header.data_block_address as usize;
|
||
let rank = chunk_dimensions.len();
|
||
|
||
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
|
||
let db_header_size = 4 + 1 + 1 + offset_size as usize;
|
||
ensure_len(file_data, db_offset, db_header_size)?;
|
||
|
||
let d = &file_data[db_offset..];
|
||
if &d[0..4] != b"FADB" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Fixed Array data block signature".into(),
|
||
));
|
||
}
|
||
|
||
// Elements start immediately after the data block prefix.
|
||
let elements_start = db_offset + db_header_size;
|
||
|
||
let num_elements = header.num_elements as usize;
|
||
// A chunk index cannot describe more elements than the file has bytes (each
|
||
// element occupies at least `offset_size` bytes). Reject a corrupt count
|
||
// before it can drive a huge loop or overflow an offset computation.
|
||
if num_elements > file_data.len() {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Fixed Array element count exceeds file size".into(),
|
||
));
|
||
}
|
||
let os = offset_size as usize;
|
||
// On-disk stride of one element. For non-filtered arrays the element is just
|
||
// the chunk address (== offset_size); for filtered arrays it is
|
||
// address + chunk_size + filter_mask (== header.element_size).
|
||
let elem_stride = (header.element_size as usize).max(os);
|
||
|
||
// Absolute file offset of element `idx` within a run starting at `base`,
|
||
// with overflow surfaced as a clean error rather than a panic/wrap.
|
||
let elem_at = |base: usize, idx: usize| -> Result<usize, FormatError> {
|
||
idx.checked_mul(elem_stride)
|
||
.and_then(|o| base.checked_add(o))
|
||
.ok_or(FormatError::ChunkedReadError(
|
||
"Fixed Array element offset overflow".into(),
|
||
))
|
||
};
|
||
|
||
// Compute chunk offsets based on index.
|
||
// Chunks are stored in row-major order within the dataset space.
|
||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||
for d_idx in 0..rank {
|
||
let ch_dim = chunk_dimensions[d_idx] as u64;
|
||
if ch_dim == 0 {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"chunk dimension is zero".into(),
|
||
));
|
||
}
|
||
let ds_dim = dataset_dims[d_idx];
|
||
num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
|
||
}
|
||
|
||
let chunk_byte_size: u64 =
|
||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||
|
||
let mut chunks = Vec::new();
|
||
let push_element =
|
||
|i: usize, abs: usize, chunks: &mut Vec<ChunkInfo>| -> Result<(), FormatError> {
|
||
if let Some((address, chunk_size, filter_mask)) = parse_fa_element(
|
||
file_data,
|
||
abs,
|
||
header.client_id,
|
||
offset_size,
|
||
header.element_size,
|
||
chunk_byte_size,
|
||
)? {
|
||
let offsets = index_to_chunk_offsets(i, &num_chunks_per_dim, chunk_dimensions);
|
||
chunks.push(ChunkInfo {
|
||
chunk_size,
|
||
filter_mask,
|
||
offsets,
|
||
address,
|
||
});
|
||
}
|
||
Ok(())
|
||
};
|
||
|
||
// A data block is paged when it holds more elements than fit in one page.
|
||
// `max_nelmts_bits` is an untrusted u8; a shift >= the pointer width would
|
||
// panic, so reject it (real page-size bits are tiny — 10 by default).
|
||
if header.max_nelmts_bits as u32 >= usize::BITS {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Fixed Array max_nelmts_bits too large".into(),
|
||
));
|
||
}
|
||
let page_nelmts = 1usize << header.max_nelmts_bits;
|
||
let is_paged = num_elements > page_nelmts;
|
||
|
||
if !is_paged {
|
||
// Non-paged: prefix, then `num_elements` elements packed directly,
|
||
// then a checksum over both.
|
||
verify_checksum(file_data, db_offset, elem_at(elements_start, num_elements)?)?;
|
||
for i in 0..num_elements {
|
||
push_element(i, elem_at(elements_start, i)?, &mut chunks)?;
|
||
}
|
||
return Ok(chunks);
|
||
}
|
||
|
||
// Paged layout: prefix, then a page-init bitmap (one bit per page, MSB-first
|
||
// within each byte), then a 4-byte checksum, then the pages. Every page
|
||
// occupies a full slot of `page_nelmts` elements plus a 4-byte checksum;
|
||
// only the final page holds fewer elements. Uninitialized pages (bit clear)
|
||
// still occupy their slot on disk but are zero-filled, so the bitmap — not a
|
||
// 0xFF sentinel — is what marks a whole page as unallocated.
|
||
let stride_overflow =
|
||
|| FormatError::ChunkedReadError("Fixed Array page offset overflow".into());
|
||
let npages = num_elements.div_ceil(page_nelmts);
|
||
let bitmap_size = npages.div_ceil(8);
|
||
let bitmap_start = elements_start;
|
||
// prefix(db_header_size) + bitmap + checksum(4)
|
||
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
||
let page_stride = page_nelmts
|
||
.checked_mul(elem_stride)
|
||
.and_then(|x| x.checked_add(4))
|
||
.ok_or_else(stride_overflow)?;
|
||
|
||
if bitmap_start + bitmap_size > file_data.len() {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: bitmap_start + bitmap_size,
|
||
available: file_data.len(),
|
||
});
|
||
}
|
||
// The prefix and page bitmap are covered by their own checksum, and each
|
||
// initialised page by one of its own.
|
||
verify_checksum(file_data, db_offset, bitmap_start + bitmap_size)?;
|
||
|
||
for p in 0..npages {
|
||
let page_first = p * page_nelmts; // < num_elements, cannot overflow
|
||
let page_count = core::cmp::min(page_nelmts, num_elements - page_first);
|
||
|
||
// Check the page-init bit (MSB-first within each byte).
|
||
let bit_byte = file_data[bitmap_start + p / 8];
|
||
let bit_mask = 1u8 << (7 - (p % 8));
|
||
if bit_byte & bit_mask == 0 {
|
||
continue; // entire page unallocated
|
||
}
|
||
|
||
let page_off = p
|
||
.checked_mul(page_stride)
|
||
.and_then(|o| pages_start.checked_add(o))
|
||
.ok_or_else(stride_overflow)?;
|
||
verify_checksum(file_data, page_off, elem_at(page_off, page_count)?)?;
|
||
for e in 0..page_count {
|
||
push_element(page_first + e, elem_at(page_off, e)?, &mut chunks)?;
|
||
}
|
||
}
|
||
|
||
Ok(chunks)
|
||
}
|
||
|
||
/// Parse a single Fixed Array element at absolute file offset `abs`.
|
||
///
|
||
/// Returns `Some((address, chunk_size, filter_mask))` for an allocated chunk, or
|
||
/// `None` if the element is undefined (an unallocated chunk, address all-`0xFF`).
|
||
fn parse_fa_element(
|
||
file_data: &[u8],
|
||
abs: usize,
|
||
client_id: u8,
|
||
offset_size: u8,
|
||
element_size: u8,
|
||
chunk_byte_size: u64,
|
||
) -> Result<Option<(u64, u32, u32)>, FormatError> {
|
||
let os = offset_size as usize;
|
||
if client_id == 0 {
|
||
// Non-filtered: element is just the chunk address.
|
||
if abs + os > file_data.len() {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: abs + os,
|
||
available: file_data.len(),
|
||
});
|
||
}
|
||
if is_undefined(file_data, abs, offset_size) {
|
||
return Ok(None);
|
||
}
|
||
let address = read_offset(file_data, abs, offset_size)?;
|
||
Ok(Some((address, chunk_byte_size as u32, 0)))
|
||
} else {
|
||
// Filtered: address(offset_size) + chunk_size(variable) + filter_mask(4)
|
||
let es = element_size as usize;
|
||
if es < os + 4 {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"element_size too small for filtered element".into(),
|
||
));
|
||
}
|
||
let chunk_size_bytes = es - os - 4;
|
||
if abs + es > file_data.len() {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: abs + es,
|
||
available: file_data.len(),
|
||
});
|
||
}
|
||
if is_undefined(file_data, abs, offset_size) {
|
||
return Ok(None);
|
||
}
|
||
let address = read_offset(file_data, abs, offset_size)?;
|
||
let chunk_size = read_variable_length(&file_data[abs + os..], chunk_size_bytes)?;
|
||
let fm_off = abs + os + chunk_size_bytes;
|
||
let filter_mask = u32::from_le_bytes([
|
||
file_data[fm_off],
|
||
file_data[fm_off + 1],
|
||
file_data[fm_off + 2],
|
||
file_data[fm_off + 3],
|
||
]);
|
||
Ok(Some((address, chunk_size as u32, filter_mask)))
|
||
}
|
||
}
|
||
|
||
/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
|
||
fn index_to_chunk_offsets(
|
||
index: usize,
|
||
num_chunks_per_dim: &[u64],
|
||
chunk_dimensions: &[u32],
|
||
) -> Vec<u64> {
|
||
let rank = num_chunks_per_dim.len();
|
||
let mut offsets = vec![0u64; rank];
|
||
let mut remaining = index as u64;
|
||
for d in (0..rank).rev() {
|
||
let nchunks = num_chunks_per_dim[d];
|
||
if nchunks == 0 {
|
||
continue;
|
||
}
|
||
let chunk_idx = remaining % nchunks;
|
||
remaining /= nchunks;
|
||
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
|
||
}
|
||
offsets
|
||
}
|
||
|
||
/// Read a variable-length little-endian unsigned integer.
|
||
fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
|
||
if size > 8 || data.len() < size {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid variable-length size".into(),
|
||
));
|
||
}
|
||
let mut val = 0u64;
|
||
for (i, &byte) in data.iter().enumerate().take(size) {
|
||
val |= (byte as u64) << (i * 8);
|
||
}
|
||
Ok(val)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Stamp the Jenkins checksum a real file would carry over
|
||
/// `data[start..end]`, writing it at `end`. Fixtures built by hand need
|
||
/// this now that the reader validates it — as every HDF5 writer does.
|
||
fn stamp_checksum(data: &mut [u8], start: usize, end: usize) {
|
||
let sum = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
data[end..end + 4].copy_from_slice(&sum.to_le_bytes());
|
||
}
|
||
|
||
#[test]
|
||
fn index_to_offsets_1d() {
|
||
let num_chunks = vec![5u64];
|
||
let chunk_dims = vec![20u32];
|
||
assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||
vec![20]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
|
||
vec![80]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn index_to_offsets_2d() {
|
||
// 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks
|
||
let num_chunks = vec![3u64, 2];
|
||
let chunk_dims = vec![4u32, 3];
|
||
assert_eq!(
|
||
index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
|
||
vec![0, 0]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||
vec![0, 3]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
|
||
vec![4, 0]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(3, &num_chunks, &chunk_dims),
|
||
vec![4, 3]
|
||
);
|
||
assert_eq!(
|
||
index_to_chunk_offsets(5, &num_chunks, &chunk_dims),
|
||
vec![8, 3]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn read_variable_length_values() {
|
||
assert_eq!(read_variable_length(&[0x78, 0x56], 2).unwrap(), 0x5678);
|
||
assert_eq!(
|
||
read_variable_length(&[0x01, 0x02, 0x03, 0x04], 4).unwrap(),
|
||
0x04030201
|
||
);
|
||
assert_eq!(read_variable_length(&[0xFF], 1).unwrap(), 0xFF);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_valid() {
|
||
let mut buf = vec![0u8; 256];
|
||
// FAHD signature
|
||
buf[0..4].copy_from_slice(b"FAHD");
|
||
buf[4] = 0; // version
|
||
buf[5] = 1; // client_id = filtered
|
||
buf[6] = 16; // element_size
|
||
buf[7] = 10; // max_nelmts_bits (page_size = 1024)
|
||
// num_elements (length_size=8)
|
||
buf[8..16].copy_from_slice(&5u64.to_le_bytes());
|
||
// data_block_address (offset_size=8)
|
||
buf[16..24].copy_from_slice(&0x1000u64.to_le_bytes());
|
||
stamp_checksum(&mut buf, 0, 24);
|
||
|
||
let header = FixedArrayHeader::parse(&buf, 0, 8, 8).unwrap();
|
||
assert_eq!(header.client_id, 1);
|
||
assert_eq!(header.element_size, 16);
|
||
assert_eq!(header.max_nelmts_bits, 10);
|
||
assert_eq!(header.num_elements, 5);
|
||
assert_eq!(header.data_block_address, 0x1000);
|
||
}
|
||
|
||
/// Corruption anywhere in the index must be an error, not a wrong
|
||
/// address. Every structure carries a checksum; flipping a bit in each in
|
||
/// turn must be caught, because the alternative is reading a chunk from
|
||
/// the wrong offset and returning it as data.
|
||
#[test]
|
||
fn corrupting_any_fixed_array_structure_is_detected() {
|
||
let build = || -> (Vec<u8>, usize) {
|
||
let (os, fahd, db) = (8usize, 0x100usize, 0x200usize);
|
||
let mut f = vec![0u8; 0x3000];
|
||
f[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
f[fahd + 6] = os as u8;
|
||
f[fahd + 7] = 10;
|
||
f[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes());
|
||
f[fahd + 16..fahd + 24].copy_from_slice(&(db as u64).to_le_bytes());
|
||
stamp_checksum(&mut f, fahd, fahd + 24);
|
||
f[db..db + 4].copy_from_slice(b"FADB");
|
||
f[db + 6..db + 14].copy_from_slice(&(fahd as u64).to_le_bytes());
|
||
let elems = db + 6 + os;
|
||
for i in 0..3usize {
|
||
let addr = 0x1000u64 + i as u64 * 0x100;
|
||
f[elems + i * os..elems + (i + 1) * os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut f, db, elems + 3 * os);
|
||
(f, fahd)
|
||
};
|
||
|
||
let read = |f: &[u8], fahd: usize| -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let h = FixedArrayHeader::parse(f, fahd, 8, 8)?;
|
||
read_fixed_array_chunks(f, &h, &[60], &[20], 8, 8, 8)
|
||
};
|
||
|
||
let (clean, fahd) = build();
|
||
assert!(read(&clean, fahd).is_ok(), "the intact fixture must read");
|
||
|
||
// A byte inside the header, and one inside a data block element.
|
||
for &at in &[0x108usize, 0x210usize] {
|
||
let (mut damaged, fahd) = build();
|
||
damaged[at] ^= 0x01;
|
||
assert!(
|
||
matches!(
|
||
read(&damaged, fahd),
|
||
Err(FormatError::ChecksumMismatch { .. })
|
||
),
|
||
"corruption at {at:#x} went undetected"
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_invalid_signature() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"XXXX");
|
||
let result = FixedArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// Malformed headers must error, never panic (shift overflow, huge counts).
|
||
#[test]
|
||
fn read_rejects_oversized_max_nelmts_bits() {
|
||
let mut buf = vec![0u8; 512];
|
||
let fahd = 0x40usize;
|
||
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
buf[fahd + 4] = 0; // version
|
||
buf[fahd + 5] = 0; // client_id
|
||
buf[fahd + 6] = 8; // element_size
|
||
buf[fahd + 7] = 200; // max_nelmts_bits — absurd, would overflow a shift
|
||
buf[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes()); // num_elements
|
||
buf[fahd + 16..fahd + 24].copy_from_slice(&0x100u64.to_le_bytes());
|
||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||
// FADB so parsing reaches the paged check
|
||
let db = 0x100usize;
|
||
buf[db..db + 4].copy_from_slice(b"FADB");
|
||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn read_rejects_num_elements_larger_than_file() {
|
||
let mut buf = vec![0u8; 256];
|
||
let fahd = 0x40usize;
|
||
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
buf[fahd + 6] = 8;
|
||
buf[fahd + 7] = 10;
|
||
buf[fahd + 8..fahd + 16].copy_from_slice(&u64::MAX.to_le_bytes()); // absurd count
|
||
buf[fahd + 16..fahd + 24].copy_from_slice(&0x80u64.to_le_bytes());
|
||
// Valid checksum, so it is the element count that must be rejected.
|
||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||
buf[0x80..0x84].copy_from_slice(b"FADB");
|
||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn parse_rejects_offset_overflow() {
|
||
let buf = vec![0u8; 64];
|
||
let result = FixedArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` data block address must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn read_rejects_data_block_offset_overflow() {
|
||
let header = FixedArrayHeader {
|
||
client_id: 0,
|
||
element_size: 8,
|
||
max_nelmts_bits: 10,
|
||
num_elements: 1,
|
||
data_block_address: (usize::MAX - 4) as u64,
|
||
};
|
||
let buf = vec![0u8; 64];
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_invalid_version() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"FAHD");
|
||
buf[4] = 1; // unsupported version
|
||
let result = FixedArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// Build a synthetic Fixed Array (non-filtered) and verify reading.
|
||
#[test]
|
||
fn read_non_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
let num_chunks = 5u64;
|
||
|
||
let mut file_data = vec![0u8; 0x3000];
|
||
|
||
// Build FAHD at offset 0x100
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x200usize;
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0; // version
|
||
file_data[fahd_offset + 5] = 0; // client_id = non-filtered
|
||
file_data[fahd_offset + 6] = os as u8; // element_size = just address
|
||
file_data[fahd_offset + 7] = 10; // max_nelmts_bits
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_chunks.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
// Build FADB at db_offset
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0; // version
|
||
file_data[db_offset + 5] = 0; // client_id
|
||
file_data[db_offset + 6..db_offset + 14]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes()); // header_address
|
||
|
||
// Elements: 5 addresses
|
||
let elem_start = db_offset + 6 + os;
|
||
let base_addr = 0x1000u64;
|
||
let chunk_byte_size = 20 * 8; // 20 elements × 8 bytes
|
||
for i in 0..5 {
|
||
let addr = base_addr + i as u64 * chunk_byte_size as u64;
|
||
let pos = elem_start + i * os;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut file_data, db_offset, elem_start + 5 * os);
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
let ds_dims = vec![100u64];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 5);
|
||
for (i, c) in chunks.iter().enumerate() {
|
||
assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size as u64);
|
||
assert_eq!(c.offsets, vec![i as u64 * 20]);
|
||
assert_eq!(c.filter_mask, 0);
|
||
assert_eq!(c.chunk_size, chunk_byte_size as u32);
|
||
}
|
||
}
|
||
|
||
/// Build a synthetic Fixed Array (filtered) and verify reading.
|
||
#[test]
|
||
fn read_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
let num_chunks = 3u64;
|
||
// element_size for filtered: offset_size + chunk_size_bytes + 4(filter_mask)
|
||
// chunk_size_bytes: let's use 4 bytes
|
||
let chunk_size_bytes = 4usize;
|
||
let elem_size = os + chunk_size_bytes + 4;
|
||
|
||
let mut file_data = vec![0u8; 0x3000];
|
||
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x200usize;
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0;
|
||
file_data[fahd_offset + 5] = 1; // client_id = filtered
|
||
file_data[fahd_offset + 6] = elem_size as u8;
|
||
file_data[fahd_offset + 7] = 10;
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_chunks.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0;
|
||
file_data[db_offset + 5] = 1;
|
||
file_data[db_offset + 6..db_offset + 14]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes());
|
||
|
||
let elem_start = db_offset + 6 + os;
|
||
let test_chunks = [
|
||
(0x1000u64, 120u32, 0u32),
|
||
(0x2000u64, 115u32, 0u32),
|
||
(0x3000u64, 100u32, 0u32),
|
||
];
|
||
|
||
for (i, &(addr, csize, fmask)) in test_chunks.iter().enumerate() {
|
||
let pos = elem_start + i * elem_size;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
// chunk_size as 4 bytes LE
|
||
file_data[pos + os..pos + os + 4].copy_from_slice(&csize.to_le_bytes());
|
||
file_data[pos + os + 4..pos + os + 8].copy_from_slice(&fmask.to_le_bytes());
|
||
}
|
||
stamp_checksum(
|
||
&mut file_data,
|
||
db_offset,
|
||
elem_start + test_chunks.len() * elem_size,
|
||
);
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
let ds_dims = vec![60u64];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 3);
|
||
assert_eq!(chunks[0].address, 0x1000);
|
||
assert_eq!(chunks[0].chunk_size, 120);
|
||
assert_eq!(chunks[0].filter_mask, 0);
|
||
assert_eq!(chunks[0].offsets, vec![0]);
|
||
assert_eq!(chunks[1].address, 0x2000);
|
||
assert_eq!(chunks[1].chunk_size, 115);
|
||
assert_eq!(chunks[2].address, 0x3000);
|
||
assert_eq!(chunks[2].chunk_size, 100);
|
||
}
|
||
|
||
/// Build a synthetic *paged* Fixed Array (non-filtered) and verify reading.
|
||
///
|
||
/// Layout reverse-engineered and confirmed against an HDF5 2.0 file:
|
||
/// after the FADB prefix comes a page-init bitmap (MSB-first within each
|
||
/// byte), a 4-byte checksum, then full-size page slots (`page_nelmts`
|
||
/// elements + a 4-byte checksum each), with only the last page shorter.
|
||
/// Uninitialized pages occupy their slot but are skipped via the bitmap.
|
||
#[test]
|
||
fn read_paged_non_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
|
||
// page_nelmts = 1 << 2 = 4. Use 11 elements => 3 pages
|
||
// (page0: 4, page1: 4, page2: 3 short). Initialize pages 0 and 2; leave
|
||
// page 1 uninitialized. 3 pages still fits one bitmap byte, but we place
|
||
// the set bits at positions 7 and 5 to lock the MSB-first ordering.
|
||
let max_nelmts_bits = 2u8;
|
||
let page_nelmts = 1usize << max_nelmts_bits; // 4
|
||
let num_elements = 11u64;
|
||
let db_header_size = 4 + 1 + 1 + os; // FADB sig+ver+client+header_addr
|
||
let bitmap_size = 1usize; // ceil(3/8)
|
||
let page_total = page_nelmts * os + 4; // elements + checksum
|
||
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x400usize;
|
||
let mut file_data = vec![0u8; 0x4000];
|
||
|
||
// FAHD
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0; // version
|
||
file_data[fahd_offset + 5] = 0; // client_id = non-filtered
|
||
file_data[fahd_offset + 6] = os as u8; // element_size = address only
|
||
file_data[fahd_offset + 7] = max_nelmts_bits;
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_elements.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
// FADB prefix
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0; // version
|
||
file_data[db_offset + 5] = 0; // client_id
|
||
file_data[db_offset + 6..db_offset + 6 + os]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes());
|
||
|
||
// Page-init bitmap: pages 0 and 2 initialized, page 1 not.
|
||
// MSB-first => page0 -> bit7 (0x80), page2 -> bit5 (0x20) => 0xA0.
|
||
let bitmap_off = db_offset + db_header_size;
|
||
file_data[bitmap_off] = 0b1010_0000;
|
||
|
||
// Pages start after bitmap + 4-byte checksum.
|
||
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
||
|
||
let base_addr = 0x1000u64;
|
||
// Page 0 (elements 0..4) and page 2 (elements 8..11) carry addresses;
|
||
// page 1's slot is left zero-filled and must be skipped.
|
||
// The prefix and bitmap carry one checksum, each initialised page
|
||
// another — as a real file does.
|
||
stamp_checksum(&mut file_data, db_offset, bitmap_off + bitmap_size);
|
||
for &p in &[0usize, 2usize] {
|
||
let page_off = pages_start + p * page_total;
|
||
let count = core::cmp::min(page_nelmts, num_elements as usize - p * page_nelmts);
|
||
for e in 0..count {
|
||
let i = p * page_nelmts + e;
|
||
let addr = base_addr + i as u64 * 0x100;
|
||
let pos = page_off + e * os;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut file_data, page_off, page_off + count * os);
|
||
}
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
assert_eq!(header.num_elements, 11);
|
||
|
||
let ds_dims = vec![11u64 * 20];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
// Page 1 (elements 4,5,6,7) is uninitialized => skipped. The remaining
|
||
// 7 chunks (0..4 and 8..11) come back with their original linear index.
|
||
assert_eq!(chunks.len(), 7);
|
||
let mut got: Vec<(u64, u64)> = chunks.iter().map(|c| (c.offsets[0], c.address)).collect();
|
||
got.sort();
|
||
let expect: Vec<(u64, u64)> = [0usize, 1, 2, 3, 8, 9, 10]
|
||
.iter()
|
||
.map(|&i| (i as u64 * 20, base_addr + i as u64 * 0x100))
|
||
.collect();
|
||
assert_eq!(got, expect);
|
||
}
|
||
}
|