osobhandClaude Opus 5.5
bba1560416
Both indexes place each chunk at a linear index computed from the dataset's maximum dimensions (libhdf5's max_down_chunks), and the Extensible Array first swizzles its unlimited dimension to the slowest position. We linearised by the current dimensions, so any dataset whose shape was smaller than its maxshape, or whose unlimited dimension was not the first, read back scrambled without an error: h5py libver="latest" files with maxshape (10, None) or (20, 10), and the libhdf5 test files h5fc_ext*.h5 and test_ld.h5. The linearisation now lives in chunk_grid (shared with the writers), and slots beyond the current extent are ignored as the library does. read_fixed_array_chunks / read_extensible_array_chunks take the dataspace's max dimensions. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
817 lines
31 KiB
Rust
817 lines
31 KiB
Rust
//! HDF5 Fixed Array index parsing for chunked datasets (v4 index type 3).
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
extern crate alloc;
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{format, vec, vec::Vec};
|
||
|
||
use crate::chunk_grid::ChunkGrid;
|
||
use crate::chunked_read::ChunkInfo;
|
||
use crate::error::FormatError;
|
||
|
||
/// Verify the Jenkins lookup3 checksum stored immediately after
|
||
/// `data[start..end]`, as every Fixed Array structure carries one.
|
||
///
|
||
/// A corrupt chunk index silently yields addresses pointing at the wrong
|
||
/// bytes, so a mismatch has to be an error rather than a shrug: without this
|
||
/// the damage surfaces as plausible-looking data from the wrong chunk.
|
||
#[cfg(feature = "checksum")]
|
||
fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> {
|
||
ensure_len(data, end, 4)?;
|
||
let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
|
||
let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
if computed != stored {
|
||
return Err(FormatError::ChecksumMismatch {
|
||
expected: stored,
|
||
computed,
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(not(feature = "checksum"))]
|
||
fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> {
|
||
Ok(())
|
||
}
|
||
|
||
/// Parsed Fixed Array header (FAHD).
|
||
#[derive(Debug, Clone)]
|
||
pub struct FixedArrayHeader {
|
||
/// Client ID: 0 = non-filtered chunks, 1 = filtered chunks.
|
||
pub client_id: u8,
|
||
/// Size of each array element in bytes.
|
||
pub element_size: u8,
|
||
/// Log2 of max number of elements in a data block page.
|
||
pub max_nelmts_bits: u8,
|
||
/// Total number of elements (chunks) in the array.
|
||
pub num_elements: u64,
|
||
/// Address of the data block.
|
||
pub data_block_address: u64,
|
||
}
|
||
|
||
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
let s = size as usize;
|
||
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: pos.saturating_add(s),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match size {
|
||
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
||
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
||
8 => u64::from_le_bytes([
|
||
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
||
]),
|
||
_ => return Err(FormatError::InvalidOffsetSize(size)),
|
||
})
|
||
}
|
||
|
||
fn read_length(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
read_offset(data, pos, size)
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
||
if offset
|
||
.checked_add(needed)
|
||
.is_none_or(|end| end > data.len())
|
||
{
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: offset.saturating_add(needed),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
|
||
let s = size as usize;
|
||
if pos + s > data.len() {
|
||
return false;
|
||
}
|
||
data[pos..pos + s].iter().all(|&b| b == 0xFF)
|
||
}
|
||
|
||
impl FixedArrayHeader {
|
||
/// Parse a Fixed Array header from file data at the given offset.
|
||
pub fn parse(
|
||
file_data: &[u8],
|
||
offset: usize,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Self, FormatError> {
|
||
// FAHD signature(4) + version(1) + client_id(1) + element_size(1) +
|
||
// max_nelmts_bits(1) + num_elements(length_size) + data_block_addr(offset_size) + checksum(4)
|
||
let min_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + offset_size as usize + 4;
|
||
ensure_len(file_data, offset, min_size)?;
|
||
|
||
let d = &file_data[offset..];
|
||
if &d[0..4] != b"FAHD" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Fixed Array header signature".into(),
|
||
));
|
||
}
|
||
|
||
let version = d[4];
|
||
if version != 0 {
|
||
return Err(FormatError::ChunkedReadError(format!(
|
||
"unsupported Fixed Array header version: {version}"
|
||
)));
|
||
}
|
||
|
||
let client_id = d[5];
|
||
let element_size = d[6];
|
||
let max_nelmts_bits = d[7];
|
||
|
||
let mut pos = 8;
|
||
let num_elements = read_length(d, pos, length_size)?;
|
||
pos += length_size as usize;
|
||
let data_block_address = read_offset(d, pos, offset_size)?;
|
||
pos += offset_size as usize;
|
||
verify_checksum(file_data, offset, offset + pos)?;
|
||
|
||
Ok(FixedArrayHeader {
|
||
client_id,
|
||
element_size,
|
||
max_nelmts_bits,
|
||
num_elements,
|
||
data_block_address,
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Read chunk records from a Fixed Array data block.
|
||
///
|
||
/// Returns a `Vec<ChunkInfo>` with one entry per allocated chunk.
|
||
/// `chunk_dimensions` should be the spatial chunk dims only (not including the element-size dim).
|
||
/// `element_size` is the datatype size in bytes.
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn read_fixed_array_chunks(
|
||
file_data: &[u8],
|
||
header: &FixedArrayHeader,
|
||
dataset_dims: &[u64],
|
||
max_dims: Option<&[u64]>,
|
||
chunk_dimensions: &[u32],
|
||
element_size: u32,
|
||
offset_size: u8,
|
||
_length_size: u8,
|
||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let db_offset = header.data_block_address as usize;
|
||
|
||
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
|
||
let db_header_size = 4 + 1 + 1 + offset_size as usize;
|
||
ensure_len(file_data, db_offset, db_header_size)?;
|
||
|
||
let d = &file_data[db_offset..];
|
||
if &d[0..4] != b"FADB" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Fixed Array data block signature".into(),
|
||
));
|
||
}
|
||
|
||
// Elements start immediately after the data block prefix.
|
||
let elements_start = db_offset + db_header_size;
|
||
|
||
let num_elements = header.num_elements as usize;
|
||
// A chunk index cannot describe more elements than the file has bytes (each
|
||
// element occupies at least `offset_size` bytes). Reject a corrupt count
|
||
// before it can drive a huge loop or overflow an offset computation.
|
||
if num_elements > file_data.len() {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Fixed Array element count exceeds file size".into(),
|
||
));
|
||
}
|
||
let os = offset_size as usize;
|
||
// On-disk stride of one element. For non-filtered arrays the element is just
|
||
// the chunk address (== offset_size); for filtered arrays it is
|
||
// address + chunk_size + filter_mask (== header.element_size).
|
||
let elem_stride = (header.element_size as usize).max(os);
|
||
|
||
// Absolute file offset of element `idx` within a run starting at `base`,
|
||
// with overflow surfaced as a clean error rather than a panic/wrap.
|
||
let elem_at = |base: usize, idx: usize| -> Result<usize, FormatError> {
|
||
idx.checked_mul(elem_stride)
|
||
.and_then(|o| base.checked_add(o))
|
||
.ok_or(FormatError::ChunkedReadError(
|
||
"Fixed Array element offset overflow".into(),
|
||
))
|
||
};
|
||
|
||
// The index is laid out over the chunk grid of the *maximum* dimensions
|
||
// (row-major), so a dataset smaller than its maxshape has gaps.
|
||
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
|
||
let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?;
|
||
|
||
let chunk_byte_size: u64 =
|
||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||
|
||
let mut chunks = Vec::new();
|
||
let push_element =
|
||
|i: usize, abs: usize, chunks: &mut Vec<ChunkInfo>| -> Result<(), FormatError> {
|
||
if let Some((address, chunk_size, filter_mask)) = parse_fa_element(
|
||
file_data,
|
||
abs,
|
||
header.client_id,
|
||
offset_size,
|
||
header.element_size,
|
||
chunk_byte_size,
|
||
)? {
|
||
// A slot beyond the current extent is ignored, as the
|
||
// library does.
|
||
let Some(offsets) = grid.offsets(i as u64) else {
|
||
return Ok(());
|
||
};
|
||
chunks.push(ChunkInfo {
|
||
chunk_size,
|
||
filter_mask,
|
||
offsets,
|
||
address,
|
||
});
|
||
}
|
||
Ok(())
|
||
};
|
||
|
||
// A data block is paged when it holds more elements than fit in one page.
|
||
// `max_nelmts_bits` is an untrusted u8; a shift >= the pointer width would
|
||
// panic, so reject it (real page-size bits are tiny — 10 by default).
|
||
if header.max_nelmts_bits as u32 >= usize::BITS {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Fixed Array max_nelmts_bits too large".into(),
|
||
));
|
||
}
|
||
let page_nelmts = 1usize << header.max_nelmts_bits;
|
||
let is_paged = num_elements > page_nelmts;
|
||
|
||
if !is_paged {
|
||
// Non-paged: prefix, then `num_elements` elements packed directly,
|
||
// then a checksum over both.
|
||
verify_checksum(file_data, db_offset, elem_at(elements_start, num_elements)?)?;
|
||
for i in 0..num_elements {
|
||
push_element(i, elem_at(elements_start, i)?, &mut chunks)?;
|
||
}
|
||
return Ok(chunks);
|
||
}
|
||
|
||
// Paged layout: prefix, then a page-init bitmap (one bit per page, MSB-first
|
||
// within each byte), then a 4-byte checksum, then the pages. Every page
|
||
// occupies a full slot of `page_nelmts` elements plus a 4-byte checksum;
|
||
// only the final page holds fewer elements. Uninitialized pages (bit clear)
|
||
// still occupy their slot on disk but are zero-filled, so the bitmap — not a
|
||
// 0xFF sentinel — is what marks a whole page as unallocated.
|
||
let stride_overflow =
|
||
|| FormatError::ChunkedReadError("Fixed Array page offset overflow".into());
|
||
let npages = num_elements.div_ceil(page_nelmts);
|
||
let bitmap_size = npages.div_ceil(8);
|
||
let bitmap_start = elements_start;
|
||
// prefix(db_header_size) + bitmap + checksum(4)
|
||
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
||
let page_stride = page_nelmts
|
||
.checked_mul(elem_stride)
|
||
.and_then(|x| x.checked_add(4))
|
||
.ok_or_else(stride_overflow)?;
|
||
|
||
if bitmap_start + bitmap_size > file_data.len() {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: bitmap_start + bitmap_size,
|
||
available: file_data.len(),
|
||
});
|
||
}
|
||
// The prefix and page bitmap are covered by their own checksum, and each
|
||
// initialised page by one of its own.
|
||
verify_checksum(file_data, db_offset, bitmap_start + bitmap_size)?;
|
||
|
||
for p in 0..npages {
|
||
let page_first = p * page_nelmts; // < num_elements, cannot overflow
|
||
let page_count = core::cmp::min(page_nelmts, num_elements - page_first);
|
||
|
||
// Check the page-init bit (MSB-first within each byte).
|
||
let bit_byte = file_data[bitmap_start + p / 8];
|
||
let bit_mask = 1u8 << (7 - (p % 8));
|
||
if bit_byte & bit_mask == 0 {
|
||
continue; // entire page unallocated
|
||
}
|
||
|
||
let page_off = p
|
||
.checked_mul(page_stride)
|
||
.and_then(|o| pages_start.checked_add(o))
|
||
.ok_or_else(stride_overflow)?;
|
||
verify_checksum(file_data, page_off, elem_at(page_off, page_count)?)?;
|
||
for e in 0..page_count {
|
||
push_element(page_first + e, elem_at(page_off, e)?, &mut chunks)?;
|
||
}
|
||
}
|
||
|
||
Ok(chunks)
|
||
}
|
||
|
||
/// Parse a single Fixed Array element at absolute file offset `abs`.
|
||
///
|
||
/// Returns `Some((address, chunk_size, filter_mask))` for an allocated chunk, or
|
||
/// `None` if the element is undefined (an unallocated chunk, address all-`0xFF`).
|
||
fn parse_fa_element(
|
||
file_data: &[u8],
|
||
abs: usize,
|
||
client_id: u8,
|
||
offset_size: u8,
|
||
element_size: u8,
|
||
chunk_byte_size: u64,
|
||
) -> Result<Option<(u64, u32, u32)>, FormatError> {
|
||
let os = offset_size as usize;
|
||
if client_id == 0 {
|
||
// Non-filtered: element is just the chunk address.
|
||
if abs + os > file_data.len() {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: abs + os,
|
||
available: file_data.len(),
|
||
});
|
||
}
|
||
if is_undefined(file_data, abs, offset_size) {
|
||
return Ok(None);
|
||
}
|
||
let address = read_offset(file_data, abs, offset_size)?;
|
||
Ok(Some((address, chunk_byte_size as u32, 0)))
|
||
} else {
|
||
// Filtered: address(offset_size) + chunk_size(variable) + filter_mask(4)
|
||
let es = element_size as usize;
|
||
if es < os + 4 {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"element_size too small for filtered element".into(),
|
||
));
|
||
}
|
||
let chunk_size_bytes = es - os - 4;
|
||
if abs + es > file_data.len() {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: abs + es,
|
||
available: file_data.len(),
|
||
});
|
||
}
|
||
if is_undefined(file_data, abs, offset_size) {
|
||
return Ok(None);
|
||
}
|
||
let address = read_offset(file_data, abs, offset_size)?;
|
||
let chunk_size = read_variable_length(&file_data[abs + os..], chunk_size_bytes)?;
|
||
let fm_off = abs + os + chunk_size_bytes;
|
||
let filter_mask = u32::from_le_bytes([
|
||
file_data[fm_off],
|
||
file_data[fm_off + 1],
|
||
file_data[fm_off + 2],
|
||
file_data[fm_off + 3],
|
||
]);
|
||
Ok(Some((address, chunk_size as u32, filter_mask)))
|
||
}
|
||
}
|
||
|
||
/// Read a variable-length little-endian unsigned integer.
|
||
fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
|
||
if size > 8 || data.len() < size {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid variable-length size".into(),
|
||
));
|
||
}
|
||
let mut val = 0u64;
|
||
for (i, &byte) in data.iter().enumerate().take(size) {
|
||
val |= (byte as u64) << (i * 8);
|
||
}
|
||
Ok(val)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Stamp the Jenkins checksum a real file would carry over
|
||
/// `data[start..end]`, writing it at `end`. Fixtures built by hand need
|
||
/// this now that the reader validates it — as every HDF5 writer does.
|
||
fn stamp_checksum(data: &mut [u8], start: usize, end: usize) {
|
||
let sum = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
data[end..end + 4].copy_from_slice(&sum.to_le_bytes());
|
||
}
|
||
|
||
#[test]
|
||
fn index_to_offsets_1d() {
|
||
let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
|
||
assert_eq!(g.offsets(0).unwrap(), vec![0]);
|
||
assert_eq!(g.offsets(1).unwrap(), vec![20]);
|
||
assert_eq!(g.offsets(4).unwrap(), vec![80]);
|
||
}
|
||
|
||
#[test]
|
||
fn index_to_offsets_2d() {
|
||
// 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks
|
||
let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
|
||
assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
|
||
assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
|
||
assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
|
||
assert_eq!(g.offsets(3).unwrap(), vec![4, 3]);
|
||
assert_eq!(g.offsets(5).unwrap(), vec![8, 3]);
|
||
}
|
||
|
||
#[test]
|
||
fn read_variable_length_values() {
|
||
assert_eq!(read_variable_length(&[0x78, 0x56], 2).unwrap(), 0x5678);
|
||
assert_eq!(
|
||
read_variable_length(&[0x01, 0x02, 0x03, 0x04], 4).unwrap(),
|
||
0x04030201
|
||
);
|
||
assert_eq!(read_variable_length(&[0xFF], 1).unwrap(), 0xFF);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_valid() {
|
||
let mut buf = vec![0u8; 256];
|
||
// FAHD signature
|
||
buf[0..4].copy_from_slice(b"FAHD");
|
||
buf[4] = 0; // version
|
||
buf[5] = 1; // client_id = filtered
|
||
buf[6] = 16; // element_size
|
||
buf[7] = 10; // max_nelmts_bits (page_size = 1024)
|
||
// num_elements (length_size=8)
|
||
buf[8..16].copy_from_slice(&5u64.to_le_bytes());
|
||
// data_block_address (offset_size=8)
|
||
buf[16..24].copy_from_slice(&0x1000u64.to_le_bytes());
|
||
stamp_checksum(&mut buf, 0, 24);
|
||
|
||
let header = FixedArrayHeader::parse(&buf, 0, 8, 8).unwrap();
|
||
assert_eq!(header.client_id, 1);
|
||
assert_eq!(header.element_size, 16);
|
||
assert_eq!(header.max_nelmts_bits, 10);
|
||
assert_eq!(header.num_elements, 5);
|
||
assert_eq!(header.data_block_address, 0x1000);
|
||
}
|
||
|
||
/// Corruption anywhere in the index must be an error, not a wrong
|
||
/// address. Every structure carries a checksum; flipping a bit in each in
|
||
/// turn must be caught, because the alternative is reading a chunk from
|
||
/// the wrong offset and returning it as data.
|
||
#[test]
|
||
fn corrupting_any_fixed_array_structure_is_detected() {
|
||
let build = || -> (Vec<u8>, usize) {
|
||
let (os, fahd, db) = (8usize, 0x100usize, 0x200usize);
|
||
let mut f = vec![0u8; 0x3000];
|
||
f[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
f[fahd + 6] = os as u8;
|
||
f[fahd + 7] = 10;
|
||
f[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes());
|
||
f[fahd + 16..fahd + 24].copy_from_slice(&(db as u64).to_le_bytes());
|
||
stamp_checksum(&mut f, fahd, fahd + 24);
|
||
f[db..db + 4].copy_from_slice(b"FADB");
|
||
f[db + 6..db + 14].copy_from_slice(&(fahd as u64).to_le_bytes());
|
||
let elems = db + 6 + os;
|
||
for i in 0..3usize {
|
||
let addr = 0x1000u64 + i as u64 * 0x100;
|
||
f[elems + i * os..elems + (i + 1) * os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut f, db, elems + 3 * os);
|
||
(f, fahd)
|
||
};
|
||
|
||
let read = |f: &[u8], fahd: usize| -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let h = FixedArrayHeader::parse(f, fahd, 8, 8)?;
|
||
read_fixed_array_chunks(f, &h, &[60], None, &[20], 8, 8, 8)
|
||
};
|
||
|
||
let (clean, fahd) = build();
|
||
assert!(read(&clean, fahd).is_ok(), "the intact fixture must read");
|
||
|
||
// A byte inside the header, and one inside a data block element.
|
||
for &at in &[0x108usize, 0x210usize] {
|
||
let (mut damaged, fahd) = build();
|
||
damaged[at] ^= 0x01;
|
||
assert!(
|
||
matches!(
|
||
read(&damaged, fahd),
|
||
Err(FormatError::ChecksumMismatch { .. })
|
||
),
|
||
"corruption at {at:#x} went undetected"
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_invalid_signature() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"XXXX");
|
||
let result = FixedArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// Malformed headers must error, never panic (shift overflow, huge counts).
|
||
#[test]
|
||
fn read_rejects_oversized_max_nelmts_bits() {
|
||
let mut buf = vec![0u8; 512];
|
||
let fahd = 0x40usize;
|
||
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
buf[fahd + 4] = 0; // version
|
||
buf[fahd + 5] = 0; // client_id
|
||
buf[fahd + 6] = 8; // element_size
|
||
buf[fahd + 7] = 200; // max_nelmts_bits — absurd, would overflow a shift
|
||
buf[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes()); // num_elements
|
||
buf[fahd + 16..fahd + 24].copy_from_slice(&0x100u64.to_le_bytes());
|
||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||
// FADB so parsing reaches the paged check
|
||
let db = 0x100usize;
|
||
buf[db..db + 4].copy_from_slice(b"FADB");
|
||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn read_rejects_num_elements_larger_than_file() {
|
||
let mut buf = vec![0u8; 256];
|
||
let fahd = 0x40usize;
|
||
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
buf[fahd + 6] = 8;
|
||
buf[fahd + 7] = 10;
|
||
buf[fahd + 8..fahd + 16].copy_from_slice(&u64::MAX.to_le_bytes()); // absurd count
|
||
buf[fahd + 16..fahd + 24].copy_from_slice(&0x80u64.to_le_bytes());
|
||
// Valid checksum, so it is the element count that must be rejected.
|
||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||
buf[0x80..0x84].copy_from_slice(b"FADB");
|
||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn parse_rejects_offset_overflow() {
|
||
let buf = vec![0u8; 64];
|
||
let result = FixedArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` data block address must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn read_rejects_data_block_offset_overflow() {
|
||
let header = FixedArrayHeader {
|
||
client_id: 0,
|
||
element_size: 8,
|
||
max_nelmts_bits: 10,
|
||
num_elements: 1,
|
||
data_block_address: (usize::MAX - 4) as u64,
|
||
};
|
||
let buf = vec![0u8; 64];
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_invalid_version() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"FAHD");
|
||
buf[4] = 1; // unsupported version
|
||
let result = FixedArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// Build a synthetic Fixed Array (non-filtered) and verify reading.
|
||
#[test]
|
||
fn read_non_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
let num_chunks = 5u64;
|
||
|
||
let mut file_data = vec![0u8; 0x3000];
|
||
|
||
// Build FAHD at offset 0x100
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x200usize;
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0; // version
|
||
file_data[fahd_offset + 5] = 0; // client_id = non-filtered
|
||
file_data[fahd_offset + 6] = os as u8; // element_size = just address
|
||
file_data[fahd_offset + 7] = 10; // max_nelmts_bits
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_chunks.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
// Build FADB at db_offset
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0; // version
|
||
file_data[db_offset + 5] = 0; // client_id
|
||
file_data[db_offset + 6..db_offset + 14]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes()); // header_address
|
||
|
||
// Elements: 5 addresses
|
||
let elem_start = db_offset + 6 + os;
|
||
let base_addr = 0x1000u64;
|
||
let chunk_byte_size = 20 * 8; // 20 elements × 8 bytes
|
||
for i in 0..5 {
|
||
let addr = base_addr + i as u64 * chunk_byte_size as u64;
|
||
let pos = elem_start + i * os;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut file_data, db_offset, elem_start + 5 * os);
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
let ds_dims = vec![100u64];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
None,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 5);
|
||
for (i, c) in chunks.iter().enumerate() {
|
||
assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size as u64);
|
||
assert_eq!(c.offsets, vec![i as u64 * 20]);
|
||
assert_eq!(c.filter_mask, 0);
|
||
assert_eq!(c.chunk_size, chunk_byte_size as u32);
|
||
}
|
||
}
|
||
|
||
/// Build a synthetic Fixed Array (filtered) and verify reading.
|
||
#[test]
|
||
fn read_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
let num_chunks = 3u64;
|
||
// element_size for filtered: offset_size + chunk_size_bytes + 4(filter_mask)
|
||
// chunk_size_bytes: let's use 4 bytes
|
||
let chunk_size_bytes = 4usize;
|
||
let elem_size = os + chunk_size_bytes + 4;
|
||
|
||
let mut file_data = vec![0u8; 0x3000];
|
||
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x200usize;
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0;
|
||
file_data[fahd_offset + 5] = 1; // client_id = filtered
|
||
file_data[fahd_offset + 6] = elem_size as u8;
|
||
file_data[fahd_offset + 7] = 10;
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_chunks.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0;
|
||
file_data[db_offset + 5] = 1;
|
||
file_data[db_offset + 6..db_offset + 14]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes());
|
||
|
||
let elem_start = db_offset + 6 + os;
|
||
let test_chunks = [
|
||
(0x1000u64, 120u32, 0u32),
|
||
(0x2000u64, 115u32, 0u32),
|
||
(0x3000u64, 100u32, 0u32),
|
||
];
|
||
|
||
for (i, &(addr, csize, fmask)) in test_chunks.iter().enumerate() {
|
||
let pos = elem_start + i * elem_size;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
// chunk_size as 4 bytes LE
|
||
file_data[pos + os..pos + os + 4].copy_from_slice(&csize.to_le_bytes());
|
||
file_data[pos + os + 4..pos + os + 8].copy_from_slice(&fmask.to_le_bytes());
|
||
}
|
||
stamp_checksum(
|
||
&mut file_data,
|
||
db_offset,
|
||
elem_start + test_chunks.len() * elem_size,
|
||
);
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
let ds_dims = vec![60u64];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
None,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 3);
|
||
assert_eq!(chunks[0].address, 0x1000);
|
||
assert_eq!(chunks[0].chunk_size, 120);
|
||
assert_eq!(chunks[0].filter_mask, 0);
|
||
assert_eq!(chunks[0].offsets, vec![0]);
|
||
assert_eq!(chunks[1].address, 0x2000);
|
||
assert_eq!(chunks[1].chunk_size, 115);
|
||
assert_eq!(chunks[2].address, 0x3000);
|
||
assert_eq!(chunks[2].chunk_size, 100);
|
||
}
|
||
|
||
/// Build a synthetic *paged* Fixed Array (non-filtered) and verify reading.
|
||
///
|
||
/// Layout reverse-engineered and confirmed against an HDF5 2.0 file:
|
||
/// after the FADB prefix comes a page-init bitmap (MSB-first within each
|
||
/// byte), a 4-byte checksum, then full-size page slots (`page_nelmts`
|
||
/// elements + a 4-byte checksum each), with only the last page shorter.
|
||
/// Uninitialized pages occupy their slot but are skipped via the bitmap.
|
||
#[test]
|
||
fn read_paged_non_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
|
||
// page_nelmts = 1 << 2 = 4. Use 11 elements => 3 pages
|
||
// (page0: 4, page1: 4, page2: 3 short). Initialize pages 0 and 2; leave
|
||
// page 1 uninitialized. 3 pages still fits one bitmap byte, but we place
|
||
// the set bits at positions 7 and 5 to lock the MSB-first ordering.
|
||
let max_nelmts_bits = 2u8;
|
||
let page_nelmts = 1usize << max_nelmts_bits; // 4
|
||
let num_elements = 11u64;
|
||
let db_header_size = 4 + 1 + 1 + os; // FADB sig+ver+client+header_addr
|
||
let bitmap_size = 1usize; // ceil(3/8)
|
||
let page_total = page_nelmts * os + 4; // elements + checksum
|
||
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x400usize;
|
||
let mut file_data = vec![0u8; 0x4000];
|
||
|
||
// FAHD
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0; // version
|
||
file_data[fahd_offset + 5] = 0; // client_id = non-filtered
|
||
file_data[fahd_offset + 6] = os as u8; // element_size = address only
|
||
file_data[fahd_offset + 7] = max_nelmts_bits;
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_elements.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
// FADB prefix
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0; // version
|
||
file_data[db_offset + 5] = 0; // client_id
|
||
file_data[db_offset + 6..db_offset + 6 + os]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes());
|
||
|
||
// Page-init bitmap: pages 0 and 2 initialized, page 1 not.
|
||
// MSB-first => page0 -> bit7 (0x80), page2 -> bit5 (0x20) => 0xA0.
|
||
let bitmap_off = db_offset + db_header_size;
|
||
file_data[bitmap_off] = 0b1010_0000;
|
||
|
||
// Pages start after bitmap + 4-byte checksum.
|
||
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
||
|
||
let base_addr = 0x1000u64;
|
||
// Page 0 (elements 0..4) and page 2 (elements 8..11) carry addresses;
|
||
// page 1's slot is left zero-filled and must be skipped.
|
||
// The prefix and bitmap carry one checksum, each initialised page
|
||
// another — as a real file does.
|
||
stamp_checksum(&mut file_data, db_offset, bitmap_off + bitmap_size);
|
||
for &p in &[0usize, 2usize] {
|
||
let page_off = pages_start + p * page_total;
|
||
let count = core::cmp::min(page_nelmts, num_elements as usize - p * page_nelmts);
|
||
for e in 0..count {
|
||
let i = p * page_nelmts + e;
|
||
let addr = base_addr + i as u64 * 0x100;
|
||
let pos = page_off + e * os;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut file_data, page_off, page_off + count * os);
|
||
}
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
assert_eq!(header.num_elements, 11);
|
||
|
||
let ds_dims = vec![11u64 * 20];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
None,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
// Page 1 (elements 4,5,6,7) is uninitialized => skipped. The remaining
|
||
// 7 chunks (0..4 and 8..11) come back with their original linear index.
|
||
assert_eq!(chunks.len(), 7);
|
||
let mut got: Vec<(u64, u64)> = chunks.iter().map(|c| (c.offsets[0], c.address)).collect();
|
||
got.sort();
|
||
let expect: Vec<(u64, u64)> = [0usize, 1, 2, 3, 8, 9, 10]
|
||
.iter()
|
||
.map(|&i| (i as u64 * 20, base_addr + i as u64 * 0x100))
|
||
.collect();
|
||
assert_eq!(got, expect);
|
||
}
|
||
}
|