osobhandClaude Opus 5.5
76c97f6c94
On a backend without the file in memory, a structure read whose length comes from untrusted header fields was clamped only by the end of the file, so a crafted size made one read (and copy) of up to the rest of the file. Each such read now covers what the parser actually uses: - local heap names: read in growing pieces (64 bytes first, then 4x) up to the end of the data segment, instead of the rest of the segment per name (quadratic for a big symbol-table group); - fractal heap indirect blocks: the doubling-table geometry locates the entry covering the object, and the first read ends at that entry; only if it is unallocated does the walk read the rest of the block (it visits every entry then). One walk implementation serves both; - paged fixed/extensible array data blocks over 1 MiB: the prefix and page bitmap, then each page in use on its own (smaller blocks are still one read); - blocks under one checksum (non-paged array data blocks, extensible array index and super blocks): the bounds check that comes first (the checksum's; the page bitmap's for a super block) is made against the file length before reading (Window::check_extent), so a block claimed past the end of the file costs no read. With the checksum feature off the parser has no such first check and the old read stands. Other windows were already bounded (the superblock and object header prefixes, the fractal heap header by a u16, SOHM tables by u8/u16 counts) or are exact reads checked against the file length first. In memory nothing changes: the pieces are borrowed slices. Tests: CountingStorage over a crafted heap (16 MiB file, width and rows 0xFFFF: under 1 KiB read, 16.7 MB before), a heap segment claiming 64 MiB (one 64-byte read per short name), long names at every piece boundary, a fixed array block claimed past the end of a 16 MiB file (under 64 bytes read), and in the equivalence harness an h5py file with a 2.4 MB fixed array block and a >1 MiB extensible array block, whole and cut at 97 points: every chunk index agrees with the slice read and the largest takes 205 KB (2.4 MB and 1.2 MB when read whole). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
997 lines
39 KiB
Rust
997 lines
39 KiB
Rust
//! HDF5 Fixed Array index parsing for chunked datasets (v4 index type 3).
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
extern crate alloc;
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{format, vec, vec::Vec};
|
||
|
||
use crate::chunk_grid::ChunkGrid;
|
||
use crate::chunked_read::ChunkInfo;
|
||
use crate::error::FormatError;
|
||
use crate::storage::{PAGED_BLOCK_ONE_READ_MAX, Storage, Window, len_usize, read_exact_at};
|
||
|
||
/// Verify the Jenkins lookup3 checksum stored immediately after
|
||
/// `data[start..end]`, as every Fixed Array structure carries one. `w` is
|
||
/// a window of the file and `start`/`end` are relative to it.
|
||
///
|
||
/// A corrupt chunk index silently yields addresses pointing at the wrong
|
||
/// bytes, so a mismatch has to be an error rather than a shrug: without this
|
||
/// the damage surfaces as plausible-looking data from the wrong chunk.
|
||
#[cfg(feature = "checksum")]
|
||
fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> {
|
||
w.ensure(end, 4)?;
|
||
let data = &w.bytes;
|
||
let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
|
||
let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
if computed != stored {
|
||
return Err(FormatError::ChecksumMismatch {
|
||
expected: stored,
|
||
computed,
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(not(feature = "checksum"))]
|
||
fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> {
|
||
Ok(())
|
||
}
|
||
|
||
/// Parsed Fixed Array header (FAHD).
|
||
#[derive(Debug, Clone)]
|
||
pub struct FixedArrayHeader {
|
||
/// Client ID: 0 = non-filtered chunks, 1 = filtered chunks.
|
||
pub client_id: u8,
|
||
/// Size of each array element in bytes.
|
||
pub element_size: u8,
|
||
/// Log2 of max number of elements in a data block page.
|
||
pub max_nelmts_bits: u8,
|
||
/// Total number of elements (chunks) in the array.
|
||
pub num_elements: u64,
|
||
/// Address of the data block.
|
||
pub data_block_address: u64,
|
||
}
|
||
|
||
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
let s = size as usize;
|
||
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: pos.saturating_add(s),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match size {
|
||
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
||
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
||
8 => u64::from_le_bytes([
|
||
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
||
]),
|
||
_ => return Err(FormatError::InvalidOffsetSize(size)),
|
||
})
|
||
}
|
||
|
||
fn read_length(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
read_offset(data, pos, size)
|
||
}
|
||
|
||
fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
|
||
let s = size as usize;
|
||
if pos + s > data.len() {
|
||
return false;
|
||
}
|
||
data[pos..pos + s].iter().all(|&b| b == 0xFF)
|
||
}
|
||
|
||
impl FixedArrayHeader {
|
||
/// Parse a Fixed Array header from file data at the given offset.
|
||
pub fn parse(
|
||
file_data: &[u8],
|
||
offset: usize,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Self, FormatError> {
|
||
Self::parse_in(file_data, offset as u64, offset_size, length_size)
|
||
}
|
||
|
||
/// [`Self::parse`] over any [`Storage`]: one read of the header.
|
||
pub fn parse_in<S: Storage + ?Sized>(
|
||
file: &S,
|
||
offset: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Self, FormatError> {
|
||
// FAHD signature(4) + version(1) + client_id(1) + element_size(1) +
|
||
// max_nelmts_bits(1) + num_elements(length_size) + data_block_addr(offset_size) + checksum(4)
|
||
let min_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + offset_size as usize + 4;
|
||
let w = Window::read(file, offset, min_size)?;
|
||
w.ensure(0, min_size)?;
|
||
|
||
let d: &[u8] = &w.bytes;
|
||
if &d[0..4] != b"FAHD" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Fixed Array header signature".into(),
|
||
));
|
||
}
|
||
|
||
let version = d[4];
|
||
if version != 0 {
|
||
return Err(FormatError::ChunkedReadError(format!(
|
||
"unsupported Fixed Array header version: {version}"
|
||
)));
|
||
}
|
||
|
||
let client_id = d[5];
|
||
let element_size = d[6];
|
||
let max_nelmts_bits = d[7];
|
||
|
||
let mut pos = 8;
|
||
let num_elements = read_length(d, pos, length_size)?;
|
||
pos += length_size as usize;
|
||
let data_block_address = read_offset(d, pos, offset_size)?;
|
||
pos += offset_size as usize;
|
||
verify_checksum(&w, 0, pos)?;
|
||
|
||
Ok(FixedArrayHeader {
|
||
client_id,
|
||
element_size,
|
||
max_nelmts_bits,
|
||
num_elements,
|
||
data_block_address,
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Read chunk records from a Fixed Array data block.
|
||
///
|
||
/// Returns a `Vec<ChunkInfo>` with one entry per allocated chunk.
|
||
/// `chunk_dimensions` should be the spatial chunk dims only (not including the element-size dim).
|
||
/// `element_size` is the datatype size in bytes.
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn read_fixed_array_chunks(
|
||
file_data: &[u8],
|
||
header: &FixedArrayHeader,
|
||
dataset_dims: &[u64],
|
||
max_dims: Option<&[u64]>,
|
||
chunk_dimensions: &[u32],
|
||
element_size: u32,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||
read_fixed_array_chunks_in(
|
||
&file_data,
|
||
header,
|
||
dataset_dims,
|
||
max_dims,
|
||
chunk_dimensions,
|
||
element_size,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
}
|
||
|
||
/// [`read_fixed_array_chunks`] over any [`Storage`]: one read of the data
|
||
/// block's prefix, one of the whole data block (pages included).
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn read_fixed_array_chunks_in<S: Storage + ?Sized>(
|
||
file: &S,
|
||
header: &FixedArrayHeader,
|
||
dataset_dims: &[u64],
|
||
max_dims: Option<&[u64]>,
|
||
chunk_dimensions: &[u32],
|
||
element_size: u32,
|
||
offset_size: u8,
|
||
_length_size: u8,
|
||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let file_len = len_usize(file);
|
||
let db_offset = header.data_block_address as usize;
|
||
|
||
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
|
||
let db_header_size = 4 + 1 + 1 + offset_size as usize;
|
||
let d = read_exact_at(file, db_offset as u64, db_header_size)?;
|
||
if &d[0..4] != b"FADB" {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid Fixed Array data block signature".into(),
|
||
));
|
||
}
|
||
|
||
// Elements start immediately after the data block prefix.
|
||
let elements_start = db_offset + db_header_size;
|
||
|
||
let num_elements = header.num_elements as usize;
|
||
// A chunk index cannot describe more elements than the file has bytes (each
|
||
// element occupies at least `offset_size` bytes). Reject a corrupt count
|
||
// before it can drive a huge loop or overflow an offset computation.
|
||
if num_elements > file_len {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Fixed Array element count exceeds file size".into(),
|
||
));
|
||
}
|
||
let os = offset_size as usize;
|
||
// On-disk stride of one element. For non-filtered arrays the element is just
|
||
// the chunk address (== offset_size); for filtered arrays it is
|
||
// address + chunk_size + filter_mask (== header.element_size).
|
||
let elem_stride = (header.element_size as usize).max(os);
|
||
|
||
// Absolute file offset of element `idx` within a run starting at `base`,
|
||
// with overflow surfaced as a clean error rather than a panic/wrap.
|
||
let elem_at = |base: usize, idx: usize| -> Result<usize, FormatError> {
|
||
idx.checked_mul(elem_stride)
|
||
.and_then(|o| base.checked_add(o))
|
||
.ok_or(FormatError::ChunkedReadError(
|
||
"Fixed Array element offset overflow".into(),
|
||
))
|
||
};
|
||
|
||
// The index is laid out over the chunk grid of the *maximum* dimensions
|
||
// (row-major), so a dataset smaller than its maxshape has gaps.
|
||
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
|
||
let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?;
|
||
|
||
let chunk_byte_size: u64 =
|
||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||
|
||
let mut chunks = Vec::new();
|
||
// `rel` is relative to the data block, whose bytes are in `w`.
|
||
let push_element = |w: &Window<'_>,
|
||
i: usize,
|
||
rel: usize,
|
||
chunks: &mut Vec<ChunkInfo>|
|
||
-> Result<(), FormatError> {
|
||
if let Some((address, chunk_size, filter_mask)) = parse_fa_element(
|
||
w,
|
||
rel,
|
||
header.client_id,
|
||
offset_size,
|
||
header.element_size,
|
||
chunk_byte_size,
|
||
)? {
|
||
// A slot beyond the current extent is ignored, as the
|
||
// library does.
|
||
let Some(offsets) = grid.offsets(i as u64) else {
|
||
return Ok(());
|
||
};
|
||
chunks.push(ChunkInfo {
|
||
chunk_size,
|
||
filter_mask,
|
||
offsets,
|
||
address,
|
||
});
|
||
}
|
||
Ok(())
|
||
};
|
||
|
||
// A data block is paged when it holds more elements than fit in one page.
|
||
// `max_nelmts_bits` is an untrusted u8; a shift >= the pointer width would
|
||
// panic, so reject it (real page-size bits are tiny — 10 by default).
|
||
if header.max_nelmts_bits as u32 >= usize::BITS {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"Fixed Array max_nelmts_bits too large".into(),
|
||
));
|
||
}
|
||
let page_nelmts = 1usize << header.max_nelmts_bits;
|
||
let is_paged = num_elements > page_nelmts;
|
||
|
||
if !is_paged {
|
||
// Non-paged: prefix, then `num_elements` elements packed directly,
|
||
// then a checksum over both. One window holds all of it (or ends at
|
||
// the end of the file), so its bounds checks are the whole-file ones.
|
||
let end = elem_at(elements_start, num_elements)?;
|
||
// The checksum's bounds check comes first: make it before reading.
|
||
#[cfg(feature = "checksum")]
|
||
Window::check_extent(file, db_offset as u64, end - db_offset, 4)?;
|
||
let w = Window::read(file, db_offset as u64, end.saturating_add(4) - db_offset)?;
|
||
verify_checksum(&w, 0, end - db_offset)?;
|
||
for i in 0..num_elements {
|
||
push_element(&w, i, elem_at(elements_start, i)? - db_offset, &mut chunks)?;
|
||
}
|
||
return Ok(chunks);
|
||
}
|
||
|
||
// Paged layout: prefix, then a page-init bitmap (one bit per page, MSB-first
|
||
// within each byte), then a 4-byte checksum, then the pages. Every page
|
||
// occupies a full slot of `page_nelmts` elements plus a 4-byte checksum;
|
||
// only the final page holds fewer elements. Uninitialized pages (bit clear)
|
||
// still occupy their slot on disk but are zero-filled, so the bitmap — not a
|
||
// 0xFF sentinel — is what marks a whole page as unallocated.
|
||
let stride_overflow =
|
||
|| FormatError::ChunkedReadError("Fixed Array page offset overflow".into());
|
||
let npages = num_elements.div_ceil(page_nelmts);
|
||
let bitmap_size = npages.div_ceil(8);
|
||
let bitmap_start = elements_start;
|
||
// prefix(db_header_size) + bitmap + checksum(4)
|
||
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
||
let page_stride = page_nelmts
|
||
.checked_mul(elem_stride)
|
||
.and_then(|x| x.checked_add(4))
|
||
.ok_or_else(stride_overflow)?;
|
||
|
||
if bitmap_start + bitmap_size > file_len {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: bitmap_start + bitmap_size,
|
||
available: file_len,
|
||
});
|
||
}
|
||
// The whole data block in one window when it is small: every page slot
|
||
// is at most `page_stride` bytes, so every position checked below lies
|
||
// inside it (or past the end of the file). A larger block is read as its
|
||
// prefix and bitmap, then each page in use on its own.
|
||
let block_len = (pages_start - db_offset).saturating_add(npages.saturating_mul(page_stride));
|
||
let whole = if block_len <= PAGED_BLOCK_ONE_READ_MAX {
|
||
Some(Window::read(file, db_offset as u64, block_len)?)
|
||
} else {
|
||
None
|
||
};
|
||
let head_w;
|
||
let head = match &whole {
|
||
Some(w) => w,
|
||
None => {
|
||
head_w = Window::read(file, db_offset as u64, pages_start - db_offset)?;
|
||
&head_w
|
||
}
|
||
};
|
||
// The prefix and page bitmap are covered by their own checksum, and each
|
||
// initialised page by one of its own.
|
||
verify_checksum(head, 0, bitmap_start + bitmap_size - db_offset)?;
|
||
|
||
for p in 0..npages {
|
||
let page_first = p * page_nelmts; // < num_elements, cannot overflow
|
||
let page_count = core::cmp::min(page_nelmts, num_elements - page_first);
|
||
|
||
// Check the page-init bit (MSB-first within each byte).
|
||
let bit_byte = head.bytes[bitmap_start + p / 8 - db_offset];
|
||
let bit_mask = 1u8 << (7 - (p % 8));
|
||
if bit_byte & bit_mask == 0 {
|
||
continue; // entire page unallocated
|
||
}
|
||
|
||
let page_off = p
|
||
.checked_mul(page_stride)
|
||
.and_then(|o| pages_start.checked_add(o))
|
||
.ok_or_else(stride_overflow)?;
|
||
let page_end = elem_at(page_off, page_count)?;
|
||
// `w` holds the page from `base` on (positions below are relative
|
||
// to it).
|
||
let page_w;
|
||
let (w, base) = match &whole {
|
||
Some(w) => (w, db_offset),
|
||
None => {
|
||
page_w =
|
||
Window::read(file, page_off as u64, page_end.saturating_add(4) - page_off)?;
|
||
(&page_w, page_off)
|
||
}
|
||
};
|
||
verify_checksum(w, page_off - base, page_end - base)?;
|
||
for e in 0..page_count {
|
||
push_element(w, page_first + e, elem_at(page_off, e)? - base, &mut chunks)?;
|
||
}
|
||
}
|
||
|
||
Ok(chunks)
|
||
}
|
||
|
||
/// Parse a single Fixed Array element at offset `abs` of the window `w`.
|
||
///
|
||
/// Returns `Some((address, chunk_size, filter_mask))` for an allocated chunk, or
|
||
/// `None` if the element is undefined (an unallocated chunk, address all-`0xFF`).
|
||
fn parse_fa_element(
|
||
w: &Window<'_>,
|
||
abs: usize,
|
||
client_id: u8,
|
||
offset_size: u8,
|
||
element_size: u8,
|
||
chunk_byte_size: u64,
|
||
) -> Result<Option<(u64, u32, u32)>, FormatError> {
|
||
let os = offset_size as usize;
|
||
if client_id == 0 {
|
||
// Non-filtered: element is just the chunk address.
|
||
w.ensure(abs, os)?;
|
||
let file_data: &[u8] = &w.bytes;
|
||
if is_undefined(file_data, abs, offset_size) {
|
||
return Ok(None);
|
||
}
|
||
let address = read_offset(file_data, abs, offset_size)?;
|
||
Ok(Some((address, chunk_byte_size as u32, 0)))
|
||
} else {
|
||
// Filtered: address(offset_size) + chunk_size(variable) + filter_mask(4)
|
||
let es = element_size as usize;
|
||
if es < os + 4 {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"element_size too small for filtered element".into(),
|
||
));
|
||
}
|
||
let chunk_size_bytes = es - os - 4;
|
||
w.ensure(abs, es)?;
|
||
let file_data: &[u8] = &w.bytes;
|
||
if is_undefined(file_data, abs, offset_size) {
|
||
return Ok(None);
|
||
}
|
||
let address = read_offset(file_data, abs, offset_size)?;
|
||
let chunk_size =
|
||
read_variable_length(&file_data[abs + os..abs + es - 4], chunk_size_bytes)?;
|
||
let fm_off = abs + os + chunk_size_bytes;
|
||
let filter_mask = u32::from_le_bytes([
|
||
file_data[fm_off],
|
||
file_data[fm_off + 1],
|
||
file_data[fm_off + 2],
|
||
file_data[fm_off + 3],
|
||
]);
|
||
Ok(Some((address, chunk_size as u32, filter_mask)))
|
||
}
|
||
}
|
||
|
||
/// Read a variable-length little-endian unsigned integer.
|
||
fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
|
||
if size > 8 || data.len() < size {
|
||
return Err(FormatError::ChunkedReadError(
|
||
"invalid variable-length size".into(),
|
||
));
|
||
}
|
||
let mut val = 0u64;
|
||
for (i, &byte) in data.iter().enumerate().take(size) {
|
||
val |= (byte as u64) << (i * 8);
|
||
}
|
||
Ok(val)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Stamp the Jenkins checksum a real file would carry over
|
||
/// `data[start..end]`, writing it at `end`. Fixtures built by hand need
|
||
/// this now that the reader validates it — as every HDF5 writer does.
|
||
fn stamp_checksum(data: &mut [u8], start: usize, end: usize) {
|
||
let sum = crate::checksum::jenkins_lookup3(&data[start..end]);
|
||
data[end..end + 4].copy_from_slice(&sum.to_le_bytes());
|
||
}
|
||
|
||
#[test]
|
||
fn index_to_offsets_1d() {
|
||
let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
|
||
assert_eq!(g.offsets(0).unwrap(), vec![0]);
|
||
assert_eq!(g.offsets(1).unwrap(), vec![20]);
|
||
assert_eq!(g.offsets(4).unwrap(), vec![80]);
|
||
}
|
||
|
||
#[test]
|
||
fn index_to_offsets_2d() {
|
||
// 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks
|
||
let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
|
||
assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
|
||
assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
|
||
assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
|
||
assert_eq!(g.offsets(3).unwrap(), vec![4, 3]);
|
||
assert_eq!(g.offsets(5).unwrap(), vec![8, 3]);
|
||
}
|
||
|
||
#[test]
|
||
fn read_variable_length_values() {
|
||
assert_eq!(read_variable_length(&[0x78, 0x56], 2).unwrap(), 0x5678);
|
||
assert_eq!(
|
||
read_variable_length(&[0x01, 0x02, 0x03, 0x04], 4).unwrap(),
|
||
0x04030201
|
||
);
|
||
assert_eq!(read_variable_length(&[0xFF], 1).unwrap(), 0xFF);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_valid() {
|
||
let mut buf = vec![0u8; 256];
|
||
// FAHD signature
|
||
buf[0..4].copy_from_slice(b"FAHD");
|
||
buf[4] = 0; // version
|
||
buf[5] = 1; // client_id = filtered
|
||
buf[6] = 16; // element_size
|
||
buf[7] = 10; // max_nelmts_bits (page_size = 1024)
|
||
// num_elements (length_size=8)
|
||
buf[8..16].copy_from_slice(&5u64.to_le_bytes());
|
||
// data_block_address (offset_size=8)
|
||
buf[16..24].copy_from_slice(&0x1000u64.to_le_bytes());
|
||
stamp_checksum(&mut buf, 0, 24);
|
||
|
||
let header = FixedArrayHeader::parse(&buf, 0, 8, 8).unwrap();
|
||
assert_eq!(header.client_id, 1);
|
||
assert_eq!(header.element_size, 16);
|
||
assert_eq!(header.max_nelmts_bits, 10);
|
||
assert_eq!(header.num_elements, 5);
|
||
assert_eq!(header.data_block_address, 0x1000);
|
||
}
|
||
|
||
/// Corruption anywhere in the index must be an error, not a wrong
|
||
/// address. Every structure carries a checksum; flipping a bit in each in
|
||
/// turn must be caught, because the alternative is reading a chunk from
|
||
/// the wrong offset and returning it as data.
|
||
#[test]
|
||
fn corrupting_any_fixed_array_structure_is_detected() {
|
||
let build = || -> (Vec<u8>, usize) {
|
||
let (os, fahd, db) = (8usize, 0x100usize, 0x200usize);
|
||
let mut f = vec![0u8; 0x3000];
|
||
f[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
f[fahd + 6] = os as u8;
|
||
f[fahd + 7] = 10;
|
||
f[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes());
|
||
f[fahd + 16..fahd + 24].copy_from_slice(&(db as u64).to_le_bytes());
|
||
stamp_checksum(&mut f, fahd, fahd + 24);
|
||
f[db..db + 4].copy_from_slice(b"FADB");
|
||
f[db + 6..db + 14].copy_from_slice(&(fahd as u64).to_le_bytes());
|
||
let elems = db + 6 + os;
|
||
for i in 0..3usize {
|
||
let addr = 0x1000u64 + i as u64 * 0x100;
|
||
f[elems + i * os..elems + (i + 1) * os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut f, db, elems + 3 * os);
|
||
(f, fahd)
|
||
};
|
||
|
||
let read = |f: &[u8], fahd: usize| -> Result<Vec<ChunkInfo>, FormatError> {
|
||
let h = FixedArrayHeader::parse(f, fahd, 8, 8)?;
|
||
read_fixed_array_chunks(f, &h, &[60], None, &[20], 8, 8, 8)
|
||
};
|
||
|
||
let (clean, fahd) = build();
|
||
assert!(read(&clean, fahd).is_ok(), "the intact fixture must read");
|
||
|
||
// A byte inside the header, and one inside a data block element.
|
||
for &at in &[0x108usize, 0x210usize] {
|
||
let (mut damaged, fahd) = build();
|
||
damaged[at] ^= 0x01;
|
||
assert!(
|
||
matches!(
|
||
read(&damaged, fahd),
|
||
Err(FormatError::ChecksumMismatch { .. })
|
||
),
|
||
"corruption at {at:#x} went undetected"
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_invalid_signature() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"XXXX");
|
||
let result = FixedArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// Malformed headers must error, never panic (shift overflow, huge counts).
|
||
#[test]
|
||
fn read_rejects_oversized_max_nelmts_bits() {
|
||
let mut buf = vec![0u8; 512];
|
||
let fahd = 0x40usize;
|
||
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
buf[fahd + 4] = 0; // version
|
||
buf[fahd + 5] = 0; // client_id
|
||
buf[fahd + 6] = 8; // element_size
|
||
buf[fahd + 7] = 200; // max_nelmts_bits — absurd, would overflow a shift
|
||
buf[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes()); // num_elements
|
||
buf[fahd + 16..fahd + 24].copy_from_slice(&0x100u64.to_le_bytes());
|
||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||
// FADB so parsing reaches the paged check
|
||
let db = 0x100usize;
|
||
buf[db..db + 4].copy_from_slice(b"FADB");
|
||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn read_rejects_num_elements_larger_than_file() {
|
||
let mut buf = vec![0u8; 256];
|
||
let fahd = 0x40usize;
|
||
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
buf[fahd + 6] = 8;
|
||
buf[fahd + 7] = 10;
|
||
buf[fahd + 8..fahd + 16].copy_from_slice(&u64::MAX.to_le_bytes()); // absurd count
|
||
buf[fahd + 16..fahd + 24].copy_from_slice(&0x80u64.to_le_bytes());
|
||
// Valid checksum, so it is the element count that must be rejected.
|
||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||
buf[0x80..0x84].copy_from_slice(b"FADB");
|
||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn parse_rejects_offset_overflow() {
|
||
let buf = vec![0u8; 64];
|
||
let result = FixedArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// A near-`usize::MAX` data block address must error cleanly, not overflow/panic.
|
||
#[test]
|
||
fn read_rejects_data_block_offset_overflow() {
|
||
let header = FixedArrayHeader {
|
||
client_id: 0,
|
||
element_size: 8,
|
||
max_nelmts_bits: 10,
|
||
num_elements: 1,
|
||
data_block_address: (usize::MAX - 4) as u64,
|
||
};
|
||
let buf = vec![0u8; 64];
|
||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||
assert!(r.is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn parse_fixed_array_header_invalid_version() {
|
||
let mut buf = vec![0u8; 256];
|
||
buf[0..4].copy_from_slice(b"FAHD");
|
||
buf[4] = 1; // unsupported version
|
||
let result = FixedArrayHeader::parse(&buf, 0, 8, 8);
|
||
assert!(result.is_err());
|
||
}
|
||
|
||
/// Build a synthetic Fixed Array (non-filtered) and verify reading.
|
||
#[test]
|
||
fn read_non_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
let num_chunks = 5u64;
|
||
|
||
let mut file_data = vec![0u8; 0x3000];
|
||
|
||
// Build FAHD at offset 0x100
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x200usize;
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0; // version
|
||
file_data[fahd_offset + 5] = 0; // client_id = non-filtered
|
||
file_data[fahd_offset + 6] = os as u8; // element_size = just address
|
||
file_data[fahd_offset + 7] = 10; // max_nelmts_bits
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_chunks.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
// Build FADB at db_offset
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0; // version
|
||
file_data[db_offset + 5] = 0; // client_id
|
||
file_data[db_offset + 6..db_offset + 14]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes()); // header_address
|
||
|
||
// Elements: 5 addresses
|
||
let elem_start = db_offset + 6 + os;
|
||
let base_addr = 0x1000u64;
|
||
let chunk_byte_size = 20 * 8; // 20 elements × 8 bytes
|
||
for i in 0..5 {
|
||
let addr = base_addr + i as u64 * chunk_byte_size as u64;
|
||
let pos = elem_start + i * os;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut file_data, db_offset, elem_start + 5 * os);
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
let ds_dims = vec![100u64];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
None,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 5);
|
||
for (i, c) in chunks.iter().enumerate() {
|
||
assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size as u64);
|
||
assert_eq!(c.offsets, vec![i as u64 * 20]);
|
||
assert_eq!(c.filter_mask, 0);
|
||
assert_eq!(c.chunk_size, chunk_byte_size as u32);
|
||
}
|
||
}
|
||
|
||
/// Build a synthetic Fixed Array (filtered) and verify reading.
|
||
#[test]
|
||
fn read_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
let num_chunks = 3u64;
|
||
// element_size for filtered: offset_size + chunk_size_bytes + 4(filter_mask)
|
||
// chunk_size_bytes: let's use 4 bytes
|
||
let chunk_size_bytes = 4usize;
|
||
let elem_size = os + chunk_size_bytes + 4;
|
||
|
||
let mut file_data = vec![0u8; 0x3000];
|
||
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x200usize;
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0;
|
||
file_data[fahd_offset + 5] = 1; // client_id = filtered
|
||
file_data[fahd_offset + 6] = elem_size as u8;
|
||
file_data[fahd_offset + 7] = 10;
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_chunks.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0;
|
||
file_data[db_offset + 5] = 1;
|
||
file_data[db_offset + 6..db_offset + 14]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes());
|
||
|
||
let elem_start = db_offset + 6 + os;
|
||
let test_chunks = [
|
||
(0x1000u64, 120u32, 0u32),
|
||
(0x2000u64, 115u32, 0u32),
|
||
(0x3000u64, 100u32, 0u32),
|
||
];
|
||
|
||
for (i, &(addr, csize, fmask)) in test_chunks.iter().enumerate() {
|
||
let pos = elem_start + i * elem_size;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
// chunk_size as 4 bytes LE
|
||
file_data[pos + os..pos + os + 4].copy_from_slice(&csize.to_le_bytes());
|
||
file_data[pos + os + 4..pos + os + 8].copy_from_slice(&fmask.to_le_bytes());
|
||
}
|
||
stamp_checksum(
|
||
&mut file_data,
|
||
db_offset,
|
||
elem_start + test_chunks.len() * elem_size,
|
||
);
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
let ds_dims = vec![60u64];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
None,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
assert_eq!(chunks.len(), 3);
|
||
assert_eq!(chunks[0].address, 0x1000);
|
||
assert_eq!(chunks[0].chunk_size, 120);
|
||
assert_eq!(chunks[0].filter_mask, 0);
|
||
assert_eq!(chunks[0].offsets, vec![0]);
|
||
assert_eq!(chunks[1].address, 0x2000);
|
||
assert_eq!(chunks[1].chunk_size, 115);
|
||
assert_eq!(chunks[2].address, 0x3000);
|
||
assert_eq!(chunks[2].chunk_size, 100);
|
||
}
|
||
|
||
/// Build a synthetic *paged* Fixed Array (non-filtered) and verify reading.
|
||
///
|
||
/// Layout reverse-engineered and confirmed against an HDF5 2.0 file:
|
||
/// after the FADB prefix comes a page-init bitmap (MSB-first within each
|
||
/// byte), a 4-byte checksum, then full-size page slots (`page_nelmts`
|
||
/// elements + a 4-byte checksum each), with only the last page shorter.
|
||
/// Uninitialized pages occupy their slot but are skipped via the bitmap.
|
||
#[test]
|
||
fn read_paged_non_filtered_chunks() {
|
||
let offset_size: u8 = 8;
|
||
let length_size: u8 = 8;
|
||
let os = offset_size as usize;
|
||
|
||
// page_nelmts = 1 << 2 = 4. Use 11 elements => 3 pages
|
||
// (page0: 4, page1: 4, page2: 3 short). Initialize pages 0 and 2; leave
|
||
// page 1 uninitialized. 3 pages still fits one bitmap byte, but we place
|
||
// the set bits at positions 7 and 5 to lock the MSB-first ordering.
|
||
let max_nelmts_bits = 2u8;
|
||
let page_nelmts = 1usize << max_nelmts_bits; // 4
|
||
let num_elements = 11u64;
|
||
let db_header_size = 4 + 1 + 1 + os; // FADB sig+ver+client+header_addr
|
||
let bitmap_size = 1usize; // ceil(3/8)
|
||
let page_total = page_nelmts * os + 4; // elements + checksum
|
||
|
||
let fahd_offset = 0x100usize;
|
||
let db_offset = 0x400usize;
|
||
let mut file_data = vec![0u8; 0x4000];
|
||
|
||
// FAHD
|
||
file_data[fahd_offset..fahd_offset + 4].copy_from_slice(b"FAHD");
|
||
file_data[fahd_offset + 4] = 0; // version
|
||
file_data[fahd_offset + 5] = 0; // client_id = non-filtered
|
||
file_data[fahd_offset + 6] = os as u8; // element_size = address only
|
||
file_data[fahd_offset + 7] = max_nelmts_bits;
|
||
file_data[fahd_offset + 8..fahd_offset + 16].copy_from_slice(&num_elements.to_le_bytes());
|
||
file_data[fahd_offset + 16..fahd_offset + 24]
|
||
.copy_from_slice(&(db_offset as u64).to_le_bytes());
|
||
stamp_checksum(&mut file_data, fahd_offset, fahd_offset + 24);
|
||
|
||
// FADB prefix
|
||
file_data[db_offset..db_offset + 4].copy_from_slice(b"FADB");
|
||
file_data[db_offset + 4] = 0; // version
|
||
file_data[db_offset + 5] = 0; // client_id
|
||
file_data[db_offset + 6..db_offset + 6 + os]
|
||
.copy_from_slice(&(fahd_offset as u64).to_le_bytes());
|
||
|
||
// Page-init bitmap: pages 0 and 2 initialized, page 1 not.
|
||
// MSB-first => page0 -> bit7 (0x80), page2 -> bit5 (0x20) => 0xA0.
|
||
let bitmap_off = db_offset + db_header_size;
|
||
file_data[bitmap_off] = 0b1010_0000;
|
||
|
||
// Pages start after bitmap + 4-byte checksum.
|
||
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
||
|
||
let base_addr = 0x1000u64;
|
||
// Page 0 (elements 0..4) and page 2 (elements 8..11) carry addresses;
|
||
// page 1's slot is left zero-filled and must be skipped.
|
||
// The prefix and bitmap carry one checksum, each initialised page
|
||
// another — as a real file does.
|
||
stamp_checksum(&mut file_data, db_offset, bitmap_off + bitmap_size);
|
||
for &p in &[0usize, 2usize] {
|
||
let page_off = pages_start + p * page_total;
|
||
let count = core::cmp::min(page_nelmts, num_elements as usize - p * page_nelmts);
|
||
for e in 0..count {
|
||
let i = p * page_nelmts + e;
|
||
let addr = base_addr + i as u64 * 0x100;
|
||
let pos = page_off + e * os;
|
||
file_data[pos..pos + os].copy_from_slice(&addr.to_le_bytes());
|
||
}
|
||
stamp_checksum(&mut file_data, page_off, page_off + count * os);
|
||
}
|
||
|
||
let header =
|
||
FixedArrayHeader::parse(&file_data, fahd_offset, offset_size, length_size).unwrap();
|
||
assert_eq!(header.num_elements, 11);
|
||
|
||
let ds_dims = vec![11u64 * 20];
|
||
let chunk_dims = vec![20u32];
|
||
let chunks = read_fixed_array_chunks(
|
||
&file_data,
|
||
&header,
|
||
&ds_dims,
|
||
None,
|
||
&chunk_dims,
|
||
8,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.unwrap();
|
||
|
||
// Page 1 (elements 4,5,6,7) is uninitialized => skipped. The remaining
|
||
// 7 chunks (0..4 and 8..11) come back with their original linear index.
|
||
assert_eq!(chunks.len(), 7);
|
||
let mut got: Vec<(u64, u64)> = chunks.iter().map(|c| (c.offsets[0], c.address)).collect();
|
||
got.sort();
|
||
let expect: Vec<(u64, u64)> = [0usize, 1, 2, 3, 8, 9, 10]
|
||
.iter()
|
||
.map(|&i| (i as u64 * 20, base_addr + i as u64 * 0x100))
|
||
.collect();
|
||
assert_eq!(got, expect);
|
||
}
|
||
|
||
/// A fixed array (header at 0x100, data block at 0x200) of `n` chunks,
|
||
/// filtered or not, paged when `n` exceeds `1 << page_bits`; every
|
||
/// page initialised except page 1.
|
||
fn build_fixed_array(n: usize, filtered: bool, page_bits: u8) -> Vec<u8> {
|
||
let os = 8usize;
|
||
let es = if filtered { os + 4 + 4 } else { os };
|
||
let (fahd, db) = (0x100usize, 0x200usize);
|
||
let mut f = vec![0u8; 0x2000];
|
||
f[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||
f[fahd + 5] = u8::from(filtered);
|
||
f[fahd + 6] = es as u8;
|
||
f[fahd + 7] = page_bits;
|
||
f[fahd + 8..fahd + 16].copy_from_slice(&(n as u64).to_le_bytes());
|
||
f[fahd + 16..fahd + 24].copy_from_slice(&(db as u64).to_le_bytes());
|
||
stamp_checksum(&mut f, fahd, fahd + 24);
|
||
f[db..db + 4].copy_from_slice(b"FADB");
|
||
f[db + 5] = u8::from(filtered);
|
||
f[db + 6..db + 14].copy_from_slice(&(fahd as u64).to_le_bytes());
|
||
let elems = db + 6 + os;
|
||
let write = |f: &mut Vec<u8>, at: usize, i: usize| {
|
||
let addr = if i == 2 {
|
||
u64::MAX
|
||
} else {
|
||
0x1000 + i as u64 * 0x100
|
||
};
|
||
f[at..at + os].copy_from_slice(&addr.to_le_bytes());
|
||
if filtered {
|
||
f[at + os..at + os + 4].copy_from_slice(&(100 + i as u32).to_le_bytes());
|
||
f[at + os + 4..at + os + 8].copy_from_slice(&(i as u32 & 1).to_le_bytes());
|
||
}
|
||
};
|
||
let page = 1usize << page_bits;
|
||
if n <= page {
|
||
for i in 0..n {
|
||
write(&mut f, elems + i * es, i);
|
||
}
|
||
stamp_checksum(&mut f, db, elems + n * es);
|
||
} else {
|
||
let npages = n.div_ceil(page);
|
||
let bitmap = npages.div_ceil(8);
|
||
for p in 0..npages {
|
||
if p != 1 {
|
||
f[elems + p / 8] |= 0x80 >> (p % 8);
|
||
}
|
||
}
|
||
stamp_checksum(&mut f, db, elems + bitmap);
|
||
let pages_start = elems + bitmap + 4;
|
||
for p in (0..npages).filter(|&p| p != 1) {
|
||
let at = pages_start + p * (page * es + 4);
|
||
let count = page.min(n - p * page);
|
||
for e in 0..count {
|
||
write(&mut f, at + e * es, p * page + e);
|
||
}
|
||
stamp_checksum(&mut f, at, at + count * es);
|
||
}
|
||
}
|
||
f
|
||
}
|
||
|
||
/// Non-paged and paged, filtered and unfiltered arrays, cut at every
|
||
/// length through the data block and with a damaged byte, read
|
||
/// identically through a `read_at`-only storage.
|
||
#[test]
|
||
fn storage_reads_match_slice_reads() {
|
||
use crate::storage::CountingStorage;
|
||
for (n, filtered, bits) in [(3, false, 10), (3, true, 10), (11, false, 2), (11, true, 2)] {
|
||
let full = build_fixed_array(n, filtered, bits);
|
||
let es = if filtered { 16 } else { 8 };
|
||
let dims = [n as u64 * 20];
|
||
let h = FixedArrayHeader::parse(&full, 0x100, 8, 8).unwrap();
|
||
let chunks = read_fixed_array_chunks(&full, &h, &dims, None, &[20], 8, 8, 8).unwrap();
|
||
// Chunk 2 is unallocated, and so is page 1 of a paged array.
|
||
let expect = if n > 4 { n - 1 - 4 } else { n - 1 };
|
||
assert_eq!(chunks.len(), expect);
|
||
let mut files = Vec::new();
|
||
for cut in (0x100..0x200 + 40 + n * (es + 4) + 16).step_by(3) {
|
||
files.push(full[..cut].to_vec());
|
||
}
|
||
for at in [0x104, 0x210, 0x21a, 0x230] {
|
||
let mut damaged = full.clone();
|
||
damaged[at] ^= 1;
|
||
files.push(damaged);
|
||
}
|
||
files.push(full);
|
||
for f in files {
|
||
let storage = CountingStorage::new(f.clone());
|
||
let want = FixedArrayHeader::parse(&f, 0x100, 8, 8);
|
||
let got = FixedArrayHeader::parse_in(&storage, 0x100, 8, 8);
|
||
assert_eq!(format!("{got:?}"), format!("{want:?}"));
|
||
let Ok(h) = want else { continue };
|
||
let want = read_fixed_array_chunks(&f, &h, &dims, None, &[20], 8, 8, 8);
|
||
let got = read_fixed_array_chunks_in(&storage, &h, &dims, None, &[20], 8, 8, 8);
|
||
assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len());
|
||
}
|
||
}
|
||
}
|
||
|
||
/// A header whose element count stretches its data block (one checksum
|
||
/// over the whole block) far past the end of a 16 MiB file: the
|
||
/// checksum's bounds check fails before the block is read, with the
|
||
/// slice read's error.
|
||
#[cfg(feature = "checksum")]
|
||
#[test]
|
||
fn oversized_block_fails_before_reading() {
|
||
use crate::storage::CountingStorage;
|
||
let mut f = build_fixed_array(3, false, 10);
|
||
f.resize(16 << 20, 0);
|
||
let mut h = FixedArrayHeader::parse(&f, 0x100, 8, 8).unwrap();
|
||
h.max_nelmts_bits = 30;
|
||
h.num_elements = 4 << 20;
|
||
let dims = [h.num_elements * 20];
|
||
let want = read_fixed_array_chunks(&f, &h, &dims, None, &[20], 8, 8, 8);
|
||
assert!(
|
||
matches!(want, Err(FormatError::UnexpectedEof { .. })),
|
||
"{want:?}"
|
||
);
|
||
let storage = CountingStorage::new(f);
|
||
let got = read_fixed_array_chunks_in(&storage, &h, &dims, None, &[20], 8, 8, 8);
|
||
assert_eq!(format!("{got:?}"), format!("{want:?}"));
|
||
assert!(storage.bytes_read() < 64, "{} bytes", storage.bytes_read());
|
||
}
|
||
}
|