quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

format: read extensible array chunk indexes over Storage

Browse Source 
ExtensibleArrayHeader::parse_in reads the header in one read, and
read_extensible_array_chunks_in reads each index block, super block and
data block as a prefix read and then one window of the whole structure
(paged data blocks included); checksums and elements are checked in the
window with bounds errors reported as the whole-file checks did. The
&[u8] functions are wrappers. New test: an array with inline elements
and a data block, cut at every length and damaged in each structure,
reads identically through a read_at-only CountingStorage.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:01:11 -05:00
co-authored by Claude Opus 5.5
parent cf2b408a63
commit ff6d644391
2 changed files with 193 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/extensible_array.rs
+183 -95
View File
@@ -12,16 +12,19 @@ use alloc::{format, vec, vec::Vec};
use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
use crate::storage::{Storage, Window, read_exact_at};
/// Verify the Jenkins lookup3 checksum stored immediately after
/// `data[start..end]`, as every Extensible Array structure carries one.
/// `data[start..end]`, as every Extensible Array structure carries one. `w`
/// is a window of the file and `start`/`end` are relative to it.
///
/// A corrupt chunk index yields addresses pointing at the wrong bytes, so a
/// mismatch is an error: otherwise the damage surfaces as plausible data read
/// from the wrong chunk.
#[cfg(feature = "checksum")]
fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> {
ensure_len(data, end, 4)?;
fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> {
w.ensure(end, 4)?;
let data: &[u8] = &w.bytes;
let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
if computed != stored {
@@ -34,7 +37,7 @@ fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatEr
}
#[cfg(not(feature = "checksum"))]
fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> {
fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> {
Ok(())
}
@@ -80,19 +83,6 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
})
}
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
if offset
.checked_add(needed)
.is_none_or(|end| end > data.len())
{
return Err(FormatError::UnexpectedEof {
expected: offset.saturating_add(needed),
available: data.len(),
});
}
Ok(())
}
fn is_undefined_addr(addr: u64, offset_size: u8) -> bool {
match offset_size {
2 => addr == 0xFFFF,
@@ -130,6 +120,16 @@ impl ExtensibleArrayHeader {
offset: usize,
offset_size: u8,
length_size: u8,
) -> Result<Self, FormatError> {
Self::parse_in(&file_data, offset as u64, offset_size, length_size)
}
/// [`Self::parse`] over any [`Storage`]: one read of the header.
pub fn parse_in(
file: &dyn Storage,
offset: u64,
offset_size: u8,
length_size: u8,
) -> Result<Self, FormatError> {
// EAHD: signature(4) + version(1) + client_id(1) + element_size(1) +
// max_nelmts_bits(1) + idx_blk_elmts(1) + min_dblk_nelmts(1) +
@@ -137,9 +137,10 @@ impl ExtensibleArrayHeader {
// 6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4)
let min_size =
4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4;
ensure_len(file_data, offset, min_size)?;
let w = Window::read(file, offset, min_size)?;
w.ensure(0, min_size)?;
let d = &file_data[offset..];
let d: &[u8] = &w.bytes;
if &d[0..4] != b"EAHD" {
return Err(FormatError::ChunkedReadError(
"invalid Extensible Array header signature".into(),
@@ -172,7 +173,7 @@ impl ExtensibleArrayHeader {
pos += ls; // skip max_idx_set (6th stats field)
let index_block_address = read_offset(d, pos, offset_size)?;
pos += offset_size as usize;
verify_checksum(file_data, offset, offset + pos)?;
verify_checksum(&w, 0, pos)?;
Ok(ExtensibleArrayHeader {
client_id,
@@ -193,11 +194,11 @@ impl ExtensibleArrayHeader {
}
}
/// Read a single element from the extensible array element data.
/// Read a single element at offset `pos` of the window `w`.
/// Returns (chunk_info, bytes_consumed) or None if unallocated.
#[allow(clippy::too_many_arguments)]
fn read_element(
data: &[u8],
w: &Window<'_>,
pos: usize,
client_id: u8,
element_size: u8,
@@ -207,15 +208,11 @@ fn read_element(
grid: &ChunkGrid,
) -> Result<(Option<ChunkInfo>, usize), FormatError> {
let os = offset_size as usize;
let data: &[u8] = &w.bytes;
if client_id == 0 {
// Non-filtered: just address
if pos + os > data.len() {
return Err(FormatError::UnexpectedEof {
expected: pos + os,
available: data.len(),
});
}
w.ensure(pos, os)?;
if is_undefined(data, pos, offset_size) {
return Ok((None, os));
}
@@ -243,15 +240,7 @@ fn read_element(
}
let chunk_size_bytes = es - os - 4;
let elem_total = os + chunk_size_bytes + 4;
if pos
.checked_add(elem_total)
.is_none_or(|end| end > data.len())
{
return Err(FormatError::UnexpectedEof {
expected: pos.saturating_add(elem_total),
available: data.len(),
});
}
w.ensure(pos, elem_total)?;
if is_undefined(data, pos, offset_size) {
return Ok((None, elem_total));
}
@@ -316,8 +305,8 @@ fn page_nelmts(header: &ExtensibleArrayHeader) -> Option<usize> {
/// stores only its prefix, then one slot per page.
#[allow(clippy::too_many_arguments)]
fn read_data_block_elements(
file_data: &[u8],
db_offset: usize,
file: &dyn Storage,
db_offset: u64,
nelmts: usize,
header: &ExtensibleArrayHeader,
offset_size: u8,
@@ -330,21 +319,28 @@ fn read_data_block_elements(
// EADB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
// + block offset(arr_off_size)
let db_header_size = 4 + 1 + 1 + offset_size as usize + arr_off_size(header);
ensure_len(file_data, db_offset, db_header_size)?;
let prefix = read_exact_at(file, db_offset, db_header_size)?;
if &file_data[db_offset..db_offset + 4] != b"EADB" {
if &prefix[0..4] != b"EADB" {
return Err(FormatError::ChunkedReadError(
"invalid Extensible Array data block signature".into(),
));
}
let mut pos = db_offset + db_header_size;
// Positions below are relative to the data block.
let mut pos = db_header_size;
let page = page_nelmts(header).ok_or_else(|| {
FormatError::Overflow("Extensible Array page element count overflows usize".into())
})?;
let elem_bytes = if header.client_id == 0 {
offset_size as usize
} else {
header.element_size as usize
};
let mut chunks = Vec::new();
let read_run = |from: usize,
let read_run = |w: &Window<'_>,
from: usize,
count: usize,
first_index: usize,
chunks: &mut Vec<ChunkInfo>|
@@ -352,7 +348,7 @@ fn read_data_block_elements(
let mut p = from;
for i in 0..count {
let (info, consumed) = read_element(
file_data,
w,
p,
header.client_id,
header.element_size,
@@ -370,18 +366,16 @@ fn read_data_block_elements(
};
if nelmts <= page {
// Prefix and elements are covered by one checksum.
let elem_bytes = if header.client_id == 0 {
offset_size as usize
} else {
header.element_size as usize
};
// Prefix and elements are covered by one checksum. One window holds
// all of it (or ends at the end of the file), so its bounds checks
// are the whole-file ones.
let end = nelmts
.checked_mul(elem_bytes)
.and_then(|b| pos.checked_add(b))
.ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?;
verify_checksum(file_data, db_offset, end)?;
read_run(pos, nelmts, start_index, &mut chunks)?;
let w = Window::read(file, db_offset, end.saturating_add(4))?;
verify_checksum(&w, 0, end)?;
read_run(&w, pos, nelmts, start_index, &mut chunks)?;
return Ok(chunks);
}
@@ -389,18 +383,19 @@ fn read_data_block_elements(
// each holding `page` elements followed by a checksum. Pages whose bit is
// clear were never written; their slot still occupies the file, so stride
// over it rather than reading zeros as addresses.
verify_checksum(file_data, db_offset, pos)?;
let npages = nelmts.div_ceil(page);
// The whole data block in one window: every position checked below lies
// inside it (or past the end of the file).
let block_len = pos
.saturating_add(4)
.saturating_add(npages.saturating_mul(page.saturating_mul(elem_bytes).saturating_add(4)));
let w = Window::read(file, db_offset, block_len)?;
verify_checksum(&w, 0, pos)?;
pos += 4;
let elem_bytes = if header.client_id == 0 {
offset_size as usize
} else {
header.element_size as usize
};
let page_stride = page
.checked_mul(elem_bytes)
.and_then(|b| b.checked_add(4))
.ok_or_else(|| FormatError::Overflow("Extensible Array page stride".into()))?;
let npages = nelmts.div_ceil(page);
for p in 0..npages {
// One bit per page across the whole super block, packed contiguously
// and MSB-first within each byte, as H5VM_bit_get reads it.
@@ -412,8 +407,8 @@ fn read_data_block_elements(
let count = core::cmp::min(page, nelmts - p * page);
// Each page carries its own checksum, over a full page's worth of
// slots even when the last one holds fewer live elements.
verify_checksum(file_data, pos, pos + page * elem_bytes)?;
read_run(pos, count, start_index + p * page, &mut chunks)?;
verify_checksum(&w, pos, pos + page * elem_bytes)?;
read_run(&w, pos, count, start_index + p * page, &mut chunks)?;
}
pos = pos
.checked_add(page_stride)
@@ -435,6 +430,32 @@ pub fn read_extensible_array_chunks(
chunk_dimensions: &[u32],
element_size: u32,
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
read_extensible_array_chunks_in(
&file_data,
header,
dataset_dims,
max_dims,
chunk_dimensions,
element_size,
offset_size,
length_size,
)
}
/// [`read_extensible_array_chunks`] over any [`Storage`]: one read of the
/// index block's prefix, one of the whole index block, and the same for
/// every super block and data block it references.
#[allow(clippy::too_many_arguments)]
pub fn read_extensible_array_chunks_in(
file: &dyn Storage,
header: &ExtensibleArrayHeader,
dataset_dims: &[u64],
max_dims: Option<&[u64]>,
chunk_dimensions: &[u32],
element_size: u32,
offset_size: u8,
_length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
let os = offset_size as usize;
@@ -451,16 +472,17 @@ pub fn read_extensible_array_chunks(
// Parse index block (EAIB): signature(4) + version(1) + client_id(1)
// + header address(offset_size), then the inline elements, then the
// direct data block addresses, then the super block addresses.
let ib_offset = header.index_block_address as usize;
// Positions below are relative to the index block.
let ib_offset = header.index_block_address;
let ib_header_size = 4 + 1 + 1 + os;
ensure_len(file_data, ib_offset, ib_header_size)?;
let prefix = read_exact_at(file, ib_offset, ib_header_size)?;
if &file_data[ib_offset..ib_offset + 4] != b"EAIB" {
if &prefix[0..4] != b"EAIB" {
return Err(FormatError::ChunkedReadError(
"invalid Extensible Array index block signature".into(),
));
}
let mut pos = ib_offset + ib_header_size;
let mut pos = ib_header_size;
let mut chunks = Vec::new();
let total_elements = header.num_elements as usize;
@@ -520,13 +542,16 @@ pub fn read_extensible_array_chunks(
.and_then(|n| n.checked_mul(os).and_then(|b| p.checked_add(b)))
})
.ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?;
verify_checksum(file_data, ib_offset, ib_end)?;
// The whole index block in one window: every position read below is
// before `ib_end`.
let w = Window::read(file, ib_offset, ib_end.saturating_add(4))?;
verify_checksum(&w, 0, ib_end)?;
// 1. Elements stored inline in the index block.
let n_inline = (header.idx_blk_elmts as usize).min(total_elements);
for i in 0..n_inline {
let (info, consumed) = read_element(
file_data,
&w,
pos,
header.client_id,
header.element_size,
@@ -550,8 +575,8 @@ pub fn read_extensible_array_chunks(
if global_index >= total_elements {
return Ok(chunks);
}
ensure_len(file_data, pos, os)?;
let addr = read_offset(file_data, pos, offset_size)?;
w.ensure(pos, os)?;
let addr = read_offset(&w.bytes, pos, offset_size)?;
pos += os;
if !is_undefined_addr(addr, offset_size) {
if dblk_nelmts > page_nelmts(header).unwrap_or(usize::MAX) {
@@ -562,8 +587,8 @@ pub fn read_extensible_array_chunks(
));
}
chunks.extend(read_data_block_elements(
file_data,
addr as usize,
file,
addr,
dblk_nelmts,
header,
offset_size,
@@ -583,16 +608,16 @@ pub fn read_extensible_array_chunks(
if global_index >= total_elements {
break;
}
ensure_len(file_data, pos, os)?;
let sb_addr = read_offset(file_data, pos, offset_size)?;
w.ensure(pos, os)?;
let sb_addr = read_offset(&w.bytes, pos, offset_size)?;
pos += os;
let (ndblks, dblk_nelmts) = sblk_info(u, dmin).ok_or_else(|| {
FormatError::Overflow("Extensible Array super block layout overflows usize".into())
})?;
if !is_undefined_addr(sb_addr, offset_size) {
chunks.extend(read_super_block(
file_data,
sb_addr as usize,
file,
sb_addr,
ndblks,
dblk_nelmts,
header,
@@ -618,8 +643,8 @@ pub fn read_extensible_array_chunks(
/// + one address per data block + checksum.
#[allow(clippy::too_many_arguments)]
fn read_super_block(
file_data: &[u8],
sb_offset: usize,
file: &dyn Storage,
sb_offset: u64,
ndblks: usize,
dblk_nelmts: usize,
header: &ExtensibleArrayHeader,
@@ -630,9 +655,9 @@ fn read_super_block(
) -> Result<Vec<ChunkInfo>, FormatError> {
let os = offset_size as usize;
let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header);
ensure_len(file_data, sb_offset, sb_header_size)?;
let prefix = read_exact_at(file, sb_offset, sb_header_size)?;
if &file_data[sb_offset..sb_offset + 4] != b"EASB" {
if &prefix[0..4] != b"EASB" {
return Err(FormatError::ChunkedReadError(
"invalid Extensible Array super block signature".into(),
));
@@ -654,9 +679,19 @@ fn read_super_block(
let bitmap_bytes = per_dblk_bitmap
.checked_mul(ndblks)
.ok_or_else(|| FormatError::Overflow("Extensible Array page bitmap size".into()))?;
let bitmap_start = sb_offset + sb_header_size;
ensure_len(file_data, bitmap_start, bitmap_bytes)?;
let bitmap = &file_data[bitmap_start..bitmap_start + bitmap_bytes];
// Positions below are relative to the super block, whose bytes (up to
// its checksum) are all in one window.
let bitmap_start = sb_header_size;
let w = Window::read(
file,
sb_offset,
bitmap_start
.saturating_add(bitmap_bytes)
.saturating_add(ndblks.saturating_mul(os))
.saturating_add(4),
)?;
w.ensure(bitmap_start, bitmap_bytes)?;
let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes];
let mut pos = bitmap_start + bitmap_bytes;
let mut chunks = Vec::new();
@@ -667,16 +702,16 @@ fn read_super_block(
.checked_mul(os)
.and_then(|b| pos.checked_add(b))
.ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?;
verify_checksum(file_data, sb_offset, sb_end)?;
verify_checksum(&w, 0, sb_end)?;
for i in 0..ndblks {
ensure_len(file_data, pos, os)?;
let addr = read_offset(file_data, pos, offset_size)?;
w.ensure(pos, os)?;
let addr = read_offset(&w.bytes, pos, offset_size)?;
pos += os;
if !is_undefined_addr(addr, offset_size) {
chunks.extend(read_data_block_elements(
file_data,
addr as usize,
file,
addr,
dblk_nelmts,
header,
offset_size,
@@ -887,11 +922,11 @@ mod tests {
assert_eq!(chunks[1].offsets, vec![20]);
}
/// Build a synthetic EA with inline elements + one direct data block.
#[test]
fn read_inline_plus_data_blocks() {
/// A synthetic EA with inline elements + one direct data block: the
/// file, with the header at 0x100 (8-byte offsets and lengths, 4 chunks
/// of 10 elements from 0x1000 on).
fn build_inline_plus_data_blocks() -> Vec<u8> {
let os: u8 = 8;
let ls: u8 = 8;
let osv = os as usize;
let chunk_byte_size = 10u64 * 8; // 10 elements × 8 bytes
let idx_blk_elmts = 2u8;
@@ -981,8 +1016,17 @@ mod tests {
dbpos += osv;
}
stamp_checksum(&mut file_data, aedb_offset, dbpos);
file_data
}
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
/// Build a synthetic EA with inline elements + one direct data block.
#[test]
fn read_inline_plus_data_blocks() {
let (os, ls) = (8u8, 8u8);
let chunk_byte_size = 10u64 * 8;
let base_addr = 0x1000u64;
let file_data = build_inline_plus_data_blocks();
let header = ExtensibleArrayHeader::parse(&file_data, 0x100, os, ls).unwrap();
let ds_dims = vec![40u64];
let chunk_dims = vec![10u32];
let chunks = read_extensible_array_chunks(
@@ -1018,7 +1062,8 @@ mod tests {
fn read_element_unallocated() {
let data = vec![0xFFu8; 16];
let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap();
let (info, consumed) =
read_element(&Window::whole(&data), 0, 0, 8, 8, 80, 0, &grid).unwrap();
assert!(info.is_none());
assert_eq!(consumed, 8);
}
@@ -1038,8 +1083,17 @@ mod tests {
data[12..16].copy_from_slice(&0u32.to_le_bytes());
let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
let (info, consumed) =
read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap();
let (info, consumed) = read_element(
&Window::whole(&data),
0,
1,
elem_size as u8,
os,
80,
2,
&grid,
)
.unwrap();
let ci = info.unwrap();
assert_eq!(ci.address, 0x2000);
assert_eq!(ci.chunk_size, 120);
@@ -1047,4 +1101,38 @@ mod tests {
assert_eq!(ci.offsets, vec![20]);
assert_eq!(consumed, elem_size);
}
/// The Storage path reads exactly what the slice path reads: the array
/// whole, cut at every length through its structures, and with a byte
/// damaged in each of them, through a read_at-only CountingStorage.
#[test]
fn storage_reads_match_slice_reads() {
use crate::storage::CountingStorage;
let full = build_inline_plus_data_blocks();
let mut files = Vec::new();
for cut in 0x100..0x340 {
files.push(full[..cut].to_vec());
}
for at in [0x104, 0x150, 0x204, 0x216, 0x230, 0x304, 0x318] {
let mut damaged = full.clone();
damaged[at] ^= 1;
files.push(damaged);
}
files.push(full);
let mut compared = 0;
for f in files {
let storage = CountingStorage::new(f.clone());
let want = ExtensibleArrayHeader::parse(&f, 0x100, 8, 8);
let got = ExtensibleArrayHeader::parse_in(&storage, 0x100, 8, 8);
assert_eq!(format!("{got:?}"), format!("{want:?}"));
let Ok(h) = want else { continue };
for dims in [&[40u64][..], &[25]] {
let want = read_extensible_array_chunks(&f, &h, dims, None, &[10], 8, 8, 8);
let got = read_extensible_array_chunks_in(&storage, &h, dims, None, &[10], 8, 8, 8);
assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len());
compared += 1;
}
}
assert!(compared > 100);
}
}
crates/clawhdf5-format/src/storage.rs
+10
View File
@@ -240,6 +240,16 @@ impl<'a> Window<'a> {
})
}
/// A whole in-memory file as one window (base 0).
#[cfg(test)]
pub fn whole(bytes: &'a [u8]) -> Self {
Window {
bytes: Cow::Borrowed(bytes),
base: 0,
file_len: bytes.len(),
}
}
/// Check that `[rel, rel + needed)` (relative to `base`) is in the file.
pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> {
match rel.checked_add(needed) {