quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): verify Fixed and Extensible Array checksums

Browse Source 
Every structure in both chunk indexes — header, index block, super
block, data block and each data block page — carries a Jenkins lookup3
checksum, and all of them were parsed past and ignored.

What that costs is not a warning but correct data. Flip one low bit of a
chunk address and the index still has the right shape, the address still
lands inside the file, and the reader returns whatever bytes now sit
there as that chunk's contents. Nothing else in the parse can tell.

Verified in both directions. The checksums accept files written by
HDF5 2.0 from 100 to 200 000 chunks — dense, sparse, gzip-filtered and
paged — which also confirms the block layouts byte for byte, since a
wrong offset would fail every file. And an interop test corrupts an
address to check the read fails instead of returning data: removing the
verification makes that test fail with "corruption produced data instead
of an error", which is what it is there to prove.

The first version of that test passed with verification disabled — it
corrupted a byte a structural check already rejected, so it proved
nothing. Worth recording, since a test that passes for the wrong reason
looks exactly like coverage.

Hand-built fixtures now stamp real checksums, as HDF5 writers do, and
the Extensible Array ones no longer describe the superseded layout.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-20 17:41:41 -07:00
co-authored by Claude Opus 5
parent 0bc7a293ae
commit b41272487a
4 changed files with 349 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/extensible_array.rs
+161 -59
View File
@@ -12,6 +12,31 @@ use alloc::{format, vec, vec::Vec};
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
/// Verify the Jenkins lookup3 checksum stored immediately after
/// `data[start..end]`, as every Extensible Array structure carries one.
///
/// A corrupt chunk index yields addresses pointing at the wrong bytes, so a
/// mismatch is an error: otherwise the damage surfaces as plausible data read
/// from the wrong chunk.
#[cfg(feature = "checksum")]
fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> {
ensure_len(data, end, 4)?;
let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
if computed != stored {
return Err(FormatError::ChecksumMismatch {
expected: stored,
computed,
});
}
Ok(())
}
#[cfg(not(feature = "checksum"))]
fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> {
Ok(())
}
/// Parsed Extensible Array header (AEHD).
#[derive(Debug, Clone)]
pub struct ExtensibleArrayHeader {
@@ -145,6 +170,8 @@ impl ExtensibleArrayHeader {
pos += ls; // skip nelmts
pos += ls; // skip max_idx_set (6th stats field)
let index_block_address = read_offset(d, pos, offset_size)?;
pos += offset_size as usize;
verify_checksum(file_data, offset, offset + pos)?;
Ok(ExtensibleArrayHeader {
client_id,
@@ -361,6 +388,17 @@ fn read_data_block_elements(
};
if nelmts <= page {
// Prefix and elements are covered by one checksum.
let elem_bytes = if header.client_id == 0 {
offset_size as usize
} else {
header.element_size as usize
};
let end = nelmts
.checked_mul(elem_bytes)
.and_then(|b| pos.checked_add(b))
.ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?;
verify_checksum(file_data, db_offset, end)?;
read_run(pos, nelmts, start_index, &mut chunks)?;
return Ok(chunks);
}
@@ -369,6 +407,7 @@ fn read_data_block_elements(
// each holding `page` elements followed by a checksum. Pages whose bit is
// clear were never written; their slot still occupies the file, so stride
// over it rather than reading zeros as addresses.
verify_checksum(file_data, db_offset, pos)?;
pos += 4;
let elem_bytes = if header.client_id == 0 {
offset_size as usize
@@ -389,6 +428,9 @@ fn read_data_block_elements(
.is_some_and(|byte| byte & (0x80 >> (bit % 8)) != 0);
if initialised {
let count = core::cmp::min(page, nelmts - p * page);
// Each page carries its own checksum, over a full page's worth of
// slots even when the last one holds fewer live elements.
verify_checksum(file_data, pos, pos + page * elem_bytes)?;
read_run(pos, count, start_index + p * page, &mut chunks)?;
}
pos = pos
@@ -447,6 +489,63 @@ pub fn read_extensible_array_chunks(
let mut chunks = Vec::new();
let total_elements = header.num_elements as usize;
let dmin = header.min_dblk_nelmts as usize;
if dmin == 0 || !dmin.is_power_of_two() {
return Err(FormatError::ChunkedReadError(
"Extensible Array data block minimum is not a power of two".into(),
));
}
// nsblks = 1 + (max_nelmts_bits - log2(data_blk_min_elmts)), and the index
// block holds 2 * (sup_blk_min_data_ptrs - 1) data block addresses.
let log2_dmin = dmin.trailing_zeros() as usize;
let nsblks = 1 + (header.max_nelmts_bits as usize).saturating_sub(log2_dmin);
let ndblk_addrs = 2 * (header.super_blk_min_nelmts as usize).saturating_sub(1);
// The data blocks listed directly in the index block are the first
// `ndblk_addrs` in super-block order, each sized by the level it belongs
// to; the super block addresses that follow resume at the next level.
let mut direct: Vec<usize> = Vec::with_capacity(ndblk_addrs);
let mut level = 0usize;
while direct.len() < ndblk_addrs {
if level >= nsblks {
return Err(FormatError::ChunkedReadError(
"Extensible Array index block claims more data blocks than the array has".into(),
));
}
let (ndblks, dblk_nelmts) = sblk_info(level, dmin).ok_or_else(|| {
FormatError::Overflow("Extensible Array super block layout overflows usize".into())
})?;
for _ in 0..ndblks {
direct.push(dblk_nelmts);
}
level += 1;
}
if direct.len() != ndblk_addrs {
// A partial level in the index block is not a layout HDF5 produces,
// and guessing where the super blocks resume would misplace elements.
return Err(FormatError::ChunkedReadError(
"Extensible Array index block ends mid super block".into(),
));
}
// One checksum covers the prefix, every inline element slot, and every
// data block and super block address.
let elem_bytes = if header.client_id == 0 {
os
} else {
header.element_size as usize
};
let ib_end = (header.idx_blk_elmts as usize)
.checked_mul(elem_bytes)
.and_then(|b| pos.checked_add(b))
.and_then(|p| {
ndblk_addrs
.checked_add(nsblks - level)
.and_then(|n| n.checked_mul(os).and_then(|b| p.checked_add(b)))
})
.ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?;
verify_checksum(file_data, ib_offset, ib_end)?;
// 1. Elements stored inline in the index block.
let n_inline = (header.idx_blk_elmts as usize).min(total_elements);
for i in 0..n_inline {
@@ -471,45 +570,7 @@ pub fn read_extensible_array_chunks(
return Ok(chunks);
}
let dmin = header.min_dblk_nelmts as usize;
if dmin == 0 || !dmin.is_power_of_two() {
return Err(FormatError::ChunkedReadError(
"Extensible Array data block minimum is not a power of two".into(),
));
}
// nsblks = 1 + (max_nelmts_bits - log2(data_blk_min_elmts)), and the index
// block holds 2 * (sup_blk_min_data_ptrs - 1) data block addresses.
let log2_dmin = dmin.trailing_zeros() as usize;
let nsblks = 1 + (header.max_nelmts_bits as usize).saturating_sub(log2_dmin);
let ndblk_addrs = 2 * (header.super_blk_min_nelmts as usize).saturating_sub(1);
// 2. Data blocks listed directly in the index block: the first
// `ndblk_addrs` data blocks in super-block order, each sized by the
// level it belongs to.
let mut direct: Vec<usize> = Vec::with_capacity(ndblk_addrs);
let mut level = 0usize;
while direct.len() < ndblk_addrs {
let (ndblks, dblk_nelmts) = sblk_info(level, dmin).ok_or_else(|| {
FormatError::Overflow("Extensible Array super block layout overflows usize".into())
})?;
if level >= nsblks {
return Err(FormatError::ChunkedReadError(
"Extensible Array index block claims more data blocks than the array has".into(),
));
}
for _ in 0..ndblks {
direct.push(dblk_nelmts);
}
level += 1;
}
if direct.len() != ndblk_addrs {
// A partial level in the index block is not a layout HDF5 produces,
// and guessing where the super blocks resume would misplace elements.
return Err(FormatError::ChunkedReadError(
"Extensible Array index block ends mid super block".into(),
));
}
// 2. Data blocks listed directly in the index block.
for &dblk_nelmts in &direct {
if global_index >= total_elements {
return Ok(chunks);
@@ -629,6 +690,13 @@ fn read_super_block(
let mut chunks = Vec::new();
let mut global_idx = start_index;
// One checksum covers the prefix, the bitmap and every data block address.
let sb_end = ndblks
.checked_mul(os)
.and_then(|b| pos.checked_add(b))
.ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?;
verify_checksum(file_data, sb_offset, sb_end)?;
for i in 0..ndblks {
ensure_len(file_data, pos, os)?;
let addr = read_offset(file_data, pos, offset_size)?;
@@ -657,6 +725,14 @@ fn read_super_block(
#[cfg(test)]
mod tests {
use super::*;
/// Stamp the Jenkins checksum a real file would carry over
/// `data[start..end]`, writing it at `end`. Hand-built fixtures need this
/// now that the reader validates it, exactly as HDF5 writes it.
fn stamp_checksum(data: &mut [u8], start: usize, end: usize) {
let sum = crate::checksum::jenkins_lookup3(&data[start..end]);
data[end..end + 4].copy_from_slice(&sum.to_le_bytes());
}
#[test]
fn index_to_offsets_1d() {
let num_chunks = vec![5u64];
@@ -712,6 +788,7 @@ mod tests {
buf[44..52].copy_from_slice(&5u64.to_le_bytes()); // stat[4] = num_elements
buf[52..60].copy_from_slice(&0u64.to_le_bytes()); // stat[5]
buf[60..68].copy_from_slice(&0x1000u64.to_le_bytes()); // index_block_address
stamp_checksum(&mut buf, 0, 68);
let hdr = ExtensibleArrayHeader::parse(&buf, 0, os, ls).unwrap();
assert_eq!(hdr.client_id, 0);
@@ -797,6 +874,7 @@ mod tests {
.copy_from_slice(&(num_chunks as u64).to_le_bytes());
file_data[aehd_offset + 60..aehd_offset + 68]
.copy_from_slice(&(aeib_offset as u64).to_le_bytes());
stamp_checksum(&mut file_data, aehd_offset, aehd_offset + 68);
// checksum (4 bytes at +68) — not validated
// Build AEIB at aeib_offset
@@ -814,6 +892,23 @@ mod tests {
let p = elem_start + i * osv;
file_data[p..p + osv].copy_from_slice(&addr.to_le_bytes());
}
// The index block's checksum covers its prefix, every inline element
// slot, and every data block and super block address slot:
// ndblk_addrs = 2 * (sup_blk_min_data_ptrs - 1), and the super block
// pointers make up the rest of nsblks levels.
let sup_ptrs = file_data[aehd_offset + 10] as usize;
let dmin = file_data[aehd_offset + 9] as usize;
let nsblks = 1 + 10 - dmin.trailing_zeros() as usize;
let ndblk_addrs = 2 * (sup_ptrs - 1);
// Levels consumed by those direct data blocks (1, 1, 2, 2, ... per level).
let mut consumed = 0usize;
let mut levels = 0usize;
while consumed < ndblk_addrs {
consumed += 1 << (levels / 2);
levels += 1;
}
let ib_end = elem_start + num_chunks * osv + (ndblk_addrs + nsblks - levels) * osv;
stamp_checksum(&mut file_data, aeib_offset, ib_end);
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
let ds_dims = vec![40u64]; // 2 chunks × 20 elements
@@ -863,6 +958,7 @@ mod tests {
// idx_blk_addr at offset 12 + 6*8 = 60
file_data[aehd_offset + 60..aehd_offset + 68]
.copy_from_slice(&(aeib_offset as u64).to_le_bytes());
stamp_checksum(&mut file_data, aehd_offset, aehd_offset + 68);
// AEIB
file_data[aeib_offset..aeib_offset + 4].copy_from_slice(b"EAIB");
@@ -881,42 +977,48 @@ mod tests {
pos += osv;
}
// Direct data block addresses: first sb_level=0 has 1 dblk, sb_level=1 has 1 dblk
// Total direct dblks for sblk_min=2: 2^0 + 2^1 = 1 + 2 = 3 (oops)
// Actually: sblk_min levels. level 0: 2^0=1 dblk, level 1: 2^1=2 dblks => 3 dblks
// But we only have 2 remaining elements.
// dblk sizes: level 0: 1 dblk of min_dblk=2; level 1: 2 dblks of 2 each (nelmts doubles at level > 0)
// Wait, re-reading the code: at level 0, nelmts=min_dblk=2, 1 dblk.
// At level 1, 1 dblk, nelmts still 2 (doubles only at level > 0... but the code says
// `if sb_level > 0 { nelmts *= 2 }` after pushing). Let me re-check.
// After push at level 0: nelmts=2. Then if 0>0 false, no double. Push 1 dblk of 2.
// Level 1: ndblks=2. Push 2 dblks of 2. Then 1>0 true, nelmts=4.
// Total: 3 dblks with sizes [2, 2, 2]. Total = 6.
// We only need 2 more elements. So only the first dblk has data.
let n_direct_dblks = 3;
// Direct data block addresses. With sup_blk_min_data_ptrs = 2 the index
// block holds 2 * (2 - 1) = 2 of them, which are the data blocks of
// super block levels 0 and 1: one of `min_dblk_nelmts` elements, then
// one of twice that (ndblks = 2^(u/2), dblk_nelmts = 2^((u+1)/2) * min).
// Only the first is allocated here; the rest of the array is empty.
let ndblk_addrs = 2 * (sblk_min as usize - 1);
file_data[pos..pos + osv].copy_from_slice(&(aedb_offset as u64).to_le_bytes());
pos += osv;
// 2 more dblk addresses - undefined
for _ in 1..n_direct_dblks {
for _ in 1..ndblk_addrs {
file_data[pos..pos + osv].copy_from_slice(&u64::MAX.to_le_bytes());
pos += osv;
}
// Super block addresses fill the remaining levels; all unallocated.
let nsblks = 1 + 10 - (min_dblk_nelmts as usize).trailing_zeros() as usize;
let mut consumed = 0usize;
let mut levels = 0usize;
while consumed < ndblk_addrs {
consumed += 1 << (levels / 2);
levels += 1;
}
for _ in 0..(nsblks - levels) {
file_data[pos..pos + osv].copy_from_slice(&u64::MAX.to_le_bytes());
pos += osv;
}
stamp_checksum(&mut file_data, aeib_offset, pos);
// EADB at aedb_offset (min_dblk_nelmts elements)
// EADB holding the first data block's `min_dblk_nelmts` elements.
file_data[aedb_offset..aedb_offset + 4].copy_from_slice(b"EADB");
file_data[aedb_offset + 4] = 0;
file_data[aedb_offset + 5] = 0;
file_data[aedb_offset + 6..aedb_offset + 14]
.copy_from_slice(&(aehd_offset as u64).to_le_bytes());
// block_offset: ceil(max_nelmts_bits/8) = ceil(10/8) = 2 bytes
// block_offset = 0 for first data block
let blk_off_size = (10usize).div_ceil(8); // max_nelmts_bits=10
let mut dbpos = aedb_offset + 6 + osv + blk_off_size;
// Block offset field: ceil(max_nelmts_bits / 8) bytes, zero here.
let blk_off_size = (10usize).div_ceil(8);
let db_elems = aedb_offset + 6 + osv + blk_off_size;
let mut dbpos = db_elems;
for i in 0..min_dblk_nelmts as usize {
let addr = base_addr + (idx_blk_elmts as u64 + i as u64) * chunk_byte_size;
file_data[dbpos..dbpos + osv].copy_from_slice(&addr.to_le_bytes());
dbpos += osv;
}
stamp_checksum(&mut file_data, aedb_offset, dbpos);
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
let ds_dims = vec![40u64];