Merge branch 'feat/p3-editor-coverage' into feat/p3-remote-editor
# Conflicts: # CHANGELOG.md # CLAUDE.md # docs/design/range-reads.md
This commit is contained in:
osobh
@@ -14,6 +14,45 @@ pub fn jenkins_lookup3(data: &[u8]) -> u32 {
|
||||
hashlittle(data, 0)
|
||||
}
|
||||
|
||||
/// HDF5's Fletcher-32 checksum, as the Fletcher-32 I/O filter (filter id 3)
|
||||
/// stores it after each chunk.
|
||||
///
|
||||
/// A line-for-line port of `H5_checksum_fletcher32` (H5checksum.c, libhdf5
|
||||
/// 1.8 through 1.14): big-endian 16-bit words summed in blocks of 360, each
|
||||
/// sum reduced after a block by the ones'-complement fold
|
||||
/// `(s & 0xffff) + (s >> 16)` rather than `% 65535`, an odd trailing byte
|
||||
/// taken as the high byte of a last word, and a final fold of both sums.
|
||||
/// The fold and `% 65535` differ whenever a sum is a non-zero multiple of
|
||||
/// 65535: the fold leaves 0xffff where the modulo gives 0, so the two
|
||||
/// disagree on about one chunk in 32768 and libhdf5 rejects the other's
|
||||
/// checksum. This must stay the only implementation.
|
||||
pub fn fletcher32(data: &[u8]) -> u32 {
|
||||
let mut sum1: u32 = 0;
|
||||
let mut sum2: u32 = 0;
|
||||
// 360 words keep both sums inside 32 bits between folds (the bound
|
||||
// libhdf5 uses: after a fold sum1 < 0x10200, so sum2 stays below
|
||||
// 360 * 361 / 2 * 0xffff + 360 * 0x10200 + 0x1fffe < 2^32). The adds wrap
|
||||
// like the C unsigned arithmetic all the same.
|
||||
let (words, odd) = data.as_chunks::<2>();
|
||||
for block in words.chunks(360) {
|
||||
for w in block {
|
||||
sum1 = sum1.wrapping_add((u32::from(w[0]) << 8) | u32::from(w[1]));
|
||||
sum2 = sum2.wrapping_add(sum1);
|
||||
}
|
||||
sum1 = (sum1 & 0xffff) + (sum1 >> 16);
|
||||
sum2 = (sum2 & 0xffff) + (sum2 >> 16);
|
||||
}
|
||||
if let [last] = odd {
|
||||
sum1 = sum1.wrapping_add(u32::from(*last) << 8);
|
||||
sum2 = sum2.wrapping_add(sum1);
|
||||
sum1 = (sum1 & 0xffff) + (sum1 >> 16);
|
||||
sum2 = (sum2 & 0xffff) + (sum2 >> 16);
|
||||
}
|
||||
sum1 = (sum1 & 0xffff) + (sum1 >> 16);
|
||||
sum2 = (sum2 & 0xffff) + (sum2 >> 16);
|
||||
(sum2 << 16) | sum1
|
||||
}
|
||||
|
||||
/// Compute CRC32 (IEEE / ISO 3309) over data.
|
||||
///
|
||||
/// When the `fast-checksum` feature is enabled, this uses hardware CRC32
|
||||
@@ -207,6 +246,19 @@ fn hashlittle(data: &[u8], initval: u32) -> u32 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Values of libhdf5's `H5_checksum_fletcher32` (h5py 3.x's bundled
|
||||
/// libhdf5, called through ctypes). The first three are sums that are
|
||||
/// multiples of 65535, where `% 65535` gave 0 instead of 0xffff.
|
||||
#[test]
|
||||
fn fletcher32_matches_libhdf5() {
|
||||
assert_eq!(fletcher32(&[0x00, 0x01, 0xff, 0xfe]), 0x0001_ffff);
|
||||
assert_eq!(fletcher32(&[0xff; 720]), 0xffff_ffff);
|
||||
assert_eq!(fletcher32(&[0xff; 721]), 0xff00_ff00);
|
||||
assert_eq!(fletcher32(&[0xff; 1441]), 0xff00_ff00);
|
||||
assert_eq!(fletcher32(&[]), 0);
|
||||
assert_eq!(fletcher32(&[7]), 0x0700_0700);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input() {
|
||||
// Empty input should return the initial state after no mixing
|
||||
|
||||
@@ -1077,16 +1077,40 @@ pub fn generate_implicit_chunks(
|
||||
dataset_dims: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
element_size: u32,
|
||||
) -> Vec<ChunkInfo> {
|
||||
generate_implicit_chunks_in_grid(
|
||||
base_address,
|
||||
dataset_dims,
|
||||
dataset_dims,
|
||||
chunk_dimensions,
|
||||
element_size,
|
||||
)
|
||||
}
|
||||
|
||||
/// [`generate_implicit_chunks`] for a dataset whose maximum dimensions
|
||||
/// (`max_dims`) exceed its current ones: libhdf5 allocates the chunks of
|
||||
/// the whole maximum extent and places chunk `scaled` at its row-major
|
||||
/// position in the *maximum* chunk grid (`H5D__none_idx_get_addr`,
|
||||
/// `max_down_chunks`), so the current extent's chunks are not contiguous.
|
||||
/// Only the chunks of the current extent are listed.
|
||||
pub fn generate_implicit_chunks_in_grid(
|
||||
base_address: u64,
|
||||
dataset_dims: &[u64],
|
||||
max_dims: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
element_size: u32,
|
||||
) -> Vec<ChunkInfo> {
|
||||
let rank = chunk_dimensions.len();
|
||||
let chunk_byte_size: u64 =
|
||||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||||
|
||||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||||
let mut grid_per_dim = Vec::with_capacity(rank);
|
||||
for d in 0..rank {
|
||||
let ds = dataset_dims[d];
|
||||
let ch = chunk_dimensions[d] as u64;
|
||||
num_chunks_per_dim.push(ds.div_ceil(ch));
|
||||
let n = dataset_dims[d].div_ceil(ch);
|
||||
num_chunks_per_dim.push(n);
|
||||
grid_per_dim.push(max_dims.get(d).map_or(n, |m| m.div_ceil(ch)).max(n));
|
||||
}
|
||||
let total_chunks: u64 = num_chunks_per_dim.iter().product();
|
||||
|
||||
@@ -1095,18 +1119,22 @@ pub fn generate_implicit_chunks(
|
||||
for linear_idx in 0..total_chunks {
|
||||
let mut offsets = vec![0u64; rank];
|
||||
let mut remaining = linear_idx;
|
||||
let mut grid_idx = 0u64;
|
||||
let mut down = 1u64;
|
||||
for d in (0..rank).rev() {
|
||||
let nchunks = num_chunks_per_dim[d];
|
||||
let chunk_idx = remaining % nchunks;
|
||||
remaining /= nchunks;
|
||||
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
|
||||
grid_idx = grid_idx.saturating_add(chunk_idx.saturating_mul(down));
|
||||
down = down.saturating_mul(grid_per_dim[d]);
|
||||
}
|
||||
|
||||
chunks.push(ChunkInfo {
|
||||
chunk_size: chunk_byte_size as u32,
|
||||
filter_mask: 0,
|
||||
offsets,
|
||||
address: base_address + linear_idx * chunk_byte_size,
|
||||
address: base_address.saturating_add(grid_idx.saturating_mul(chunk_byte_size)),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1320,9 +1348,13 @@ pub fn list_chunks_in<S: Storage + ?Sized>(
|
||||
(4, Some(2)) => {
|
||||
// Implicit index — use spatial chunk dims only
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
generate_implicit_chunks(
|
||||
generate_implicit_chunks_in_grid(
|
||||
addr,
|
||||
&dataspace.dimensions,
|
||||
dataspace
|
||||
.max_dimensions
|
||||
.as_deref()
|
||||
.unwrap_or(&dataspace.dimensions),
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
)
|
||||
@@ -3335,6 +3367,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// A dataset below its maximum extent: libhdf5 lays chunks out over the
|
||||
/// maximum chunk grid, so row 1 starts after a whole maximum row (here
|
||||
/// 4 chunks), not after the current row of 3.
|
||||
#[test]
|
||||
fn implicit_chunks_use_the_maximum_grid() {
|
||||
let chunks = generate_implicit_chunks_in_grid(0x100, &[2, 3], &[4, 4], &[1, 1], 4);
|
||||
let addrs: Vec<u64> = chunks.iter().map(|c| (c.address - 0x100) / 4).collect();
|
||||
assert_eq!(addrs, vec![0, 1, 2, 4, 5, 6]);
|
||||
assert_eq!(chunks[3].offsets, vec![1, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn implicit_chunks_partial_last() {
|
||||
// 25 elements, chunk size 10 => 3 chunks (last partial)
|
||||
|
||||
@@ -1716,56 +1716,6 @@ fn shuffle_compress_general(data: &[u8], n: usize, element_size: usize, result:
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute HDF5 Fletcher32 checksum over data.
|
||||
/// HDF5 uses a modified Fletcher32 that operates on 16-bit words.
|
||||
///
|
||||
/// Optimized with wider accumulators: processes blocks of 360 words before
|
||||
/// taking the modulo, reducing the number of expensive modulo operations.
|
||||
/// (360 is the maximum block size that avoids u32 overflow for sum2.)
|
||||
fn fletcher32_compute(data: &[u8]) -> u32 {
|
||||
let mut sum1: u32 = 0;
|
||||
let mut sum2: u32 = 0;
|
||||
|
||||
// Process in blocks of 360 16-bit words (720 bytes) to delay modulo.
|
||||
// Max sum1 before mod: 360 * 65535 = 23_592_600 < u32::MAX
|
||||
// Max sum2 before mod: 360 * 23_592_600 ~ 8.5B > u32::MAX, but actual
|
||||
// sum2 accumulates incrementally, so worst case is 360*360*65535/2 which
|
||||
// fits in u64. We use u32 with block size 360 which is safe.
|
||||
const BLOCK_WORDS: usize = 360;
|
||||
const BLOCK_BYTES: usize = BLOCK_WORDS * 2;
|
||||
|
||||
let mut offset = 0;
|
||||
let len = data.len();
|
||||
|
||||
while offset + BLOCK_BYTES <= len {
|
||||
let end = offset + BLOCK_BYTES;
|
||||
let mut i = offset;
|
||||
while i < end {
|
||||
let val = ((data[i] as u32) << 8) | (data[i + 1] as u32);
|
||||
sum1 += val;
|
||||
sum2 += sum1;
|
||||
i += 2;
|
||||
}
|
||||
sum1 %= 65535;
|
||||
sum2 %= 65535;
|
||||
offset = end;
|
||||
}
|
||||
|
||||
// Handle remaining bytes
|
||||
while offset < len {
|
||||
let val = if offset + 1 < len {
|
||||
((data[offset] as u32) << 8) | (data[offset + 1] as u32)
|
||||
} else {
|
||||
(data[offset] as u32) << 8
|
||||
};
|
||||
sum1 = (sum1 + val) % 65535;
|
||||
sum2 = (sum2 + sum1) % 65535;
|
||||
offset += 2;
|
||||
}
|
||||
|
||||
(sum2 << 16) | sum1
|
||||
}
|
||||
|
||||
/// Verify Fletcher32 checksum and strip it from the data.
|
||||
/// The last 4 bytes are the stored checksum.
|
||||
fn fletcher32_verify(data: &[u8]) -> Result<Vec<u8>, FormatError> {
|
||||
@@ -1787,8 +1737,18 @@ fn fletcher32_payload(data: &[u8]) -> Result<usize, FormatError> {
|
||||
data[data.len() - 2],
|
||||
data[data.len() - 1],
|
||||
]);
|
||||
let computed = fletcher32_compute(payload);
|
||||
if stored != computed {
|
||||
let computed = crate::checksum::fletcher32(payload);
|
||||
// libhdf5 also accepts the checksum with the bytes of each 16-bit half
|
||||
// swapped, which is how 1.6.2 and earlier stored it
|
||||
// (H5Z__filter_fletcher32's `reversed_fletcher`).
|
||||
let reversed = ((computed & 0x00ff_00ff) << 8) | ((computed >> 8) & 0x00ff_00ff);
|
||||
// clawhdf5 v2.7.0 and earlier reduced the sums `% 65535`, which gives 0
|
||||
// where libhdf5's fold gives 0xffff; accept that form too, so that files
|
||||
// those releases wrote can still be read (and rewritten for libhdf5).
|
||||
// It differs from `computed` only in a half that is 0xffff.
|
||||
let half = |h: u32| if h == 0xffff { 0 } else { h };
|
||||
let legacy = (half(computed >> 16) << 16) | half(computed & 0xffff);
|
||||
if stored != computed && stored != reversed && stored != legacy {
|
||||
return Err(FormatError::Fletcher32Mismatch {
|
||||
expected: stored,
|
||||
computed,
|
||||
@@ -1799,7 +1759,7 @@ fn fletcher32_payload(data: &[u8]) -> Result<usize, FormatError> {
|
||||
|
||||
/// Append Fletcher32 checksum to data.
|
||||
fn fletcher32_append(data: &[u8]) -> Result<Vec<u8>, FormatError> {
|
||||
let checksum = fletcher32_compute(data);
|
||||
let checksum = crate::checksum::fletcher32(data);
|
||||
let mut result = data.to_vec();
|
||||
result.extend_from_slice(&checksum.to_le_bytes());
|
||||
Ok(result)
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
//! HDF5 Object Header parsing (v1 and v2).
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
use alloc::{collections::BTreeSet, vec, vec::Vec};
|
||||
#[cfg(feature = "std")]
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
|
||||
@@ -196,7 +198,6 @@ impl ObjectHeader {
|
||||
header_data_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
MAX_V1_CONTINUATION_DEPTH,
|
||||
&mut messages,
|
||||
)?;
|
||||
// libhdf5 reads every message in the first chunk and refuses a header
|
||||
@@ -230,79 +231,84 @@ impl ObjectHeader {
|
||||
/// 8; libhdf5 refuses a message that is not aligned, that runs past the
|
||||
/// end of the chunk, or leftover bytes too few for a message header (a
|
||||
/// "gap", which only version 2 allows).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
///
|
||||
/// Continuation chunks are read in the order their messages are found,
|
||||
/// as `H5O_protect` loads them (so the messages keep libhdf5's order):
|
||||
/// a queue of (address, length) pairs, each chunk read, parsed and
|
||||
/// released before the next, so only one chunk buffer is alive at a
|
||||
/// time whatever the storage. Every chunk must start at a new address
|
||||
/// (else a cycle), and the chunks together may be no larger than the
|
||||
/// file, so the bytes read stay within the file's size; a header of
|
||||
/// more than [`MAX_V1_CHUNKS`] chunks is refused.
|
||||
fn parse_v1_chunk<S: Storage + ?Sized>(
|
||||
file: &S,
|
||||
offset: u64,
|
||||
length: usize,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
depth_remaining: u16,
|
||||
messages: &mut Vec<HeaderMessage>,
|
||||
) -> Result<usize, FormatError> {
|
||||
if depth_remaining == 0 {
|
||||
return Err(FormatError::NestingDepthExceeded);
|
||||
let mut spans = ChunkSpans::new(file.len(), offset, length)?;
|
||||
let mut queue: Vec<(u64, usize)> = vec![(offset, length)];
|
||||
let mut chunk0_count = 0usize;
|
||||
let mut next = 0usize;
|
||||
while let Some(&(chunk_offset, chunk_length)) = queue.get(next) {
|
||||
let chunk = read_exact_at(file, chunk_offset, chunk_length)?;
|
||||
let data: &[u8] = &chunk;
|
||||
let end = data.len();
|
||||
let mut pos = 0usize;
|
||||
let mut count = 0usize;
|
||||
while pos < end {
|
||||
if end - pos < V1_MSG_HEADER_SIZE {
|
||||
return Err(FormatError::InvalidObjectHeader(
|
||||
"gap found in early version of file format",
|
||||
));
|
||||
}
|
||||
let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
|
||||
let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
|
||||
let msg_flags = data[pos + 4];
|
||||
// reserved(3) at pos+5..pos+8
|
||||
pos += V1_MSG_HEADER_SIZE;
|
||||
|
||||
if !msg_data_size.is_multiple_of(8) {
|
||||
return Err(FormatError::InvalidObjectHeader("message not aligned"));
|
||||
}
|
||||
if msg_data_size > end - pos {
|
||||
return Err(FormatError::InvalidObjectHeader(
|
||||
"message size exceeds buffer end",
|
||||
));
|
||||
}
|
||||
let body = &data[pos..pos + msg_data_size];
|
||||
check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
|
||||
count += 1;
|
||||
let msg_type = MessageType::from_u16(msg_type_raw);
|
||||
if msg_type != MessageType::Nil {
|
||||
messages.push(HeaderMessage {
|
||||
msg_type,
|
||||
size: msg_data_size,
|
||||
flags: msg_flags,
|
||||
creation_order: None,
|
||||
data: body.to_vec(),
|
||||
});
|
||||
}
|
||||
// Queue continuations (v1 continuation chunks are just raw
|
||||
// messages, no signature); check_message has checked the body.
|
||||
if msg_type == MessageType::ObjectHeaderContinuation {
|
||||
let cont_offset = read_offset(body, 0, offset_size)?;
|
||||
let cont_length =
|
||||
to_usize(read_offset(body, offset_size as usize, length_size)?)?;
|
||||
spans.add(cont_offset, cont_length)?;
|
||||
queue.push((cont_offset, cont_length));
|
||||
}
|
||||
pos += msg_data_size;
|
||||
}
|
||||
// Only the first chunk's messages are held to the prefix count.
|
||||
if next == 0 {
|
||||
chunk0_count = count;
|
||||
}
|
||||
next += 1;
|
||||
}
|
||||
let chunk = read_exact_at(file, offset, length)?;
|
||||
let data: &[u8] = &chunk;
|
||||
let end = length;
|
||||
let mut pos = 0usize;
|
||||
let mut count = 0usize;
|
||||
|
||||
while pos < end {
|
||||
if end - pos < V1_MSG_HEADER_SIZE {
|
||||
return Err(FormatError::InvalidObjectHeader(
|
||||
"gap found in early version of file format",
|
||||
));
|
||||
}
|
||||
let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
|
||||
let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
|
||||
let msg_flags = data[pos + 4];
|
||||
// reserved(3) at pos+5..pos+8
|
||||
pos += V1_MSG_HEADER_SIZE;
|
||||
|
||||
if !msg_data_size.is_multiple_of(8) {
|
||||
return Err(FormatError::InvalidObjectHeader("message not aligned"));
|
||||
}
|
||||
if msg_data_size > end - pos {
|
||||
return Err(FormatError::InvalidObjectHeader(
|
||||
"message size exceeds buffer end",
|
||||
));
|
||||
}
|
||||
let body = &data[pos..pos + msg_data_size];
|
||||
check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
|
||||
count += 1;
|
||||
|
||||
let msg_type = MessageType::from_u16(msg_type_raw);
|
||||
if msg_type != MessageType::Nil {
|
||||
messages.push(HeaderMessage {
|
||||
msg_type,
|
||||
size: msg_data_size,
|
||||
flags: msg_flags,
|
||||
creation_order: None,
|
||||
data: body.to_vec(),
|
||||
});
|
||||
}
|
||||
pos += msg_data_size;
|
||||
|
||||
// Follow continuations (v1 continuation chunks are just raw
|
||||
// messages, no signature); check_message has checked the body.
|
||||
if msg_type == MessageType::ObjectHeaderContinuation {
|
||||
let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?;
|
||||
let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
|
||||
Self::parse_v1_chunk(
|
||||
file,
|
||||
cont_offset as u64,
|
||||
cont_length,
|
||||
offset_size,
|
||||
length_size,
|
||||
depth_remaining - 1,
|
||||
messages,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(count)
|
||||
Ok(chunk0_count)
|
||||
}
|
||||
|
||||
fn parse_v2<S: Storage + ?Sized>(
|
||||
@@ -425,13 +431,14 @@ impl ObjectHeader {
|
||||
&mut continuations,
|
||||
)?;
|
||||
|
||||
// Follow continuations (limit to prevent cycles in malformed data)
|
||||
let mut cont_remaining = 256u16;
|
||||
// Follow continuations, one chunk buffer at a time. A chunk address
|
||||
// seen twice is a cycle in malformed data, and the chunks may add up
|
||||
// to no more than the file; a valid header can have many chunks (libhdf5 adds one
|
||||
// whenever a message no longer fits), up to the same bound as a
|
||||
// version-1 header.
|
||||
let mut spans = ChunkSpans::new(file.len(), base as u64, chunk0_msg_end.saturating_add(4))?;
|
||||
while let Some((cont_offset, cont_length)) = continuations.pop() {
|
||||
if cont_remaining == 0 {
|
||||
return Err(FormatError::NestingDepthExceeded);
|
||||
}
|
||||
cont_remaining -= 1;
|
||||
spans.add(cont_offset as u64, cont_length)?;
|
||||
Self::parse_v2_continuation(
|
||||
file,
|
||||
cont_offset as u64,
|
||||
@@ -594,8 +601,49 @@ const V2_PREFIX_MAX: usize = 34;
|
||||
/// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3).
|
||||
const V1_MSG_HEADER_SIZE: usize = 8;
|
||||
|
||||
/// How deep version-1 continuation chunks may chain (malformed-data guard).
|
||||
const MAX_V1_CONTINUATION_DEPTH: u16 = 32;
|
||||
/// The chunks of one object header read so far. A chunk starting where
|
||||
/// another did is a cycle. Chunks of a valid header do not overlap, so
|
||||
/// together they are no larger than the file; a header whose chunks add up
|
||||
/// to more is refused, which bounds what its chunks can make a reader read
|
||||
/// (a crafted chain of chunks each nested in the last would otherwise read
|
||||
/// the file over and over). Overlap itself is not refused: libhdf5 reads
|
||||
/// such headers (`cve-2025-7067.h5` has one).
|
||||
struct ChunkSpans {
|
||||
starts: BTreeSet<u64>,
|
||||
/// Bytes of the chunks so far, and the most they may add up to.
|
||||
total: u64,
|
||||
budget: u64,
|
||||
}
|
||||
|
||||
impl ChunkSpans {
|
||||
fn new(file_len: u64, start: u64, len: usize) -> Result<Self, FormatError> {
|
||||
let mut s = Self {
|
||||
starts: BTreeSet::new(),
|
||||
total: 0,
|
||||
budget: file_len,
|
||||
};
|
||||
s.add(start, len)?;
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
fn add(&mut self, start: u64, len: usize) -> Result<(), FormatError> {
|
||||
if !self.starts.insert(start) || self.starts.len() > MAX_V1_CHUNKS {
|
||||
return Err(FormatError::NestingDepthExceeded);
|
||||
}
|
||||
self.total = self.total.saturating_add(len as u64);
|
||||
if self.total > self.budget {
|
||||
return Err(FormatError::InvalidObjectHeader(
|
||||
"object header chunks larger than the file",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Most chunks a version-1 object header may have (malformed-data guard;
|
||||
/// libhdf5 has no limit, and a header that gains one continuation chunk per
|
||||
/// attribute added can have many).
|
||||
const MAX_V1_CHUNKS: usize = 1 << 16;
|
||||
|
||||
/// Every defined version-2 object header status flag (libhdf5
|
||||
/// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation
|
||||
@@ -901,6 +949,169 @@ mod tests {
|
||||
assert_eq!(hdr.messages[1].data[..2], [5, 6]);
|
||||
}
|
||||
|
||||
/// A version-1 header whose continuation chunks form a chain: chunk k
|
||||
/// holds a Dataspace message `[k]` and the continuation to chunk k + 1.
|
||||
/// With `cycle`, the last chunk points back at the first continuation
|
||||
/// chunk.
|
||||
fn v1_chain(n: usize, cycle: bool) -> Vec<u8> {
|
||||
// Each continuation chunk: dataspace (8 + 8) + continuation (8 + 16).
|
||||
let chunk_len = 40u64;
|
||||
let first = 64u64;
|
||||
let cont = |addr: u64| {
|
||||
let mut b = addr.to_le_bytes().to_vec();
|
||||
b.extend_from_slice(&chunk_len.to_le_bytes());
|
||||
b
|
||||
};
|
||||
let mut data = build_v1_header(&[(0x0010, &cont(first)[..], 0)], 8, 8);
|
||||
data.resize(first as usize, 0);
|
||||
for k in 0..n {
|
||||
let mut c = Vec::new();
|
||||
c.extend_from_slice(&1u16.to_le_bytes());
|
||||
c.extend_from_slice(&8u16.to_le_bytes());
|
||||
c.extend_from_slice(&[0; 4]);
|
||||
c.extend_from_slice(&(k as u64).to_le_bytes());
|
||||
let next = if k + 1 < n {
|
||||
first + (k as u64 + 1) * chunk_len
|
||||
} else if cycle {
|
||||
first
|
||||
} else {
|
||||
// The last chunk ends in a NIL message instead.
|
||||
c.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]);
|
||||
c.extend_from_slice(&[0; 16]);
|
||||
data.extend_from_slice(&c);
|
||||
continue;
|
||||
};
|
||||
c.extend_from_slice(&0x10u16.to_le_bytes());
|
||||
c.extend_from_slice(&16u16.to_le_bytes());
|
||||
c.extend_from_slice(&[0; 4]);
|
||||
c.extend_from_slice(&cont(next));
|
||||
data.extend_from_slice(&c);
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
/// libhdf5 reads any chain of continuation chunks (a header grows one
|
||||
/// per attribute added when full); the reader used to stop at 32.
|
||||
#[test]
|
||||
fn long_v1_continuation_chains_are_read() {
|
||||
let data = v1_chain(200, false);
|
||||
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
|
||||
let spaces: Vec<u8> = hdr
|
||||
.messages
|
||||
.iter()
|
||||
.filter(|m| m.msg_type == MessageType::Dataspace)
|
||||
.map(|m| m.data[0])
|
||||
.collect();
|
||||
assert_eq!(spaces, (0..200).map(|k| k as u8).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// A crafted version-1 header whose continuation chunks nest: each
|
||||
/// chunk's continuation message points at the rest of that chunk. Read
|
||||
/// depth-first with every enclosing chunk kept alive, from storage that
|
||||
/// hands out owned buffers, it read n^2 bytes and held them all at once
|
||||
/// (a 192 KB file read 768 MB). Chunks adding up to more than the file
|
||||
/// are refused, and the bytes read stay within the file's size.
|
||||
#[test]
|
||||
fn nested_v1_continuation_chunks_are_bounded() {
|
||||
use crate::storage::CountingStorage;
|
||||
let n = 2000u64;
|
||||
let a = 64u64;
|
||||
let cont = |addr: u64, len: u64| {
|
||||
let mut m = vec![0x10, 0, 16, 0, 0, 0, 0, 0];
|
||||
m.extend_from_slice(&addr.to_le_bytes());
|
||||
m.extend_from_slice(&len.to_le_bytes());
|
||||
m
|
||||
};
|
||||
// Prefix: version 1, one message, reference count 1, 24 bytes.
|
||||
let mut buf = vec![1, 0, 1, 0, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0];
|
||||
buf.extend_from_slice(&cont(a, 24 * n));
|
||||
buf.resize(a as usize, 0);
|
||||
for k in 0..n {
|
||||
if k + 1 < n {
|
||||
buf.extend_from_slice(&cont(a + 24 * (k + 1), 24 * (n - k - 1)));
|
||||
} else {
|
||||
buf.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]);
|
||||
buf.extend_from_slice(&[0; 16]);
|
||||
}
|
||||
}
|
||||
let len = buf.len() as u64;
|
||||
let s = CountingStorage::new(buf);
|
||||
assert!(matches!(
|
||||
ObjectHeader::parse_in(&s, 0, 8, 8),
|
||||
Err(FormatError::InvalidObjectHeader(
|
||||
"object header chunks larger than the file"
|
||||
))
|
||||
));
|
||||
assert!(
|
||||
s.bytes_read() <= 2 * len,
|
||||
"read {} of {len}",
|
||||
s.bytes_read()
|
||||
);
|
||||
}
|
||||
|
||||
/// libhdf5 reads a continuation chunk that overlaps the chunk holding
|
||||
/// its message (`cve-2025-7067.h5` has one), and so does this reader.
|
||||
#[test]
|
||||
fn overlapping_v1_continuation_chunk_is_read() {
|
||||
// Chunk 0 (at 16): continuation (24 bytes), then a NIL message at
|
||||
// 40; the continuation chunk is that NIL message's 8-byte header.
|
||||
let mut cont = 40u64.to_le_bytes().to_vec();
|
||||
cont.extend_from_slice(&8u64.to_le_bytes());
|
||||
let data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0000, &[][..], 0)], 8, 8);
|
||||
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
|
||||
assert_eq!(hdr.messages.len(), 1);
|
||||
}
|
||||
|
||||
/// A valid chain over owned-buffer storage reads each chunk once.
|
||||
#[test]
|
||||
fn long_v1_chain_reads_each_chunk_once() {
|
||||
use crate::storage::CountingStorage;
|
||||
let data = v1_chain(3000, false);
|
||||
let len = data.len() as u64;
|
||||
let s = CountingStorage::new(data);
|
||||
let hdr = ObjectHeader::parse_in(&s, 0, 8, 8).unwrap();
|
||||
assert_eq!(
|
||||
hdr.messages
|
||||
.iter()
|
||||
.filter(|m| m.msg_type == MessageType::Dataspace)
|
||||
.count(),
|
||||
3000
|
||||
);
|
||||
assert!(s.bytes_read() <= len, "read {} of {len}", s.bytes_read());
|
||||
}
|
||||
|
||||
/// Continuation chunks are read in the order their messages are found
|
||||
/// (libhdf5's `H5O_protect`), so a chunk's messages follow every
|
||||
/// message of the chunk before, not the continuation message.
|
||||
#[test]
|
||||
fn v1_continuation_messages_keep_libhdf5_order() {
|
||||
// Chunk 0: continuation to A, dataspace [1]; A: dataspace [2].
|
||||
let a = 64u64;
|
||||
let mut cont = a.to_le_bytes().to_vec();
|
||||
cont.extend_from_slice(&16u64.to_le_bytes());
|
||||
let mut data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0001, &[1; 8][..], 0)], 8, 8);
|
||||
data.resize(a as usize, 0);
|
||||
data.extend_from_slice(&[1, 0, 8, 0, 0, 0, 0, 0]);
|
||||
data.extend_from_slice(&[2; 8]);
|
||||
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
|
||||
let spaces: Vec<u8> = hdr
|
||||
.messages
|
||||
.iter()
|
||||
.filter(|m| m.msg_type == MessageType::Dataspace)
|
||||
.map(|m| m.data[0])
|
||||
.collect();
|
||||
assert_eq!(spaces, [1, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v1_continuation_cycles_are_refused() {
|
||||
let data = v1_chain(5, true);
|
||||
assert!(matches!(
|
||||
ObjectHeader::parse(&data, 0, 8, 8),
|
||||
Err(FormatError::NestingDepthExceeded)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_v1_unknown_message_ok() {
|
||||
let messages = [(0x00FFu16, &[0xAA, 0xBB][..], 0u8)];
|
||||
|
||||
Reference in New Issue
Block a user