quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases 1 Wiki Activity

format: bound Storage reads that hostile size fields could stretch

Browse Source 
On a backend without the file in memory, a structure read whose length
comes from untrusted header fields was clamped only by the end of the
file, so a crafted size made one read (and copy) of up to the rest of
the file. Each such read now covers what the parser actually uses:

- local heap names: read in growing pieces (64 bytes first, then 4x)
  up to the end of the data segment, instead of the rest of the segment
  per name (quadratic for a big symbol-table group);
- fractal heap indirect blocks: the doubling-table geometry locates the
  entry covering the object, and the first read ends at that entry; only
  if it is unallocated does the walk read the rest of the block (it
  visits every entry then). One walk implementation serves both;
- paged fixed/extensible array data blocks over 1 MiB: the prefix and
  page bitmap, then each page in use on its own (smaller blocks are
  still one read);
- blocks under one checksum (non-paged array data blocks, extensible
  array index and super blocks): the bounds check that comes first (the
  checksum's; the page bitmap's for a super block) is made against the
  file length before reading (Window::check_extent), so a block claimed
  past the end of the file costs no read. With the checksum feature off
  the parser has no such first check and the old read stands.

Other windows were already bounded (the superblock and object header
prefixes, the fractal heap header by a u16, SOHM tables by u8/u16
counts) or are exact reads checked against the file length first.
In memory nothing changes: the pieces are borrowed slices.

Tests: CountingStorage over a crafted heap (16 MiB file, width and rows
0xFFFF: under 1 KiB read, 16.7 MB before), a heap segment claiming 64 MiB
(one 64-byte read per short name), long names at every piece boundary,
a fixed array block claimed past the end of a 16 MiB file (under 64
bytes read), and in the equivalence harness an h5py file with a 2.4 MB
fixed array block and a >1 MiB extensible array block, whole and cut at
97 points: every chunk index agrees with the slice read and the largest
takes 205 KB (2.4 MB and 1.2 MB when read whole).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 14:31:46 -05:00
co-authored by Claude Opus 5.5
parent e5359354b7
commit 76c97f6c94
6 changed files with 467 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/extensible_array.rs
+48 -20
View File
@@ -12,7 +12,7 @@ use alloc::{format, vec, vec::Vec};
use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
use crate::storage::{Storage, Window, read_exact_at};
use crate::storage::{PAGED_BLOCK_ONE_READ_MAX, Storage, Window, read_exact_at};
/// Verify the Jenkins lookup3 checksum stored immediately after
/// `data[start..end]`, as every Extensible Array structure carries one. `w`
@@ -373,6 +373,9 @@ fn read_data_block_elements<S: Storage + ?Sized>(
.checked_mul(elem_bytes)
.and_then(|b| pos.checked_add(b))
.ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?;
// The checksum's bounds check comes first: make it before reading.
#[cfg(feature = "checksum")]
Window::check_extent(file, db_offset, end, 4)?;
let w = Window::read(file, db_offset, end.saturating_add(4))?;
verify_checksum(&w, 0, end)?;
read_run(&w, pos, nelmts, start_index, &mut chunks)?;
@@ -384,13 +387,26 @@ fn read_data_block_elements<S: Storage + ?Sized>(
// clear were never written; their slot still occupies the file, so stride
// over it rather than reading zeros as addresses.
let npages = nelmts.div_ceil(page);
// The whole data block in one window: every position checked below lies
// inside it (or past the end of the file).
// The whole data block in one window when it is small: every position
// checked below lies inside it (or past the end of the file). A larger
// block is read as its prefix, then each page in use on its own.
let block_len = pos
.saturating_add(4)
.saturating_add(npages.saturating_mul(page.saturating_mul(elem_bytes).saturating_add(4)));
let w = Window::read(file, db_offset, block_len)?;
verify_checksum(&w, 0, pos)?;
let whole = if block_len <= PAGED_BLOCK_ONE_READ_MAX {
Some(Window::read(file, db_offset, block_len)?)
} else {
None
};
let head_w;
let head = match &whole {
Some(w) => w,
None => {
head_w = Window::read(file, db_offset, pos + 4)?;
&head_w
}
};
verify_checksum(head, 0, pos)?;
pos += 4;
let page_stride = page
.checked_mul(elem_bytes)
@@ -405,10 +421,20 @@ fn read_data_block_elements<S: Storage + ?Sized>(
.is_some_and(|byte| byte & (0x80 >> (bit % 8)) != 0);
if initialised {
let count = core::cmp::min(page, nelmts - p * page);
// `w` holds the page from `base` on (positions below are
// relative to it, and `pos` to the data block).
let page_w;
let (w, base) = match &whole {
Some(w) => (w, 0),
None => {
page_w = Window::read(file, db_offset.saturating_add(pos as u64), page_stride)?;
(&page_w, pos)
}
};
// Each page carries its own checksum, over a full page's worth of
// slots even when the last one holds fewer live elements.
verify_checksum(&w, pos, pos + page * elem_bytes)?;
read_run(&w, pos, count, start_index + p * page, &mut chunks)?;
verify_checksum(w, pos - base, pos - base + page * elem_bytes)?;
read_run(w, pos - base, count, start_index + p * page, &mut chunks)?;
}
pos = pos
.checked_add(page_stride)
@@ -544,6 +570,9 @@ pub fn read_extensible_array_chunks_in<S: Storage + ?Sized>(
.ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?;
// The whole index block in one window: every position read below is
// before `ib_end`.
// The checksum's bounds check comes first: make it before reading.
#[cfg(feature = "checksum")]
Window::check_extent(file, ib_offset, ib_end, 4)?;
let w = Window::read(file, ib_offset, ib_end.saturating_add(4))?;
verify_checksum(&w, 0, ib_end)?;
@@ -682,26 +711,25 @@ fn read_super_block<S: Storage + ?Sized>(
// Positions below are relative to the super block, whose bytes (up to
// its checksum) are all in one window.
let bitmap_start = sb_header_size;
let w = Window::read(
file,
sb_offset,
bitmap_start
.saturating_add(bitmap_bytes)
.saturating_add(ndblks.saturating_mul(os))
.saturating_add(4),
)?;
w.ensure(bitmap_start, bitmap_bytes)?;
let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes];
// The bitmap's bounds check, then (with checksums) the checksum's, come
// before anything else is read from the block: make them before reading
// it, so size fields stretching it past the end of the file cost no read.
Window::check_extent(file, sb_offset, bitmap_start, bitmap_bytes)?;
let mut pos = bitmap_start + bitmap_bytes;
let mut chunks = Vec::new();
let mut global_idx = start_index;
// One checksum covers the prefix, the bitmap and every data block address.
let sb_end = ndblks
.checked_mul(os)
.and_then(|b| pos.checked_add(b))
.ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?;
#[cfg(feature = "checksum")]
Window::check_extent(file, sb_offset, sb_end, 4)?;
let w = Window::read(file, sb_offset, sb_end.saturating_add(4))?;
w.ensure(bitmap_start, bitmap_bytes)?;
let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes];
let mut chunks = Vec::new();
let mut global_idx = start_index;
verify_checksum(&w, 0, sb_end)?;
for i in 0..ndblks {