From a0160730f26643203402a3ba95aa6a635d6ee0d9 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:54:48 -0500 Subject: [PATCH] format: read fractal heaps over Storage FractalHeapHeader::parse_in reads the header as one window (a second, longer one when it holds an I/O filter pipeline); read_managed_object_in reads direct blocks, indirect blocks (one window up to the last child entry) and huge objects with bounded reads. The &[u8] methods are wrappers. A huge object indexed by the huge-object v2 B-tree, which is not converted yet, is a clean ContiguousStorageRequired error on a backend without the whole file in memory (after the "no index" check, so the error order is unchanged). storage::Window (crate-internal) reads a window of a structure and reports bounds failures exactly as the whole-file ensure_len did, and a short read inside the file is now a Storage error rather than an EOF. New tests: headers (with and without a filter pipeline) cut at every length, and managed objects in a direct root and through an indirect root, huge objects with direct IDs and tiny objects, give identical results through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/fractal_heap.rs | 228 ++++++++++++++++++--- crates/clawhdf5-format/src/storage.rs | 47 ++++- 2 files changed, 242 insertions(+), 33 deletions(-) diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index 9bc4ba4..1c6a77c 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -9,6 +9,7 @@ use byteorder::{ByteOrder, LittleEndian}; use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; +use crate::storage::{Storage, Window, len_usize, read_exact_at, require_contiguous}; /// Parsed fractal heap header (signature "FRHP"). #[derive(Debug, Clone)] @@ -138,6 +139,36 @@ impl FractalHeapHeader { offset_size: u8, length_size: u8, ) -> Result { + Self::parse_in(&file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the header (two + /// when it holds an I/O filter pipeline). + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + length_size: u8, + ) -> Result { + // Every field up to the checksum, without and with the filter + // information; the window holds all of it (or ends at the end of + // the file), so its bounds checks are the whole-file ones. + let (os, ls) = (usize::from(offset_size), usize::from(length_size)); + let unfiltered_len = 26 + 12 * ls + 3 * os; + let mut w = Window::read(file, offset, unfiltered_len)?; + if w.bytes.len() == unfiltered_len { + let filter_len = usize::from(u16::from_le_bytes([w.bytes[7], w.bytes[8]])); + if filter_len > 0 { + w = Window::read(file, offset, unfiltered_len + ls + 4 + filter_len)?; + } + } + let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed); + let read_offset = |_: &[u8], pos: usize, size: u8| { + w.ensure(pos, usize::from(size))?; + read_offset(&w.bytes, pos, size) + }; + let file_data: &[u8] = &w.bytes; + let offset = 0usize; ensure_len(file_data, offset, 5)?; if &file_data[offset..offset + 4] != b"FRHP" { return Err(FormatError::InvalidFractalHeapSignature); @@ -148,9 +179,6 @@ impl FractalHeapHeader { return Err(FormatError::InvalidFractalHeapVersion(version)); } - let os = offset_size as usize; - let ls = length_size as usize; - let mut pos = offset + 5; ensure_len(file_data, pos, 2)?; let heap_id_length = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); @@ -353,6 +381,18 @@ impl FractalHeapHeader { file_data: &[u8], id_bytes: &[u8], offset_size: u8, + ) -> Result, FormatError> { + self.read_managed_object_in(&file_data, id_bytes, offset_size) + } + + /// [`Self::read_managed_object`] over any [`Storage`]. A huge object + /// found through the huge-object v2 B-tree still needs the whole file + /// in memory ([`FormatError::ContiguousStorageRequired`] otherwise). + pub fn read_managed_object_in( + &self, + file_data: &dyn Storage, + id_bytes: &[u8], + offset_size: u8, ) -> Result, FormatError> { let Some(&first) = id_bytes.first() else { return Err(FormatError::UnexpectedEof { @@ -383,7 +423,7 @@ impl FractalHeapHeader { } /// Read a huge object (heap ID type 1). - fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result, FormatError> { + fn read_huge_object(&self, file: &dyn Storage, id: &[u8]) -> Result, FormatError> { let os = usize::from(self.offset_size); let ls = usize::from(self.length_size); // (address, stored length, filter mask, decoded length); the last two @@ -414,18 +454,17 @@ impl FractalHeapHeader { let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8); ensure_len(id, 1, key_len)?; let key = le_uint(&id[1..1 + key_len]); - self.find_huge_record(file_data, key)? + self.find_huge_record(file, key)? }; let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?; let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?; - ensure_len(file_data, start, len)?; - let stored = &file_data[start..start + len]; + let stored = read_exact_at(file, start as u64, len)?; match &self.filter_pipeline { - None => Ok(stored.to_vec()), + None => Ok(stored.into_owned()), Some(pipeline) => { let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?; - let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?; + let out = crate::filters::decompress_chunk_masked(&stored, pipeline, mem, 1, mask)?; if out.len() != mem { return Err(heap_error("filtered huge object decoded to the wrong size")); } @@ -438,7 +477,7 @@ impl FractalHeapHeader { /// (address, stored length, filter mask, decoded length). fn find_huge_record( &self, - file_data: &[u8], + file: &dyn Storage, key: u64, ) -> Result<(u64, u64, u32, u64), FormatError> { if is_undefined(self.huge_btree_address, self.offset_size) { @@ -446,6 +485,8 @@ impl FractalHeapHeader { "huge object ID but the heap has no huge-object index", )); } + // The v2 B-tree is read from a slice until it is converted. + let file_data = require_contiguous(file, "a huge fractal-heap object's B-tree")?; let hdr = BTreeV2Header::parse( file_data, self.huge_btree_address as usize, @@ -515,7 +556,7 @@ impl FractalHeapHeader { /// Read a managed object (heap ID type 0). fn read_heap_managed( &self, - file_data: &[u8], + file_data: &dyn Storage, id_bytes: &[u8], offset_size: u8, ) -> Result, FormatError> { @@ -565,7 +606,7 @@ impl FractalHeapHeader { /// through its filter pipeline, so the block is decoded first. fn read_from_direct_block( &self, - file_data: &[u8], + file: &dyn Storage, block: DirectBlock, target_offset: u64, length: usize, @@ -581,9 +622,9 @@ impl FractalHeapHeader { let stored_len = usize::try_from(block.filtered_size) .map_err(|_| heap_error("direct block size"))?; let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?; - ensure_len(file_data, block.addr, stored_len)?; + let stored = read_exact_at(file, block.addr as u64, stored_len)?; let decoded = crate::filters::decompress_chunk_masked( - &file_data[block.addr..block.addr + stored_len], + &stored, pipeline, size, 1, @@ -597,17 +638,16 @@ impl FractalHeapHeader { .checked_add(local_offset) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: len_usize(file), })?; - ensure_len(file_data, pos, length)?; - Ok(file_data[pos..pos + length].to_vec()) + Ok(read_exact_at(file, pos as u64, length)?.into_owned()) } /// Read an object by traversing an indirect block to find the right direct block. #[allow(clippy::too_many_arguments)] fn read_from_indirect_block( &self, - file_data: &[u8], + file: &dyn Storage, iblock_addr: usize, nrows: u16, iblock_heap_offset: u64, @@ -621,16 +661,8 @@ impl FractalHeapHeader { "fractal heap: maximum recursion depth exceeded".into(), )); } - // Parse indirect block header - ensure_len(file_data, iblock_addr, 4)?; - if &file_data[iblock_addr..iblock_addr + 4] != b"FHIB" { - return Err(FormatError::InvalidFractalHeapSignature); - } - let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8); let iblock_header = 5 + offset_size as usize + block_offset_bytes; - let mut pos = iblock_addr + iblock_header; - let tw = self.table_width as u64; let nrows_usize = nrows as usize; let mut current_heap_offset = iblock_heap_offset; @@ -640,6 +672,38 @@ impl FractalHeapHeader { let start_indirect = self.max_direct_rows(); let max_direct_rows = nrows_usize.min(start_indirect); + // The block up to its last child entry, in one window: every + // position read below lies inside it, so its bounds checks are the + // whole-file ones. + let direct_entry = usize::from(offset_size) + + if self.filter_pipeline.is_some() { + usize::from(self.length_size) + 4 + } else { + 0 + }; + let entries = + |rows: usize, entry: usize| rows.saturating_mul(tw as usize).saturating_mul(entry); + let block_len = iblock_header + .saturating_add(entries(max_direct_rows, direct_entry)) + .saturating_add(entries( + nrows_usize.saturating_sub(start_indirect), + usize::from(offset_size), + )); + let w = Window::read(file, iblock_addr as u64, block_len)?; + let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed); + let read_offset = |_: &[u8], pos: usize, size: u8| { + w.ensure(pos, usize::from(size))?; + read_offset(&w.bytes, pos, size) + }; + let file_data: &[u8] = &w.bytes; + + // Parse indirect block header + ensure_len(file_data, 0, 4)?; + if &file_data[..4] != b"FHIB" { + return Err(FormatError::InvalidFractalHeapSignature); + } + let mut pos = iblock_header; + for row in 0..max_direct_rows { let block_size = self.block_size_for_row(row); @@ -671,7 +735,7 @@ impl FractalHeapHeader { && target_offset < block_end { return self.read_from_direct_block( - file_data, + file, DirectBlock { addr: child_addr as usize, size: block_size, @@ -704,7 +768,7 @@ impl FractalHeapHeader { && target_offset < block_end { return self.read_from_indirect_block( - file_data, + file, child_addr as usize, child_nrows, current_heap_offset, @@ -720,7 +784,7 @@ impl FractalHeapHeader { Err(FormatError::UnexpectedEof { expected: target_offset as usize + length, - available: file_data.len(), + available: len_usize(file), }) } @@ -1024,4 +1088,110 @@ mod tests { let id = [0x40u8, 0, 0, 0, 0, 0, 0]; assert!(hdr.read_managed_object(&file_data, &id, 8).is_err()); } + + /// Headers, and managed (in a direct root and through an indirect + /// root), huge and tiny objects read identically through a + /// `read_at`-only storage, for every truncation of the file. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + let (mut file, header_end) = build_simple_heap(8, 8); + // An indirect root block at 600: row 0 holds the direct block at + // 256, then three undefined blocks. + file[600..604].copy_from_slice(b"FHIB"); + let mut at = 600 + 5 + 8 + 2; + for addr in [256u64, u64::MAX, u64::MAX, u64::MAX] { + file[at..at + 8].copy_from_slice(&addr.to_le_bytes()); + at += 8; + } + file[900..905].copy_from_slice(b"huge!"); + let managed_id = |offset: u64, len: u64| { + let payload = offset | (len << 16); + let mut id = vec![0u8]; + id.extend_from_slice(&payload.to_le_bytes()[..6]); + id + }; + let mut huge = vec![0x10u8]; + huge.extend_from_slice(&900u64.to_le_bytes()); + huge.extend_from_slice(&5u64.to_le_bytes()); + let ids = [ + managed_id(15, 13), + managed_id(15, 200), + managed_id(130, 4), + huge, + vec![0x22, b'a', b'b', b'c', 0, 0, 0], + ]; + let mut cuts: Vec = (0..=header_end + 1).collect(); + cuts.extend([256, 260, 271, 280, 600, 610, 620, 640, 900, 903, file.len()]); + for cut in cuts { + let f = &file[..cut]; + let storage = CountingStorage::new(f.to_vec()); + let want = FractalHeapHeader::parse(f, 0, 8, 8); + let got = FractalHeapHeader::parse_in(&storage, 0, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "cut {cut}"); + let Ok(direct) = want else { continue }; + let mut indirect = direct.clone(); + indirect.root_block_address = 600; + indirect.current_rows_in_root_indirect_block = 1; + let mut huge_ids = direct.clone(); + huge_ids.heap_id_length = 17; + for hdr in [&direct, &indirect, &huge_ids] { + for id in &ids { + assert_eq!( + hdr.read_managed_object_in(&storage, id, 8), + hdr.read_managed_object(f, id, 8), + "cut {cut}" + ); + } + } + } + } + + /// A huge object found through the huge-object B-tree needs the whole + /// file in memory until the B-tree reader is converted: a clean error + /// on other storage. + #[test] + fn huge_object_btree_needs_contiguous_storage() { + use crate::storage::CountingStorage; + let (file, _) = build_simple_heap(8, 8); + let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap(); + hdr.huge_btree_address = 700; + let storage = CountingStorage::new(file); + assert_eq!( + hdr.read_managed_object_in(&storage, &[0x10, 1, 0, 0, 0, 0, 0], 8), + Err(FormatError::ContiguousStorageRequired( + "a huge fractal-heap object's B-tree" + )) + ); + } + + /// A header with an I/O filter pipeline (read in a second, longer + /// window) parses identically through a `read_at`-only storage, for + /// every truncation. + #[test] + fn filtered_header_parses_identically_through_storage() { + use crate::storage::CountingStorage; + let (simple, header_end) = build_simple_heap(8, 8); + let pipeline = [2u8, 1, 1, 0, 0, 0, 1, 0, 6, 0, 0, 0]; // deflate, level 6 + let mut header = simple[..header_end - 4].to_vec(); + header[7..9].copy_from_slice(&(pipeline.len() as u16).to_le_bytes()); + header.extend_from_slice(&100u64.to_le_bytes()); // root block's stored size + header.extend_from_slice(&0u32.to_le_bytes()); // its filter mask + header.extend_from_slice(&pipeline); + let sum = crate::checksum::jenkins_lookup3(&header); + header.extend_from_slice(&sum.to_le_bytes()); + let mut file = header.clone(); + file.resize(256, 0); + let hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap(); + assert!(hdr.filter_pipeline.is_some()); + for cut in 0..=file.len() { + let f = &file[..cut]; + let storage = CountingStorage::new(f.to_vec()); + assert_eq!( + format!("{:?}", FractalHeapHeader::parse_in(&storage, 0, 8, 8)), + format!("{:?}", FractalHeapHeader::parse(f, 0, 8, 8)), + "cut {cut}" + ); + } + } } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index fb19f32..4608b2d 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -206,11 +206,52 @@ pub fn read_exact_at( if bytes.len() < len { // The storage shrank or the backend served a short read inside the // file: never parse a partial structure. - return Err(eof()); + return Err(short_read()); } Ok(bytes) } +fn short_read() -> FormatError { + FormatError::Storage( + "short read inside the file (the storage shrank or the backend failed)".into(), + ) +} + +/// A window of the file: up to `max` bytes read at `base`, fewer only at +/// the end of the file. Its [`Window::ensure`] reports a bounds failure +/// exactly as the whole-file check `ensure_len(file_data, base + rel, n)` +/// did — with the absolute position and the file's length — as long as +/// every position checked lies within the `max` bytes the window was asked +/// for: then a position past the window is past the end of the file. +pub(crate) struct Window<'a> { + /// The bytes, from `base` on. + pub bytes: Cow<'a, [u8]>, + base: usize, + file_len: usize, +} + +impl<'a> Window<'a> { + /// Read up to `max` bytes at `base`. + pub fn read(file: &'a dyn Storage, base: u64, max: usize) -> Result { + Ok(Window { + bytes: read_upto(file, base, max)?, + base: usize::try_from(base).unwrap_or(usize::MAX), + file_len: len_usize(file), + }) + } + + /// Check that `[rel, rel + needed)` (relative to `base`) is in the file. + pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> { + match rel.checked_add(needed) { + Some(end) if end <= self.bytes.len() => Ok(()), + _ => Err(FormatError::UnexpectedEof { + expected: self.base.saturating_add(rel).saturating_add(needed), + available: self.file_len, + }), + } + } +} + /// Up to `max` bytes from `offset` on: fewer only at the end of the /// storage. For structures whose size is only known once their prefix has /// been parsed and whose parsers bound-check what they are given. @@ -224,9 +265,7 @@ pub fn read_upto( let len = usize::try_from(avail).map_or(max, |a| a.min(max)); let bytes = file.read_at(offset, len)?; if bytes.len() < len { - return Err(FormatError::Storage( - "short read inside the file (the storage shrank or the backend failed)".into(), - )); + return Err(short_read()); } Ok(bytes) }