quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

format: read fractal heaps over Storage

Browse Source 
FractalHeapHeader::parse_in reads the header as one window (a second,
longer one when it holds an I/O filter pipeline); read_managed_object_in
reads direct blocks, indirect blocks (one window up to the last child
entry) and huge objects with bounded reads. The &[u8] methods are
wrappers. A huge object indexed by the huge-object v2 B-tree, which is
not converted yet, is a clean ContiguousStorageRequired error on a
backend without the whole file in memory (after the "no index" check,
so the error order is unchanged).

storage::Window (crate-internal) reads a window of a structure and
reports bounds failures exactly as the whole-file ensure_len did, and a
short read inside the file is now a Storage error rather than an EOF.

New tests: headers (with and without a filter pipeline) cut at every
length, and managed objects in a direct root and through an indirect
root, huge objects with direct IDs and tiny objects, give identical
results through a read_at-only CountingStorage.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 12:54:48 -05:00
co-authored by Claude Opus 5.5
parent 24cbf12f16
commit a0160730f2
2 changed files with 242 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/fractal_heap.rs
+199 -29
View File
@@ -9,6 +9,7 @@ use byteorder::{ByteOrder, LittleEndian};
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
use crate::storage::{Storage, Window, len_usize, read_exact_at, require_contiguous};
/// Parsed fractal heap header (signature "FRHP").
#[derive(Debug, Clone)]
@@ -138,6 +139,36 @@ impl FractalHeapHeader {
offset_size: u8,
length_size: u8,
) -> Result<FractalHeapHeader, FormatError> {
Self::parse_in(&file_data, offset as u64, offset_size, length_size)
}
/// [`Self::parse`] over any [`Storage`]: one read of the header (two
/// when it holds an I/O filter pipeline).
pub fn parse_in(
file: &dyn Storage,
offset: u64,
offset_size: u8,
length_size: u8,
) -> Result<FractalHeapHeader, FormatError> {
// Every field up to the checksum, without and with the filter
// information; the window holds all of it (or ends at the end of
// the file), so its bounds checks are the whole-file ones.
let (os, ls) = (usize::from(offset_size), usize::from(length_size));
let unfiltered_len = 26 + 12 * ls + 3 * os;
let mut w = Window::read(file, offset, unfiltered_len)?;
if w.bytes.len() == unfiltered_len {
let filter_len = usize::from(u16::from_le_bytes([w.bytes[7], w.bytes[8]]));
if filter_len > 0 {
w = Window::read(file, offset, unfiltered_len + ls + 4 + filter_len)?;
}
}
let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
let read_offset = |_: &[u8], pos: usize, size: u8| {
w.ensure(pos, usize::from(size))?;
read_offset(&w.bytes, pos, size)
};
let file_data: &[u8] = &w.bytes;
let offset = 0usize;
ensure_len(file_data, offset, 5)?;
if &file_data[offset..offset + 4] != b"FRHP" {
return Err(FormatError::InvalidFractalHeapSignature);
@@ -148,9 +179,6 @@ impl FractalHeapHeader {
return Err(FormatError::InvalidFractalHeapVersion(version));
}
let os = offset_size as usize;
let ls = length_size as usize;
let mut pos = offset + 5;
ensure_len(file_data, pos, 2)?;
let heap_id_length = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
@@ -353,6 +381,18 @@ impl FractalHeapHeader {
file_data: &[u8],
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
self.read_managed_object_in(&file_data, id_bytes, offset_size)
}
/// [`Self::read_managed_object`] over any [`Storage`]. A huge object
/// found through the huge-object v2 B-tree still needs the whole file
/// in memory ([`FormatError::ContiguousStorageRequired`] otherwise).
pub fn read_managed_object_in(
&self,
file_data: &dyn Storage,
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
let Some(&first) = id_bytes.first() else {
return Err(FormatError::UnexpectedEof {
@@ -383,7 +423,7 @@ impl FractalHeapHeader {
}
/// Read a huge object (heap ID type 1).
fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
fn read_huge_object(&self, file: &dyn Storage, id: &[u8]) -> Result<Vec<u8>, FormatError> {
let os = usize::from(self.offset_size);
let ls = usize::from(self.length_size);
// (address, stored length, filter mask, decoded length); the last two
@@ -414,18 +454,17 @@ impl FractalHeapHeader {
let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
ensure_len(id, 1, key_len)?;
let key = le_uint(&id[1..1 + key_len]);
self.find_huge_record(file_data, key)?
self.find_huge_record(file, key)?
};
let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
ensure_len(file_data, start, len)?;
let stored = &file_data[start..start + len];
let stored = read_exact_at(file, start as u64, len)?;
match &self.filter_pipeline {
None => Ok(stored.to_vec()),
None => Ok(stored.into_owned()),
Some(pipeline) => {
let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
let out = crate::filters::decompress_chunk_masked(&stored, pipeline, mem, 1, mask)?;
if out.len() != mem {
return Err(heap_error("filtered huge object decoded to the wrong size"));
}
@@ -438,7 +477,7 @@ impl FractalHeapHeader {
/// (address, stored length, filter mask, decoded length).
fn find_huge_record(
&self,
file_data: &[u8],
file: &dyn Storage,
key: u64,
) -> Result<(u64, u64, u32, u64), FormatError> {
if is_undefined(self.huge_btree_address, self.offset_size) {
@@ -446,6 +485,8 @@ impl FractalHeapHeader {
"huge object ID but the heap has no huge-object index",
));
}
// The v2 B-tree is read from a slice until it is converted.
let file_data = require_contiguous(file, "a huge fractal-heap object's B-tree")?;
let hdr = BTreeV2Header::parse(
file_data,
self.huge_btree_address as usize,
@@ -515,7 +556,7 @@ impl FractalHeapHeader {
/// Read a managed object (heap ID type 0).
fn read_heap_managed(
&self,
file_data: &[u8],
file_data: &dyn Storage,
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
@@ -565,7 +606,7 @@ impl FractalHeapHeader {
/// through its filter pipeline, so the block is decoded first.
fn read_from_direct_block(
&self,
file_data: &[u8],
file: &dyn Storage,
block: DirectBlock,
target_offset: u64,
length: usize,
@@ -581,9 +622,9 @@ impl FractalHeapHeader {
let stored_len = usize::try_from(block.filtered_size)
.map_err(|_| heap_error("direct block size"))?;
let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
ensure_len(file_data, block.addr, stored_len)?;
let stored = read_exact_at(file, block.addr as u64, stored_len)?;
let decoded = crate::filters::decompress_chunk_masked(
&file_data[block.addr..block.addr + stored_len],
&stored,
pipeline,
size,
1,
@@ -597,17 +638,16 @@ impl FractalHeapHeader {
.checked_add(local_offset)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
available: len_usize(file),
})?;
ensure_len(file_data, pos, length)?;
Ok(file_data[pos..pos + length].to_vec())
Ok(read_exact_at(file, pos as u64, length)?.into_owned())
}
/// Read an object by traversing an indirect block to find the right direct block.
#[allow(clippy::too_many_arguments)]
fn read_from_indirect_block(
&self,
file_data: &[u8],
file: &dyn Storage,
iblock_addr: usize,
nrows: u16,
iblock_heap_offset: u64,
@@ -621,16 +661,8 @@ impl FractalHeapHeader {
"fractal heap: maximum recursion depth exceeded".into(),
));
}
// Parse indirect block header
ensure_len(file_data, iblock_addr, 4)?;
if &file_data[iblock_addr..iblock_addr + 4] != b"FHIB" {
return Err(FormatError::InvalidFractalHeapSignature);
}
let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
let iblock_header = 5 + offset_size as usize + block_offset_bytes;
let mut pos = iblock_addr + iblock_header;
let tw = self.table_width as u64;
let nrows_usize = nrows as usize;
let mut current_heap_offset = iblock_heap_offset;
@@ -640,6 +672,38 @@ impl FractalHeapHeader {
let start_indirect = self.max_direct_rows();
let max_direct_rows = nrows_usize.min(start_indirect);
// The block up to its last child entry, in one window: every
// position read below lies inside it, so its bounds checks are the
// whole-file ones.
let direct_entry = usize::from(offset_size)
+ if self.filter_pipeline.is_some() {
usize::from(self.length_size) + 4
} else {
0
};
let entries =
|rows: usize, entry: usize| rows.saturating_mul(tw as usize).saturating_mul(entry);
let block_len = iblock_header
.saturating_add(entries(max_direct_rows, direct_entry))
.saturating_add(entries(
nrows_usize.saturating_sub(start_indirect),
usize::from(offset_size),
));
let w = Window::read(file, iblock_addr as u64, block_len)?;
let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
let read_offset = |_: &[u8], pos: usize, size: u8| {
w.ensure(pos, usize::from(size))?;
read_offset(&w.bytes, pos, size)
};
let file_data: &[u8] = &w.bytes;
// Parse indirect block header
ensure_len(file_data, 0, 4)?;
if &file_data[..4] != b"FHIB" {
return Err(FormatError::InvalidFractalHeapSignature);
}
let mut pos = iblock_header;
for row in 0..max_direct_rows {
let block_size = self.block_size_for_row(row);
@@ -671,7 +735,7 @@ impl FractalHeapHeader {
&& target_offset < block_end
{
return self.read_from_direct_block(
file_data,
file,
DirectBlock {
addr: child_addr as usize,
size: block_size,
@@ -704,7 +768,7 @@ impl FractalHeapHeader {
&& target_offset < block_end
{
return self.read_from_indirect_block(
file_data,
file,
child_addr as usize,
child_nrows,
current_heap_offset,
@@ -720,7 +784,7 @@ impl FractalHeapHeader {
Err(FormatError::UnexpectedEof {
expected: target_offset as usize + length,
available: file_data.len(),
available: len_usize(file),
})
}
@@ -1024,4 +1088,110 @@ mod tests {
let id = [0x40u8, 0, 0, 0, 0, 0, 0];
assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
}
/// Headers, and managed (in a direct root and through an indirect
/// root), huge and tiny objects read identically through a
/// `read_at`-only storage, for every truncation of the file.
#[test]
fn storage_reads_match_slice_reads() {
use crate::storage::CountingStorage;
let (mut file, header_end) = build_simple_heap(8, 8);
// An indirect root block at 600: row 0 holds the direct block at
// 256, then three undefined blocks.
file[600..604].copy_from_slice(b"FHIB");
let mut at = 600 + 5 + 8 + 2;
for addr in [256u64, u64::MAX, u64::MAX, u64::MAX] {
file[at..at + 8].copy_from_slice(&addr.to_le_bytes());
at += 8;
}
file[900..905].copy_from_slice(b"huge!");
let managed_id = |offset: u64, len: u64| {
let payload = offset | (len << 16);
let mut id = vec![0u8];
id.extend_from_slice(&payload.to_le_bytes()[..6]);
id
};
let mut huge = vec![0x10u8];
huge.extend_from_slice(&900u64.to_le_bytes());
huge.extend_from_slice(&5u64.to_le_bytes());
let ids = [
managed_id(15, 13),
managed_id(15, 200),
managed_id(130, 4),
huge,
vec![0x22, b'a', b'b', b'c', 0, 0, 0],
];
let mut cuts: Vec<usize> = (0..=header_end + 1).collect();
cuts.extend([256, 260, 271, 280, 600, 610, 620, 640, 900, 903, file.len()]);
for cut in cuts {
let f = &file[..cut];
let storage = CountingStorage::new(f.to_vec());
let want = FractalHeapHeader::parse(f, 0, 8, 8);
let got = FractalHeapHeader::parse_in(&storage, 0, 8, 8);
assert_eq!(format!("{got:?}"), format!("{want:?}"), "cut {cut}");
let Ok(direct) = want else { continue };
let mut indirect = direct.clone();
indirect.root_block_address = 600;
indirect.current_rows_in_root_indirect_block = 1;
let mut huge_ids = direct.clone();
huge_ids.heap_id_length = 17;
for hdr in [&direct, &indirect, &huge_ids] {
for id in &ids {
assert_eq!(
hdr.read_managed_object_in(&storage, id, 8),
hdr.read_managed_object(f, id, 8),
"cut {cut}"
);
}
}
}
}
/// A huge object found through the huge-object B-tree needs the whole
/// file in memory until the B-tree reader is converted: a clean error
/// on other storage.
#[test]
fn huge_object_btree_needs_contiguous_storage() {
use crate::storage::CountingStorage;
let (file, _) = build_simple_heap(8, 8);
let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
hdr.huge_btree_address = 700;
let storage = CountingStorage::new(file);
assert_eq!(
hdr.read_managed_object_in(&storage, &[0x10, 1, 0, 0, 0, 0, 0], 8),
Err(FormatError::ContiguousStorageRequired(
"a huge fractal-heap object's B-tree"
))
);
}
/// A header with an I/O filter pipeline (read in a second, longer
/// window) parses identically through a `read_at`-only storage, for
/// every truncation.
#[test]
fn filtered_header_parses_identically_through_storage() {
use crate::storage::CountingStorage;
let (simple, header_end) = build_simple_heap(8, 8);
let pipeline = [2u8, 1, 1, 0, 0, 0, 1, 0, 6, 0, 0, 0]; // deflate, level 6
let mut header = simple[..header_end - 4].to_vec();
header[7..9].copy_from_slice(&(pipeline.len() as u16).to_le_bytes());
header.extend_from_slice(&100u64.to_le_bytes()); // root block's stored size
header.extend_from_slice(&0u32.to_le_bytes()); // its filter mask
header.extend_from_slice(&pipeline);
let sum = crate::checksum::jenkins_lookup3(&header);
header.extend_from_slice(&sum.to_le_bytes());
let mut file = header.clone();
file.resize(256, 0);
let hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
assert!(hdr.filter_pipeline.is_some());
for cut in 0..=file.len() {
let f = &file[..cut];
let storage = CountingStorage::new(f.to_vec());
assert_eq!(
format!("{:?}", FractalHeapHeader::parse_in(&storage, 0, 8, 8)),
format!("{:?}", FractalHeapHeader::parse(f, 0, 8, 8)),
"cut {cut}"
);
}
}
}
crates/clawhdf5-format/src/storage.rs
+43 -4
View File
@@ -206,11 +206,52 @@ pub fn read_exact_at(
if bytes.len() < len {
// The storage shrank or the backend served a short read inside the
// file: never parse a partial structure.
return Err(eof());
return Err(short_read());
}
Ok(bytes)
}
fn short_read() -> FormatError {
FormatError::Storage(
"short read inside the file (the storage shrank or the backend failed)".into(),
)
}
/// A window of the file: up to `max` bytes read at `base`, fewer only at
/// the end of the file. Its [`Window::ensure`] reports a bounds failure
/// exactly as the whole-file check `ensure_len(file_data, base + rel, n)`
/// did — with the absolute position and the file's length — as long as
/// every position checked lies within the `max` bytes the window was asked
/// for: then a position past the window is past the end of the file.
pub(crate) struct Window<'a> {
/// The bytes, from `base` on.
pub bytes: Cow<'a, [u8]>,
base: usize,
file_len: usize,
}
impl<'a> Window<'a> {
/// Read up to `max` bytes at `base`.
pub fn read(file: &'a dyn Storage, base: u64, max: usize) -> Result<Self, FormatError> {
Ok(Window {
bytes: read_upto(file, base, max)?,
base: usize::try_from(base).unwrap_or(usize::MAX),
file_len: len_usize(file),
})
}
/// Check that `[rel, rel + needed)` (relative to `base`) is in the file.
pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> {
match rel.checked_add(needed) {
Some(end) if end <= self.bytes.len() => Ok(()),
_ => Err(FormatError::UnexpectedEof {
expected: self.base.saturating_add(rel).saturating_add(needed),
available: self.file_len,
}),
}
}
}
/// Up to `max` bytes from `offset` on: fewer only at the end of the
/// storage. For structures whose size is only known once their prefix has
/// been parsed and whose parsers bound-check what they are given.
@@ -224,9 +265,7 @@ pub fn read_upto(
let len = usize::try_from(avail).map_or(max, |a| a.min(max));
let bytes = file.read_at(offset, len)?;
if bytes.len() < len {
return Err(FormatError::Storage(
"short read inside the file (the storage shrank or the backend failed)".into(),
));
return Err(short_read());
}
Ok(bytes)
}