quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p3-storage-trait' into feat/p3-range-zfp-edit

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/btree_v1.rs
#	crates/clawhdf5-format/src/data_layout.rs
#	crates/clawhdf5-format/src/extensible_array.rs
#	crates/clawhdf5-format/src/fixed_array.rs
#	crates/clawhdf5-format/src/fractal_heap.rs
#	crates/clawhdf5-format/src/local_heap.rs
#	crates/clawhdf5-format/src/shared_message.rs
This commit is contained in:
osobh
2026-09-26 14:51:51 -05:00
parent 234dd3e36c 895c79a2fe
commit 437e81cfff
26 changed files with 3480 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/fractal_heap.rs
+394 -60
View File
@@ -10,6 +10,7 @@ use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, find_btree_v2_records};
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
use crate::storage::{Storage, Window, len_usize, read_exact_at, require_contiguous};
/// Parsed fractal heap header (signature "FRHP").
#[derive(Debug, Clone)]
@@ -139,6 +140,36 @@ impl FractalHeapHeader {
offset_size: u8,
length_size: u8,
) -> Result<FractalHeapHeader, FormatError> {
Self::parse_in(file_data, offset as u64, offset_size, length_size)
}
/// [`Self::parse`] over any [`Storage`]: one read of the header (two
/// when it holds an I/O filter pipeline).
pub fn parse_in<S: Storage + ?Sized>(
file: &S,
offset: u64,
offset_size: u8,
length_size: u8,
) -> Result<FractalHeapHeader, FormatError> {
// Every field up to the checksum, without and with the filter
// information; the window holds all of it (or ends at the end of
// the file), so its bounds checks are the whole-file ones.
let (os, ls) = (usize::from(offset_size), usize::from(length_size));
let unfiltered_len = 26 + 12 * ls + 3 * os;
let mut w = Window::read(file, offset, unfiltered_len)?;
if w.bytes.len() == unfiltered_len {
let filter_len = usize::from(u16::from_le_bytes([w.bytes[7], w.bytes[8]]));
if filter_len > 0 {
w = Window::read(file, offset, unfiltered_len + ls + 4 + filter_len)?;
}
}
let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
let read_offset = |_: &[u8], pos: usize, size: u8| {
w.ensure(pos, usize::from(size))?;
read_offset(&w.bytes, pos, size)
};
let file_data: &[u8] = &w.bytes;
let offset = 0usize;
ensure_len(file_data, offset, 5)?;
if &file_data[offset..offset + 4] != b"FRHP" {
return Err(FormatError::InvalidFractalHeapSignature);
@@ -149,9 +180,6 @@ impl FractalHeapHeader {
return Err(FormatError::InvalidFractalHeapVersion(version));
}
let os = offset_size as usize;
let ls = length_size as usize;
let mut pos = offset + 5;
ensure_len(file_data, pos, 2)?;
let heap_id_length = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
@@ -354,6 +382,18 @@ impl FractalHeapHeader {
file_data: &[u8],
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
self.read_managed_object_in(file_data, id_bytes, offset_size)
}
/// [`Self::read_managed_object`] over any [`Storage`]. A huge object
/// found through the huge-object v2 B-tree still needs the whole file
/// in memory ([`FormatError::ContiguousStorageRequired`] otherwise).
pub fn read_managed_object_in<S: Storage + ?Sized>(
&self,
file_data: &S,
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
crate::lookup_stats::heap_object_read();
let Some(&first) = id_bytes.first() else {
@@ -385,7 +425,11 @@ impl FractalHeapHeader {
}
/// Read a huge object (heap ID type 1).
fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
fn read_huge_object<S: Storage + ?Sized>(
&self,
file: &S,
id: &[u8],
) -> Result<Vec<u8>, FormatError> {
let os = usize::from(self.offset_size);
let ls = usize::from(self.length_size);
// (address, stored length, filter mask, decoded length); the last two
@@ -416,18 +460,17 @@ impl FractalHeapHeader {
let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
ensure_len(id, 1, key_len)?;
let key = le_uint(&id[1..1 + key_len]);
self.find_huge_record(file_data, key)?
self.find_huge_record(file, key)?
};
let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
ensure_len(file_data, start, len)?;
let stored = &file_data[start..start + len];
let stored = read_exact_at(file, start as u64, len)?;
match &self.filter_pipeline {
None => Ok(stored.to_vec()),
None => Ok(stored.into_owned()),
Some(pipeline) => {
let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
let out = crate::filters::decompress_chunk_masked(&stored, pipeline, mem, 1, mask)?;
if out.len() != mem {
return Err(heap_error("filtered huge object decoded to the wrong size"));
}
@@ -438,9 +481,9 @@ impl FractalHeapHeader {
/// Look up huge object `key` in the huge-object v2 B-tree, returning
/// (address, stored length, filter mask, decoded length).
fn find_huge_record(
fn find_huge_record<S: Storage + ?Sized>(
&self,
file_data: &[u8],
file: &S,
key: u64,
) -> Result<(u64, u64, u32, u64), FormatError> {
if is_undefined(self.huge_btree_address, self.offset_size) {
@@ -448,6 +491,8 @@ impl FractalHeapHeader {
"huge object ID but the heap has no huge-object index",
));
}
// The v2 B-tree is read from a slice until it is converted.
let file_data = require_contiguous(file, "a huge fractal-heap object's B-tree")?;
let hdr = BTreeV2Header::parse(
file_data,
to_usize(self.huge_btree_address)?,
@@ -519,9 +564,9 @@ impl FractalHeapHeader {
}
/// Read a managed object (heap ID type 0).
fn read_heap_managed(
fn read_heap_managed<S: Storage + ?Sized>(
&self,
file_data: &[u8],
file_data: &S,
id_bytes: &[u8],
offset_size: u8,
) -> Result<Vec<u8>, FormatError> {
@@ -569,9 +614,9 @@ impl FractalHeapHeader {
/// header), so we just add it to the block address minus the block's heap
/// offset. A filtered heap stores each direct block (header included)
/// through its filter pipeline, so the block is decoded first.
fn read_from_direct_block(
fn read_from_direct_block<S: Storage + ?Sized>(
&self,
file_data: &[u8],
file: &S,
block: DirectBlock,
target_offset: u64,
length: usize,
@@ -587,9 +632,9 @@ impl FractalHeapHeader {
let stored_len = usize::try_from(block.filtered_size)
.map_err(|_| heap_error("direct block size"))?;
let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
ensure_len(file_data, block.addr, stored_len)?;
let stored = read_exact_at(file, block.addr as u64, stored_len)?;
let decoded = crate::filters::decompress_chunk_masked(
&file_data[block.addr..block.addr + stored_len],
&stored,
pipeline,
size,
1,
@@ -603,17 +648,16 @@ impl FractalHeapHeader {
.checked_add(local_offset)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: file_data.len(),
available: len_usize(file),
})?;
ensure_len(file_data, pos, length)?;
Ok(file_data[pos..pos + length].to_vec())
Ok(read_exact_at(file, pos as u64, length)?.into_owned())
}
/// Read an object by traversing an indirect block to find the right direct block.
#[allow(clippy::too_many_arguments)]
fn read_from_indirect_block(
fn read_from_indirect_block<S: Storage + ?Sized>(
&self,
file_data: &[u8],
file: &S,
iblock_addr: usize,
nrows: u16,
iblock_heap_offset: u64,
@@ -627,29 +671,169 @@ impl FractalHeapHeader {
"fractal heap: maximum recursion depth exceeded".into(),
));
}
// Parse indirect block header
ensure_len(file_data, iblock_addr, 4)?;
if &file_data[iblock_addr..iblock_addr + 4] != b"FHIB" {
return Err(FormatError::InvalidFractalHeapSignature);
}
let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
let iblock_header = 5 + offset_size as usize + block_offset_bytes;
let mut pos = iblock_addr + iblock_header;
let tw = self.table_width as u64;
let nrows_usize = nrows as usize;
let mut current_heap_offset = iblock_heap_offset;
// Rows below max_direct_rows hold direct blocks; rows at/above hold
// child indirect blocks. (NOT the FRHP "starting rows" field.)
let start_indirect = self.max_direct_rows();
let max_direct_rows = nrows_usize.min(start_indirect);
// The block up to its last child entry. The walk below reads
// entries in order and stops at the one covering the target, which
// the geometry alone locates, so the first window ends there: a
// header claiming a huge table costs a read of the entries in front
// of the target, not of the rest of the file. Only when that entry
// is unallocated (or none covers the target) does the walk go on,
// over the whole block. Either window holds what it was asked for or
// ends at the end of the file, so its bounds checks are the
// whole-file ones.
let direct_entry = usize::from(offset_size)
+ if self.filter_pipeline.is_some() {
usize::from(self.length_size) + 4
} else {
0
};
let direct_entries = max_direct_rows.saturating_mul(usize::from(self.table_width));
let entries_len = |n: usize| {
n.min(direct_entries)
.saturating_mul(direct_entry)
.saturating_add(
n.saturating_sub(direct_entries)
.saturating_mul(usize::from(offset_size)),
)
};
let all_entries = direct_entries.saturating_add(
nrows_usize
.saturating_sub(start_indirect)
.saturating_mul(usize::from(self.table_width)),
);
let block_len = iblock_header.saturating_add(entries_len(all_entries));
let target_entry = self.indirect_entry_for(nrows_usize, iblock_heap_offset, target_offset);
let first_len = target_entry.map_or(block_len, |i| {
iblock_header
.saturating_add(entries_len(i.saturating_add(1)))
.min(block_len)
});
let mut next = self.walk_indirect_block(
&Window::read(file, iblock_addr as u64, first_len)?,
nrows_usize,
iblock_heap_offset,
target_offset,
offset_size,
target_entry.map_or(usize::MAX, |i| i.saturating_add(1)),
)?;
if next.is_none() && first_len < block_len {
next = self.walk_indirect_block(
&Window::read(file, iblock_addr as u64, block_len)?,
nrows_usize,
iblock_heap_offset,
target_offset,
offset_size,
usize::MAX,
)?;
}
match next {
Some(IndirectChild::Direct(block)) => {
self.read_from_direct_block(file, block, target_offset, length)
}
Some(IndirectChild::Indirect {
addr,
nrows,
heap_offset,
}) => self.read_from_indirect_block(
file,
addr,
nrows,
heap_offset,
target_offset,
length,
offset_size,
depth_remaining - 1,
),
None => Err(FormatError::UnexpectedEof {
expected: to_usize(target_offset)?.saturating_add(length),
available: len_usize(file),
}),
}
}
/// Which child entry of an indirect block (numbered in walk order:
/// direct rows, then indirect rows) covers `target_offset`, from the
/// doubling-table geometry alone — the entry
/// [`Self::walk_indirect_block`] stops at if it is allocated. `None`
/// when no entry does.
fn indirect_entry_for(&self, nrows: usize, heap_offset: u64, target: u64) -> Option<usize> {
// The walk adds block sizes with saturation; in u128 the same test
// is `cur <= target < cur + size` without it (a target of u64::MAX
// is never inside a saturated range).
if target == u64::MAX {
return None;
}
let (tw, target) = (u128::from(self.table_width), u128::from(target));
let mut cur = u128::from(heap_offset);
let mut before = 0usize;
for row in 0..nrows {
if target < cur {
return None;
}
// Direct and indirect rows alike span this row's block size per
// entry.
let size = u128::from(self.block_size_for_row(row));
let span = size * tw;
if size > 0 && target < cur + span {
let col = usize::try_from((target - cur) / size).ok()?;
return before.checked_add(col);
}
cur += span;
before = before.saturating_add(self.table_width as usize);
}
None
}
/// Walk an indirect block's child entries in order, in the window `w`
/// (the block from its signature on), and return the allocated child
/// covering `target_offset`, or `None` when no entry among the first
/// `limit` does.
fn walk_indirect_block(
&self,
w: &Window<'_>,
nrows: usize,
iblock_heap_offset: u64,
target_offset: u64,
offset_size: u8,
limit: usize,
) -> Result<Option<IndirectChild>, FormatError> {
let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
let read_offset = |_: &[u8], pos: usize, size: u8| {
w.ensure(pos, usize::from(size))?;
read_offset(&w.bytes, pos, size)
};
let file_data: &[u8] = &w.bytes;
let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
let iblock_header = 5 + offset_size as usize + block_offset_bytes;
let tw = self.table_width as u64;
let mut current_heap_offset = iblock_heap_offset;
let start_indirect = self.max_direct_rows();
let max_direct_rows = nrows.min(start_indirect);
let mut walked = 0usize;
// Parse indirect block header
ensure_len(file_data, 0, 4)?;
if &file_data[..4] != b"FHIB" {
return Err(FormatError::InvalidFractalHeapSignature);
}
let mut pos = iblock_header;
for row in 0..max_direct_rows {
let block_size = self.block_size_for_row(row);
for _col in 0..tw {
if walked == limit {
return Ok(None);
}
walked += 1;
let child_addr = read_offset(file_data, pos, offset_size)?;
pos += offset_size as usize;
@@ -676,18 +860,13 @@ impl FractalHeapHeader {
&& target_offset >= current_heap_offset
&& target_offset < block_end
{
return self.read_from_direct_block(
file_data,
DirectBlock {
addr: to_usize(child_addr)?,
size: block_size,
heap_offset: current_heap_offset,
filtered_size,
filter_mask,
},
target_offset,
length,
);
return Ok(Some(IndirectChild::Direct(DirectBlock {
addr: to_usize(child_addr)?,
size: block_size,
heap_offset: current_heap_offset,
filtered_size,
filter_mask,
})));
}
current_heap_offset = block_end;
}
@@ -696,11 +875,15 @@ impl FractalHeapHeader {
// Rows at and above `start_indirect` hold child indirect blocks. A
// child in row r spans exactly that row's block size of heap space,
// so it has as many rows as a table of that total size needs.
for row in start_indirect..nrows_usize {
for row in start_indirect..nrows {
let child_space = self.block_size_for_row(row);
let child_nrows = self.rows_for_size(child_space);
for _col in 0..tw {
if walked == limit {
return Ok(None);
}
walked += 1;
let child_addr = read_offset(file_data, pos, offset_size)?;
pos += offset_size as usize;
@@ -709,25 +892,16 @@ impl FractalHeapHeader {
&& target_offset >= current_heap_offset
&& target_offset < block_end
{
return self.read_from_indirect_block(
file_data,
to_usize(child_addr)?,
child_nrows,
current_heap_offset,
target_offset,
length,
offset_size,
depth_remaining - 1,
);
return Ok(Some(IndirectChild::Indirect {
addr: to_usize(child_addr)?,
nrows: child_nrows,
heap_offset: current_heap_offset,
}));
}
current_heap_offset = block_end;
}
}
Err(FormatError::UnexpectedEof {
expected: to_usize(target_offset)?.saturating_add(length),
available: file_data.len(),
})
Ok(None)
}
/// Number of rows in the doubling table whose block size is at most the
@@ -771,6 +945,16 @@ impl FractalHeapHeader {
/// A managed direct block's location, extent and (for a filtered heap) its
/// stored size and filter mask.
/// The child of an indirect block that covers a heap offset.
enum IndirectChild {
Direct(DirectBlock),
Indirect {
addr: usize,
nrows: u16,
heap_offset: u64,
},
}
struct DirectBlock {
addr: usize,
size: u64,
@@ -1030,4 +1214,154 @@ mod tests {
let id = [0x40u8, 0, 0, 0, 0, 0, 0];
assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
}
/// Headers, and managed (in a direct root and through an indirect
/// root), huge and tiny objects read identically through a
/// `read_at`-only storage, for every truncation of the file.
#[test]
fn storage_reads_match_slice_reads() {
use crate::storage::CountingStorage;
let (mut file, header_end) = build_simple_heap(8, 8);
// An indirect root block at 600: row 0 holds the direct block at
// 256, then three undefined blocks.
file[600..604].copy_from_slice(b"FHIB");
let mut at = 600 + 5 + 8 + 2;
for addr in [256u64, u64::MAX, u64::MAX, u64::MAX] {
file[at..at + 8].copy_from_slice(&addr.to_le_bytes());
at += 8;
}
file[900..905].copy_from_slice(b"huge!");
let managed_id = |offset: u64, len: u64| {
let payload = offset | (len << 16);
let mut id = vec![0u8];
id.extend_from_slice(&payload.to_le_bytes()[..6]);
id
};
let mut huge = vec![0x10u8];
huge.extend_from_slice(&900u64.to_le_bytes());
huge.extend_from_slice(&5u64.to_le_bytes());
let ids = [
managed_id(15, 13),
managed_id(15, 200),
managed_id(130, 4),
huge,
vec![0x22, b'a', b'b', b'c', 0, 0, 0],
];
let mut cuts: Vec<usize> = (0..=header_end + 1).collect();
cuts.extend([256, 260, 271, 280, 600, 610, 620, 640, 900, 903, file.len()]);
for cut in cuts {
let f = &file[..cut];
let storage = CountingStorage::new(f.to_vec());
let want = FractalHeapHeader::parse(f, 0, 8, 8);
let got = FractalHeapHeader::parse_in(&storage, 0, 8, 8);
assert_eq!(format!("{got:?}"), format!("{want:?}"), "cut {cut}");
let Ok(direct) = want else { continue };
let mut indirect = direct.clone();
indirect.root_block_address = 600;
indirect.current_rows_in_root_indirect_block = 1;
let mut huge_ids = direct.clone();
huge_ids.heap_id_length = 17;
for hdr in [&direct, &indirect, &huge_ids] {
for id in &ids {
assert_eq!(
hdr.read_managed_object_in(&storage, id, 8),
hdr.read_managed_object(f, id, 8),
"cut {cut}"
);
}
}
}
}
/// A header claiming a huge doubling table (width 0xFFFF, 0xFFFF rows in
/// the root indirect block) in a 16 MiB file: reading an object from the
/// table's first block reads the entries up to it, not the rest of the
/// file, and gives what the slice read gives. When the covering entry is
/// unallocated the walk goes on over the whole block, still identically.
#[test]
fn huge_table_claims_read_only_what_the_walk_needs() {
use crate::storage::CountingStorage;
let (mut file, _) = build_simple_heap(8, 8);
file.resize(16 << 20, 0);
file[600..604].copy_from_slice(b"FHIB");
let first_entry = 600 + 5 + 8 + 2;
file[first_entry..first_entry + 8].copy_from_slice(&256u64.to_le_bytes());
let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
hdr.table_width = 0xFFFF;
hdr.root_block_address = 600;
hdr.current_rows_in_root_indirect_block = 0xFFFF;
let managed_id = |offset: u64, len: u64| {
let payload = offset | (len << 16);
let mut id = vec![0u8];
id.extend_from_slice(&payload.to_le_bytes()[..6]);
id
};
let storage = CountingStorage::new(file.clone());
let id = managed_id(15, 13);
let want = hdr.read_managed_object(&file, &id, 8);
assert!(want.is_ok(), "{want:?}");
storage.reset();
assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want);
assert!(
storage.bytes_read() < 1024,
"{} bytes in {} reads",
storage.bytes_read(),
storage.reads()
);
// The second entry (heap offsets 128..256) is unallocated (zero is
// not the undefined address, so make it all ones).
file[first_entry + 8..first_entry + 16].fill(0xFF);
let storage = CountingStorage::new(file.clone());
let id = managed_id(130, 4);
let want = hdr.read_managed_object(&file, &id, 8);
assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want);
}
/// A huge object found through the huge-object B-tree needs the whole
/// file in memory until the B-tree reader is converted: a clean error
/// on other storage.
#[test]
fn huge_object_btree_needs_contiguous_storage() {
use crate::storage::CountingStorage;
let (file, _) = build_simple_heap(8, 8);
let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
hdr.huge_btree_address = 700;
let storage = CountingStorage::new(file);
assert_eq!(
hdr.read_managed_object_in(&storage, &[0x10, 1, 0, 0, 0, 0, 0], 8),
Err(FormatError::ContiguousStorageRequired(
"a huge fractal-heap object's B-tree"
))
);
}
/// A header with an I/O filter pipeline (read in a second, longer
/// window) parses identically through a `read_at`-only storage, for
/// every truncation.
#[test]
fn filtered_header_parses_identically_through_storage() {
use crate::storage::CountingStorage;
let (simple, header_end) = build_simple_heap(8, 8);
let pipeline = [2u8, 1, 1, 0, 0, 0, 1, 0, 6, 0, 0, 0]; // deflate, level 6
let mut header = simple[..header_end - 4].to_vec();
header[7..9].copy_from_slice(&(pipeline.len() as u16).to_le_bytes());
header.extend_from_slice(&100u64.to_le_bytes()); // root block's stored size
header.extend_from_slice(&0u32.to_le_bytes()); // its filter mask
header.extend_from_slice(&pipeline);
let sum = crate::checksum::jenkins_lookup3(&header);
header.extend_from_slice(&sum.to_le_bytes());
let mut file = header.clone();
file.resize(256, 0);
let hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
assert!(hdr.filter_pipeline.is_some());
for cut in 0..=file.len() {
let f = &file[..cut];
let storage = CountingStorage::new(f.to_vec());
assert_eq!(
format!("{:?}", FractalHeapHeader::parse_in(&storage, 0, 8, 8)),
format!("{:?}", FractalHeapHeader::parse(f, 0, 8, 8)),
"cut {cut}"
);
}
}
}