From 42894bf93b4685945bc3401f5e13811ab231facc Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:13:01 -0500 Subject: [PATCH 01/10] format: v2 B-trees, dense groups and group listings over Storage BTreeV2Header::parse_in, collect_btree_v2_records_in and find_btree_v2_records_in read one bounded window per node (its size is known from the parent before the node is read; a count stretched past node_size is checked against the end of the file first), with the whole-file bounds errors unchanged. With them, dense attributes, a SOHM B-tree index and huge fractal-heap objects no longer answer ContiguousStorageRequired, and group_v1/group_v2 listings, lookups and path resolution get *_in cores (resolve_group_children_in, resolve_child_in, resolve_path_any_in, ...). The &[u8] functions are thin wrappers, as in M1. The equivalence harness now fails on any ContiguousStorageRequired and compares v2 B-tree headers, records and descents, group listings, child lookups and paths; a unit test compares a two-level tree through a read_at-only storage truncated at every length and with every node byte flipped. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/addr.rs | 9 + crates/clawhdf5-format/src/attribute.rs | 55 ++-- crates/clawhdf5-format/src/btree_v2.rs | 306 ++++++++++++------ crates/clawhdf5-format/src/btree_v2_write.rs | 51 +++ crates/clawhdf5-format/src/fractal_heap.rs | 36 +-- crates/clawhdf5-format/src/group_v1.rs | 85 +++-- crates/clawhdf5-format/src/group_v2.rs | 138 +++++--- crates/clawhdf5-format/src/shared_message.rs | 23 +- .../tests/storage_equivalence.rs | 138 ++++---- 9 files changed, 544 insertions(+), 297 deletions(-) diff --git a/crates/clawhdf5-format/src/addr.rs b/crates/clawhdf5-format/src/addr.rs index 3198f1f..e6471d0 100644 --- a/crates/clawhdf5-format/src/addr.rs +++ b/crates/clawhdf5-format/src/addr.rs @@ -23,6 +23,15 @@ pub fn to_usize(value: u64) -> Result { to_index::(value) } +/// A file address for a [`crate::storage::Storage`] read, checked as +/// [`to_usize`] checks it: the parsers read through 64-bit offsets, but an +/// address that could not index an in-memory file on this platform is the +/// same [`FormatError::Overflow`] the slice parsers gave for it. +#[inline] +pub fn checked_addr(value: u64) -> Result { + to_usize(value).map(|_| value) +} + /// [`to_usize`] for an index type of any width. `usize` is 64 bits wide on /// the hosts CI tests on, where the error path cannot be reached through /// `usize`; tests run the same code with `u32` in its place, as on a 32-bit diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 78aaf12..4b30fac 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -7,7 +7,7 @@ use std::borrow::Cow; use crate::addr::to_usize; use crate::attribute_info::AttributeInfoMessage; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in}; use crate::checksum::jenkins_lookup3; use crate::data_read; use crate::dataspace::Dataspace; @@ -17,7 +17,7 @@ use crate::fractal_heap::FractalHeapHeader; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; use crate::shared_message; -use crate::storage::{Storage, require_contiguous}; +use crate::storage::Storage; use crate::vl_data; /// A parsed HDF5 attribute message. @@ -578,9 +578,12 @@ pub fn find_attribute_in( .find(|a| a.name == name), ); }; - let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?; - let btree_hdr = - BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + to_usize(btree_addr)? as u64, + offset_size, + length_size, + )?; let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?; if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 { return Ok( @@ -610,7 +613,7 @@ pub fn find_attribute_in( // hash is the last field. let hash = jenkins_lookup3(name.as_bytes()); let hash_at = usize::from(btree_hdr.record_size) - 4; - let records = find_btree_v2_records(contiguous, &btree_hdr, offset_size, &mut |r| match r + let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| match r .get(hash_at..hash_at + 4) { Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), @@ -722,10 +725,13 @@ fn extract_dense_attributes( expected: 1, available: 0, })?; - let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?; - let btree_hdr = - BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?; - let records = collect_btree_v2_records(contiguous, &btree_hdr, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + to_usize(btree_addr)? as u64, + offset_size, + length_size, + )?; + let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { // Per HDF5 spec, both type 8 and type 9 records start with heap_id: @@ -1156,11 +1162,9 @@ mod tests { } /// Every object's attributes in h5py-written files read identically - /// through a read_at-only CountingStorage — compact ones, shared ones - /// and those behind an Attribute Info message — except dense storage, - /// whose v2 B-tree index is not read over Storage yet: that is the clean - /// ContiguousStorageRequired error, never a partial list. Through a - /// slice as Storage every object matches. + /// through a read_at-only CountingStorage — compact ones, shared ones, + /// those behind an Attribute Info message and dense storage (its v2 + /// B-tree name index included) — and through a slice as Storage. #[test] fn storage_reads_match_slice_reads() { use crate::storage::CountingStorage; @@ -1206,18 +1210,17 @@ mod tests { .unwrap() .is_some_and(|i| i.fractal_heap_address.is_some()); if is_dense { - let e = FormatError::ContiguousStorageRequired( - "dense attribute storage (a v2 B-tree)", - ); - assert_eq!(got.unwrap_err(), e, "{name}"); - assert_eq!(got_t.unwrap_err(), e, "{name}"); dense += 1; - } else { - attrs += want.as_ref().map_or(0, Vec::len); - assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}"); - let want_t = extract_attributes_tolerant(file, &header, os, ls); - assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}"); - same += 1; + } + attrs += want.as_ref().map_or(0, Vec::len); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}"); + let want_t = extract_attributes_tolerant(file, &header, os, ls); + assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}"); + same += 1; + for a in want.iter().flatten() { + let one = find_attribute_in(&storage, &header, &a.name, os, ls); + let want_one = find_attribute_in_file(file, &header, &a.name, os, ls); + assert_eq!(format!("{one:?}"), format!("{want_one:?}"), "{name}"); } } } diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index d3f6250..5641eb7 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -9,6 +9,7 @@ use byteorder::{ByteOrder, LittleEndian}; use crate::addr::to_usize; use crate::error::FormatError; +use crate::storage::{Storage, Window, len_usize}; /// Parsed B-tree v2 header (signature "BTHD"). #[derive(Debug, Clone)] @@ -99,38 +100,52 @@ impl BTreeV2Header { offset_size: u8, length_size: u8, ) -> Result { - ensure_len(file_data, offset, 4)?; - if &file_data[offset..offset + 4] != b"BTHD" { + Self::parse_in(file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one bounded read of the + /// header. + pub fn parse_in( + file: &S, + offset: u64, + offset_size: u8, + length_size: u8, + ) -> Result { + // Every field and the checksum; the window holds all of it or ends + // at the end of the file, so its bounds checks are the whole-file + // ones. + let full = 16 + usize::from(offset_size) + 2 + usize::from(length_size) + 4; + let w = Window::read(file, offset, full)?; + let d = &w.bytes; + w.ensure(0, 4)?; + if &d[..4] != b"BTHD" { return Err(FormatError::InvalidBTreeV2Signature); } - ensure_len(file_data, offset, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?; - let version = file_data[offset + 4]; + w.ensure(0, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?; + let version = d[4]; if version != 0 { return Err(FormatError::InvalidBTreeV2Version(version)); } - let tree_type = file_data[offset + 5]; - let node_size = u32::from_le_bytes([ - file_data[offset + 6], - file_data[offset + 7], - file_data[offset + 8], - file_data[offset + 9], - ]); - let record_size = u16::from_le_bytes([file_data[offset + 10], file_data[offset + 11]]); - let depth = u16::from_le_bytes([file_data[offset + 12], file_data[offset + 13]]); - let _split_percent = file_data[offset + 14]; - let _merge_percent = file_data[offset + 15]; + let tree_type = d[5]; + let node_size = u32::from_le_bytes([d[6], d[7], d[8], d[9]]); + let record_size = u16::from_le_bytes([d[10], d[11]]); + let depth = u16::from_le_bytes([d[12], d[13]]); + let _split_percent = d[14]; + let _merge_percent = d[15]; - let mut pos = offset + 16; - let root_node_address = read_offset(file_data, pos, offset_size)?; + let mut pos = 16; + w.ensure(pos, usize::from(offset_size))?; + let root_node_address = read_offset(d, pos, offset_size)?; pos += offset_size as usize; - ensure_len(file_data, pos, 2)?; - let num_records_in_root = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); + w.ensure(pos, 2)?; + let num_records_in_root = u16::from_le_bytes([d[pos], d[pos + 1]]); pos += 2; - let total_records = read_offset(file_data, pos, length_size)?; + w.ensure(pos, usize::from(length_size))?; + let total_records = read_offset(d, pos, length_size)?; #[allow(unused_assignments)] { pos += length_size as usize; @@ -139,9 +154,9 @@ impl BTreeV2Header { // Validate header checksum #[cfg(feature = "checksum")] { - ensure_len(file_data, pos, 4)?; - let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]); - let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]); + w.ensure(pos, 4)?; + let stored = LittleEndian::read_u32(&d[pos..pos + 4]); + let computed = crate::checksum::jenkins_lookup3(&d[..pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, @@ -191,6 +206,17 @@ pub fn collect_btree_v2_records( header: &BTreeV2Header, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + collect_btree_v2_records_in(file_data, header, offset_size, length_size) +} + +/// [`collect_btree_v2_records`] over any [`Storage`]: one bounded read per +/// node. +pub fn collect_btree_v2_records_in( + file: &S, + header: &BTreeV2Header, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { if header.total_records == 0 || header.num_records_in_root == 0 { return Ok(Vec::new()); @@ -210,23 +236,24 @@ pub fn collect_btree_v2_records( // millions of records from a few kilobytes. Counting against what the // file could physically contain bounds that without trusting the // header's own `total_records`. - let mut budget = file_data.len() / usize::from(header.record_size.max(1)); + let mut budget = len_usize(file) / usize::from(header.record_size.max(1)); let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size); if header.depth == 0 { // Root is a leaf parse_leaf_records( - file_data, + file, to_usize(header.root_node_address)?, header.num_records_in_root, header.record_size, + header.node_size, ) } else { // Root is internal; traverse recursively let mut records = Vec::new(); collect_internal_records( - file_data, + file, to_usize(header.root_node_address)?, header.num_records_in_root, header.depth, @@ -242,36 +269,72 @@ pub fn collect_btree_v2_records( } } +/// A node's bytes: `want` bytes at `offset` (fewer only at the end of the +/// file), after checking its 4-byte signature. A node is read in one piece +/// when it fits in `node_size` (every valid node does); a larger claimed +/// extent — record counts from a damaged parent — is first checked against +/// the end of the file, so it costs a read only of bytes the file has. +/// Bounds errors are the whole-file ones: the signature check needs the +/// first 6 bytes, then `checks` — `(position, length)` pairs relative to +/// the node, in the order the parser checks them — must lie in the file. +fn read_node<'a, S: Storage + ?Sized>( + file: &'a S, + offset: usize, + want: usize, + node_size: u32, + signature: &[u8; 4], + checks: &[(usize, usize)], +) -> Result, FormatError> { + let one_read = usize::try_from(node_size).unwrap_or(usize::MAX).max(6); + let w = Window::read(file, offset as u64, want.min(one_read))?; + w.ensure(0, 6)?; + if &w.bytes[..4] != signature { + return Err(FormatError::InvalidBTreeV2Signature); + } + if want <= one_read { + return Ok(w); + } + for &(rel, len) in checks { + Window::check_extent(file, offset as u64, rel, len)?; + } + Window::read(file, offset as u64, want) +} + /// Parse records from a leaf node (signature "BTLF"). -fn parse_leaf_records( - file_data: &[u8], +fn parse_leaf_records( + file: &S, offset: usize, num_records: u16, record_size: u16, + node_size: u32, ) -> Result, FormatError> { // signature(4) + version(1) + type(1) = 6 bytes header - ensure_len(file_data, offset, 6)?; - if &file_data[offset..offset + 4] != b"BTLF" { - return Err(FormatError::InvalidBTreeV2Signature); - } - - let pos = offset + 6; + let pos = 6; let rs = record_size as usize; let total = (num_records as usize) .checked_mul(rs) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: len_usize(file), })?; - ensure_len(file_data, pos, total)?; + let w = read_node( + file, + offset, + pos + total + 4, + node_size, + b"BTLF", + &[(pos, total)], + )?; + let d = &w.bytes; + w.ensure(pos, total)?; // Validate checksum: 4 bytes after records + padding #[cfg(feature = "checksum")] { let checksum_pos = pos + total; - if file_data.len() >= checksum_pos + 4 { - let stored = LittleEndian::read_u32(&file_data[checksum_pos..checksum_pos + 4]); - let computed = crate::checksum::jenkins_lookup3(&file_data[offset..checksum_pos]); + if d.len() >= checksum_pos + 4 { + let stored = LittleEndian::read_u32(&d[checksum_pos..checksum_pos + 4]); + let computed = crate::checksum::jenkins_lookup3(&d[..checksum_pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, @@ -285,17 +348,41 @@ fn parse_leaf_records( for i in 0..num_records as usize { let start = pos + i * rs; records.push(BTreeV2Record { - data: file_data[start..start + rs].to_vec(), + data: d[start..start + rs].to_vec(), }); } Ok(records) } +/// An internal node read from the file: its bytes (from the signature on), +/// where its records start, and its children as `(address, record count)`. +struct InternalNode<'a> { + node: Window<'a>, + records_start: usize, + children: Vec<(u64, u16)>, +} + +impl InternalNode<'_> { + /// Record `i`, `rs` bytes long. + fn record(&self, i: usize, rs: usize) -> Result<&[u8], FormatError> { + let overflow = || FormatError::UnexpectedEof { + expected: usize::MAX, + available: usize::MAX, + }; + let rec_start = i + .checked_mul(rs) + .and_then(|o| self.records_start.checked_add(o)) + .ok_or_else(overflow)?; + self.node.ensure(rec_start, rs)?; + Ok(&self.node.bytes[rec_start..rec_start + rs]) + } +} + /// An internal node's layout: where its records start, and its children as /// `(address, record count)`. #[allow(clippy::too_many_arguments)] -fn read_internal_node( - file_data: &[u8], +fn read_internal_node( + file: &S, offset: usize, num_records: u16, depth: u16, @@ -303,25 +390,15 @@ fn read_internal_node( node_size: u32, offset_size: u8, max_leaf_nrec: u64, -) -> Result<(usize, Vec<(u64, u16)>), FormatError> { - // signature(4) + version(1) + type(1) = 6 - ensure_len(file_data, offset, 6)?; - if &file_data[offset..offset + 4] != b"BTIN" { - return Err(FormatError::InvalidBTreeV2Signature); - } - +) -> Result, FormatError> { let nr = num_records as usize; let rs = record_size as usize; - let mut pos = offset + 6; // Records first let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: len_usize(file), })?; - ensure_len(file_data, pos, records_total)?; - let records_start = pos; - pos += records_total; // Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the // child's record count is always encoded in the width needed for a @@ -344,13 +421,30 @@ fn read_internal_node( let num_children = nr + 1; let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width; - ensure_len(file_data, pos, num_children * child_ptr_size)?; + let pointers = num_children * child_ptr_size; + + // signature(4) + version(1) + type(1) = 6, records, pointers, checksum. + let w = read_node( + file, + offset, + 6 + records_total + pointers + 4, + node_size, + b"BTIN", + &[(6, records_total), (6 + records_total, pointers)], + )?; + let d = &w.bytes; + let mut pos = 6; + w.ensure(pos, records_total)?; + let records_start = pos; + pos += records_total; + + w.ensure(pos, pointers)?; let mut children = Vec::with_capacity(num_children); for _ in 0..num_children { - let addr = read_offset(file_data, pos, offset_size)?; + let addr = read_offset(d, pos, offset_size)?; pos += offset_size as usize; - let child_nrec = read_var_uint(file_data, pos, nrec_width)? as u16; + let child_nrec = read_var_uint(d, pos, nrec_width)? as u16; pos += nrec_width; pos += total_nrec_width; // skip total records in subtree children.push((addr, child_nrec)); @@ -362,9 +456,9 @@ fn read_internal_node( // a mismatch here, and so does this. #[cfg(feature = "checksum")] { - ensure_len(file_data, pos, 4)?; - let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]); - let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]); + w.ensure(pos, 4)?; + let stored = LittleEndian::read_u32(&d[pos..pos + 4]); + let computed = crate::checksum::jenkins_lookup3(&d[..pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, @@ -372,37 +466,17 @@ fn read_internal_node( }); } } - Ok((records_start, children)) -} - -/// Record `i` of an internal node whose records start at `records_start`. -fn internal_record( - file_data: &[u8], - records_start: usize, - i: usize, - rs: usize, -) -> Result<&[u8], FormatError> { - let overflow = || FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - }; - let rec_start = i - .checked_mul(rs) - .and_then(|o| records_start.checked_add(o)) - .ok_or_else(overflow)?; - let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?; - file_data - .get(rec_start..rec_end) - .ok_or(FormatError::UnexpectedEof { - expected: rec_end, - available: file_data.len(), - }) + Ok(InternalNode { + node: w, + records_start, + children, + }) } /// Recursively collect records from an internal node. #[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)] -fn collect_internal_records( - file_data: &[u8], +fn collect_internal_records( + file: &S, offset: usize, num_records: u16, depth: u16, @@ -416,8 +490,8 @@ fn collect_internal_records( ) -> Result<(), FormatError> { let nr = num_records as usize; let rs = record_size as usize; - let (records_start, children) = read_internal_node( - file_data, + let node = read_internal_node( + file, offset, num_records, depth, @@ -430,16 +504,21 @@ fn collect_internal_records( // Interleave: child[0], record[0], child[1], record[1], ..., child[nr] // We collect child[0] records, then record[0], then child[1], etc. - for (i, &(child_addr, child_nrec)) in children.iter().enumerate() { + for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() { if child_depth == 0 { // Before parsing, so a refused tree is not also a large allocation. spend(budget, usize::from(child_nrec))?; - let leaf_recs = - parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?; + let leaf_recs = parse_leaf_records( + file, + to_usize(child_addr)?, + child_nrec, + record_size, + node_size, + )?; out.extend(leaf_recs); } else { collect_internal_records( - file_data, + file, to_usize(child_addr)?, child_nrec, child_depth, @@ -455,7 +534,7 @@ fn collect_internal_records( // Add record[i] (except after the last child) if i < nr { - let data = internal_record(file_data, records_start, i, rs)?; + let data = node.record(i, rs)?; spend(budget, 1)?; out.push(BTreeV2Record { data: data.to_vec(), @@ -481,6 +560,17 @@ pub fn find_btree_v2_records( header: &BTreeV2Header, offset_size: u8, cmp: &mut dyn FnMut(&[u8]) -> Ordering, +) -> Result, FormatError> { + find_btree_v2_records_in(file_data, header, offset_size, cmp) +} + +/// [`find_btree_v2_records`] over any [`Storage`]: one bounded read per +/// node visited. +pub fn find_btree_v2_records_in( + file: &S, + header: &BTreeV2Header, + offset_size: u8, + cmp: &mut dyn FnMut(&[u8]) -> Ordering, ) -> Result, FormatError> { if header.total_records == 0 || header.num_records_in_root == 0 { return Ok(Vec::new()); @@ -490,11 +580,11 @@ pub fn find_btree_v2_records( } // As in `collect_btree_v2_records`: a valid tree cannot hold more // records than the file has room for, however its children are shared. - let mut budget = file_data.len() / usize::from(header.record_size.max(1)); + let mut budget = len_usize(file) / usize::from(header.record_size.max(1)); let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size); let mut out = Vec::new(); find_in_node( - file_data, + file, header, to_usize(header.root_node_address)?, header.num_records_in_root, @@ -509,8 +599,8 @@ pub fn find_btree_v2_records( } #[allow(clippy::too_many_arguments)] -fn find_in_node( - file_data: &[u8], +fn find_in_node( + file: &S, header: &BTreeV2Header, offset: usize, num_records: u16, @@ -523,7 +613,13 @@ fn find_in_node( ) -> Result<(), FormatError> { spend(budget, usize::from(num_records))?; if depth == 0 { - let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?; + let records = parse_leaf_records( + file, + offset, + num_records, + header.record_size, + header.node_size, + )?; out.extend( records .into_iter() @@ -532,8 +628,8 @@ fn find_in_node( return Ok(()); } let rs = usize::from(header.record_size); - let (records_start, children) = read_internal_node( - file_data, + let node = read_internal_node( + file, offset, num_records, depth, @@ -545,17 +641,17 @@ fn find_in_node( let nr = usize::from(num_records); let mut order = Vec::with_capacity(nr); for i in 0..nr { - order.push(cmp(internal_record(file_data, records_start, i, rs)?)); + order.push(cmp(node.record(i, rs)?)); } // Child `i` holds the keys between record `i - 1` and record `i`: it can // hold a match unless the record before it is already past the range or // the record after it is still before it. - for (i, &(child_addr, child_nrec)) in children.iter().enumerate() { + for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() { let after_left = i == 0 || order[i - 1] != Ordering::Greater; let before_right = i == nr || order[i] != Ordering::Less; if after_left && before_right { find_in_node( - file_data, + file, header, to_usize(child_addr)?, child_nrec, @@ -569,7 +665,7 @@ fn find_in_node( } if i < nr && order[i] == Ordering::Equal { out.push(BTreeV2Record { - data: internal_record(file_data, records_start, i, rs)?.to_vec(), + data: node.record(i, rs)?.to_vec(), }); } } diff --git a/crates/clawhdf5-format/src/btree_v2_write.rs b/crates/clawhdf5-format/src/btree_v2_write.rs index 38fa7c1..dd8e9cf 100644 --- a/crates/clawhdf5-format/src/btree_v2_write.rs +++ b/crates/clawhdf5-format/src/btree_v2_write.rs @@ -441,6 +441,57 @@ mod tests { } } + /// A two-level tree read through a `read_at`-only storage gives what + /// the slice gives — records, descents and errors — whole, truncated + /// at every length, and with each byte of its nodes flipped, and each + /// node costs one read. + #[test] + fn storage_reads_match_slice_reads() { + use crate::btree_v2::{ + collect_btree_v2_records_in, find_btree_v2_records, find_btree_v2_records_in, + }; + use crate::storage::CountingStorage; + let (rs, n, base) = (11usize, 120usize, 64usize); + let recs = records(n, rs); + let tree = build_btree_v2(params(128, 11), &recs, base as u64, 8, 8).unwrap(); + let mut whole = vec![0u8; base]; + whole.extend_from_slice(&tree); + let hdr = BTreeV2Header::parse(&whole, base, 8, 8).unwrap(); + assert!(hdr.depth >= 1, "{hdr:?}"); + let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap()); + let mut files = Vec::new(); + for cut in base..=whole.len() { + files.push(whole[..cut].to_vec()); + } + for at in base..whole.len() { + let mut bad = whole.clone(); + bad[at] ^= 0x5a; + files.push(bad); + } + let mut ok = 0; + for f in &files { + let st = CountingStorage::new(f.clone()); + let want_h = BTreeV2Header::parse(f, base, 8, 8); + let got_h = BTreeV2Header::parse_in(&st, base as u64, 8, 8); + assert_eq!(format!("{got_h:?}"), format!("{want_h:?}")); + // The nodes of the intact header, over each damaged file. + let want = collect_btree_v2_records(f, &hdr, 8, 8); + st.reset(); + let got = collect_btree_v2_records_in(&st, &hdr, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + if want.is_ok() { + ok += 1; + assert!(st.reads() <= 1 + n as u64 / 3, "{} reads", st.reads()); + } + for k in [0u64, 7, 60, 119, 500] { + let want = find_btree_v2_records(f, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)); + let got = find_btree_v2_records_in(&st, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + } + } + assert!(ok > 1); + } + #[test] fn a_node_too_small_or_too_big_is_an_error() { assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err()); diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index 7bdce6f..83cfd33 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -7,10 +7,10 @@ use alloc::{format, vec::Vec}; use byteorder::{ByteOrder, LittleEndian}; use crate::addr::to_usize; -use crate::btree_v2::{BTreeV2Header, find_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, find_btree_v2_records_in}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; -use crate::storage::{Storage, Window, len_usize, read_exact_at, require_contiguous}; +use crate::storage::{Storage, Window, len_usize, read_exact_at}; /// Parsed fractal heap header (signature "FRHP"). #[derive(Debug, Clone)] @@ -386,9 +386,7 @@ impl FractalHeapHeader { self.read_managed_object_in(file_data, id_bytes, offset_size) } - /// [`Self::read_managed_object`] over any [`Storage`]. A huge object - /// found through the huge-object v2 B-tree still needs the whole file - /// in memory ([`FormatError::ContiguousStorageRequired`] otherwise). + /// [`Self::read_managed_object`] over any [`Storage`]. pub fn read_managed_object_in( &self, file_data: &S, @@ -491,11 +489,9 @@ impl FractalHeapHeader { "huge object ID but the heap has no huge-object index", )); } - // The v2 B-tree is read from a slice until it is converted. - let file_data = require_contiguous(file, "a huge fractal-heap object's B-tree")?; - let hdr = BTreeV2Header::parse( - file_data, - to_usize(self.huge_btree_address)?, + let hdr = BTreeV2Header::parse_in( + file, + self.huge_btree_address, self.offset_size, self.length_size, )?; @@ -513,7 +509,7 @@ impl FractalHeapHeader { // Records are ordered by ID (the last field): descend to the ones // equal to `key` instead of reading the whole index. let id_at = rec_len - ls; - let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| { + let records = find_btree_v2_records_in(file, &hdr, self.offset_size, &mut |r| { le_uint(&r[id_at..id_at + ls]).cmp(&key) })?; for rec in &records { @@ -1317,22 +1313,20 @@ mod tests { assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want); } - /// A huge object found through the huge-object B-tree needs the whole - /// file in memory until the B-tree reader is converted: a clean error - /// on other storage. + /// A huge object found through the huge-object B-tree reads the + /// B-tree through Storage: the same result (here an error, there is no + /// B-tree at that address) as from the slice. #[test] - fn huge_object_btree_needs_contiguous_storage() { + fn huge_object_btree_reads_through_storage() { use crate::storage::CountingStorage; let (file, _) = build_simple_heap(8, 8); let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap(); hdr.huge_btree_address = 700; + let id = [0x10, 1, 0, 0, 0, 0, 0]; + let want = hdr.read_managed_object(&file, &id, 8); + assert!(want.is_err()); let storage = CountingStorage::new(file); - assert_eq!( - hdr.read_managed_object_in(&storage, &[0x10, 1, 0, 0, 0, 0, 0], 8), - Err(FormatError::ContiguousStorageRequired( - "a huge fractal-heap object's B-tree" - )) - ); + assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want); } /// A header with an I/O filter pipeline (read in a second, longer diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index 3dff9cd..ebd4d32 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -3,12 +3,13 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; -use crate::addr::to_usize; -use crate::btree_v1::collect_symbol_table_nodes; +use crate::addr::checked_addr; +use crate::btree_v1::collect_symbol_table_nodes_in; use crate::error::FormatError; use crate::local_heap::LocalHeap; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; +use crate::storage::Storage; use crate::symbol_table::{SymbolTableMessage, SymbolTableNode}; /// A resolved group entry (child name + object header address). @@ -36,6 +37,16 @@ pub fn resolve_v1_group_entries( sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + resolve_v1_group_entries_in(file_data, sym_table_msg, offset_size, length_size) +} + +/// [`resolve_v1_group_entries`] over any [`Storage`]. +pub fn resolve_v1_group_entries_in( + file_data: &S, + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size)?; if entries.iter().any(|e| e.name.is_empty()) { @@ -46,22 +57,22 @@ pub fn resolve_v1_group_entries( /// Every entry of a v1 group, empty names included — for looking a name up, /// which never matches an empty name. -pub(crate) fn v1_group_entries( - file_data: &[u8], +pub(crate) fn v1_group_entries( + file_data: &S, sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, ) -> Result, FormatError> { // Parse local heap - let heap = LocalHeap::parse( + let heap = LocalHeap::parse_in( file_data, - to_usize(sym_table_msg.local_heap_address)?, + checked_addr(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; // Collect all SNOD addresses from B-tree - let snod_addrs = collect_symbol_table_nodes( + let snod_addrs = collect_symbol_table_nodes_in( file_data, sym_table_msg.btree_address, offset_size, @@ -71,15 +82,15 @@ pub(crate) fn v1_group_entries( let mut entries = Vec::new(); let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; + let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?; for entry in &snod.entries { // Like libhdf5, look at the heap's free list only once a name is // needed: an empty group with a damaged heap still lists. if !heap_checked { - heap.validate_free_list(file_data, length_size)?; + heap.validate_free_list_in(file_data, length_size)?; heap_checked = true; } - let name = heap.read_string(file_data, entry.link_name_offset)?; + let name = heap.read_string_in(file_data, entry.link_name_offset)?; entries.push(GroupEntry { name, object_header_address: entry.object_header_address, @@ -103,6 +114,17 @@ pub fn find_v1_soft_link( name: &str, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + find_v1_soft_link_in(file_data, sym_table_msg, name, offset_size, length_size) +} + +/// [`find_v1_soft_link`] over any [`Storage`]. +pub fn find_v1_soft_link_in( + file_data: &S, + sym_table_msg: &SymbolTableMessage, + name: &str, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let mut found = None; for_each_v1_soft_link( @@ -125,6 +147,16 @@ pub fn v1_soft_links( sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + v1_soft_links_in(file_data, sym_table_msg, offset_size, length_size) +} + +/// [`v1_soft_links`] over any [`Storage`]. +pub fn v1_soft_links_in( + file_data: &S, + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let mut links = Vec::new(); for_each_v1_soft_link( @@ -143,21 +175,21 @@ pub fn v1_soft_links( /// Visit the soft links of a v1 group whose name passes `wanted`, with their /// target paths, until `visit` returns false. -fn for_each_v1_soft_link( - file_data: &[u8], +fn for_each_v1_soft_link( + file_data: &S, sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, wanted: impl Fn(&str) -> bool, mut visit: impl FnMut(&str, String) -> bool, ) -> Result<(), FormatError> { - let heap = LocalHeap::parse( + let heap = LocalHeap::parse_in( file_data, - to_usize(sym_table_msg.local_heap_address)?, + checked_addr(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; - let snod_addrs = collect_symbol_table_nodes( + let snod_addrs = collect_symbol_table_nodes_in( file_data, sym_table_msg.btree_address, offset_size, @@ -165,16 +197,16 @@ fn for_each_v1_soft_link( )?; let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; + let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?; for entry in &snod.entries { if entry.cache_type != CACHE_TYPE_SOFT_LINK { continue; } if !heap_checked { - heap.validate_free_list(file_data, length_size)?; + heap.validate_free_list_in(file_data, length_size)?; heap_checked = true; } - let name = heap.read_string(file_data, entry.link_name_offset)?; + let name = heap.read_string_in(file_data, entry.link_name_offset)?; if !wanted(&name) { continue; } @@ -184,7 +216,7 @@ fn for_each_v1_soft_link( entry.scratch_pad[2], entry.scratch_pad[3], ]); - let target = heap.read_string(file_data, u64::from(value_offset))?; + let target = heap.read_string_in(file_data, u64::from(value_offset))?; if !visit(&name, target) { return Ok(()); } @@ -222,6 +254,17 @@ pub fn resolve_path( path: &str, offset_size: u8, length_size: u8, +) -> Result { + resolve_path_in(file_data, root_sym_table, path, offset_size, length_size) +} + +/// [`resolve_path`] over any [`Storage`]. +pub fn resolve_path_in( + file_data: &S, + root_sym_table: &SymbolTableMessage, + path: &str, + offset_size: u8, + length_size: u8, ) -> Result { let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); if components.is_empty() { @@ -241,9 +284,9 @@ pub fn resolve_path( return Ok(entry.object_header_address); } // Not last — must be a group, parse its object header to get symbol table - let obj_header = ObjectHeader::parse( + let obj_header = ObjectHeader::parse_in( file_data, - to_usize(entry.object_header_address)?, + checked_addr(entry.object_header_address)?, offset_size, length_size, )?; diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index 0edf972..9dda843 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -11,8 +11,8 @@ use alloc::collections::BTreeSet; #[cfg(feature = "std")] use std::collections::BTreeSet; -use crate::addr::to_usize; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::addr::checked_addr; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in}; use crate::checksum::jenkins_lookup3; use crate::error::FormatError; use crate::fractal_heap::FractalHeapHeader; @@ -21,6 +21,7 @@ use crate::link_info::LinkInfoMessage; use crate::link_message::{LinkMessage, LinkTarget}; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; +use crate::storage::Storage; use crate::superblock::Superblock; use crate::symbol_table::SymbolTableMessage; @@ -32,6 +33,16 @@ pub fn resolve_v2_group_entries( object_header: &ObjectHeader, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size) +} + +/// [`resolve_v2_group_entries`] over any [`Storage`]. +pub fn resolve_v2_group_entries_in( + file_data: &S, + object_header: &ObjectHeader, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { // Look for Link Info message to determine storage type let link_info = find_link_info(object_header, offset_size)?; @@ -91,8 +102,8 @@ fn resolve_compact_entries( } /// Visit every link in dense storage (fractal heap + B-tree v2 name index). -fn for_each_dense_link( - file_data: &[u8], +fn for_each_dense_link( + file_data: &S, link_info: &LinkInfoMessage, fh_addr: u64, offset_size: u8, @@ -100,15 +111,20 @@ fn for_each_dense_link( mut visit: impl FnMut(LinkMessage), ) -> Result<(), FormatError> { // Parse fractal heap - let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + let fh = + FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?; // Parse B-tree v2 for name index let btree_addr = link_info .btree_name_index_address .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; - let btree_hdr = - BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; - let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + checked_addr(btree_addr)?, + offset_size, + length_size, + )?; + let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { // For type 5 (name index): hash(4) + heap_id(heap_id_length) @@ -125,7 +141,7 @@ fn for_each_dense_link( let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize]; // Read managed object from fractal heap - let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; + let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?; if let Some(link) = parse_link(&link_data, offset_size)? { visit(link); } @@ -134,8 +150,8 @@ fn for_each_dense_link( } /// Resolve entries from dense storage (fractal heap + B-tree v2). -fn resolve_dense_entries( - file_data: &[u8], +fn resolve_dense_entries( + file_data: &S, link_info: &LinkInfoMessage, fh_addr: u64, offset_size: u8, @@ -167,8 +183,8 @@ fn resolve_dense_entries( /// The soft link called `name` in a v1 (symbol table) group, if there is /// one. Hard links are what `resolve_group_entries` returns; this is /// consulted only when a path component isn't among them. -fn find_v1_symbolic_link( - file_data: &[u8], +fn find_v1_symbolic_link( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -182,7 +198,7 @@ fn find_v1_symbolic_link( return Ok(None); }; let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::find_v1_soft_link(file_data, &stm, name, offset_size, length_size) + group_v1::find_v1_soft_link_in(file_data, &stm, name, offset_size, length_size) .map(|target| target.map(|target_path| LinkTarget::Soft { target_path })) } @@ -199,8 +215,8 @@ const LINK_NAME_INDEX: u8 = 5; /// link. libhdf5 orders records with equal hashes by name; all of them are /// read and compared here, so that order does not matter. An index of /// another type is scanned in full. -fn links_named( - file_data: &[u8], +fn links_named( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -220,12 +236,17 @@ fn links_named( return Ok(found); }; - let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + let fh = + FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?; let btree_addr = link_info .btree_name_index_address .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; - let btree_hdr = - BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + checked_addr(btree_addr)?, + offset_size, + length_size, + )?; if btree_hdr.tree_type != LINK_NAME_INDEX { for_each_dense_link( file_data, @@ -244,7 +265,7 @@ fn links_named( // Record: hash(4) + heap ID. let hash = jenkins_lookup3(name.as_bytes()); - let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| { + let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| { match r.get(..4) { Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), // Too short to hold a hash (a corrupt record size): never a match. @@ -256,7 +277,7 @@ fn links_named( let Some(id_bytes) = record.data.get(4..4 + id_len) else { continue; }; - let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; + let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?; if let Some(link) = parse_link(&link_data, offset_size)? && link.name == name { @@ -278,8 +299,8 @@ fn links_named( /// and may land on another of several exact duplicates. The listing /// ([`resolve_group_children`]), [`resolve_child`] and path resolution all /// apply this rule, so they agree. -fn first_link_named( - file_data: &[u8], +fn first_link_named( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -296,8 +317,8 @@ fn first_link_named( /// the group with header `object_header`: a hard link (as `Hard`), else a /// soft or external link of that name, else `None`. Fails with /// `PathNotFound` if the object is not a group. -fn lookup_link( - file_data: &[u8], +fn lookup_link( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -346,13 +367,23 @@ pub fn resolve_child( superblock: &Superblock, group_address: u64, name: &str, +) -> Result { + resolve_child_in(file_data, superblock, group_address, name) +} + +/// [`resolve_child`] over any [`Storage`]. +pub fn resolve_child_in( + file_data: &S, + superblock: &Superblock, + group_address: u64, + name: &str, ) -> Result { let os = superblock.offset_size; let ls = superblock.length_size; let not_found = || FormatError::PathNotFound(String::from(name)); - let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; + let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?; if !is_v2_group(&header) || is_v1_group(&header) { - return resolve_group_children(file_data, superblock, group_address)? + return resolve_group_children_in(file_data, superblock, group_address)? .into_iter() .find(|e| e.name == name) .map(|e| e.object_header_address) @@ -365,7 +396,7 @@ pub fn resolve_child( object_header_address, }) => Ok(object_header_address), Some(LinkTarget::Soft { target_path }) => { - match resolve_path_from(file_data, superblock, group_address, &target_path) { + match resolve_path_from_in(file_data, superblock, group_address, &target_path) { // Left out of the listing: dangling, cyclic, or in another file. Err( FormatError::PathNotFound(_) @@ -421,6 +452,15 @@ pub fn resolve_path_any( file_data: &[u8], superblock: &Superblock, path: &str, +) -> Result { + resolve_path_any_in(file_data, superblock, path) +} + +/// [`resolve_path_any`] over any [`Storage`]. +pub fn resolve_path_any_in( + file_data: &S, + superblock: &Superblock, + path: &str, ) -> Result { resolve_path_following_links( file_data, @@ -439,6 +479,16 @@ pub fn resolve_path_from( superblock: &Superblock, group_address: u64, path: &str, +) -> Result { + resolve_path_from_in(file_data, superblock, group_address, path) +} + +/// [`resolve_path_from`] over any [`Storage`]. +pub fn resolve_path_from_in( + file_data: &S, + superblock: &Superblock, + group_address: u64, + path: &str, ) -> Result { let start = if path.starts_with('/') { superblock.root_group_address @@ -462,10 +512,19 @@ pub fn resolve_group_children( file_data: &[u8], superblock: &Superblock, group_address: u64, +) -> Result, FormatError> { + resolve_group_children_in(file_data, superblock, group_address) +} + +/// [`resolve_group_children`] over any [`Storage`]. +pub fn resolve_group_children_in( + file_data: &S, + superblock: &Superblock, + group_address: u64, ) -> Result, FormatError> { let os = superblock.offset_size; let ls = superblock.length_size; - let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; + let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?; let mut entries = Vec::new(); let mut soft = Vec::new(); @@ -476,9 +535,9 @@ pub fn resolve_group_children( .find(|m| m.msg_type == MessageType::SymbolTable) .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?; let stm = SymbolTableMessage::parse(&sym_msg.data, os)?; - let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?; + let all = group_v1::resolve_v1_group_entries_in(file_data, &stm, os, ls)?; if all.iter().any(group_v1::is_v1_soft_link) { - soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?; + soft = group_v1::v1_soft_links_in(file_data, &stm, os, ls)?; } entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e))); } else if is_v2_group(&header) { @@ -515,7 +574,7 @@ pub fn resolve_group_children( } for (name, target) in soft { - match resolve_path_from(file_data, superblock, group_address, &target) { + match resolve_path_from_in(file_data, superblock, group_address, &target) { Ok(object_header_address) => entries.push(GroupEntry { name, object_header_address, @@ -538,8 +597,8 @@ pub fn resolve_group_children( const MAX_SOFT_LINK_DEPTH: u8 = 16; /// Walk `path` from the group at `start`, following soft links. -fn resolve_path_following_links( - file_data: &[u8], +fn resolve_path_following_links( + file_data: &S, superblock: &Superblock, start: u64, path: &str, @@ -557,7 +616,7 @@ fn resolve_path_following_links( let ls = superblock.length_size; let mut current_addr = start; - let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?; + let mut current_header = ObjectHeader::parse_in(file_data, checked_addr(start)?, os, ls)?; for (i, component) in components.iter().enumerate() { match lookup_link(file_data, ¤t_header, component, os, ls)? { @@ -568,7 +627,8 @@ fn resolve_path_following_links( return Ok(object_header_address); } current_addr = object_header_address; - current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?; + current_header = + ObjectHeader::parse_in(file_data, checked_addr(current_addr)?, os, ls)?; } found => { return match found { @@ -607,8 +667,8 @@ fn resolve_path_following_links( } /// Resolve group entries from an object header, auto-detecting v1 vs v2. -fn resolve_group_entries( - file_data: &[u8], +fn resolve_group_entries( + file_data: &S, object_header: &ObjectHeader, offset_size: u8, length_size: u8, @@ -625,7 +685,7 @@ fn resolve_group_entries( // skipped by the name comparison, as in libhdf5. group_v1::v1_group_entries(file_data, &stm, offset_size, length_size) } else if is_v2_group(object_header) { - resolve_v2_group_entries(file_data, object_header, offset_size, length_size) + resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size) } else { Err(FormatError::PathNotFound(String::from( "object header is not a group", diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 05ee649..5f0fc2e 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -23,12 +23,12 @@ use alloc::vec::Vec; #[cfg(feature = "std")] use std::borrow::Cow; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in}; use crate::error::FormatError; use crate::fractal_heap::FractalHeapHeader; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; -use crate::storage::{Storage, Window, read_exact_at, require_contiguous}; +use crate::storage::{Storage, Window, read_exact_at}; /// Fractal heap ID length for SOHM entries (fixed at 8 bytes). const FHEAP_ID_LEN: usize = 8; @@ -423,19 +423,15 @@ pub fn parse_sohm_btree_entries( parse_sohm_btree_entries_in(file_data, btree_addr as u64, offset_size, length_size) } -/// [`parse_sohm_btree_entries`] over any [`Storage`]. The v2 B-tree is not -/// read over [`Storage`] yet, so this needs the whole file in memory -/// ([`FormatError::ContiguousStorageRequired`] otherwise). +/// [`parse_sohm_btree_entries`] over any [`Storage`]. pub fn parse_sohm_btree_entries_in( file: &S, btree_addr: u64, offset_size: u8, length_size: u8, ) -> Result, FormatError> { - let file_data = require_contiguous(file, "a shared-message B-tree index")?; - let btree_addr = usize::try_from(btree_addr).unwrap_or(usize::MAX); - let header = BTreeV2Header::parse(file_data, btree_addr, offset_size, length_size)?; - let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?; + let header = BTreeV2Header::parse_in(file, btree_addr, offset_size, length_size)?; + let records = collect_btree_v2_records_in(file, &header, offset_size, length_size)?; let mut entries = Vec::with_capacity(records.len()); for rec in &records { let entry = parse_sohm_entry(&rec.data, offset_size)?; @@ -1243,11 +1239,12 @@ mod tests { } } assert!(compared > 200); - // The B-tree index is not read over Storage yet: a clean error. - let st = CountingStorage::new(vec![0u8; 64]); + // The B-tree index reads through Storage too, errors included. + let junk = vec![0u8; 64]; + let st = CountingStorage::new(junk.clone()); assert_eq!( - parse_sohm_btree_entries_in(&st, 0, 8, 8).unwrap_err(), - FormatError::ContiguousStorageRequired("a shared-message B-tree index") + format!("{:?}", parse_sohm_btree_entries_in(&st, 0, 8, 8)), + format!("{:?}", parse_sohm_btree_entries(&junk, 0, 8, 8)) ); } } diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index 5cc71b9..02a4f17 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -1,24 +1,19 @@ //! Equivalence harness for the range-read migration -//! (`docs/design/range-reads.md`, milestone M1). +//! (`docs/design/range-reads.md`, milestones M1 and M2). //! -//! Every metadata parser converted to [`Storage`] must give exactly what its -//! `&[u8]` form gives. This walks real files — the fixtures, files h5py -//! writes to exercise the less common structures, and optionally the -//! conformance corpus — and, for every object, runs each converted parser -//! twice: over the file as a slice, and over a [`CountingStorage`] that -//! serves the same bytes through `read_at` only (`as_contiguous()` is -//! `None`, so no parser can fall back to the whole slice). The results must -//! be identical, value for value and error for error. +//! Every parser converted to [`Storage`] must give exactly what its `&[u8]` +//! form gives. This walks real files — the fixtures, files h5py writes to +//! exercise the less common structures, and optionally the conformance +//! corpus — and, for every object, runs each converted parser twice: over +//! the file as a slice, and over a [`CountingStorage`] that serves the same +//! bytes through `read_at` only (`as_contiguous()` is `None`, so no parser +//! can fall back to the whole slice). The results must be identical, value +//! for value and error for error. //! -//! The one allowed difference is [`FormatError::ContiguousStorageRequired`] -//! from the storage path, and only from the structures still indexed by a v2 -//! B-tree (dense attributes, a SOHM B-tree index, huge fractal-heap objects; -//! see `CONTIGUOUS_REQUIRED`), which fail cleanly instead of reading the -//! whole file. Those are counted; the error from any other site or check -//! fails the harness. -//! -//! Milestones M2/M3 extend `check_object` with the raw-data and group -//! parsers as they are converted. +//! Nothing may answer [`FormatError::ContiguousStorageRequired`] any more: +//! since milestone M2 every read path works over `read_at` alone, the v2 +//! B-tree structures (dense groups and attributes, a SOHM B-tree index, +//! huge fractal-heap objects) included. //! //! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/ //! `.nc`/`.h5ad` file under those directories (the conformance corpus is @@ -38,7 +33,10 @@ use clawhdf5_format::attribute::{ }; use clawhdf5_format::attribute_info::AttributeInfoMessage; use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in}; -use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use clawhdf5_format::btree_v2::{ + BTreeV2Header, collect_btree_v2_records, collect_btree_v2_records_in, find_btree_v2_records, + find_btree_v2_records_in, +}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; @@ -51,6 +49,7 @@ use clawhdf5_format::fixed_array::{ FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in, }; use clawhdf5_format::fractal_heap::FractalHeapHeader; +use clawhdf5_format::group_v2; use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::local_heap::LocalHeap; use clawhdf5_format::message_type::MessageType; @@ -73,44 +72,11 @@ use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode}; const MAX_OBJECTS: usize = 1500; const MAX_HEAP_IDS: usize = 200; -/// The structures that still need the whole file in memory, because they -/// are found through a version-2 B-tree (not converted yet), and the checks -/// that can reach each of them. Anything else answering -/// [`FormatError::ContiguousStorageRequired`] is a converted parser falling -/// back to the whole file, and fails the harness. -const CONTIGUOUS_REQUIRED: &[(&str, &[&str])] = &[ - ( - "dense attribute storage (a v2 B-tree)", - &["attributes", "attributes (tolerant)"], - ), - ( - "a shared-message B-tree index", - &[ - "SOHM B-tree", - "shared message", - "fill value", - "attributes", - "attributes (tolerant)", - ], - ), - ( - "a huge fractal-heap object's B-tree", - &["heap object", "attributes", "attributes (tolerant)"], - ), -]; - -fn may_require_contiguous(check: &str, site: &str) -> bool { - CONTIGUOUS_REQUIRED - .iter() - .any(|(s, checks)| *s == site && checks.contains(&check)) -} - #[derive(Default, Debug)] struct Tally { files: usize, objects: usize, checks: usize, - contiguous_required: usize, reads: u64, bytes: u64, /// Chunk indexes (fixed and extensible arrays) read, and the most bytes @@ -127,8 +93,8 @@ struct Walk<'a> { } impl Walk<'_> { - /// The storage result must equal the slice result, or be the clean - /// "needs the whole file" error. + /// The storage result must equal the slice result; no parser may ask + /// for the whole file. fn same( &mut self, what: &str, @@ -137,14 +103,11 @@ impl Walk<'_> { ) { self.tally.checks += 1; if let Err(FormatError::ContiguousStorageRequired(site)) = got { - assert!( - may_require_contiguous(what, site), - "{}: {what} fell back to the whole file ({site}), which only the \ - v2-B-tree-indexed structures may do", + panic!( + "{}: {what} fell back to the whole file ({site}); every read path \ + must work through read_at", self.name ); - self.tally.contiguous_required += 1; - return; } let (w, g) = (format!("{want:?}"), format!("{got:?}")); assert!( @@ -206,13 +169,34 @@ impl Walk<'_> { } self.tally.objects += 1; self.check_object(&sb, addr); - // Traversal only (group lookups are milestone M0/M3 work). - if let Ok(children) = - clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr) - { + let children = group_v2::resolve_group_children(slice, &sb, addr); + let got = group_v2::resolve_group_children_in(self.st(), &sb, addr); + self.same("group listing", &children, &got); + if let Ok(children) = children { + for c in children.iter().take(MAX_HEAP_IDS) { + let want = group_v2::resolve_child(slice, &sb, addr, &c.name); + let got = group_v2::resolve_child_in(self.st(), &sb, addr, &c.name); + self.same("child lookup", &want, &got); + } + // A name no group has: the lookup's not-found path. + let want = group_v2::resolve_child(slice, &sb, addr, "no such child"); + let got = group_v2::resolve_child_in(self.st(), &sb, addr, "no such child"); + self.same("child lookup (missing)", &want, &got); queue.extend(children.iter().map(|c| c.object_header_address)); } } + // Paths: every listed name from the root, and one that is missing. + if let Ok(children) = group_v2::resolve_group_children(slice, &sb, sb.root_group_address) { + for c in children.iter().take(MAX_HEAP_IDS) { + let path = format!("/{}", c.name); + let want = group_v2::resolve_path_any(slice, &sb, &path); + let got = group_v2::resolve_path_any_in(self.st(), &sb, &path); + self.same("path", &want, &got); + } + } + let want = group_v2::resolve_path_any(slice, &sb, "/no/such/path"); + let got = group_v2::resolve_path_any_in(self.st(), &sb, "/no/such/path"); + self.same("path (missing)", &want, &got); } fn check_object(&mut self, sb: &Superblock, addr: u64) { @@ -336,12 +320,24 @@ impl Walk<'_> { let (Ok(fh), Some(index)) = (fh, index) else { return; }; - let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else { - return; - }; - let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else { - return; - }; + let bt = BTreeV2Header::parse(slice, index as usize, os, ls); + self.same( + "v2 B-tree header", + &bt, + &BTreeV2Header::parse_in(self.st(), index, os, ls), + ); + let Ok(bt) = bt else { return }; + let records = collect_btree_v2_records(slice, &bt, os, ls); + let got = collect_btree_v2_records_in(self.st(), &bt, os, ls); + self.same("v2 B-tree records", &records, &got); + let Ok(records) = records else { return }; + // Descents to single records (by their bytes), as name lookups do. + for rec in records.iter().take(8) { + let key = rec.data.clone(); + let want = find_btree_v2_records(slice, &bt, os, &mut |r| r.cmp(&key[..])); + let got = find_btree_v2_records_in(self.st(), &bt, os, &mut |r| r.cmp(&key[..])); + self.same("v2 B-tree descent", &want, &got); + } let id_len = fh.heap_id_length as usize; for rec in records.iter().take(MAX_HEAP_IDS) { let Some(id) = rec.data.get(id_at..id_at + id_len) else { @@ -696,6 +692,4 @@ fn h5py_files_parse_identically_through_storage() { } eprintln!("h5py files: {tally:?}"); assert!(tally.objects >= 700, "{tally:?}"); - // Dense attributes and the SOHM B-tree are the known clean errors. - assert!(tally.contiguous_required > 0, "{tally:?}"); } From 3fa5ed1dda7088c6a90757e399142bc789859545 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:28:01 -0500 Subject: [PATCH 02/10] format: raw data, VDS and VL data over Storage Every raw-data path has a generic *_in core, with the &[u8] functions as thin wrappers: data_read (read_raw_data*, read_raw_data_selection, read_chunked_native), chunked_read (the v1 B-tree chunk index, list_chunks, the full, cached, sweep and indexed reads), parallel_read, partial_read, fill_value (read_full_with_fill, apply_to_unallocated_chunks; and dataset_fill_value_from_storage is now generic), vds (the virtual file through Storage, external sources still through the resolver), vl_data (VlResolver<'a, S = [u8]>, read_vl_strings_in, read_vl_bytes_in), AttributeMessage::read_vl_strings_in and provenance::verify_dataset_in. With the whole file in memory nothing changes: chunks and contiguous data are sliced from it as before. Otherwise a chunked read lists its chunks, fetches their stored bytes with one Storage::read_ranges call per 64 MiB batch (chunks the cache already holds are not fetched), then decodes as today; a selection fetches only the chunks it overlaps, and a contiguous selection only its runs. Each extent's bounds error is the one the slice code gave, reported when that extent is reached, so errors keep their order. Tests: the equivalence harness now reads every dataset's values (whole, fill-aware, cached, indexed, three selections, VDS, VL strings and sequences) through the read_at-only storage and requires the slice results (all 653 corpus files agree); a misbehaving storage (a failing Nth read, short reads) only ever yields errors or the right values; and chunked reads are checked to use one read_ranges call. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/attribute.rs | 12 +- crates/clawhdf5-format/src/chunked_read.rs | 437 ++++++++++++--- crates/clawhdf5-format/src/data_read.rs | 215 +++++++- crates/clawhdf5-format/src/fill_value.rs | 65 ++- crates/clawhdf5-format/src/gather.rs | 129 +++++ crates/clawhdf5-format/src/global_heap.rs | 2 +- crates/clawhdf5-format/src/parallel_read.rs | 105 ++-- crates/clawhdf5-format/src/partial_read.rs | 106 +++- crates/clawhdf5-format/src/provenance.rs | 17 +- crates/clawhdf5-format/src/storage.rs | 153 ++++++ crates/clawhdf5-format/src/vds.rs | 123 ++++- crates/clawhdf5-format/src/vl_data.rs | 204 ++++--- .../tests/storage_equivalence.rs | 514 +++++++++++++++++- 13 files changed, 1803 insertions(+), 279 deletions(-) diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 4b30fac..c8f69b5 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -336,9 +336,19 @@ impl AttributeMessage { file_data: &[u8], offset_size: u8, length_size: u8, + ) -> Result, FormatError> { + self.read_vl_strings_in(file_data, offset_size, length_size) + } + + /// [`Self::read_vl_strings`] over any [`Storage`]. + pub fn read_vl_strings_in( + &self, + file_data: &S, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let num_elements = self.dataspace.num_elements(); - vl_data::read_vl_strings( + vl_data::read_vl_strings_in( file_data, &self.raw_data, num_elements, diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index e249168..140cf37 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -6,19 +6,20 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; -use crate::addr::to_usize; +use crate::addr::{checked_addr, to_usize}; #[cfg(feature = "std")] use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::datatype::Datatype; use crate::error::FormatError; -use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks}; +use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks_in}; use crate::filter_pipeline::FilterPipeline; use crate::filters::{DecodeScratch, decompress_chunk_exact_with}; #[cfg(feature = "std")] use crate::filters::{all_filters_skipped, decompress_chunk_exact}; -use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks}; +use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks_in}; +use crate::storage::{ExtentBytes, Storage, Window, raw_batches, read_exact_at}; #[cfg(feature = "std")] use std::sync::Arc; @@ -238,22 +239,101 @@ type CacheUse<'a> = Option<&'a core::convert::Infallible>; /// ([`parallel_read::run_with_helpers`]), so the caller never waits on a /// busy pool. The error returned is the first failing chunk's, in `chunks` /// order. +/// +/// With the whole file in memory the chunks are sliced from it. Otherwise +/// their stored bytes are fetched first, with one +/// [`Storage::read_ranges`] call per batch of up to +/// [`crate::storage::RAW_BATCH_BYTES`] (all of them, for most datasets), +/// and then decoded as above; chunks the cache already holds are not +/// fetched. #[allow(clippy::too_many_arguments)] -fn fill_from_chunks( - file_data: &[u8], +fn fill_from_chunks( + file_data: &S, chunks: &[ChunkInfo], pipeline: Option<&FilterPipeline>, placer: &ChunkPlacer, chunk_total_bytes: usize, cache: CacheUse<'_>, output: &mut [u8], +) -> Result<(), FormatError> { + let contiguous = file_data.as_contiguous().is_some(); + let out = OutBuf::new(output); + let batches = raw_batches(chunks.len(), contiguous, |i| chunks[i].chunk_size as usize); + for batch in batches { + fill_batch( + file_data, + &chunks[batch], + pipeline, + placer, + chunk_total_bytes, + cache, + &out, + )?; + } + Ok(()) +} + +/// Whether a full read looks chunk `c` up in the cache (see +/// [`fill_from_chunks`]): a filtered chunk of a dataset the cache keeps. +#[cfg(feature = "std")] +fn uses_cache(cache: CacheUse<'_>, pipeline: Option<&FilterPipeline>, c: &ChunkInfo) -> bool { + matches!(cache, Some((_, _, true))) + && pipeline.is_some_and(|pl| !all_filters_skipped(pl, c.filter_mask)) +} + +/// [`fill_from_chunks`] for one batch of chunks. +#[allow(clippy::too_many_arguments)] +fn fill_batch( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: Option<&FilterPipeline>, + placer: &ChunkPlacer, + chunk_total_bytes: usize, + cache: CacheUse<'_>, + out: &OutBuf<'_>, ) -> Result<(), FormatError> { let rank = placer.rank; let elem_size = placer.elem_size as u32; - let out = OutBuf::new(output); #[cfg(not(feature = "std"))] let _ = cache; + // Over a backend without the whole file in memory, the decoded chunks + // the cache holds are taken now (so a chunk evicted before it is placed + // is not left without bytes), and only the others are fetched. + #[cfg(feature = "std")] + let hits: Vec>> = match cache { + Some((cache, key, true)) if file_data.as_contiguous().is_none() => chunks + .iter() + .map(|c| { + if c.offsets.len() >= rank && uses_cache(Some((cache, key, true)), pipeline, c) { + cache.get_decompressed_in(key, &c.offsets[..rank]) + } else { + None + } + }) + .collect(), + _ => Vec::new(), + }; + let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { + Vec::new() + } else { + chunks + .iter() + .enumerate() + .map(|(i, c)| { + #[cfg(feature = "std")] + let wanted = hits.get(i).is_none_or(Option::is_none); + #[cfg(not(feature = "std"))] + let wanted = { + let _ = i; + true + }; + (c.address, c.chunk_size as usize, wanted) + }) + .collect() + }; + let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + // Decode chunk `i` and place it. Its callers below run it either on one // thread, or on several for chunks whose regions are pairwise disjoint, // each chunk once: no two threads ever write the same bytes. @@ -266,13 +346,18 @@ fn fill_from_chunks( ))); } let offsets = &c.offsets[..rank]; - let c_addr = to_usize(c.address)?; let size = c.chunk_size as usize; - ensure_len(file_data, c_addr, size)?; - let raw = &file_data[c_addr..c_addr + size]; + #[cfg(feature = "std")] + if let Some(Some(hit)) = hits.get(i) { + raw_bytes.check(i, c.address, size)?; + // SAFETY: see above. + unsafe { placer.place(hit, offsets, out) }; + return Ok(()); + } + let raw = raw_bytes.get(i, c.address, size)?; let Some(pl) = pipeline else { // SAFETY: see above. - unsafe { placer.place(raw, offsets, &out) }; + unsafe { placer.place(raw, offsets, out) }; return Ok(()); }; // A chunk stored as-is (every filter skipped) is checked and placed @@ -296,7 +381,7 @@ fn fill_from_chunks( } }; // SAFETY: see above. - unsafe { placer.place(&cached, offsets, &out) }; + unsafe { placer.place(&cached, offsets, out) }; return Ok(()); } let data = decompress_chunk_exact_with( @@ -309,7 +394,7 @@ fn fill_from_chunks( scratch, )?; // SAFETY: see above. - unsafe { placer.place(data, offsets, &out) }; + unsafe { placer.place(data, offsets, out) }; Ok(()) }; @@ -370,7 +455,29 @@ pub fn decompress_all_chunks_with_stats( seed: u64, num_lanes: Option, ) -> Result<(Vec>, PartitionStats), FormatError> { - parallel_read::decompress_chunks_lane_partitioned( + decompress_all_chunks_with_stats_in( + file_data, + chunks, + pipeline, + chunk_total_bytes, + element_size, + seed, + num_lanes, + ) +} + +/// [`decompress_all_chunks_with_stats`] over any [`Storage`]. +#[cfg(feature = "parallel")] +pub fn decompress_all_chunks_with_stats_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: &FilterPipeline, + chunk_total_bytes: usize, + element_size: u32, + seed: u64, + num_lanes: Option, +) -> Result<(Vec>, PartitionStats), FormatError> { + parallel_read::decompress_chunks_lane_partitioned_in( file_data, chunks, pipeline, @@ -394,21 +501,6 @@ pub struct ChunkInfo { pub address: u64, } -/// Checks that `[offset, offset + needed)` fits within `data`, guarding the -/// addition against `usize` overflow from a crafted near-`usize::MAX` offset. -fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { - if offset - .checked_add(needed) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: offset.saturating_add(needed), - available: data.len(), - }); - } - Ok(()) -} - /// `elements * elem_size` for sizes that come from the file. Dataspace and /// chunk dimensions are untrusted 64-bit fields, so a crafted file can make /// the plain product wrap to a small number (or to something enormous). @@ -588,6 +680,17 @@ pub fn collect_chunk_info( ndims: usize, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + collect_chunk_info_in(file_data, btree_address, ndims, offset_size, length_size) +} + +/// [`collect_chunk_info`] over any [`Storage`]. +pub fn collect_chunk_info_in( + file_data: &S, + btree_address: u64, + ndims: usize, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let _ = length_size; let mut chunks = Vec::new(); @@ -630,6 +733,23 @@ pub fn collect_chunk_info_checked( chunk_dimensions: &[u32], offset_size: u8, length_size: u8, +) -> Result, FormatError> { + collect_chunk_info_checked_in( + file_data, + btree_address, + chunk_dimensions, + offset_size, + length_size, + ) +} + +/// [`collect_chunk_info_checked`] over any [`Storage`]. +pub fn collect_chunk_info_checked_in( + file_data: &S, + btree_address: u64, + chunk_dimensions: &[u32], + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let _ = length_size; let ndims = chunk_dimensions.len(); @@ -813,8 +933,8 @@ const MAX_CHUNK_BTREE_DEPTH: usize = 64; /// Parse the v1 B-tree chunk index node at `btree_address` and its /// subtree, appending its chunks to `stored` in tree order. -fn parse_chunk_node( - file_data: &[u8], +fn parse_chunk_node( + file_data: &S, btree_address: u64, ndims: usize, chunk_dimensions: Option<&[u32]>, @@ -831,21 +951,24 @@ fn parse_chunk_node( // Parse B-tree v1 header let header_size = 8 + os * 2; - ensure_len(file_data, offset, header_size)?; + let head = Window::read(file_data, offset as u64, header_size)?; + head.ensure(0, header_size)?; + let h = &head.bytes; - if &file_data[offset..offset + 4] != b"TREE" { + if &h[..4] != b"TREE" { return Err(FormatError::InvalidBTreeSignature); } - let node_type = file_data[offset + 4]; + let node_type = h[4]; if node_type != 1 { return Err(FormatError::InvalidBTreeNodeType(node_type)); } - let node_level = file_data[offset + 5]; - let entries_used = u16::from_le_bytes([file_data[offset + 6], file_data[offset + 7]]) as usize; + let node_level = h[5]; + let entries_used = u16::from_le_bytes([h[6], h[7]]) as usize; - let mut pos = offset + 8 + os * 2; // skip left/right sibling + // Positions below are relative to the node's start. + let mut pos = header_size; // skip left/right sibling // Key: chunk_size(4) + filter_mask(4) + one offset per dimension. The // offsets are always 8 bytes each — they are dataset coordinates, not file @@ -858,28 +981,20 @@ fn parse_chunk_node( // key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N] let needed = entries_used * (key_size + os) + key_size; - ensure_len(file_data, pos, needed)?; + let node = Window::read(file_data, offset as u64, header_size + needed)?; + node.ensure(pos, needed)?; + let d = &node.bytes; let mut keys = Vec::with_capacity((entries_used + 1) * ndims); let mut chunks = Vec::new(); let mut child_addrs = Vec::new(); for _ in 0..entries_used { - let chunk_size = u32::from_le_bytes([ - file_data[pos], - file_data[pos + 1], - file_data[pos + 2], - file_data[pos + 3], - ]); - let filter_mask = u32::from_le_bytes([ - file_data[pos + 4], - file_data[pos + 5], - file_data[pos + 6], - file_data[pos + 7], - ]); + let chunk_size = u32::from_le_bytes([d[pos], d[pos + 1], d[pos + 2], d[pos + 3]]); + let filter_mask = u32::from_le_bytes([d[pos + 4], d[pos + 5], d[pos + 6], d[pos + 7]]); let k = keys.len(); - read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?; + read_key_offsets(d, pos, ndims, chunk_dimensions, &mut keys)?; pos += key_size; - let address = read_offset(file_data, pos, offset_size)?; + let address = read_offset(d, pos, offset_size)?; pos += os; if node_level == 0 { chunks.push(stored.len()); @@ -894,7 +1009,7 @@ fn parse_chunk_node( } } // The final key only bounds the node; libhdf5 still checks it. - read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?; + read_key_offsets(d, pos, ndims, chunk_dimensions, &mut keys)?; let children = if node_level == 0 { ChunkChildren::Chunks(chunks) @@ -984,18 +1099,18 @@ const BT2_CHUNK_FILTERED: u8 = 11; /// The width of the stored-size field depends on the largest possible chunk; /// rather than re-derive the library's formula it is taken from the record /// size the tree header declares, which is what actually governs the bytes. -fn read_btree_v2_chunks( - file_data: &[u8], +fn read_btree_v2_chunks( + file_data: &S, addr: u64, chunk_dims: &[usize], elem_size: usize, offset_size: u8, length_size: u8, ) -> Result, FormatError> { - use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; + use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in}; let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}")); - let header = BTreeV2Header::parse(file_data, to_usize(addr)?, offset_size, length_size)?; + let header = BTreeV2Header::parse_in(file_data, checked_addr(addr)?, offset_size, length_size)?; let rank = chunk_dims.len(); let os = offset_size as usize; let record_size = header.record_size as usize; @@ -1022,7 +1137,7 @@ fn read_btree_v2_chunks( let unfiltered_bytes = u32::try_from(unfiltered_bytes).map_err(|_| bad("chunk larger than 4 GiB"))?; - let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?; + let records = collect_btree_v2_records_in(file_data, &header, offset_size, length_size)?; let mut chunks = Vec::with_capacity(records.len()); for record in &records { let data = record.data.as_slice(); @@ -1085,6 +1200,25 @@ pub fn list_chunks( elem_size: usize, offset_size: u8, length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + list_chunks_in( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + ) +} + +/// [`list_chunks`] over any [`Storage`]. +pub fn list_chunks_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + offset_size: u8, + length_size: u8, ) -> Result<(Vec, Vec), FormatError> { let ( chunk_dimensions, @@ -1132,9 +1266,13 @@ pub fn list_chunks( // Collect chunks based on version and index type let mut chunks = match (version, chunk_index_type) { - (3, _) => { - collect_chunk_info_checked(file_data, addr, chunk_dimensions, offset_size, length_size)? - } + (3, _) => collect_chunk_info_checked_in( + file_data, + addr, + chunk_dimensions, + offset_size, + length_size, + )?, (4, Some(1)) => { // Single chunk — one chunk covering the entire dataset let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?; @@ -1163,9 +1301,13 @@ pub fn list_chunks( (4, Some(3)) => { // Fixed Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = - FixedArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; - read_fixed_array_chunks( + let header = FixedArrayHeader::parse_in( + file_data, + checked_addr(addr)?, + offset_size, + length_size, + )?; + read_fixed_array_chunks_in( file_data, &header, &dataspace.dimensions, @@ -1179,9 +1321,13 @@ pub fn list_chunks( (4, Some(4)) => { // Extensible Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = - ExtensibleArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; - read_extensible_array_chunks( + let header = ExtensibleArrayHeader::parse_in( + file_data, + checked_addr(addr)?, + offset_size, + length_size, + )?; + read_extensible_array_chunks_in( file_data, &header, &dataspace.dimensions, @@ -1248,7 +1394,28 @@ pub fn list_chunks_for_read( offset_size: u8, length_size: u8, ) -> Result<(Vec, Vec), FormatError> { - let (chunks, chunk_dims) = list_chunks( + list_chunks_for_read_in( + file_data, + layout, + dataspace, + elem_size, + pipeline, + offset_size, + length_size, + ) +} + +/// [`list_chunks_for_read`] over any [`Storage`]. +pub fn list_chunks_for_read_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + let (chunks, chunk_dims) = list_chunks_in( file_data, layout, dataspace, @@ -1284,8 +1451,8 @@ pub(crate) type CacheRef<'a> = Option<&'a core::convert::Infallible>; /// output with `alloc` (zeroed, `total_bytes` long, as bytes through /// `bytes`), and decode every chunk straight into it. #[allow(clippy::too_many_arguments)] -pub(crate) fn read_chunked_full( - file_data: &[u8], +pub(crate) fn read_chunked_full( + file_data: &S, layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, @@ -1299,7 +1466,7 @@ pub(crate) fn read_chunked_full( check_chunk_element_size(layout, datatype, offset_size)?; let elem_size = datatype.type_size() as usize; let list = || { - list_chunks_for_read( + list_chunks_for_read_in( file_data, layout, dataspace, @@ -1390,6 +1557,27 @@ pub fn read_chunked_data( pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + read_chunked_data_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + ) +} + +/// [`read_chunked_data`] over any [`Storage`]. +pub fn read_chunked_data_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { read_chunked_full( file_data, @@ -1422,6 +1610,31 @@ pub fn read_chunked_data_cached( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_chunked_data_cached_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_chunked_data_cached`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_data_cached_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { read_chunked_full( file_data, @@ -1592,6 +1805,33 @@ pub fn read_chunked_data_sweep( length_size: u8, cache: &ChunkCache, sweep: &mut SweepContext, +) -> Result, FormatError> { + read_chunked_data_sweep_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + sweep, + ) +} + +/// [`read_chunked_data_sweep`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_data_sweep_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, + sweep: &mut SweepContext, ) -> Result, FormatError> { let (chunk_dimensions, version, addr_opt) = match layout { DataLayout::Chunked { @@ -1623,7 +1863,7 @@ pub fn read_chunked_data_sweep( // lookup is keyed by this dataset's chunk-index address, so another // dataset's index or chunks are never used for this read. let chunks = cache.chunks_for(addr, rank, || { - list_chunks_for_read( + list_chunks_for_read_in( file_data, layout, dataspace, @@ -1675,8 +1915,8 @@ pub fn read_chunked_data_sweep( // Decompress from file let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; - ensure_len(file_data, c_addr, size)?; - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = read_exact_at(file_data, c_addr as u64, size)?; + let raw_chunk = &*raw_chunk; let dec = if let Some(pl) = pipeline { decompress_chunk_exact( raw_chunk, @@ -1736,6 +1976,31 @@ pub fn read_chunked_data_indexed( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_chunked_data_indexed_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_chunked_data_indexed`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_data_indexed_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { let (chunk_dimensions, version, addr_opt) = match layout { DataLayout::Chunked { @@ -1769,7 +2034,7 @@ pub fn read_chunked_data_indexed( addr, rank, || { - list_chunks_for_read( + list_chunks_for_read_in( file_data, layout, dataspace, @@ -1786,18 +2051,30 @@ pub fn read_chunked_data_indexed( )?; let chunk_total_bytes = plan.chunk_total_bytes; + // The decoded chunks the cache holds, and the stored bytes of the + // others: fetched in one batch when the file is not in memory. + let hits: Vec>> = plan + .mappings + .iter() + .map(|m| cache.get_decompressed_in(addr, &m.coord)) + .collect(); + let extents: Vec<(u64, usize, bool)> = plan + .mappings + .iter() + .zip(&hits) + .map(|(m, hit)| (m.file_offset, m.file_size as usize, hit.is_none())) + .collect(); + let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + // Decompress chunks (using LRU cache where possible) let mut chunk_buffers: Vec> = Vec::with_capacity(plan.mappings.len()); - for m in &plan.mappings { + for (i, (m, hit)) in plan.mappings.iter().zip(hits).enumerate() { let (coord, file_offset, file_size, filter_mask) = (&m.coord, &m.file_offset, &m.file_size, &m.filter_mask); - if let Some(cached) = cache.get_decompressed_in(addr, coord) { + if let Some(cached) = hit { chunk_buffers.push(cached); } else { - let c_addr = to_usize(*file_offset)?; - let size = *file_size as usize; - ensure_len(file_data, c_addr, size)?; - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(i, *file_offset, *file_size as usize)?; let decompressed = if let Some(pl) = pipeline { decompress_chunk_exact( raw_chunk, diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index bcf1a18..28bc4f2 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -1,7 +1,9 @@ //! Raw data reading and typed conversion for HDF5 datasets. #[cfg(not(feature = "std"))] -use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; +use alloc::{borrow::Cow, collections::BTreeMap, format, string::String, vec, vec::Vec}; +#[cfg(feature = "std")] +use std::borrow::Cow; #[cfg(feature = "std")] use std::collections::BTreeMap; @@ -9,14 +11,15 @@ use std::collections::BTreeMap; use crate::addr::to_usize; #[cfg(feature = "std")] use crate::chunk_cache::ChunkCache; -use crate::chunked_read::read_chunked_data; +use crate::chunked_read::read_chunked_data_in; #[cfg(feature = "std")] -use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed}; +use crate::chunked_read::{read_chunked_data_cached_in, read_chunked_data_indexed_in}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::datatype::{Datatype, DatatypeByteOrder}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; +use crate::storage::{Storage, read_exact_at}; /// Checks that `[offset, offset + needed)` fits within `data`, guarding the /// addition against `usize` overflow from a crafted near-`usize::MAX` offset. @@ -149,7 +152,17 @@ pub fn read_raw_data( dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { - read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8) + read_raw_data_in(file_data, layout, dataspace, datatype) +} + +/// [`read_raw_data`] over any [`Storage`]. +pub fn read_raw_data_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, +) -> Result, FormatError> { + read_raw_data_full_in(file_data, layout, dataspace, datatype, None, 8, 8) } /// Resolves a Virtual Dataset source **file name** (as stored in the mapping, @@ -171,6 +184,27 @@ pub fn read_raw_data_full( pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + read_raw_data_full_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + ) +} + +/// [`read_raw_data_full`] over any [`Storage`]. +pub fn read_raw_data_full_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, @@ -196,6 +230,30 @@ pub fn read_raw_data_full_with_resolver( offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, +) -> Result, FormatError> { + read_raw_data_full_with_resolver_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + resolver, + ) +} + +/// [`read_raw_data_full_with_resolver`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_full_with_resolver_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, @@ -210,8 +268,8 @@ pub fn read_raw_data_full_with_resolver( } #[allow(clippy::too_many_arguments)] -fn read_raw_data_full_impl( - file_data: &[u8], +fn read_raw_data_full_impl( + file_data: &S, layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, @@ -242,12 +300,17 @@ fn read_raw_data_full_impl( let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = to_usize(addr)?; let sz = contiguous_read_len(*size, expected_size)?; - ensure_len(file_data, addr, sz)?; - let mut out = crate::bulk_alloc::vec_for_bulk(sz); - out.extend_from_slice(&file_data[addr..addr + sz]); - Ok(out) + match read_exact_at(file_data, addr as u64, sz)? { + Cow::Borrowed(bytes) => { + let mut out = crate::bulk_alloc::vec_for_bulk(sz); + out.extend_from_slice(bytes); + Ok(out) + } + // Fetched for this read: already the caller's copy. + Cow::Owned(out) => Ok(out), + } } - DataLayout::Chunked { .. } => read_chunked_data( + DataLayout::Chunked { .. } => read_chunked_data_in( file_data, layout, dataspace, @@ -284,9 +347,34 @@ pub fn read_raw_data_cached( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_raw_data_cached_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_raw_data_cached`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_cached_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { match layout { - DataLayout::Chunked { .. } => read_chunked_data_cached( + DataLayout::Chunked { .. } => read_chunked_data_cached_in( file_data, layout, dataspace, @@ -296,7 +384,7 @@ pub fn read_raw_data_cached( length_size, cache, ), - _ => read_raw_data_full( + _ => read_raw_data_full_in( file_data, layout, dataspace, @@ -325,9 +413,34 @@ pub fn read_raw_data_indexed( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_raw_data_indexed_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_raw_data_indexed`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_indexed_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { match layout { - DataLayout::Chunked { .. } => read_chunked_data_indexed( + DataLayout::Chunked { .. } => read_chunked_data_indexed_in( file_data, layout, dataspace, @@ -337,7 +450,7 @@ pub fn read_raw_data_indexed( length_size, cache, ), - _ => read_raw_data_full( + _ => read_raw_data_full_in( file_data, layout, dataspace, @@ -366,6 +479,30 @@ pub fn read_raw_data_selection( offset_size: u8, length_size: u8, selection: &crate::selection::Selection, +) -> Result, FormatError> { + read_raw_data_selection_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + selection, + ) +} + +/// [`read_raw_data_selection`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_selection_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + selection: &crate::selection::Selection, ) -> Result, FormatError> { use crate::selection::Selection; @@ -375,7 +512,7 @@ pub fn read_raw_data_selection( // Read only what the selection's bounding box touches when that is // possible; everything below is the decode-everything-then-pick path, // kept for the cases `partial_read` declines. - if let Some(selected) = crate::partial_read::read_selection( + if let Some(selected) = crate::partial_read::read_selection_in( file_data, layout, dataspace, @@ -390,7 +527,7 @@ pub fn read_raw_data_selection( match selection { Selection::All => { - return read_raw_data_full( + return read_raw_data_full_in( file_data, layout, dataspace, @@ -410,7 +547,7 @@ pub fn read_raw_data_selection( match layout { DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => { // Read all data, then extract the selection - let full_data = read_raw_data_full( + let full_data = read_raw_data_full_in( file_data, layout, dataspace, @@ -434,7 +571,7 @@ pub fn read_raw_data_selection( // implicit-index generator, which then indexed past the rank and // panicked — only to decode the full dataset anyway. crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?; - let full_data = read_raw_data_full( + let full_data = read_raw_data_full_in( file_data, layout, dataspace, @@ -447,7 +584,7 @@ pub fn read_raw_data_selection( } DataLayout::Virtual { .. } => { // Assemble the full virtual dataset, then apply the read selection. - let full_data = read_raw_data_full( + let full_data = read_raw_data_full_in( file_data, layout, dataspace, @@ -471,8 +608,8 @@ pub fn read_raw_data_selection( /// would report differently from the stored dataspace (unlimited mappings). /// Use [`crate::vds::read_virtual_dataset`] to read those. #[allow(clippy::too_many_arguments)] -fn read_virtual_data( - file_data: &[u8], +fn read_virtual_data( + file_data: &S, layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, @@ -483,7 +620,7 @@ fn read_virtual_data( let wrapped = resolver.map(|r| move |name: &str| -> Result>, FormatError> { Ok(r(name)) }); let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver); - let v = crate::vds::read_virtual_dataset( + let v = crate::vds::read_virtual_dataset_in( file_data, layout, dataspace, @@ -885,6 +1022,33 @@ pub fn read_chunked_native( offset_size: u8, length_size: u8, cache: Option<&ChunkCache>, +) -> Result>, FormatError> { + read_chunked_native_in( + messages, + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_chunked_native`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_native_in( + messages: &[crate::object_header::HeaderMessage], + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: Option<&ChunkCache>, ) -> Result>, FormatError> { use crate::fill_value; use crate::message_type::MessageType; @@ -919,8 +1083,9 @@ pub fn read_chunked_native( }, |values| bytes_of_mut(values), )?; - let fill = fill_value::dataset_fill_value_in(file_data, messages, offset_size, length_size)?; - fill_value::apply_to_unallocated_chunks( + let fill = + fill_value::dataset_fill_value_from_storage(file_data, messages, offset_size, length_size)?; + fill_value::apply_to_unallocated_chunks_in( bytes_of_mut(&mut values), file_data, layout, diff --git a/crates/clawhdf5-format/src/fill_value.rs b/crates/clawhdf5-format/src/fill_value.rs index f0f442e..ac23044 100644 --- a/crates/clawhdf5-format/src/fill_value.rs +++ b/crates/clawhdf5-format/src/fill_value.rs @@ -13,7 +13,7 @@ use alloc::{format, vec, vec::Vec}; use crate::addr::to_usize; -use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks}; +use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_in}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::error::FormatError; @@ -122,10 +122,11 @@ pub fn dataset_fill_value_in( } /// [`dataset_fill_value_in`] with the file behind any -/// [`Storage`](crate::storage::Storage). (The trait is not imported here: -/// its `len` would shadow the slice method in this module.) -pub fn dataset_fill_value_from_storage( - file: &dyn crate::storage::Storage, +/// [`Storage`](crate::storage::Storage) (a `&dyn Storage` too). (The trait +/// is not imported here: its `len` would shadow the slice method in this +/// module.) +pub fn dataset_fill_value_from_storage( + file: &S, messages: &[HeaderMessage], offset_size: u8, length_size: u8, @@ -212,6 +213,30 @@ pub fn read_full_with_fill>( offset_size: u8, length_size: u8, read: impl FnOnce() -> Result, E>, +) -> Result, E> { + read_full_with_fill_in( + messages, + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + read, + ) +} + +/// [`read_full_with_fill`] over any [`Storage`](crate::storage::Storage). +#[allow(clippy::too_many_arguments)] +pub fn read_full_with_fill_in, S: crate::storage::Storage + ?Sized>( + messages: &[HeaderMessage], + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + offset_size: u8, + length_size: u8, + read: impl FnOnce() -> Result, E>, ) -> Result, E> { // A dataset with external raw data also has no data address in this // file. It is NOT unallocated — its values live elsewhere — so it must @@ -222,12 +247,12 @@ pub fn read_full_with_fill>( { return Err(FormatError::ExternalDataFilesUnsupported.into()); } - let fill = dataset_fill_value_in(file_data, messages, offset_size, length_size)?; + let fill = dataset_fill_value_from_storage(file_data, messages, offset_size, length_size)?; if !has_storage(layout) { return Ok(filled_dataset(dataspace, elem_size, fill.as_deref())?); } let mut output = read()?; - apply_to_unallocated_chunks( + apply_to_unallocated_chunks_in( &mut output, file_data, layout, @@ -253,6 +278,30 @@ pub fn apply_to_unallocated_chunks( fill: Option<&[u8]>, offset_size: u8, length_size: u8, +) -> Result<(), FormatError> { + apply_to_unallocated_chunks_in( + output, + file_data, + layout, + dataspace, + elem_size, + fill, + offset_size, + length_size, + ) +} + +/// [`apply_to_unallocated_chunks`] over any [`Storage`](crate::storage::Storage). +#[allow(clippy::too_many_arguments)] +pub fn apply_to_unallocated_chunks_in( + output: &mut [u8], + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + fill: Option<&[u8]>, + offset_size: u8, + length_size: u8, ) -> Result<(), FormatError> { let Some(fill) = fill.filter(|f| f.len() == elem_size && !is_default(Some(f))) else { return Ok(()); @@ -260,7 +309,7 @@ pub fn apply_to_unallocated_chunks( if !matches!(layout, DataLayout::Chunked { .. }) || elem_size == 0 { return Ok(()); } - let (chunks, chunk_dims) = list_chunks( + let (chunks, chunk_dims) = list_chunks_in( file_data, layout, dataspace, diff --git a/crates/clawhdf5-format/src/gather.rs b/crates/clawhdf5-format/src/gather.rs index 22d2ae7..2b8a4ab 100644 --- a/crates/clawhdf5-format/src/gather.rs +++ b/crates/clawhdf5-format/src/gather.rs @@ -13,6 +13,7 @@ use alloc::{vec, vec::Vec}; use crate::data_read::NativeElement; use crate::error::FormatError; use crate::selection::Selection; +use crate::storage::Storage; /// Row-major element strides of `dims` (the last dimension has stride 1). fn strides(dims: &[u64]) -> Vec { @@ -261,6 +262,134 @@ pub(crate) fn gather( Ok(out) } +/// [`gather`] of bytes (`T = u8`) from a dataset that is not in memory: the +/// dataset's `src_len` bytes start at `base` in `file`, which must hold all +/// of them (the caller checks). The selection's runs are collected first, +/// adjacent ones merged, and fetched with one [`Storage::read_ranges`] call, +/// so only the selected bytes are read. Same checks and errors as +/// [`gather`]. +pub(crate) fn gather_storage( + file: &S, + base: u64, + src_len: usize, + dims: &[u64], + elem_size: usize, + selection: &Selection, +) -> Result, FormatError> { + if elem_size == 0 { + return Err(FormatError::DataSizeMismatch { + expected: 1, + actual: elem_size, + }); + } + let n_elements = match selection { + Selection::None => 0, + Selection::Hyperslab { count, block, .. } => count + .iter() + .zip(block) + .try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?)) + .ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?, + Selection::Points(points) => points.len() as u64, + Selection::All => { + return Err(FormatError::SelectionOutOfBounds( + "gather does not take Selection::All".into(), + )); + } + }; + let out_bytes = crate::chunked_read::checked_byte_len(n_elements, elem_size)?; + // Runs as (byte offset in the dataset, byte length), in output order. + let mut runs: Vec<(usize, usize)> = Vec::new(); + let mut total = 0usize; + let mut failed = false; + let mut collect = |first: u64, n: u64| { + if failed { + return; + } + let range = usize::try_from(first) + .ok() + .and_then(|f| f.checked_mul(elem_size)) + .zip( + usize::try_from(n) + .ok() + .and_then(|n| n.checked_mul(elem_size)), + ) + .and_then(|(at, len)| Some((at, len, at.checked_add(len)?))); + match range { + Some((at, len, end)) if end <= src_len && total + len <= out_bytes => { + match runs.last_mut() { + Some((a, l)) if *a + *l == at => *l += len, + _ => runs.push((at, len)), + } + total += len; + } + _ => failed = true, + } + }; + let mut bad_point = false; + match selection { + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let rank = dims.len(); + if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { + return Err(FormatError::SelectionOutOfBounds( + "hyperslab rank does not match dataset rank".into(), + )); + } + hyperslab_runs(dims, start, stride, count, block, &mut collect); + } + Selection::Points(points) => { + let strides = strides(dims); + let mut coalesce = Coalesce { + start: 0, + len: 0, + emit: &mut collect, + }; + for p in points { + if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) { + bad_point = true; + break; + } + let at = p + .iter() + .zip(&strides) + .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s))); + coalesce.push(at, 1); + } + coalesce.flush(); + } + Selection::None | Selection::All => {} + } + if failed || bad_point || total != out_bytes { + return Err(FormatError::SelectionOutOfBounds( + "selection addresses elements outside the dataset".into(), + )); + } + let ranges: Vec> = runs + .iter() + .map(|&(at, len)| base + at as u64..base + (at + len) as u64) + .collect(); + let fetched = file.read_ranges(&ranges)?; + if fetched.len() != ranges.len() { + return Err(FormatError::Storage( + "read_ranges returned the wrong number of ranges".into(), + )); + } + let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes); + for (bytes, &(_, len)) in fetched.iter().zip(&runs) { + let Some(b) = bytes.get(..len) else { + return Err(FormatError::Storage( + "short read inside the file (the storage shrank or the backend failed)".into(), + )); + }; + out.extend_from_slice(b); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index 9861e8f..be69189 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -152,7 +152,7 @@ impl GlobalHeapCollection { /// Read the collection at `offset` and index its objects: the /// collection's bytes, its offset as a `usize`, and the index (with /// file offsets). - fn read_collection( + pub(crate) fn read_collection( file: &S, offset: u64, length_size: u8, diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 92fb1fa..75a4bd3 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -7,12 +7,30 @@ //! The lane assignment is seeded by dataset metadata so repeated reads of //! the same region produce identical partitions (cache-friendly, reproducible). -use crate::addr::to_usize; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::decompress_chunk_exact; use crate::lane_partition::{self, LaneStats, PartitionStats}; +use crate::storage::{ExtentBytes, Storage}; + +/// The stored bytes of every chunk in `chunks`, fetched in one +/// [`Storage::read_ranges`] call when the file is not in memory (each +/// chunk's bounds error is reported when that chunk is decoded, as before). +fn fetch_all<'a, S: Storage + ?Sized>( + file_data: &'a S, + chunks: &[ChunkInfo], +) -> Result, FormatError> { + let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { + Vec::new() + } else { + chunks + .iter() + .map(|c| (c.address, c.chunk_size as usize, true)) + .collect() + }; + ExtentBytes::fetch(file_data, &extents) +} /// Threshold: only use parallel decompression when chunk count exceeds this. const PARALLEL_THRESHOLD: usize = 4; @@ -190,6 +208,27 @@ pub fn decompress_chunks_lane_partitioned( element_size: u32, seed: u64, num_lanes: Option, +) -> Result<(Vec>, PartitionStats), FormatError> { + decompress_chunks_lane_partitioned_in( + file_data, + chunks, + pipeline, + chunk_total_bytes, + element_size, + seed, + num_lanes, + ) +} + +/// [`decompress_chunks_lane_partitioned`] over any [`Storage`]. +pub fn decompress_chunks_lane_partitioned_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: &FilterPipeline, + chunk_total_bytes: usize, + element_size: u32, + seed: u64, + num_lanes: Option, ) -> Result<(Vec>, PartitionStats), FormatError> { use rayon::prelude::*; @@ -199,6 +238,7 @@ pub fn decompress_chunks_lane_partitioned( .unwrap_or(1) }); + let raw_bytes = fetch_all(file_data, chunks)?; let assignments = lane_partition::partition_chunks(chunks.len(), lanes, seed); let num_lanes = assignments.len(); @@ -211,19 +251,8 @@ pub fn decompress_chunks_lane_partitioned( for &index in &indices { let chunk_info = &chunks[index]; - let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; - - if c_addr - .checked_add(size) - .is_none_or(|end| end > file_data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: c_addr.saturating_add(size), - available: file_data.len(), - }); - } - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; let decompressed = decompress_chunk_exact( raw_chunk, @@ -282,25 +311,27 @@ pub fn decompress_chunks_parallel( pipeline: &FilterPipeline, chunk_total_bytes: usize, element_size: u32, +) -> Result>, FormatError> { + decompress_chunks_parallel_in(file_data, chunks, pipeline, chunk_total_bytes, element_size) +} + +/// [`decompress_chunks_parallel`] over any [`Storage`]. +pub fn decompress_chunks_parallel_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: &FilterPipeline, + chunk_total_bytes: usize, + element_size: u32, ) -> Result>, FormatError> { use rayon::prelude::*; + let raw_bytes = fetch_all(file_data, chunks)?; let results: Result, FormatError> = chunks .par_iter() .enumerate() .map(|(index, chunk_info)| { - let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; - if c_addr - .checked_add(size) - .is_none_or(|end| end > file_data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: c_addr.saturating_add(size), - available: file_data.len(), - }); - } - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; let decompressed = decompress_chunk_exact( raw_chunk, @@ -331,20 +362,22 @@ pub fn decompress_chunks_sequential( chunk_total_bytes: usize, element_size: u32, ) -> Result>, FormatError> { + decompress_chunks_sequential_in(file_data, chunks, pipeline, chunk_total_bytes, element_size) +} + +/// [`decompress_chunks_sequential`] over any [`Storage`]. +pub fn decompress_chunks_sequential_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: Option<&FilterPipeline>, + chunk_total_bytes: usize, + element_size: u32, +) -> Result>, FormatError> { + let raw_bytes = fetch_all(file_data, chunks)?; let mut result = Vec::with_capacity(chunks.len()); - for chunk_info in chunks { - let c_addr = to_usize(chunk_info.address)?; + for (i, chunk_info) in chunks.iter().enumerate() { let size = chunk_info.chunk_size as usize; - if c_addr - .checked_add(size) - .is_none_or(|end| end > file_data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: c_addr.saturating_add(size), - available: file_data.len(), - }); - } - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(i, chunk_info.address, size)?; let decompressed = if let Some(pl) = pipeline { decompress_chunk_exact( diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index ad5dc8e..a9f630c 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -18,7 +18,7 @@ use alloc::{format, vec, vec::Vec}; #[cfg(feature = "std")] use std::string as alloc_or_std; -use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read}; +use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read_in}; use crate::data_layout::DataLayout; use crate::data_read::extract_selection_from_buffer; use crate::dataspace::Dataspace; @@ -26,6 +26,7 @@ use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::{all_filters_skipped, decompress_chunk_exact_with}; use crate::selection::Selection; +use crate::storage::{ExtentBytes, Storage}; /// The smallest axis-aligned box containing every selected element, as /// `(start, extent)` per dimension. `None` when there is nothing to gain or @@ -256,6 +257,30 @@ pub fn read_selection( offset_size: u8, length_size: u8, selection: &Selection, +) -> Result>, FormatError> { + read_selection_in( + file_data, + layout, + dataspace, + elem_size, + pipeline, + offset_size, + length_size, + selection, + ) +} + +/// [`read_selection`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_selection_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + selection: &Selection, ) -> Result>, FormatError> { let dims = &dataspace.dimensions; if dims.is_empty() || elem_size == 0 { @@ -276,14 +301,33 @@ pub fn read_selection( validate(selection, dims)?; let base = usize::try_from(*address) .map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?; - let data = file_data - .get(base..) - .and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?)) - .ok_or(FormatError::UnexpectedEof { - expected: base, - available: file_data.len(), - })?; - return crate::gather::gather::(data, dims, elem_size, selection).map(Some); + let file_len = crate::storage::len_usize(file_data); + let eof = FormatError::UnexpectedEof { + expected: base, + available: file_len, + }; + if let Some(all) = file_data.as_contiguous() { + let data = all + .get(base..) + .and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?)) + .ok_or(eof)?; + return crate::gather::gather::(data, dims, elem_size, selection).map(Some); + } + // Not in memory: the same bounds check, then only the selected runs + // are read. + let len = checked_byte_len(total, elem_size) + .ok() + .filter(|&len| base <= file_len && len <= file_len - base) + .ok_or(eof)?; + return crate::gather::gather_storage( + file_data, + base as u64, + len, + dims, + elem_size, + selection, + ) + .map(Some); } let Some((box_start, box_extent)) = bounding_box(selection, dims) else { return Ok(None); @@ -303,7 +347,7 @@ pub fn read_selection( btree_address: Some(_), .. } => { - let (chunks, chunk_dims) = list_chunks_for_read( + let (chunks, chunk_dims) = list_chunks_for_read_in( file_data, layout, dataspace, @@ -315,29 +359,37 @@ pub fn read_selection( let rank = dims.len(); let chunk_shape: Vec = chunk_dims.iter().map(|&d| d as u64).collect(); let chunk_bytes = crate::chunked_read::checked_chunk_byte_len(&chunk_dims, elem_size)?; - // Chunks are decoded into this thread's reusable buffers. - crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { - for chunk in &chunks { + // The chunks overlapping the box, in index order. + let wanted: Vec<&crate::chunked_read::ChunkInfo> = chunks + .iter() + .filter(|chunk| { if chunk.offsets.len() < rank || chunk.address == u64::MAX { - continue; + return false; } let origin = &chunk.offsets[..rank]; - let overlaps = (0..rank).all(|d| { + (0..rank).all(|d| { origin[d] < box_start[d] + box_extent[d] && origin[d].saturating_add(chunk_shape[d]) > box_start[d] - }); - if !overlaps { - continue; - } - let at = usize::try_from(chunk.address) + }) + }) + .collect(); + // Their stored bytes, in one batch when the file is not in memory. + let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { + Vec::new() + } else { + wanted + .iter() + .map(|c| (c.address, c.chunk_size as usize, true)) + .collect() + }; + let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + // Chunks are decoded into this thread's reusable buffers. + crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { + for (i, chunk) in wanted.iter().enumerate() { + let origin = &chunk.offsets[..rank]; + usize::try_from(chunk.address) .map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?; - let raw = at - .checked_add(chunk.chunk_size as usize) - .and_then(|end| file_data.get(at..end)) - .ok_or(FormatError::UnexpectedEof { - expected: at.saturating_add(chunk.chunk_size as usize), - available: file_data.len(), - })?; + let raw = raw_bytes.get(i, chunk.address, chunk.chunk_size as usize)?; // Mirrors the full-read path: filter-mask bit i set means // filter i was not applied to this chunk. let data: &[u8] = match pipeline { diff --git a/crates/clawhdf5-format/src/provenance.rs b/crates/clawhdf5-format/src/provenance.rs index ee74994..0bdccab 100644 --- a/crates/clawhdf5-format/src/provenance.rs +++ b/crates/clawhdf5-format/src/provenance.rs @@ -13,7 +13,6 @@ use sha2::{Digest, Sha256}; use crate::attribute::AttributeMessage; use crate::data_layout::DataLayout; -use crate::data_read::read_raw_data; use crate::dataspace::Dataspace; use crate::datatype::Datatype; use crate::error::FormatError; @@ -128,10 +127,20 @@ pub fn verify_dataset( header: &ObjectHeader, offset_size: u8, length_size: u8, +) -> Result { + verify_dataset_in(file_data, header, offset_size, length_size) +} + +/// [`verify_dataset`] over any [`Storage`](crate::storage::Storage). +pub fn verify_dataset_in( + file_data: &S, + header: &ObjectHeader, + offset_size: u8, + length_size: u8, ) -> Result { // 1. Extract all attributes (compact + dense). let attrs = - crate::attribute::extract_attributes_full(file_data, header, offset_size, length_size)?; + crate::attribute::extract_attributes_full_in(file_data, header, offset_size, length_size)?; // 2. Find the stored hash. let stored_hash = attrs @@ -174,7 +183,7 @@ pub fn verify_dataset( .transpose()?; let raw = match &dl { - DataLayout::Chunked { .. } => crate::chunked_read::read_chunked_data( + DataLayout::Chunked { .. } => crate::chunked_read::read_chunked_data_in( file_data, &dl, &ds, @@ -183,7 +192,7 @@ pub fn verify_dataset( offset_size, length_size, )?, - _ => read_raw_data(file_data, &dl, &ds, &dt)?, + _ => crate::data_read::read_raw_data_in(file_data, &dl, &ds, &dt)?, }; // 4. Compare. diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 555ac78..0dd0540 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -336,6 +336,159 @@ pub fn read_upto( Ok(bytes) } +/// Most stored bytes fetched by one [`Storage::read_ranges`] call when a +/// read gathers many extents (a chunked dataset's chunks): a larger read is +/// fetched and decoded batch by batch, so a remote backend never holds more +/// than this much undecoded data per read. +pub(crate) const RAW_BATCH_BYTES: usize = 64 << 20; + +/// The stored bytes of a list of extents (chunks, contiguous runs), fetched +/// together: [`Storage::read_ranges`] is called once for all of them, so a +/// remote backend can coalesce and parallelise the requests. +/// +/// With the whole file in memory nothing is fetched: [`Self::get`] slices +/// it, as the slice readers did. Either way an extent that does not lie in +/// the file is the error the slice readers gave for it +/// ([`FormatError::UnexpectedEof`] with its end and the file length, or +/// [`FormatError::Overflow`] for an address past this platform's `usize`), +/// reported when that extent is asked for — so a read reports the first +/// failing extent in its own order, whatever fails after it. +pub(crate) enum ExtentBytes<'a> { + /// The whole file. + Contiguous(&'a [u8]), + /// Each extent's bytes, or its bounds error. + Fetched(Vec>), +} + +/// One extent of [`ExtentBytes::Fetched`]. +pub(crate) enum Extent<'a> { + /// Its bytes. + Bytes(Cow<'a, [u8]>), + /// In the file, but not fetched (the caller did not want its bytes). + NotFetched, + /// The error reading it gives. + Err(FormatError), +} + +impl<'a> ExtentBytes<'a> { + /// Fetch `extents` (`(address, length, wanted)`): the bytes of those + /// `wanted`, and the bounds check of all of them. + pub(crate) fn fetch( + file: &'a S, + extents: &[(u64, usize, bool)], + ) -> Result { + if let Some(all) = file.as_contiguous() { + return Ok(ExtentBytes::Contiguous(all)); + } + let file_len = len_usize(file); + let mut ranges = Vec::new(); + let mut out = Vec::with_capacity(extents.len()); + // Positions in `out` of the extents being read, in `ranges` order. + let mut slots = Vec::new(); + for &(addr, len, wanted) in extents { + let checked = + crate::addr::to_usize(addr).and_then(|start| match start.checked_add(len) { + Some(end) if end <= file_len => Ok(()), + _ => Err(FormatError::UnexpectedEof { + expected: start.saturating_add(len), + available: file_len, + }), + }); + match checked { + Ok(()) if wanted => { + slots.push(out.len()); + ranges.push(addr..addr + len as u64); + out.push(Extent::NotFetched); + } + Ok(()) => out.push(Extent::NotFetched), + Err(e) => out.push(Extent::Err(e)), + } + } + if !ranges.is_empty() { + let got = file.read_ranges(&ranges)?; + if got.len() != ranges.len() { + return Err(FormatError::Storage( + "read_ranges returned the wrong number of ranges".into(), + )); + } + for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) { + if (bytes.len() as u64) < r.end - r.start { + return Err(short_read()); + } + out[slot] = Extent::Bytes(bytes); + } + } + Ok(ExtentBytes::Fetched(out)) + } + + /// Whether extent `i` (at `addr`, `len` bytes, as passed to + /// [`Self::fetch`]) lies in the file: its bounds error if not. + pub(crate) fn check(&self, i: usize, addr: u64, len: usize) -> Result<(), FormatError> { + match self { + ExtentBytes::Contiguous(_) => self.get(i, addr, len).map(|_| ()), + ExtentBytes::Fetched(v) => match v.get(i) { + Some(Extent::Err(e)) => Err(e.clone()), + Some(_) => Ok(()), + None => Err(not_fetched()), + }, + } + } + + /// Extent `i`'s bytes (at `addr`, `len` bytes, as passed to + /// [`Self::fetch`]). + pub(crate) fn get(&self, i: usize, addr: u64, len: usize) -> Result<&[u8], FormatError> { + match self { + ExtentBytes::Contiguous(all) => { + let start = crate::addr::to_usize(addr)?; + start + .checked_add(len) + .and_then(|end| all.get(start..end)) + .ok_or(FormatError::UnexpectedEof { + expected: start.saturating_add(len), + available: <[u8]>::len(all), + }) + } + ExtentBytes::Fetched(v) => match v.get(i) { + Some(Extent::Bytes(b)) => Ok(b), + Some(Extent::Err(e)) => Err(e.clone()), + _ => Err(not_fetched()), + }, + } + } +} + +#[cold] +fn not_fetched() -> FormatError { + FormatError::Storage("an extent that was not fetched was asked for".into()) +} + +/// Split `n` extents, whose sizes `size(i)` gives, into consecutive batches +/// of at most [`RAW_BATCH_BYTES`] (at least one extent each): the ranges of +/// `0..n` to fetch together. With the whole file in memory (`contiguous`) +/// there is nothing to fetch, and one batch. +pub(crate) fn raw_batches( + n: usize, + contiguous: bool, + size: impl Fn(usize) -> usize, +) -> Vec> { + if contiguous || n == 0 { + return core::iter::once(0..n).collect(); + } + let mut out = Vec::new(); + let (mut start, mut bytes) = (0, 0usize); + for i in 0..n { + let s = size(i); + if i > start && bytes.saturating_add(s) > RAW_BATCH_BYTES { + out.push(start..i); + start = i; + bytes = 0; + } + bytes = bytes.saturating_add(s); + } + out.push(start..n); + out +} + /// Borrow the whole file for a code path that has not been converted to /// [`Storage`] yet. On a backend without a contiguous view this is the /// clean [`FormatError::ContiguousStorageRequired`] error, never a guess. diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs index df1b068..92ad4e7 100644 --- a/crates/clawhdf5-format/src/vds.rs +++ b/crates/clawhdf5-format/src/vds.rs @@ -15,12 +15,13 @@ #[cfg(not(feature = "std"))] use alloc::{format, string::String, vec, vec::Vec}; -use crate::addr::to_usize; +use crate::addr::{checked_addr, to_usize}; use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings}; use crate::dataspace::Dataspace; use crate::datatype::Datatype; use crate::error::FormatError; use crate::selection::{SerializedSelection, UNLIMITED}; +use crate::storage::Storage; /// Resolves the name of an external VDS source file, as stored in the /// mapping, to that file's bytes. @@ -192,8 +193,8 @@ fn non_unlimited_elements(sel: &SerializedSelection, skip: usize) -> Option } /// Load and decode the mapping list of a virtual layout. -fn load_mappings( - file_data: &[u8], +fn load_mappings( + file_data: &S, layout: &DataLayout, length_size: u8, ) -> Result, FormatError> { @@ -208,8 +209,11 @@ fn load_mappings( let Some(addr) = *global_heap_address else { return Ok(Vec::new()); }; - let coll = - crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?; + let coll = crate::global_heap::GlobalHeapCollection::parse_in( + file_data, + checked_addr(addr)?, + length_size, + )?; let index = u16::try_from(*global_heap_index) .map_err(|_| vds_err("VDS mapping heap index out of range"))?; let obj = coll @@ -330,7 +334,11 @@ enum Step { /// Work out the extent libhdf5 gives the virtual dataset /// (`H5D__virtual_set_extent_unlim`, default view `H5D_VDS_LAST_AVAILABLE` /// with a printf gap of 0) and how much of each unlimited mapping is read. -fn plan(mappings: &[Mapping], stored: &[u64], sources: &mut Sources) -> Result { +fn plan( + mappings: &[Mapping], + stored: &[u64], + sources: &mut Sources<'_, '_, S>, +) -> Result { let overflow = || FormatError::Overflow("VDS extent overflow".into()); let rank = stored.len(); let mut new_dims: Vec> = vec![None; rank]; @@ -472,6 +480,25 @@ pub fn virtual_dataset_extent( _offset_size: u8, length_size: u8, resolver: Option<&VdsFileResolver>, +) -> Result, FormatError> { + virtual_dataset_extent_in( + file_data, + layout, + dataspace, + _offset_size, + length_size, + resolver, + ) +} + +/// [`virtual_dataset_extent`] over any [`Storage`]. +pub fn virtual_dataset_extent_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + _offset_size: u8, + length_size: u8, + resolver: Option<&VdsFileResolver>, ) -> Result, FormatError> { let mappings = load_mappings(file_data, layout, length_size)?; if mappings.iter().all(|m| m.kind == Kind::Fixed) { @@ -498,6 +525,30 @@ pub fn read_virtual_dataset( _offset_size: u8, length_size: u8, resolver: Option<&VdsFileResolver>, +) -> Result { + read_virtual_dataset_in( + file_data, + layout, + dataspace, + datatype, + fill, + _offset_size, + length_size, + resolver, + ) +} + +/// [`read_virtual_dataset`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_virtual_dataset_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + fill: Option<&[u8]>, + _offset_size: u8, + length_size: u8, + resolver: Option<&VdsFileResolver>, ) -> Result { let mappings = load_mappings(file_data, layout, length_size)?; let mut sources = Sources::new(file_data, resolver); @@ -511,7 +562,7 @@ pub fn read_virtual_dataset( let mut data = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len( total, elem_size, )?)?; - if let Some(fill) = fill.filter(|f| f.len() == elem_size && f.iter().any(|&b| b != 0)) { + if let Some(fill) = fill.filter(|f| <[u8]>::len(f) == elem_size && f.iter().any(|&b| b != 0)) { for element in data.chunks_exact_mut(elem_size) { element.copy_from_slice(fill); } @@ -780,14 +831,17 @@ struct SourceData { /// Source files and datasets, fetched on demand. The most recently used /// external file is kept, since consecutive mappings usually share one. -struct Sources<'a, 'r> { - file_data: &'a [u8], +/// +/// The virtual dataset's own file (`"."`) is read through its [`Storage`]; +/// an external source file is loaded whole, through the resolver. +struct Sources<'a, 'r, S: Storage + ?Sized> { + file_data: &'a S, resolver: Option<&'r VdsFileResolver<'r>>, cached_file: Option<(String, Option>)>, } -impl<'a, 'r> Sources<'a, 'r> { - fn new(file_data: &'a [u8], resolver: Option<&'r VdsFileResolver<'r>>) -> Self { +impl<'a, 'r, S: Storage + ?Sized> Sources<'a, 'r, S> { + fn new(file_data: &'a S, resolver: Option<&'r VdsFileResolver<'r>>) -> Self { Sources { file_data, resolver, @@ -795,11 +849,9 @@ impl<'a, 'r> Sources<'a, 'r> { } } - /// The bytes of source file `name`, or `None` if it does not exist. - fn file(&mut self, name: &str) -> Result, FormatError> { - if name == "." { - return Ok(Some(self.file_data)); - } + /// The bytes of external source file `name` (not `"."`), or `None` if it + /// does not exist. + fn external(&mut self, name: &str) -> Result, FormatError> { if self.cached_file.as_ref().is_none_or(|(n, _)| n != name) { let resolver = self.resolver.ok_or_else(|| { vds_err("external-file virtual dataset sources require a file resolver") @@ -821,7 +873,10 @@ impl<'a, 'r> Sources<'a, 'r> { /// The extent of source dataset `path` in file `file`, or `None` when /// either does not exist. fn dims(&mut self, file: &str, path: &str) -> Result>, FormatError> { - let Some(bytes) = self.file(file)? else { + if file == "." { + return Ok(open_source(self.file_data, path)?.map(|s| s.dataspace.dimensions)); + } + let Some(bytes) = self.external(file)? else { return Ok(None); }; Ok(open_source(bytes, path)?.map(|s| s.dataspace.dimensions)) @@ -846,7 +901,13 @@ impl<'a, 'r> Sources<'a, 'r> { from another file is not supported" ))); } - let Some(bytes) = self.file(file)? else { + if file == "." { + let Some(src) = open_source(self.file_data, path)? else { + return Ok(None); + }; + return read_source(self.file_data, src, path, datatype).map(Some); + } + let Some(bytes) = self.external(file)? else { return Ok(None); }; let Some(src) = open_source(bytes, path)? else { @@ -910,19 +971,23 @@ fn source_message<'h>( /// Open source dataset `path` of the file in `file_data`, or `None` if there /// is no such object (libhdf5 reads a missing source as fill). -fn open_source(file_data: &[u8], path: &str) -> Result, FormatError> { +fn open_source( + file_data: &S, + path: &str, +) -> Result, FormatError> { use crate::message_type::MessageType; - use crate::shared_message::message_data_with_sohm; + use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm; - // `file_data` starts at the superblock (see `Sources::file`). - let sb = crate::superblock::Superblock::parse(file_data, 0)?; + // `file_data` starts at the superblock (see `Sources::external`). + let sb = crate::superblock::Superblock::parse_in(file_data, 0)?; let (os, ls) = (sb.offset_size, sb.length_size); - let addr = match crate::group_v2::resolve_path_any(file_data, &sb, path) { + let addr = match crate::group_v2::resolve_path_any_in(file_data, &sb, path) { Ok(a) => a, Err(FormatError::PathNotFound(_)) => return Ok(None), Err(e) => return Err(e), }; - let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?; + let header = + crate::object_header::ObjectHeader::parse_in(file_data, checked_addr(addr)?, os, ls)?; let mut src = OpenSource { offset_size: os, length_size: ls, @@ -941,15 +1006,15 @@ fn open_source(file_data: &[u8], path: &str) -> Result, Forma /// Read an opened source dataset in full (its own fill value applied to /// unallocated chunks). -fn read_source( - file_data: &[u8], +fn read_source( + file_data: &S, src: OpenSource, path: &str, datatype: &Datatype, ) -> Result { use crate::filter_pipeline::FilterPipeline; use crate::message_type::MessageType; - use crate::shared_message::message_data_with_sohm; + use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm; let (os, ls) = (src.offset_size, src.length_size); let dt_msg = source_message(&src, path, MessageType::Datatype)?; @@ -987,7 +1052,7 @@ fn read_source( message_data_with_sohm(file_data, m, os, ls).and_then(|d| FilterPipeline::parse(&d)) }) .transpose()?; - let raw = crate::fill_value::read_full_with_fill( + let raw = crate::fill_value::read_full_with_fill_in( &src.header.messages, file_data, &layout, @@ -996,7 +1061,7 @@ fn read_source( os, ls, || { - crate::data_read::read_raw_data_full( + crate::data_read::read_raw_data_full_in( file_data, &layout, &src.dataspace, diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index a998a62..f37638b 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -5,9 +5,9 @@ //! `sequence_length(4 LE) + collection_address(offset_size LE) + object_index(4 LE)`. #[cfg(not(feature = "std"))] -use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; +use alloc::{borrow::Cow, collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] -use std::collections::BTreeMap; +use std::{borrow::Cow, collections::BTreeMap}; use crate::addr::to_usize; use crate::error::FormatError; @@ -137,13 +137,15 @@ pub fn check_element_size(stored_size: u32, offset_size: u8) -> Result<(), Forma /// A collection's objects, located in the file data but not copied: /// `(index, offset, size)` of the first object with each index, sorted by -/// index. -struct CachedCollection { +/// index. Over a storage without the whole file in memory, also the +/// collection's bytes (`(offset, bytes)`), read once when it is indexed. +struct CachedCollection<'a> { objects: Vec<(u16, usize, usize)>, + bytes: Option<(usize, Cow<'a, [u8]>)>, } -impl CachedCollection { - fn new(index: GlobalHeapIndex) -> Self { +impl<'a> CachedCollection<'a> { + fn new(index: GlobalHeapIndex, bytes: Option<(usize, Cow<'a, [u8]>)>) -> Self { let mut objects: Vec<(u16, usize, usize)> = index .objects .iter() @@ -152,12 +154,16 @@ impl CachedCollection { // Stable, so the first object with a repeated index is kept. objects.sort_by_key(|o| o.0); objects.dedup_by_key(|o| o.0); - Self { objects } + Self { objects, bytes } } /// What this entry costs to keep, in bytes (roughly). fn cost(&self) -> usize { - 64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>() + let held = match &self.bytes { + Some((_, Cow::Owned(b))) => b.len(), + _ => 0, + }; + 64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>() + held } fn get(&self, index: u32) -> Option<(usize, usize)> { @@ -170,6 +176,8 @@ impl CachedCollection { /// How many bytes of collection indexes a [`VlResolver`] keeps before it /// drops them and starts again. Values are never copied into the cache, so /// this bounds what a read retains however many collections it visits. +/// (Over a storage without the whole file in memory the collections' bytes +/// are kept too, and count against this.) const CACHE_BUDGET: usize = 32 << 20; /// Resolves variable-length elements against a file's global heap, parsing @@ -185,11 +193,18 @@ const CACHE_BUDGET: usize = 32 << 20; /// that overlap one another are refused (libhdf5 never writes them), so a /// file cannot make the resolver parse the same bytes as the objects of /// many collections. -pub struct VlResolver<'a> { - file_data: &'a [u8], +/// +/// The file is any [`Storage`](crate::storage::Storage) (`S`, a slice by default). Over one without +/// the whole file in memory each collection is read once, when first used, +/// and kept (within the budget above); [`Self::strings`], +/// [`Self::string_bytes`] and [`Self::sequences`] work over any storage, +/// [`Self::element`] and [`Self::string_element`], which borrow from the +/// file, over a slice. +pub struct VlResolver<'a, S: crate::storage::Storage + ?Sized = [u8]> { + file_data: &'a S, offset_size: u8, length_size: u8, - cache: BTreeMap, + cache: BTreeMap>, cached_bytes: usize, budget: usize, /// Start → end of every collection parsed so far (kept when the cache @@ -201,6 +216,56 @@ impl<'a> VlResolver<'a> { /// A resolver over `file_data` (the file from its superblock on), with /// the superblock's offset and length sizes. pub fn new(file_data: &'a [u8], offset_size: u8, length_size: u8) -> Self { + Self::new_in(file_data, offset_size, length_size) + } + + /// One element (the first [`element_size`](Self::element_size) bytes of + /// `elem`) of a variable-length sequence whose base type is `base_size` + /// bytes: its `length × base_size` bytes, or `None` for a null element + /// (heap address 0). + pub fn element( + &mut self, + elem: &[u8], + base_size: usize, + ) -> Result, FormatError> { + let vl = parse_vl_references(elem, 1, self.offset_size)?; + let vl = &vl[0]; + if vl.collection_address == 0 { + return Ok(None); + } + let (start, size) = self.locate(vl)?; + let data = &self.file_data[start..start + size]; + check_object_size(vl, data.len(), base_size)?; + Ok(Some(data)) + } + + /// One variable-length string element: its bytes up to the first NUL, + /// or `None` for a null element (h5dump prints it as `NULL`, h5py + /// returns it as empty). + pub fn string_element(&mut self, elem: &[u8]) -> Result, FormatError> { + Ok(self.element(elem, 1)?.map(cut_at_nul)) + } +} + +/// `data_len`, the size of `vl`'s heap object, against the `length × +/// base_size` bytes the element says it holds. +fn check_object_size(vl: &VlElement, data_len: usize, base_size: usize) -> Result<(), FormatError> { + let expected = (vl.length as usize) + .checked_mul(base_size) + .ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?; + if data_len != expected { + return Err(FormatError::VlDataError(format!( + "global heap object {} in the collection at {} holds {data_len} bytes; the element \ + says {} × {base_size}", + vl.object_index, vl.collection_address, vl.length + ))); + } + Ok(()) +} + +impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> { + /// [`VlResolver::new`] over any [`Storage`](crate::storage::Storage). + pub fn new_in(file_data: &'a S, offset_size: u8, length_size: u8) -> Self { Self { file_data, offset_size, @@ -231,51 +296,15 @@ impl<'a> VlResolver<'a> { /// The bytes of one element: `length × base_size` bytes from the heap, /// or `None` for a null element. - fn resolve( - &mut self, - vl: &VlElement, - base_size: usize, - ) -> Result, FormatError> { - let addr = vl.collection_address; - if addr == 0 { + fn resolve(&mut self, vl: &VlElement, base_size: usize) -> Result, FormatError> { + if vl.collection_address == 0 { return Ok(None); } let data = self.object(vl)?; - let expected = (vl.length as usize) - .checked_mul(base_size) - .ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?; - if data.len() != expected { - return Err(FormatError::VlDataError(format!( - "global heap object {} in the collection at {addr} holds {} bytes; the element \ - says {} × {base_size}", - vl.object_index, - data.len(), - vl.length - ))); - } + check_object_size(vl, data.len(), base_size)?; Ok(Some(data)) } - /// One element (the first [`element_size`](Self::element_size) bytes of - /// `elem`) of a variable-length sequence whose base type is `base_size` - /// bytes: its `length × base_size` bytes, or `None` for a null element - /// (heap address 0). - pub fn element( - &mut self, - elem: &[u8], - base_size: usize, - ) -> Result, FormatError> { - let vl = parse_vl_references(elem, 1, self.offset_size)?; - self.resolve(&vl[0], base_size) - } - - /// One variable-length string element: its bytes up to the first NUL, - /// or `None` for a null element (h5dump prints it as `NULL`, h5py - /// returns it as empty). - pub fn string_element(&mut self, elem: &[u8]) -> Result, FormatError> { - Ok(self.element(elem, 1)?.map(cut_at_nul)) - } - /// The strings of the variable-length string elements in `raw`, as /// bytes. A string ends at its first NUL, as libhdf5 returns it (it /// converts each to a C string); a null element is empty. @@ -330,9 +359,20 @@ pub fn read_vl_strings( num_elements: u64, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + read_vl_strings_in(file_data, raw_data, num_elements, offset_size, length_size) +} + +/// [`read_vl_strings`] over any [`Storage`](crate::storage::Storage). +pub fn read_vl_strings_in( + file_data: &S, + raw_data: &[u8], + num_elements: u64, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let raw = first_elements(raw_data, num_elements, offset_size)?; - VlResolver::new(file_data, offset_size, length_size).strings(raw) + VlResolver::new_in(file_data, offset_size, length_size).strings(raw) } /// The first `num_elements` elements of `raw`, or an error if it is shorter. @@ -364,9 +404,20 @@ pub fn read_vl_bytes( num_elements: u64, offset_size: u8, length_size: u8, +) -> Result>, FormatError> { + read_vl_bytes_in(file_data, raw_data, num_elements, offset_size, length_size) +} + +/// [`read_vl_bytes`] over any [`Storage`](crate::storage::Storage). +pub fn read_vl_bytes_in( + file_data: &S, + raw_data: &[u8], + num_elements: u64, + offset_size: u8, + length_size: u8, ) -> Result>, FormatError> { let refs = parse_vl_references(raw_data, num_elements, offset_size)?; - let mut resolver = VlResolver::new(file_data, offset_size, length_size); + let mut resolver = VlResolver::new_in(file_data, offset_size, length_size); let mut result = Vec::with_capacity(refs.len()); for vl in &refs { @@ -385,10 +436,10 @@ pub fn read_vl_bytes( Ok(result) } -impl<'a> VlResolver<'a> { - /// The heap object `vl` points to, whatever its size; its collection is - /// parsed on first use. - fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> { +impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> { + /// Where the heap object `vl` points to lies in the file, whatever its + /// size (`(offset, size)`); its collection is parsed on first use. + fn locate(&mut self, vl: &VlElement) -> Result<(usize, usize), FormatError> { let addr = vl.collection_address; // libhdf5 writes a null element with address 0, never the undefined // address, and fails to read one ("addr undefined") even when its @@ -402,14 +453,20 @@ impl<'a> VlResolver<'a> { if !self.cache.contains_key(&addr) { let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof { expected: usize::MAX, - available: self.file_data.len(), + available: crate::storage::len_usize(self.file_data), })?; - let index = - GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?; - // parse_index checked that the collection lies in the file. + let (bytes, base, index) = + GlobalHeapCollection::read_collection(self.file_data, addr, self.length_size)?; + // read_collection checked that the collection lies in the file. let end = offset + to_usize(index.collection_size)?; self.check_overlap(offset, end)?; - let coll = CachedCollection::new(index); + // With the whole file in memory the objects are sliced from it; + // otherwise the collection's bytes are kept. + let bytes = match self.file_data.as_contiguous() { + Some(_) => None, + None => Some((base, bytes)), + }; + let coll = CachedCollection::new(index, bytes); if self.cached_bytes.saturating_add(coll.cost()) > self.budget { self.cache.clear(); self.cached_bytes = 0; @@ -417,13 +474,27 @@ impl<'a> VlResolver<'a> { self.cached_bytes += coll.cost(); self.cache.insert(addr, coll); } - let (start, size) = self.cache[&addr].get(vl.object_index).ok_or( - FormatError::GlobalHeapObjectNotFound { + self.cache[&addr] + .get(vl.object_index) + .ok_or(FormatError::GlobalHeapObjectNotFound { collection_address: addr, index: vl.object_index as u16, - }, - )?; - Ok(&self.file_data[start..start + size]) + }) + } + + /// The heap object `vl` points to, whatever its size; its collection is + /// parsed on first use. + fn object(&mut self, vl: &VlElement) -> Result<&[u8], FormatError> { + let (start, size) = self.locate(vl)?; + if let Some(all) = self.file_data.as_contiguous() { + return Ok(&all[start..start + size]); + } + match &self.cache[&vl.collection_address].bytes { + Some((base, bytes)) => Ok(&bytes[start - base..start - base + size]), + None => Err(FormatError::Storage( + "global heap collection bytes were not kept".into(), + )), + } } /// Record the collection at `start..end`, refusing one that overlaps a @@ -709,6 +780,7 @@ mod tests { let mut r = VlResolver::new(&file_data, 8, 8); let one = CachedCollection { objects: vec![(0, 0, 0); 3], + bytes: None, } .cost(); r.budget = 2 * one + 1; diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index 02a4f17..71b0dc0 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -37,14 +37,25 @@ use clawhdf5_format::btree_v2::{ BTreeV2Header, collect_btree_v2_records, collect_btree_v2_records_in, find_btree_v2_records, find_btree_v2_records_in, }; +use clawhdf5_format::chunk_cache::ChunkCache; +use clawhdf5_format::chunked_read::{list_chunks, list_chunks_in}; use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read::{ + read_raw_data_cached, read_raw_data_cached_in, read_raw_data_full, read_raw_data_full_in, + read_raw_data_indexed, read_raw_data_indexed_in, read_raw_data_selection, + read_raw_data_selection_in, +}; use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; use clawhdf5_format::extensible_array::{ ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in, }; -use clawhdf5_format::fill_value::{dataset_fill_value_from_storage, dataset_fill_value_in}; +use clawhdf5_format::fill_value::{ + dataset_fill_value_from_storage, dataset_fill_value_in, read_full_with_fill, + read_full_with_fill_in, +}; +use clawhdf5_format::filter_pipeline::FilterPipeline; use clawhdf5_format::fixed_array::{ FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in, }; @@ -54,6 +65,7 @@ use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::local_heap::LocalHeap; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::selection::Selection; use clawhdf5_format::shared_message::{ self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in, parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in, @@ -66,11 +78,18 @@ use clawhdf5_format::superblock_ext::{ read_superblock_extension_in, }; use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode}; +use clawhdf5_format::vds::{ + read_virtual_dataset, read_virtual_dataset_in, virtual_dataset_extent, + virtual_dataset_extent_in, +}; +use clawhdf5_format::vl_data::{VlResolver, read_vl_bytes, read_vl_bytes_in}; /// Objects visited per file, heap objects read per heap: enough to cover /// every structure kind while keeping a 35 000-group file fast. const MAX_OBJECTS: usize = 1500; const MAX_HEAP_IDS: usize = 200; +/// Datasets larger than this are not read (their chunk indexes still are). +const MAX_DATA_BYTES: u64 = 16 << 20; #[derive(Default, Debug)] struct Tally { @@ -89,6 +108,8 @@ struct Walk<'a> { slice: &'a [u8], storage: &'a CountingStorage, name: String, + /// The file's directory, for external VDS sources. + dir: Option, tally: &'a mut Tally, } @@ -264,6 +285,7 @@ impl Walk<'_> { } } self.check_layout(&header, os, ls); + self.check_data(&header, os, ls); } /// A symbol-table group: its local heap, B-tree, nodes and names. @@ -352,6 +374,179 @@ impl Walk<'_> { } } + /// A dataset's values through every raw-data path: whole reads (plain, + /// cached, indexed, fill-aware, virtual), chunk listings, selections + /// (a box, a strided hyperslab, points), VL strings and sequences. + fn check_data(&mut self, header: &ObjectHeader, os: u8, ls: u8) { + let slice = self.slice; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok()) + }; + let (Some(dt), Some(ds), Some(dl)) = ( + find(MessageType::Datatype), + find(MessageType::Dataspace), + find(MessageType::DataLayout), + ) else { + return; + }; + let (Ok((dt, _)), Ok(ds), Ok(dl)) = ( + Datatype::parse(&dt), + Dataspace::parse(&ds, ls), + DataLayout::parse(&dl, os, ls), + ) else { + return; + }; + let pipeline = match find(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p)) { + Some(Ok(p)) => Some(p), + Some(Err(_)) => return, + None => None, + }; + let pl = pipeline.as_ref(); + let elem = dt.type_size() as u64; + let bytes = ds + .dimensions + .iter() + .try_fold(elem, |a, &d| a.checked_mul(d)); + if bytes.is_none_or(|b| b > MAX_DATA_BYTES) { + return; + } + + if matches!(dl, DataLayout::Virtual { .. }) { + let resolver = self.resolver(); + let r: &clawhdf5_format::vds::VdsFileResolver = &resolver; + let want = virtual_dataset_extent(slice, &dl, &ds, os, ls, Some(r)); + let got = virtual_dataset_extent_in(self.st(), &dl, &ds, os, ls, Some(r)); + self.same("VDS extent", &want, &got); + let want = read_virtual_dataset(slice, &dl, &ds, &dt, None, os, ls, Some(r)); + let got = read_virtual_dataset_in(self.st(), &dl, &ds, &dt, None, os, ls, Some(r)); + self.same("VDS read", &want, &got); + return; + } + + let want = read_raw_data_full(slice, &dl, &ds, &dt, pl, os, ls); + let got = read_raw_data_full_in(self.st(), &dl, &ds, &dt, pl, os, ls); + self.same("raw data", &want, &got); + let want_fill = read_full_with_fill( + &header.messages, + slice, + &dl, + &ds, + elem as usize, + os, + ls, + || read_raw_data_full(slice, &dl, &ds, &dt, pl, os, ls), + ); + let got_fill = read_full_with_fill_in( + &header.messages, + self.st(), + &dl, + &ds, + elem as usize, + os, + ls, + || read_raw_data_full_in(self.st(), &dl, &ds, &dt, pl, os, ls), + ); + self.same("raw data with fill", &want_fill, &got_fill); + + if matches!(dl, DataLayout::Chunked { .. }) { + let want = list_chunks(slice, &dl, &ds, elem as usize, os, ls); + let got = list_chunks_in(self.st(), &dl, &ds, elem as usize, os, ls); + self.same("chunk list", &want, &got); + // Through a chunk cache, twice (the second read is served from + // it), and through the indexed path. + let (c1, c2) = (ChunkCache::new(), ChunkCache::new()); + for _ in 0..2 { + let want = read_raw_data_cached(slice, &dl, &ds, &dt, pl, os, ls, &c1); + let got = read_raw_data_cached_in(self.st(), &dl, &ds, &dt, pl, os, ls, &c2); + // A cache lists the chunks in hash-map order (see below). + if want.is_err() && got.is_err() { + self.tally.checks += 1; + } else { + self.same("raw data (cached)", &want, &got); + } + } + let (c1, c2) = (ChunkCache::new(), ChunkCache::new()); + let want = read_raw_data_indexed(slice, &dl, &ds, &dt, pl, os, ls, &c1); + let got = read_raw_data_indexed_in(self.st(), &dl, &ds, &dt, pl, os, ls, &c2); + // The indexed path decodes chunks in hash-map order, so which + // failing chunk it reports varies between two caches (with the + // slice alone, too): only whether it fails must agree. + if want.is_err() && got.is_err() { + self.tally.checks += 1; + } else { + self.same("raw data (indexed)", &want, &got); + } + } + + let dims = &ds.dimensions; + if !dims.is_empty() && dims.iter().all(|&d| d > 0) { + let rank = dims.len(); + let ones = vec![1u64; rank]; + let quarter = Selection::Hyperslab { + start: dims.iter().map(|&d| d / 4).collect(), + stride: ones.clone(), + count: dims.iter().map(|&d| (d / 3).max(1)).collect(), + block: ones.clone(), + }; + let mut stride = ones.clone(); + stride[rank - 1] = 2; + let mut count = dims.clone(); + count[rank - 1] = dims[rank - 1].div_ceil(2); + let strided = Selection::Hyperslab { + start: vec![0; rank], + stride, + count, + block: ones.clone(), + }; + let points = Selection::Points(vec![ + dims.iter().map(|&d| d - 1).collect(), + vec![0; rank], + dims.iter().map(|&d| d / 2).collect(), + ]); + for (what, sel) in [ + ("selection (box)", &quarter), + ("selection (strided)", &strided), + ("selection (points)", &points), + ] { + let want = read_raw_data_selection(slice, &dl, &ds, &dt, pl, os, ls, sel); + let got = read_raw_data_selection_in(self.st(), &dl, &ds, &dt, pl, os, ls, sel); + self.same(what, &want, &got); + } + } + + // Variable-length strings and sequences, resolved in the global heap. + if let (Datatype::VariableLength { base_type, .. }, Ok(raw)) = (&dt, &want) { + let n = raw.len() / clawhdf5_format::vl_data::element_size(os).max(1); + let raw = &raw[..n * clawhdf5_format::vl_data::element_size(os)]; + let want = VlResolver::new(slice, os, ls).string_bytes(raw); + let got = VlResolver::new_in(self.st(), os, ls).string_bytes(raw); + self.same("VL strings", &want, &got); + let base = base_type.type_size() as usize; + let want = VlResolver::new(slice, os, ls).sequences(raw, base); + let got = VlResolver::new_in(self.st(), os, ls).sequences(raw, base); + self.same("VL sequences", &want, &got); + let want = read_vl_bytes(slice, raw, n as u64, os, ls); + let got = read_vl_bytes_in(self.st(), raw, n as u64, os, ls); + self.same("VL bytes", &want, &got); + } + } + + /// External VDS source files: siblings of the file being walked. + fn resolver(&self) -> impl Fn(&str) -> Result>, FormatError> + use<> { + let dir = self.dir.clone(); + move |name: &str| { + let (Some(dir), false) = (dir.as_ref(), name.contains("..") || name.starts_with('/')) + else { + return Ok(None); + }; + Ok(std::fs::read(dir.join(name)).ok()) + } + } + /// A dataset's layout: VDS mappings, and fixed/extensible array chunk /// indexes. fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) { @@ -461,10 +656,19 @@ fn check_file(path: &Path, tally: &mut Tally) { let Ok(bytes) = std::fs::read(path) else { return; }; - check_bytes(&path.display().to_string(), &bytes, tally); + check_bytes_in( + &path.display().to_string(), + &bytes, + path.parent().map(Path::to_path_buf), + tally, + ); } fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) { + check_bytes_in(name, bytes, None, tally); +} + +fn check_bytes_in(name: &str, bytes: &[u8], dir: Option, tally: &mut Tally) { let Ok((_, hdf5)) = split_user_block(bytes) else { return; }; @@ -474,6 +678,7 @@ fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) { slice: hdf5, storage: &storage, name: name.to_string(), + dir, tally, }; walk.run(); @@ -548,6 +753,311 @@ fn corpus_parses_identically_through_storage() { assert!(tally.files > 0); } +/// A storage that misbehaves: fails its `fail_at`-th read (1-based, `0` +/// never), and, with `short`, serves one byte less than asked for inside +/// the file (a truncated response). +struct Adversary { + data: Vec, + reads: std::sync::atomic::AtomicUsize, + fail_at: usize, + short: bool, +} + +impl Storage for Adversary { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let n = self + .reads + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1; + if n == self.fail_at { + return Err(FormatError::Storage(format!( + "injected failure of read {n}" + ))); + } + let got = self.data.as_slice().read_at(offset, len)?; + let mut v = got.into_owned(); + if self.short && v.len() > 1 { + v.pop(); + } + Ok(std::borrow::Cow::Owned(v)) + } + + fn len(&self) -> u64 { + self.data.len() as u64 + } +} + +/// Every group listing and every dataset's values (whole, fill-aware and +/// through a selection) read through a storage that fails or serves short +/// reads: each result is an error or exactly the in-memory result, never +/// other data; and a failing read is reported as that failure. +#[test] +fn misbehaving_storage_never_returns_wrong_data() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut files = Vec::new(); + hdf5_files(&dir, &mut files); + files.sort(); + let (mut compared, mut failures_seen) = (0usize, 0usize); + for path in &files { + let Ok(bytes) = std::fs::read(path) else { + continue; + }; + let Ok((_, hdf5)) = split_user_block(&bytes) else { + continue; + }; + let Ok(sb) = Superblock::parse(hdf5, 0) else { + continue; + }; + // The whole read of every object, as one result to compare. + let everything = |file: &dyn Storage| -> Result { + let (os, ls) = (sb.offset_size, sb.length_size); + let mut out = String::new(); + let mut queue = VecDeque::from([sb.root_group_address]); + let mut seen = HashSet::new(); + while let Some(addr) = queue.pop_front() { + if seen.len() > 200 || !seen.insert(addr) { + continue; + } + let header = ObjectHeader::parse_in(file, addr, os, ls)?; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| message_data_with_sohm_in(file, m, os, ls)) + .transpose() + }; + if let (Some(dt), Some(ds), Some(dl)) = ( + find(MessageType::Datatype)?, + find(MessageType::Dataspace)?, + find(MessageType::DataLayout)?, + ) { + let dt = Datatype::parse(&dt)?.0; + let ds = Dataspace::parse(&ds, ls)?; + let dl = DataLayout::parse(&dl, os, ls)?; + let pl = find(MessageType::FilterPipeline)? + .map(|p| FilterPipeline::parse(&p)) + .transpose()?; + let data = read_full_with_fill_in( + &header.messages, + file, + &dl, + &ds, + dt.type_size() as usize, + os, + ls, + || read_raw_data_full_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls), + ); + out.push_str(&format!("{addr}: {data:?}\n")); + if let Some(&d0) = ds.dimensions.first() { + let rank = ds.dimensions.len(); + let sel = Selection::Hyperslab { + start: vec![0; rank], + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(2)) + .chain(ds.dimensions[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + let part = read_raw_data_selection_in( + file, + &dl, + &ds, + &dt, + pl.as_ref(), + os, + ls, + &sel, + ); + out.push_str(&format!("{addr} half: {part:?}\n")); + } + } + let children = group_v2::resolve_group_children_in(file, &sb, addr); + out.push_str(&format!("{addr} children: {children:?}\n")); + if let Ok(c) = children { + queue.extend(c.iter().map(|c| c.object_header_address)); + } + } + Ok(out) + }; + let want = everything(&hdf5); + let counting = CountingStorage::new(hdf5.to_vec()); + assert_eq!( + format!("{:?}", everything(&counting)), + format!("{want:?}"), + "{}", + path.display() + ); + let total = counting.reads() as usize; + let step = (total / 25).max(1); + for fail_at in (1..=total).step_by(step) { + for short in [false, true] { + if short && fail_at != 1 { + continue; + } + let adv = Adversary { + data: hdf5.to_vec(), + reads: Default::default(), + fail_at: if short { 0 } else { fail_at }, + short, + }; + let got = everything(&adv); + compared += 1; + match (&got, &want) { + (Ok(g), Ok(w)) => { + // Per-object results inside may be errors; values + // that were read must be the right ones. + for (gl, wl) in g.lines().zip(w.lines()) { + if gl != wl { + assert!( + gl.contains("Err("), + "{}: fail_at {fail_at} short {short}:\n got {gl}\n want {wl}", + path.display() + ); + failures_seen += 1; + // A listing that failed ends the walk + // differently from here on. + if gl.contains("children: Err(") { + break; + } + } + } + } + (Err(FormatError::Storage(_)), _) => failures_seen += 1, + (Err(e), Ok(_)) => panic!( + "{}: fail_at {fail_at} short {short}: {e:?} instead of a storage error", + path.display() + ), + (Err(_), Err(_)) => {} + // A listing failed, so the walk never reached the + // object that fails in memory. + (Ok(g), Err(e)) => assert!( + g.contains("children: Err(Storage"), + "{}: fail_at {fail_at} short {short}: read where memory fails ({e:?})", + path.display() + ), + } + } + } + } + eprintln!("misbehaving storage: {compared} runs, {failures_seen} failures reported"); + assert!( + compared > 500 && failures_seen > 100, + "{compared} {failures_seen}" + ); +} + +/// A read_at-only storage that also counts `read_ranges` calls and ranges. +struct BatchCounting { + inner: CountingStorage, + batches: std::sync::atomic::AtomicUsize, + ranges: std::sync::atomic::AtomicUsize, +} + +impl Storage for BatchCounting { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + self.inner.read_at(offset, len) + } + + fn len(&self) -> u64 { + self.inner.len() + } + + fn read_ranges( + &self, + ranges: &[std::ops::Range], + ) -> Result>, FormatError> { + use std::sync::atomic::Ordering::Relaxed; + self.batches.fetch_add(1, Relaxed); + self.ranges.fetch_add(ranges.len(), Relaxed); + ranges + .iter() + .map(|r| self.inner.read_at(r.start, (r.end - r.start) as usize)) + .collect() + } +} + +/// A chunked read lists its chunks, then fetches all their bytes with one +/// `read_ranges` call (a remote backend coalesces and parallelises it), and +/// a selection fetches only the chunks it overlaps, in one call too. +#[test] +fn chunked_reads_fetch_their_chunks_in_one_batch() { + use std::sync::atomic::Ordering::Relaxed; + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut datasets = 0; + for name in [ + "chunked_large.h5", + "chunked_deflate.h5", + "chunked_2d.h5", + "v4_fixed_array.h5", + ] { + let bytes = std::fs::read(dir.join(name)).unwrap(); + let sb = Superblock::parse(&bytes, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + let st = BatchCounting { + inner: CountingStorage::new(bytes.clone()), + batches: Default::default(), + ranges: Default::default(), + }; + for child in group_v2::resolve_group_children(&bytes, &sb, sb.root_group_address).unwrap() { + let header = + ObjectHeader::parse(&bytes, child.object_header_address as usize, os, ls).unwrap(); + let msg = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| m.data.clone()) + }; + let Some(dl) = msg(MessageType::DataLayout) else { + continue; + }; + let dl = DataLayout::parse(&dl, os, ls).unwrap(); + if !matches!(dl, DataLayout::Chunked { .. }) { + continue; + } + let dt = Datatype::parse(&msg(MessageType::Datatype).unwrap()) + .unwrap() + .0; + let ds = Dataspace::parse(&msg(MessageType::Dataspace).unwrap(), ls).unwrap(); + let pl = msg(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p).unwrap()); + let es = dt.type_size() as usize; + let (chunks, _) = list_chunks(&bytes, &dl, &ds, es, os, ls).unwrap(); + let want = read_raw_data_full(&bytes, &dl, &ds, &dt, pl.as_ref(), os, ls).unwrap(); + st.batches.store(0, Relaxed); + st.ranges.store(0, Relaxed); + let got = read_raw_data_full_in(&st, &dl, &ds, &dt, pl.as_ref(), os, ls).unwrap(); + assert_eq!(got, want, "{name} {}", child.name); + assert_eq!(st.batches.load(Relaxed), 1, "{name} {}", child.name); + assert_eq!( + st.ranges.load(Relaxed), + chunks.len(), + "{name} {}", + child.name + ); + // The first chunk only. + let rank = ds.dimensions.len(); + let sel = Selection::Hyperslab { + start: vec![0; rank], + stride: vec![1; rank], + count: vec![1; rank], + block: vec![1; rank], + }; + let want = read_raw_data_selection(&bytes, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel); + st.batches.store(0, Relaxed); + st.ranges.store(0, Relaxed); + let got = read_raw_data_selection_in(&st, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel); + assert_eq!(got, want, "{name} {}", child.name); + if chunks.len() > 2 { + assert_eq!(st.batches.load(Relaxed), 1, "{name} {}", child.name); + assert_eq!(st.ranges.load(Relaxed), 1, "{name} {}", child.name); + } + datasets += 1; + } + } + assert!(datasets >= 4, "{datasets}"); +} + fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) } From 1c3ef98828e5bcd104063f0a84c6eb66af41a3ad Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:41:47 -0500 Subject: [PATCH 03/10] clawhdf5: File::open_storage reads any Storage through the full API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File::open_storage(Arc) opens a file served by any backend: groups, datasets, attributes, read_*, selections, VL data and virtual datasets (external sources through the new File::set_vds_resolver) all read through Storage::read_at/read_ranges. The file's view (user block skipped, bounded by the recorded end of file) is itself a Storage; File::open and File::from_bytes keep their mmap and in-memory paths, now as that view's as_contiguous() fast path. A storage-backed file's metadata cache image is laid over each read it covers (new CacheImage::entries), as libhdf5 loads it. The typed readers keep their fast paths over any storage: a contiguous dataset is read in one piece and converted (read_f64 and friends), and a contiguous native selection reads only its runs (data_read::read_selection_native_in). Zero-copy methods answer ContiguousStorageRequired when the bytes are not in memory, and File::as_bytes panics there (File::contiguous_bytes is the fallible form). Test: tests/storage_equivalence.rs reads every fixture, and with CLAWHDF5_STORAGE_CORPUS every conformance-corpus file, through File::open and through open_storage over a read_at-only CountingStorage — tree, attributes, and every dataset's values several ways — and requires identical transcripts; it prints the read_at calls and bytes a one-pass read costs per file. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 72 +++ crates/clawhdf5-format/src/superblock_ext.rs | 17 + crates/clawhdf5/src/lib.rs | 3 +- crates/clawhdf5/src/reader.rs | 395 ++++++++++++++--- crates/clawhdf5/src/types.rs | 22 +- crates/clawhdf5/src/vlen.rs | 19 +- crates/clawhdf5/tests/storage_equivalence.rs | 434 +++++++++++++++++++ 7 files changed, 869 insertions(+), 93 deletions(-) create mode 100644 crates/clawhdf5/tests/storage_equivalence.rs diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 28bc4f2..40826c1 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -956,6 +956,78 @@ pub fn read_selection_native( crate::gather::gather::(raw, dims, elem_size, selection).map(Some) } +/// [`read_selection_native`] of a contiguous dataset in any [`Storage`], +/// reading only the selected elements' runs (adjacent ones merged, one +/// [`Storage::read_ranges`] call) instead of the whole dataset. +/// +/// `Ok(None)` wherever the in-memory fast path does not apply and the +/// caller converts through the byte readers instead: `datatype` is not +/// `T`'s native representation, the layout is not contiguous, or the +/// dataset's bytes cannot be located in the file (no address, storage too +/// small, past the end of file: the cases [`read_raw_data_zerocopy`] +/// fails). Otherwise the result and errors are [`read_selection_native`]'s +/// over those bytes. +pub fn read_selection_native_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + selection: &crate::selection::Selection, +) -> Result>, FormatError> { + if !T::is_native(datatype) { + return Ok(None); + } + let DataLayout::Contiguous { + address: Some(address), + size, + } = layout + else { + return Ok(None); + }; + // Where the dataset's bytes are, as `read_raw_data_zerocopy` finds them. + let located = to_usize(dataspace.num_elements()) + .ok() + .and_then(|n| n.checked_mul(datatype.type_size() as usize)) + .filter(|&len| contiguous_read_len(*size, len).is_ok()) + .filter(|&len| { + address + .checked_add(len as u64) + .is_some_and(|end| end <= file_data.len()) + }); + let Some(len) = located else { + return Ok(None); + }; + if let Some(all) = file_data.as_contiguous() { + let start = to_usize(*address)?; + return read_selection_native( + &all[start..start + len], + &dataspace.dimensions, + datatype, + selection, + ); + } + let dims = &dataspace.dimensions; + let elem_size = core::mem::size_of::(); + let total = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?; + let expected = crate::chunked_read::checked_byte_len(total, elem_size)?; + if len != expected { + return Err(FormatError::DataSizeMismatch { + expected, + actual: len, + }); + } + let bytes = if let crate::selection::Selection::All = selection { + read_exact_at(file_data, *address, len)?.into_owned() + } else { + crate::partial_read::validate(selection, dims)?; + crate::gather::gather_storage(file_data, *address, len, dims, elem_size, selection)? + }; + Ok(Some(native_to_vec(&bytes, bytes.len() / elem_size))) +} + /// The bytes of a slice of [`NativeElement`]s. #[cfg(feature = "std")] fn bytes_of_mut(values: &mut [T]) -> &mut [u8] { diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index 4a66473..7edbda6 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -490,6 +490,23 @@ impl CacheImage { image_block_in(file, self.location) } + /// Every entry as `(file address, its bytes)`, taken from `block` (the + /// image block, see [`Self::block_in`]), in the order [`Self::apply`] + /// writes them: for a reader that cannot write the image over the + /// file's bytes and lays the entries over each read instead. + pub fn entries<'b>(&self, block: &'b [u8]) -> Result, FormatError> { + let short = || FormatError::InvalidCacheImage("image applied to the wrong file"); + self.entries + .iter() + .map(|e| { + let src = block + .get(e.image_offset..e.image_offset + e.len) + .ok_or_else(short)?; + Ok((e.address, src)) + }) + .collect() + } + /// Write every entry over `dst`, the file's bytes from the superblock /// on (as long as the `data` the image was decoded from), taking the /// entries from `block` (the image block, see [`Self::block`]). `block` diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 280f64b..5ceec87 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -56,7 +56,7 @@ pub use error::Error; pub use lazy::{LazyDataset, LazyFile, LazyGroup}; #[cfg(feature = "mmap")] pub use mmap_file::{MmapDataset, MmapFile, MmapGroup}; -pub use reader::{Dataset, File, Group}; +pub use reader::{Dataset, File, Group, SharedStorage, VdsResolver}; pub use types::{AttrValue, DType}; pub use vlen::VlenValue; pub use writer::FileBuilder; @@ -72,6 +72,7 @@ pub use clawhdf5_format::property_list::{ #[cfg(feature = "provenance")] pub use clawhdf5_format::provenance; pub use clawhdf5_format::selection::Selection; +pub use clawhdf5_format::storage::Storage; pub use clawhdf5_format::superblock::swmr_flags; pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime}; diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index fcd3b67..6bd7b35 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -5,7 +5,10 @@ //! the traditional read-into-`Vec` fallback. [`File::from_bytes`] remains //! available for in-memory usage (tests, etc.). +use std::borrow::Cow; use std::collections::HashMap; +use std::ops::Range; +use std::sync::Arc; use clawhdf5_format::chunk_cache::ChunkCache; use clawhdf5_format::data_layout::DataLayout; @@ -19,29 +22,44 @@ use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; +use clawhdf5_format::storage::Storage; use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext::{self, CacheImageState}; use crate::cache_image::{self, ImageView}; use crate::error::Error; use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; // --------------------------------------------------------------------------- -// FileData — internal storage for either owned bytes or an mmap +// FileData — internal storage for owned bytes, an mmap, or any Storage // --------------------------------------------------------------------------- -/// Internal storage: either an owned `Vec` or a memory-mapped region. +/// A [`Storage`] a [`File`] can be opened over: any backend that can be +/// shared between threads (see [`File::open_storage`]). +pub type SharedStorage = Arc; + +/// Resolves an external Virtual Dataset source file name to that file's +/// bytes (`Ok(None)`: the file does not exist, its mappings read as the +/// fill value). See [`File::set_vds_resolver`]. +pub type VdsResolver = Arc Result>, FormatError> + Send + Sync>; + +/// Internal storage: an owned `Vec`, a memory-mapped region, or a +/// [`Storage`] backend. enum Backing { Owned(Vec), #[cfg(feature = "mmap")] Mmap(clawhdf5_io::MmapReader), + Storage(SharedStorage), } impl Backing { - fn whole_file(&self) -> &[u8] { + /// The whole file, when it is in memory. + fn whole_file(&self) -> Option<&[u8]> { match self { - Backing::Owned(v) => v, + Backing::Owned(v) => Some(v), #[cfg(feature = "mmap")] - Backing::Mmap(r) => r.as_bytes(), + Backing::Mmap(r) => Some(r.as_bytes()), + Backing::Storage(s) => s.as_contiguous(), } } } @@ -49,13 +67,16 @@ impl Backing { /// The file's bytes, viewed from the superblock on and up to the end of /// file the superblock records. A file may start with a user block (the /// superblock at 512, 1024, …); every HDF5 address is relative to the -/// superblock, so all parsing goes through [`Self::as_bytes`]. +/// superblock, so all parsing goes through this view, which is a +/// [`Storage`]: in memory (a `Vec`, an mmap) its reads are slices of the +/// file, as before; over any other storage they are reads of that storage, +/// shifted by the user block and bounded by the end of file. struct FileData { backing: Backing, /// Offset of the superblock in the file (the user-block size). - base: usize, + base: u64, /// End of the HDF5 data in the file (`Superblock::data_end`, absolute). - end: usize, + end: u64, /// A mapped file that holds a metadata cache image, with the image /// written in: a private copy-on-write mapping of the whole file, so /// only the pages the image's entries land on are copied (see @@ -63,6 +84,10 @@ struct FileData { /// an image is read straight from the mapping, and an owned buffer has /// the image written into it in place. patched: Option, + /// A [`Storage`]-backed file's metadata cache image: its entries + /// (address relative to the superblock, bytes), laid over every read in + /// order, as libhdf5 reads them instead of the file's own bytes. + overlay: Vec<(u64, Vec)>, /// The file has a metadata cache image libhdf5 cannot load. libhdf5 /// opens such a file and fails its first metadata read (the image loads /// then); every object lookup here fails with this error, and no @@ -74,7 +99,10 @@ impl FileData { /// Locate the superblock and parse it. A truncated file is refused, and /// bytes past the recorded end of file are not read, as in libhdf5. fn new(mut backing: Backing) -> Result<(Self, Superblock), Error> { - let whole = backing.whole_file(); + if let Backing::Storage(storage) = backing { + return Self::new_storage(storage); + } + let whole = backing.whole_file().unwrap_or_default(); let (user_block, hdf5) = signature::split_user_block(whole)?; let base = user_block.len(); let superblock = Superblock::parse(hdf5, 0)?; @@ -92,6 +120,7 @@ impl FileData { clawhdf5_io::HDF5Read::private_copy(r) })? } + Backing::Storage(_) => unreachable!("handled above"), }; let (patched, image_error) = match view { ImageView::Plain => (None, None), @@ -101,34 +130,140 @@ impl FileData { Ok(( Self { backing, - base, - end, + base: base as u64, + end: end as u64, patched, + overlay: Vec::new(), image_error, }, superblock, )) } - fn as_bytes(&self) -> &[u8] { - match &self.patched { - Some(p) => &p[self.base..self.end], - None => &self.backing.whole_file()[self.base..self.end], + /// [`Self::new`] for a [`Storage`] backend: the same checks, through + /// reads of the storage. + fn new_storage(storage: SharedStorage) -> Result<(Self, Superblock), Error> { + let file_len = storage.len(); + let base = signature::find_signature_in(&*storage)?; + let mut data = Self { + backing: Backing::Storage(storage), + base, + end: file_len, + patched: None, + overlay: Vec::new(), + image_error: None, + }; + let superblock = Superblock::parse_in(&data, 0)?; + data.end = base + superblock.data_end(base, file_len)?; + match superblock_ext::cache_image_state_in(&data, &superblock)? { + CacheImageState::Absent => {} + CacheImageState::Unloadable(e) => data.image_error = Some(e), + CacheImageState::Loaded(image) => { + let block = image.block_in(&data)?.into_owned(); + data.overlay = image + .entries(&block)? + .into_iter() + .map(|(addr, bytes)| (addr, bytes.to_vec())) + .collect(); + } } + Ok((data, superblock)) } - fn len(&self) -> usize { - self.as_bytes().len() + /// The HDF5 data as one slice, when the file is in memory (a `Vec`, an + /// mmap, or a storage that holds it all and has no cache image to lay + /// over it). + fn contiguous(&self) -> Option<&[u8]> { + if let Some(p) = &self.patched { + return p.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?); + } + if !self.overlay.is_empty() { + return None; + } + self.backing + .whole_file()? + .get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?) } /// The bytes to read metadata from; fails for a file whose cache image /// cannot be loaded (see [`Self::image_error`]). - fn meta(&self) -> Result<&[u8], FormatError> { + fn meta(&self) -> Result<&Self, FormatError> { match &self.image_error { Some(e) => Err(e.clone()), - None => Ok(self.as_bytes()), + None => Ok(self), } } + + /// The backend of a file that is not in memory. + fn remote(&self) -> Result<&SharedStorage, FormatError> { + match &self.backing { + Backing::Storage(s) => Ok(s), + _ => Err(FormatError::Storage( + "in-memory file has no contiguous view".into(), + )), + } + } + + /// `bytes`, read at `offset`, with the cache image entries they overlap + /// written over them. + fn with_overlay<'a>(&self, offset: u64, mut bytes: Cow<'a, [u8]>) -> Cow<'a, [u8]> { + let end = offset + bytes.len() as u64; + for (addr, entry) in &self.overlay { + let entry_end = addr + entry.len() as u64; + if *addr >= end || entry_end <= offset { + continue; + } + let from = (*addr).max(offset); + let to = entry_end.min(end); + let dst = bytes.to_mut(); + dst[(from - offset) as usize..(to - offset) as usize] + .copy_from_slice(&entry[(from - addr) as usize..(to - addr) as usize]); + } + bytes + } +} + +impl Storage for FileData { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if let Some(all) = self.contiguous() { + return all.read_at(offset, len); + } + let size = self.end - self.base; + let len = usize::try_from(size.saturating_sub(offset)).map_or(len, |avail| avail.min(len)); + if len == 0 { + return Ok(Cow::Borrowed(&[])); + } + let bytes = self.remote()?.read_at(self.base + offset, len)?; + Ok(self.with_overlay(offset, bytes)) + } + + fn len(&self) -> u64 { + self.end - self.base + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if let Some(all) = self.contiguous() { + return all.read_ranges(ranges); + } + let size = self.end - self.base; + let shifted: Vec> = ranges + .iter() + .map(|r| { + let (s, e) = (r.start.min(size), r.end.min(size)); + self.base + s..self.base + e.max(s) + }) + .collect(); + let got = self.remote()?.read_ranges(&shifted)?; + Ok(got + .into_iter() + .zip(ranges) + .map(|(bytes, r)| self.with_overlay(r.start, bytes)) + .collect()) + } + + fn as_contiguous(&self) -> Option<&[u8]> { + self.contiguous() + } } // --------------------------------------------------------------------------- @@ -149,6 +284,9 @@ pub struct File { /// Directory the file was opened from, used to resolve external Virtual /// Dataset source files relative to this file. `None` for in-memory files. base_dir: Option, + /// Resolves external Virtual Dataset source files instead of + /// `base_dir` (see [`File::set_vds_resolver`]). + vds_resolver: Option, } impl File { @@ -167,6 +305,7 @@ impl File { superblock, chunk_cache: ChunkCache::new(), base_dir, + vds_resolver: None, }) } #[cfg(not(feature = "mmap"))] @@ -200,9 +339,53 @@ impl File { superblock, chunk_cache: ChunkCache::new(), base_dir: None, + vds_resolver: None, }) } + /// Open an HDF5 file served by any [`Storage`]: a range-reading remote + /// backend, a block cache, or an in-memory buffer — through the same + /// read API as [`File::open`] (groups, datasets, attributes, `read_*`, + /// selections, variable-length data, virtual datasets). + /// + /// Every read goes through the storage's [`Storage::read_at`] and + /// [`Storage::read_ranges`] (a chunked read fetches all the chunks it + /// needs with one `read_ranges` call per 64 MiB), so nothing is read that + /// the operation does not need. A storage that has the whole file in + /// memory ([`Storage::as_contiguous`]) is read as [`File::from_bytes`] + /// reads its buffer. The storage holds the whole file: a user block is + /// found and skipped, and bytes past the end of file the superblock + /// records are not read. A metadata cache image is laid over the reads + /// it covers, as libhdf5 loads it. + /// + /// The zero-copy methods ([`Dataset::read_raw_ref`], + /// [`Dataset::read_as_slice`], `read_*_zerocopy`) borrow the file's bytes + /// and return [`FormatError::ContiguousStorageRequired`] over a storage + /// that does not hold them; [`File::as_bytes`] panics there (use + /// [`File::contiguous_bytes`]). External virtual-dataset source files + /// are read through [`File::set_vds_resolver`]; without one they cannot + /// be resolved. + pub fn open_storage(storage: SharedStorage) -> Result { + let (data, superblock) = FileData::new(Backing::Storage(storage))?; + Ok(Self { + data, + superblock, + chunk_cache: ChunkCache::new(), + base_dir: None, + vds_resolver: None, + }) + } + + /// Resolve external Virtual Dataset source files (their names as the + /// mappings store them) with `resolver`, instead of reading them from + /// the directory of the file (for [`File::open`]) or refusing them (for + /// in-memory and [`Storage`]-backed files). `Ok(None)` means the source + /// file does not exist, and its mappings read as the fill value, as in + /// libhdf5; `Err` fails the read. + pub fn set_vds_resolver(&mut self, resolver: VdsResolver) { + self.vds_resolver = Some(resolver); + } + /// Returns a handle to the root group. pub fn root(&self) -> Group<'_> { Group { @@ -216,7 +399,7 @@ impl File { /// The path uses `/` separators (e.g., `"group1/values"`). pub fn dataset(&self, path: &str) -> Result, Error> { let data = self.data.meta()?; - let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; + let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); @@ -263,7 +446,7 @@ impl File { /// Use `"/"` or `""` for the root group. pub fn group(&self, path: &str) -> Result, Error> { let data = self.data.meta()?; - let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; + let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; Ok(Group { file: self, address: addr, @@ -322,8 +505,22 @@ impl File { /// with a metadata cache image these are the bytes with the image /// applied; when the image cannot be loaded they are the file's own /// bytes, whose metadata may be stale (every object lookup fails then). + /// + /// # Panics + /// + /// For a file opened with [`File::open_storage`] over a storage that + /// does not hold the whole file in memory; use + /// [`contiguous_bytes`](Self::contiguous_bytes) there. pub fn as_bytes(&self) -> &[u8] { - self.data.as_bytes() + self.data + .contiguous() + .expect("File::as_bytes: the file is not in memory (see File::contiguous_bytes)") + } + + /// [`as_bytes`](Self::as_bytes), or `None` for a file whose bytes are + /// not all in memory ([`File::open_storage`]). + pub fn contiguous_bytes(&self) -> Option<&[u8]> { + self.data.contiguous() } /// The error of a metadata cache image libhdf5 cannot load, when the @@ -338,7 +535,7 @@ impl File { /// Size of the user block before the superblock (0 for most files). /// Matches h5py's `File.userblock_size`. pub fn user_block_size(&self) -> u64 { - self.data.base as u64 + self.data.base } /// Returns a reference to the parsed superblock. @@ -349,7 +546,7 @@ impl File { /// Returns `true` when the file is backed by memory-mapped I/O. pub fn is_mmap(&self) -> bool { match &self.data.backing { - Backing::Owned(_) => false, + Backing::Owned(_) | Backing::Storage(_) => false, #[cfg(feature = "mmap")] Backing::Mmap(_) => true, } @@ -362,7 +559,7 @@ impl File { /// this file's global heap; see [`Dataset::read_string`] for the values. pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result, Error> { crate::vlen::decode_strings( - self.as_bytes(), + &self.data, datatype, raw, self.offset_size(), @@ -406,9 +603,9 @@ impl File { } fn parse_header(&self, address: u64) -> Result { - ObjectHeader::parse( + ObjectHeader::parse_in( self.data.meta()?, - address as usize, + address, self.superblock.offset_size, self.superblock.length_size, ) @@ -426,7 +623,7 @@ impl File { impl std::fmt::Debug for File { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("File") - .field("size", &self.data.len()) + .field("size", &Storage::len(&self.data)) .field("superblock_version", &self.superblock.version) .field("mmap", &self.is_mmap()) .finish() @@ -489,7 +686,7 @@ impl<'f> Group<'f> { &self, ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.parse_header(self.address)?; - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) } @@ -520,7 +717,7 @@ impl<'f> Group<'f> { /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { let hdr = self.file.parse_header(self.address)?; - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attr( data, &hdr, @@ -536,7 +733,7 @@ impl<'f> Group<'f> { /// [`group_v2::resolve_child`]). fn child_address(&self, name: &str) -> Result { let data = self.file.data.meta()?; - group_v2::resolve_child(data, &self.file.superblock, self.address, name) + group_v2::resolve_child_in(data, &self.file.superblock, self.address, name) .map_err(Error::Format) } @@ -559,7 +756,7 @@ impl<'f> Group<'f> { /// user-defined links are left out. fn children(&self) -> Result, Error> { let data = self.file.data.meta()?; - group_v2::resolve_group_children(data, &self.file.superblock, self.address) + group_v2::resolve_group_children_in(data, &self.file.superblock, self.address) .map_err(Error::Format) } } @@ -589,7 +786,7 @@ impl<'f> Dataset<'f> { Ok((self.datatype()?, ds, self.data_layout()?)) })(); if let Ok((dt, ds, dl)) = decoded { - data_read::check_dataset_storage(&dl, &ds, &dt, self.file.data.len() as u64)?; + data_read::check_dataset_storage(&dl, &ds, &dt, Storage::len(&self.file.data))?; } Ok(self) } @@ -629,8 +826,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_f64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_f64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -649,8 +846,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_f32(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_f32(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -664,8 +861,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_i32(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_i32(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -679,8 +876,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_i64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_i64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -694,8 +891,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_u64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_u64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -793,8 +990,8 @@ impl<'f> Dataset<'f> { // sparse) dataset — select from a fill-aware full read instead. (The // selection reader currently decodes the full dataset too, so this // costs nothing extra.) - let fill = clawhdf5_format::fill_value::dataset_fill_value_in( - self.file.data.as_bytes(), + let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage( + &self.file.data, &self.header.messages, self.file.offset_size(), self.file.length_size(), @@ -813,8 +1010,8 @@ impl<'f> Dataset<'f> { selection, )?); } - Ok(data_read::read_raw_data_selection( - self.file.data.as_bytes(), + Ok(data_read::read_raw_data_selection_in( + &self.file.data, &dl, &ds, &dt, @@ -871,12 +1068,26 @@ impl<'f> Dataset<'f> { return full(); } let dt = self.datatype()?; - if T::is_native(&dt) - && let Ok(Some(raw)) = self.read_raw_ref() + if T::is_native(&dt) && self.file.data.contiguous().is_some() { + if let Ok(Some(raw)) = self.read_raw_ref() { + let dims = self.dataspace()?.dimensions; + if let Some(values) = + data_read::read_selection_native::(raw, &dims, &dt, selection)? + { + return Ok(values); + } + } + } else if T::is_native(&dt) + && let (Ok(dl), Ok(ds)) = (self.data_layout(), self.dataspace()) { - let dims = self.dataspace()?.dimensions; - if let Some(values) = data_read::read_selection_native::(raw, &dims, &dt, selection)? - { + // Not in memory: only the selected runs are read. + if let Some(values) = data_read::read_selection_native_in::( + &self.file.data, + &dl, + &ds, + &dt, + selection, + )? { return Ok(values); } } @@ -893,10 +1104,46 @@ impl<'f> Dataset<'f> { let dl = self.data_layout()?; let ds = self.dataspace()?; let dt = self.datatype()?; - let slice = data_read::read_raw_data_zerocopy(self.file.data.as_bytes(), &dl, &ds, &dt)?; + let Some(bytes) = self.file.data.contiguous() else { + // The bytes are not in memory to borrow. + return match dl { + DataLayout::Contiguous { .. } => { + Err(Error::Format(FormatError::ContiguousStorageRequired( + "a zero-copy read (the file is not in memory)", + ))) + } + _ => Ok(None), + }; + }; + let slice = data_read::read_raw_data_zerocopy(bytes, &dl, &ds, &dt)?; Ok(slice) } + /// A contiguous dataset's stored bytes, for the typed readers' fast + /// path: borrowed from the file when it is in memory + /// ([`read_raw_ref`](Self::read_raw_ref)), read in one piece otherwise. + /// `Ok(None)` for other layouts; an error where `read_raw_ref` fails. + fn contiguous_raw(&self) -> Result>, Error> { + if self.file.data.contiguous().is_some() { + return Ok(self.read_raw_ref()?.map(Cow::Borrowed)); + } + let dl = self.data_layout()?; + if !matches!(dl, DataLayout::Contiguous { .. }) { + return Ok(None); + } + let ds = self.dataspace()?; + let dt = self.datatype()?; + Ok(Some(Cow::Owned(data_read::read_raw_data_full_in( + &self.file.data, + &dl, + &ds, + &dt, + None, + self.file.offset_size(), + self.file.length_size(), + )?))) + } + /// Zero-copy typed read of contiguous data as `&[T]`. /// /// Returns a borrowed slice of `T` directly from the file buffer with @@ -1083,7 +1330,7 @@ impl<'f> Dataset<'f> { pub fn attrs_with_errors( &self, ) -> Result<(HashMap, Vec), Error> { - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attrs( data, &self.header, @@ -1097,7 +1344,7 @@ impl<'f> Dataset<'f> { /// that name, found without reading the other attributes when they are /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attr( data, &self.header, @@ -1124,8 +1371,8 @@ impl<'f> Dataset<'f> { /// result is not a tamper-evidence or authenticity guarantee. #[cfg(feature = "provenance")] pub fn verify_provenance(&self) -> Result { - Ok(clawhdf5_format::provenance::verify_dataset( - self.file.as_bytes(), + Ok(clawhdf5_format::provenance::verify_dataset_in( + &self.file.data, &self.header, self.file.offset_size(), self.file.length_size(), @@ -1144,8 +1391,8 @@ impl<'f> Dataset<'f> { .iter() .find(|m| m.msg_type == msg_type) .map(|msg| { - clawhdf5_format::shared_message::message_data( - self.file.as_bytes(), + clawhdf5_format::shared_message::message_data_in( + &self.file.data, msg, self.file.offset_size(), self.file.length_size(), @@ -1179,8 +1426,8 @@ impl<'f> Dataset<'f> { // one (`H5Dget_space`). if let Ok(dl @ DataLayout::Virtual { .. }) = self.data_layout() { let resolver = self.vds_resolver(); - ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent( - self.file.data.as_bytes(), + ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent_in( + &self.file.data, &dl, &ds, self.file.offset_size(), @@ -1225,9 +1472,9 @@ impl<'f> Dataset<'f> { } let ds = self.dataspace()?; let pipeline = self.filter_pipeline()?; - Ok(data_read::read_chunked_native::( + Ok(data_read::read_chunked_native_in::( &self.header.messages, - self.file.data.as_bytes(), + &self.file.data, &dl, &ds, &dt, @@ -1251,17 +1498,17 @@ impl<'f> Dataset<'f> { } // Unallocated storage reads as the dataset's fill value. - clawhdf5_format::fill_value::read_full_with_fill( + clawhdf5_format::fill_value::read_full_with_fill_in( &self.header.messages, - self.file.data.as_bytes(), + &self.file.data, &dl, &ds, dt.type_size() as usize, self.file.offset_size(), self.file.length_size(), || { - Ok(data_read::read_raw_data_cached( - self.file.data.as_bytes(), + Ok(data_read::read_raw_data_cached_in( + &self.file.data, &dl, &ds, &dt, @@ -1281,7 +1528,11 @@ impl<'f> Dataset<'f> { /// refused with an error rather than read as fill. fn vds_resolver(&self) -> impl Fn(&str) -> Result>, FormatError> + use<> { let base_dir = self.file.base_dir.clone(); + let custom = self.file.vds_resolver.clone(); move |name: &str| { + if let Some(resolve) = &custom { + return resolve(name); + } let Some(dir) = base_dir.as_ref() else { return Err(FormatError::ChunkedReadError(format!( "virtual dataset source file {name:?} cannot be resolved for an in-memory file" @@ -1310,15 +1561,15 @@ impl<'f> Dataset<'f> { ds: &Dataspace, dt: &Datatype, ) -> Result, Error> { - let fill = clawhdf5_format::fill_value::dataset_fill_value_in( - self.file.data.as_bytes(), + let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage( + &self.file.data, &self.header.messages, self.file.offset_size(), self.file.length_size(), )?; let resolver = self.vds_resolver(); - let v = clawhdf5_format::vds::read_virtual_dataset( - self.file.data.as_bytes(), + let v = clawhdf5_format::vds::read_virtual_dataset_in( + &self.file.data, dl, ds, dt, @@ -1439,7 +1690,7 @@ mod zero_copy_tests { let Backing::Mmap(r) = &f.data.backing else { return None; }; - let mapped = r.as_bytes()[f.data.base..].as_ptr(); + let mapped = r.as_bytes()[f.data.base as usize..].as_ptr(); match &f.data.patched { None => Some(std::ptr::eq(f.as_bytes().as_ptr(), mapped)), Some(p) => { diff --git a/crates/clawhdf5/src/types.rs b/crates/clawhdf5/src/types.rs index acae280..b3f7823 100644 --- a/crates/clawhdf5/src/types.rs +++ b/crates/clawhdf5/src/types.rs @@ -158,8 +158,8 @@ pub(crate) fn classify_datatype(dt: &clawhdf5_format::datatype::Datatype) -> DTy /// The attributes of the object with header `header` that could be read, /// and one error for each that could not (see /// [`extract_attributes_tolerant`](clawhdf5_format::attribute::extract_attributes_tolerant)). -pub(crate) fn read_attrs( - file_data: &[u8], +pub(crate) fn read_attrs( + file_data: &S, header: &clawhdf5_format::object_header::ObjectHeader, offset_size: u8, length_size: u8, @@ -170,7 +170,7 @@ pub(crate) fn read_attrs( ), crate::Error, > { - let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant( + let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant_in( file_data, header, offset_size, @@ -185,14 +185,14 @@ pub(crate) fn read_attrs( /// The attribute called `name` on the object with header `header`, decoded /// as [`read_attrs`] decodes it, or `None` (see /// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)). -pub(crate) fn read_attr( - file_data: &[u8], +pub(crate) fn read_attr( + file_data: &S, header: &clawhdf5_format::object_header::ObjectHeader, name: &str, offset_size: u8, length_size: u8, ) -> Result, crate::Error> { - let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file( + let Some(msg) = clawhdf5_format::attribute::find_attribute_in( file_data, header, name, @@ -211,9 +211,9 @@ pub(crate) fn read_attr( .remove(name)) } -pub(crate) fn attrs_to_map( +pub(crate) fn attrs_to_map( attrs: &[clawhdf5_format::attribute::AttributeMessage], - file_data: &[u8], + file_data: &S, offset_size: u8, length_size: u8, ) -> HashMap { @@ -266,9 +266,9 @@ fn decode_bool_enum(attr: &clawhdf5_format::attribute::AttributeMessage) -> Opti values.iter().all(|v| *v == 0 || *v == 1).then_some(values) } -fn decode_attr_value( +fn decode_attr_value( attr: &clawhdf5_format::attribute::AttributeMessage, - file_data: &[u8], + file_data: &S, offset_size: u8, length_size: u8, ) -> Option { @@ -311,7 +311,7 @@ fn decode_attr_value( is_string: true, .. } => { let strings = attr - .read_vl_strings(file_data, offset_size, length_size) + .read_vl_strings_in(file_data, offset_size, length_size) .ok()?; if strings.len() == 1 { Some(AttrValue::String(strings[0].clone())) diff --git a/crates/clawhdf5/src/vlen.rs b/crates/clawhdf5/src/vlen.rs index 01ba02d..f471b24 100644 --- a/crates/clawhdf5/src/vlen.rs +++ b/crates/clawhdf5/src/vlen.rs @@ -14,6 +14,7 @@ use clawhdf5_format::data_read; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; use clawhdf5_format::vl_data::{VlResolver, check_element_size}; use crate::error::Error; @@ -68,8 +69,8 @@ fn class_name(dt: &Datatype) -> &'static str { /// The strings in `raw`, elements of `dt`: fixed-length strings decoded as /// `read_string` always has, variable-length strings resolved in the heap. -pub(crate) fn decode_strings( - file_data: &[u8], +pub(crate) fn decode_strings( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -82,15 +83,15 @@ pub(crate) fn decode_strings( .. } => { check_element_size(*size, offset_size)?; - Ok(VlResolver::new(file_data, offset_size, length_size).strings(raw)?) + Ok(VlResolver::new_in(file_data, offset_size, length_size).strings(raw)?) } _ => Ok(data_read::read_as_strings(raw, dt)?), } } /// The exact bytes of the variable-length strings in `raw`. -pub(crate) fn decode_string_bytes( - file_data: &[u8], +pub(crate) fn decode_string_bytes( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -103,7 +104,7 @@ pub(crate) fn decode_string_bytes( .. } => { check_element_size(*size, offset_size)?; - Ok(VlResolver::new(file_data, offset_size, length_size).string_bytes(raw)?) + Ok(VlResolver::new_in(file_data, offset_size, length_size).string_bytes(raw)?) } other => Err(Error::Format(FormatError::TypeMismatch { expected: "variable-length string", @@ -114,8 +115,8 @@ pub(crate) fn decode_string_bytes( /// The sequences in `raw`, elements of the variable-length sequence type /// `dt`, converted to `T`. -pub(crate) fn decode_vlen( - file_data: &[u8], +pub(crate) fn decode_vlen( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -135,7 +136,7 @@ pub(crate) fn decode_vlen( }; check_element_size(*size, offset_size)?; let base_size = base_type.type_size() as usize; - VlResolver::new(file_data, offset_size, length_size) + VlResolver::new_in(file_data, offset_size, length_size) .sequences(raw, base_size)? .iter() .map(|bytes| Ok(T::decode(bytes, base_type)?)) diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs new file mode 100644 index 0000000..7a2749b --- /dev/null +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -0,0 +1,434 @@ +//! `File::open_storage` over a read_at-only storage reads every file as +//! `File::open` does (range reads, milestone M2 in +//! `docs/design/range-reads.md`). +//! +//! Each file is read end to end twice — through `File::open` (the mmap +//! fast path) and through `File::open_storage` over a +//! [`CountingStorage`], which serves the file through `read_at` only +//! (`as_contiguous()` is `None`, so no reader can fall back to a slice of +//! the whole file) — and the two transcripts must be identical: the tree +//! (every group's entries, followed by address), every object's attributes +//! (the whole map and each one by name), and every dataset's shape, types +//! and values (all bytes, as `f64`, a hyperslab of them, strings and +//! variable-length sequences). +//! +//! The storage also counts its `read_at` calls and bytes: what a remote +//! backend without a cache would be asked for. The totals and the files +//! that cost most are printed. +//! +//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every HDF5 file under those +//! directories (the conformance corpus is `conformance/.cache/corpus`); +//! `CLAWHDF5_STORAGE_REPORT=1` prints every file's counts. + +use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clawhdf5::{DType, File, Selection}; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::CountingStorage; + +/// Objects visited per file. +const MAX_OBJECTS: usize = 2000; +/// Datasets with more bytes than this are not read (their metadata is). +const MAX_DATA_BYTES: u64 = 64 << 20; + +/// A short, stable digest of a value's `Debug` form. +fn digest(v: &T) -> String { + let s = format!("{v:?}"); + if s.len() <= 200 { + return s; + } + let mut h = 0xcbf2_9ce4_8422_2325u64; + for b in s.bytes() { + h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); + } + format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len()) +} + +/// A data read's result: its value, or just `Err` — a full read goes +/// through the file's chunk cache, which lists a damaged dataset's chunks +/// in hash-map order, so which failing chunk it reports varies from one +/// `File` to the next (the format crate's harness compares these errors on +/// the uncached path). +fn value(r: &Result) -> String { + match r { + Ok(v) => digest(v), + Err(_) => "Err".into(), + } +} + +fn transcript(file: &File) -> String { + let mut out = String::new(); + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]); + while let Some((path, addr)) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let entries = group.entries(); + writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap(); + match file.dataset_at(addr) { + Ok(ds) => dataset(&mut out, &path, &ds), + Err(e) => writeln!(out, "{path} dataset_at {e:?}").unwrap(), + } + let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap(); + if let Ok((attrs, _)) = &attrs { + for name in attrs.keys().take(50) { + writeln!(out, "{path} attr {name:?} {}", digest(&group.attr(name))).unwrap(); + } + } + if let Ok(entries) = entries { + for (name, child) in entries { + queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child)); + // Name lookups (through the name index of a dense group). + if queue.len() < 64 { + writeln!( + out, + "{path} group({name:?}) {}", + digest(&group.group(&name).map(|_| ())) + ) + .unwrap(); + } + } + } + } + out +} + +fn sorted(m: std::collections::HashMap) -> BTreeMap { + m.into_iter().collect() +} + +fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { + let shape = ds.shape(); + let dtype = ds.dtype(); + writeln!( + out, + "{path} shape {} max {} dtype {} raw {}", + digest(&shape), + digest(&ds.max_dimensions()), + digest(&dtype), + digest(&ds.raw_datatype()) + ) + .unwrap(); + let attrs = ds.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} dataset attrs {}", digest(&attrs)).unwrap(); + let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else { + return; + }; + let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d)); + let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size()))); + if bytes.is_none_or(|b| b > MAX_DATA_BYTES) { + writeln!(out, "{path} too large to read").unwrap(); + return; + } + writeln!( + out, + "{path} all {}", + value(&ds.read_selection(&Selection::All)) + ) + .unwrap(); + let numeric = matches!( + dtype, + DType::F32 + | DType::F64 + | DType::I8 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::U8 + | DType::U16 + | DType::U32 + | DType::U64 + ); + if numeric { + writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap(); + writeln!(out, "{path} f32 {}", value(&ds.read_f32())).unwrap(); + writeln!(out, "{path} i64 {}", value(&ds.read_i64())).unwrap(); + if let Some(&d0) = shape.first() { + let rank = shape.len(); + let sel = Selection::Hyperslab { + start: std::iter::once(d0 / 3) + .chain(std::iter::repeat_n(0, rank - 1)) + .collect(), + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(3)) + .chain(shape[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + writeln!( + out, + "{path} f64 third {}", + value(&ds.read_f64_selection(&sel)) + ) + .unwrap(); + writeln!( + out, + "{path} bytes third {}", + value(&ds.read_selection(&sel)) + ) + .unwrap(); + } + } + match &raw_dt { + clawhdf5_format::datatype::Datatype::String { .. } + | clawhdf5_format::datatype::Datatype::VariableLength { + is_string: true, .. + } => { + writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap(); + writeln!(out, "{path} string {}", value(&ds.read_string())).unwrap(); + } + clawhdf5_format::datatype::Datatype::VariableLength { .. } => { + writeln!(out, "{path} vlen {}", value(&ds.read_vlen::())).unwrap(); + } + _ => {} + } +} + +/// External virtual-dataset sources as `File::open` finds them: files in +/// the same directory. +fn sibling_resolver(dir: Option) -> clawhdf5::VdsResolver { + Arc::new(move |name: &str| { + let Some(dir) = dir.as_ref() else { + return Err(FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} cannot be resolved for an in-memory file" + ))); + }; + let p = Path::new(name); + if name.is_empty() + || !p.components().all(|c| { + matches!( + c, + std::path::Component::Normal(_) | std::path::Component::CurDir + ) + }) + { + return Err(FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} is outside the virtual file's \ + directory and is not followed" + ))); + } + match std::fs::read(dir.join(p)) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(FormatError::ChunkedReadError(format!( + "cannot read virtual dataset source file {name:?}: {e}" + ))), + } + }) +} + +#[derive(Default)] +struct Totals { + files: usize, + opened: usize, + /// The comparison's reads (each dataset read several ways). + reads: u64, + bytes: u64, + /// One pass: open, list every group, read every attribute and every + /// dataset once (`read_selection(All)`). + pass_reads: u64, + pass_bytes: u64, + file_bytes: u64, + /// (one-pass reads, one-pass bytes, file size, name) per file. + per_file: Vec<(u64, u64, u64, String)>, +} + +/// Open the file and read everything once, as a tree viewer that then +/// shows every value would. +fn one_pass(file: &File) { + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([file.superblock().root_group_address]); + while let Some(addr) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let _ = group.attrs(); + if let Ok(ds) = file.dataset_at(addr) { + let small = ds.shape().ok().and_then(|s| { + let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; + let size = u64::from(ds.raw_datatype().ok()?.type_size()); + n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) + }); + if small.is_some() { + let _ = ds.read_selection(&Selection::All); + } + } + if let Ok(entries) = group.entries() { + queue.extend(entries.into_iter().map(|(_, a)| a)); + } + } +} + +fn check(path: &Path, totals: &mut Totals) { + let Ok(bytes) = std::fs::read(path) else { + return; + }; + let name = path.display().to_string(); + let local = File::open(path); + let storage = Arc::new(CountingStorage::new(bytes.clone())); + let resolver = sibling_resolver(path.parent().map(Path::to_path_buf)); + let remote = File::open_storage(storage.clone()).map(|mut f| { + f.set_vds_resolver(resolver.clone()); + f + }); + totals.files += 1; + let (local, remote) = match (local, remote) { + (Ok(l), Ok(r)) => (l, r), + (l, r) => { + // Both refuse the file, with the same error. + assert_eq!( + format!("{:?}", l.map(|_| ())), + format!("{:?}", r.map(|_| ())), + "{name}: open" + ); + return; + } + }; + totals.opened += 1; + assert!(remote.contiguous_bytes().is_none(), "{name}"); + assert_eq!(local.user_block_size(), remote.user_block_size(), "{name}"); + let want = transcript(&local); + let got = transcript(&remote); + if want != got { + let first = want + .lines() + .zip(got.lines()) + .find(|(w, g)| w != g) + .map(|(w, g)| format!("\n local: {w}\n storage: {g}")) + .unwrap_or_else(|| { + format!( + "\n {} vs {} lines", + want.lines().count(), + got.lines().count() + ) + }); + panic!("{name}: File::open_storage differs from File::open{first}"); + } + totals.reads += storage.reads(); + totals.bytes += storage.bytes_read(); + totals.file_bytes += bytes.len() as u64; + + let pass = Arc::new(CountingStorage::new(bytes.clone())); + if let Ok(mut f) = File::open_storage(pass.clone()) { + f.set_vds_resolver(resolver); + one_pass(&f); + } + let (reads, read_bytes) = (pass.reads(), pass.bytes_read()); + totals.pass_reads += reads; + totals.pass_bytes += read_bytes; + if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") { + eprintln!( + "{reads:>9} reads {read_bytes:>12} bytes {:>12} file {name}", + bytes.len() + ); + } + totals + .per_file + .push((reads, read_bytes, bytes.len() as u64, name)); +} + +fn report(what: &str, totals: &mut Totals) { + eprintln!( + "{what}: {} files ({} open, {} bytes); comparison: {} read_at calls, {} bytes; \ + one pass (list, attributes, every dataset once): {} read_at calls, {} bytes", + totals.files, + totals.opened, + totals.file_bytes, + totals.reads, + totals.bytes, + totals.pass_reads, + totals.pass_bytes + ); + totals.per_file.sort_by_key(|a| std::cmp::Reverse(a.0)); + for (reads, bytes, size, name) in totals.per_file.iter().take(10) { + eprintln!(" {reads:>9} reads {bytes:>12} bytes (file {size:>11}) {name}"); + } +} +fn hdf5_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + hdf5_files(&p, out); + } else if p + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf")) + { + out.push(p); + } + } +} + +#[test] +fn fixtures_read_identically_through_open_storage() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + files.sort(); + assert!(files.len() >= 45, "{} fixtures", files.len()); + let mut totals = Totals::default(); + for f in &files { + check(f, &mut totals); + } + report("fixtures", &mut totals); + assert!(totals.opened >= 40, "{}", totals.opened); + assert!(totals.reads > 0); +} + +#[test] +fn corpus_reads_identically_through_open_storage() { + let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else { + eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus"); + return; + }; + let mut files = Vec::new(); + for d in std::env::split_paths(&dirs) { + hdf5_files(&d, &mut files); + } + files.sort(); + let mut totals = Totals::default(); + for f in &files { + check(f, &mut totals); + } + report("corpus", &mut totals); + assert!(totals.files > 0); +} + +/// A user block, a metadata cache image and a Storage that is itself in +/// memory: the in-memory view of a storage that has one is used as is. +#[test] +fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let path = root.join("tests/fixtures/h5clear_mdc_image.h5"); + let bytes = std::fs::read(&path).unwrap(); + let local = File::open(&path).unwrap(); + // A Vec is a Storage with a contiguous view; the image still has + // to be laid over it, so the view is not used. + let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap(); + assert!(in_memory.contiguous_bytes().is_none()); + assert_eq!(transcript(&local), transcript(&in_memory)); + + let plain = root.join("../clawhdf5-format/tests/fixtures/chunked_2d.h5"); + let bytes = std::fs::read(&plain).unwrap(); + let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap(); + assert_eq!(in_memory.contiguous_bytes(), Some(&bytes[..])); + let counting = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap(); + assert!(counting.contiguous_bytes().is_none()); + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| counting.as_bytes().len())); + assert!( + result.is_err(), + "as_bytes over a range storage must not answer" + ); +} From f191dc09d5537e90eb141a0f04077cdbf0cef64f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:49:42 -0500 Subject: [PATCH 04/10] =?UTF-8?q?docs:=20range-read=20milestone=20M2=20?= =?UTF-8?q?=E2=80=94=20changelog,=20limits,=20design=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHANGELOG (Unreleased): File::open_storage, raw data and v2 B-trees over Storage, the tests and their corpus results (2026-09-26, tank; conformance 600 of 697, results.json identical to 8f59b2e). known-issues: what open_storage does not do yet (no remote backend or block cache, read_at counts of a one-pass read, v1 group lookups, whole-file VDS sources, zero-copy methods, SWMR growth, hash-order error choice on damaged chunked datasets). Design: M2 status and the choices made. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 94 +++++++++++++++++++++++++++++++++++++- docs/design/range-reads.md | 35 ++++++++++++-- docs/known-issues.md | 38 +++++++++++++++ 3 files changed, 163 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa0a30f..661a2eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,97 @@ ## Unreleased +### Range reads, milestone M2: raw data and `File::open_storage` (2026-09-26) +- **`clawhdf5::File::open_storage(Arc)`** opens + a file served by any `clawhdf5_format::storage::Storage` and gives the + whole read API over it: groups and paths, datasets, attributes, the + `read_*` methods, selections, variable-length strings and sequences, and + virtual datasets. Every byte comes through `Storage::read_at` / + `read_ranges`; a user block is found and skipped, nothing past the + superblock's end of file is read, and a metadata cache image is laid over + the reads it covers (new `CacheImage::entries`). External virtual-dataset + sources are read through the new `File::set_vds_resolver` (any `File`; + without one a storage-backed file cannot follow them). `File::open` and + `File::from_bytes` keep their mmap and in-memory paths: the file's view + is now a `Storage` whose `as_contiguous()` is that buffer, and every hot + loop takes it. New exports: `clawhdf5::{Storage, SharedStorage, + VdsResolver}`, `File::contiguous_bytes()`. + - Over a storage without the whole file in memory the zero-copy methods + (`read_raw_ref`, `read_as_slice`, `read_*_zerocopy`) answer + `FormatError::ContiguousStorageRequired`, and `File::as_bytes` panics + (documented; use `contiguous_bytes`). The typed readers keep their fast + paths: a contiguous dataset is read in one piece and converted, and a + contiguous selection of a native type reads only its runs + (`data_read::read_selection_native_in`). +- **Nothing in the format crate needs the whole file any more.** The + structures M1 left to `ContiguousStorageRequired` read through `Storage`: + v2 B-trees (`BTreeV2Header::parse_in`, `collect_btree_v2_records_in`, + `find_btree_v2_records_in`; one bounded read per node, whose size is known + before it is read), hence dense attributes, a SOHM B-tree index, huge + fractal-heap objects, and dense groups; v1 and v2 group listings, lookups + and paths (`group_v2::resolve_group_children_in`, `resolve_child_in`, + `resolve_path_any_in`, `group_v1::*_in`). +- **Raw data reads through `Storage`**, each with a generic `*_in` core and + its `&[u8]` function as a thin wrapper (callers do not change): + `data_read` (`read_raw_data*_in`, `read_raw_data_selection_in`, + `read_chunked_native_in`), `chunked_read` (the v1 B-tree chunk index — + one read of each node's header, one of its entries — `list_chunks_in`, + and the full, cached, sweep and indexed reads), `parallel_read`, + `partial_read`, `fill_value` (`read_full_with_fill_in`, + `apply_to_unallocated_chunks_in`; `dataset_fill_value_from_storage` is + now generic, so a `&dyn Storage` still works), `vds` + (`read_virtual_dataset_in`, `virtual_dataset_extent_in`: the virtual + file through `Storage`, external source files still loaded whole through + the resolver), `vl_data` (`VlResolver<'a, S = [u8]>` with `new_in`; + `read_vl_strings_in`, `read_vl_bytes_in`), + `AttributeMessage::read_vl_strings_in`, `provenance::verify_dataset_in`. + - A chunked read first lists the chunks it needs, then fetches all their + stored bytes with **one `read_ranges` call** (per 64 MiB of stored + data), so a remote backend can coalesce and parallelise them, then + decodes as before (in parallel with the `parallel` feature). Chunks the + file's chunk cache already holds are not fetched. A selection fetches + only the chunks its bounding box overlaps; a contiguous selection only + its runs (adjacent ones merged). A global-heap collection is read once + per resolver and kept (within the resolver's 32 MiB budget). + - Each extent's bounds error is the one the slice readers gave, reported + when the read reaches that extent, so a damaged file fails with the + same error, in the same order, through either path. +- **No behaviour change for in-memory and mapped files:** with + `as_contiguous()` every path slices the file as before (checked below). +- Tests (2026-09-26, tank): + - `clawhdf5-format/tests/storage_equivalence.rs` now also reads every + dataset — whole, fill-aware, through a chunk cache (twice) and the + indexed path, three selections, virtual datasets with their sibling + sources, VL strings, sequences and bytes — through the read_at-only + `CountingStorage` and requires the slice results, and fails on any + `ContiguousStorageRequired`. With + `CLAWHDF5_STORAGE_CORPUS=conformance/.cache/corpus`, all 653 HDF5 + files of the corpus agree (82 396 checks). The cached and indexed + paths are compared on values only when a read fails: they order + chunks by hash map, so which failing chunk a damaged dataset reports + varies between two caches even for the same slice (seen on + `cve-2025-2310.h5`; see `docs/known-issues.md`). + - A misbehaving storage (fails its N-th read; serves short reads) over + every fixture: each listing and dataset read is an error or exactly the + in-memory result, never other data (1 137 runs). + - A chunked read issues one `read_ranges` call with one range per chunk, + and a one-chunk selection one call with one range. + - `clawhdf5/tests/storage_equivalence.rs` reads every fixture (61 files) + and, with `CLAWHDF5_STORAGE_CORPUS`, every corpus file (701 files, 621 + that open) through `File::open` and through `File::open_storage` over + `CountingStorage`: the tree, every attribute (all, and each by name), + every dataset's shape, types and values (all bytes, `f64`, `f32`, + `i64`, a hyperslab, strings, VL sequences) must be identical, and are. + It also counts what one pass — open, list, read every attribute and + every dataset once — asks of a storage with no cache: 176 092 `read_at` + calls and 208 MB for the 621 corpus files (254 MB of files); the most + are `h5stat_newgrat.h5` (35 001 groups: 92 489 calls) and + `ref_hdf5_compat1.nc` (16 062). A remote backend needs the block cache + of milestone M3. Command: `CLAWHDF5_STORAGE_CORPUS=… cargo test + --release -p clawhdf5 --test storage_equivalence -- --nocapture`. + - Conformance sweep (`conformance/run.sh --no-fetch`): 600 of 697 files + ok, `results.json` byte-identical to `8f59b2e`. + ### Name lookups through the name index (2026-09-26) - **Finding one link or attribute by name reads the name index, not every entry.** In a dense group (links in a fractal heap) the v2 B-tree name @@ -108,7 +199,8 @@ v2 B-tree and dense groups come with milestone M3); over a backend without the whole file in memory they are the clean `ContiguousStorageRequired` error, never a partial result. Raw data, chunk B-tree (v1) indexes and VL - data are milestone M2. + data are milestone M2. (All of them read through `Storage` since M2, + above.) - **No behaviour change**, checked three ways (2026-09-26, tank): every existing test passes unchanged; the conformance sweep (`conformance/run.sh --no-fetch`) gives a byte-identical `results.json` diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1c7fa55..1098922 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -1,9 +1,12 @@ # Design: range reads (reading HDF5 without holding the whole file) Status: proposal, 2026-09-26; the plan for Phase 3's largest architectural -change. Progress: M1, first part (the `Storage` trait and the metadata -parsers listed in `CHANGELOG.md` under "Range reads, milestone M1") is done; -group B-tree v2 lookups, dense groups and the facade are not converted yet. Every count below was +change. Progress: M0 and M1 are done, and so is M2 (branch +`feat/p3-m2-raw-data`): every read path of the format crate works through +`Storage`, v2 B-trees, dense groups and raw data included, and +`File::open_storage` gives the facade's read API over any `Storage` (see +`CHANGELOG.md`, "Range reads, milestone M2"). M3 (a remote backend with +its block cache) is next. Every count in §1–§2 was taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given next to it. No timing numbers appear here on purpose: the machine was shared with other build jobs when this was written. @@ -421,6 +424,32 @@ fast path within benchmark noise. on other backends (they already return `Option`/`Result`). - Facade: `File::open_storage(Box)`; `File::open` keeps mmap and `from_bytes` keeps `Vec`, both through `impl Storage for [u8]`. +- *Status 2026-09-26:* done on branch `feat/p3-m2-raw-data`. As planned, + with these choices: + - `File::open_storage` takes an `Arc` (the + file handle is shared by its datasets and may be sent across threads). + The file's view (user block skipped, bounded by the recorded end of + file, cache image laid over its reads) is itself a `Storage`, and the + facade calls the generic cores with it; for a `Vec` or an mmap its + `as_contiguous()` is the buffer, so the local paths are the slice code + (checked: identical conformance results; the bench gate below still + has to be run on an idle machine). + - Chunked reads fetch in batches of at most 64 MiB of stored bytes, one + `read_ranges` call each, so a remote read never holds more than that + undecoded; chunks already in the chunk cache are not fetched. + - The typed readers' zero-copy fast path became "read the contiguous + bytes once": over a range storage a contiguous `read_f64` is one read, + and a native contiguous selection reads only its runs. + - External VDS source files stay whole-file, through a resolver that + returns bytes (`File::set_vds_resolver`). + - The v2 B-tree and dense groups were done here rather than in M3, so no + format-crate path answers `ContiguousStorageRequired` any more; only the + facade's zero-copy methods do. + - Measured with the M2 harness (`crates/clawhdf5/tests/storage_equivalence.rs`, + tank, 2026-09-26): one pass over the 621 corpus files that open — list, + every attribute, every dataset once — is 176 092 `read_at` calls and + 208 MB through a storage with no cache (254 MB of files). The block + cache of M3 is what turns that into requests (§2). **M3 — HTTP/S3 backend (1–2 weeks).** - `clawhdf5-io`, feature `remote` (off by default, so the default tree stays diff --git a/docs/known-issues.md b/docs/known-issues.md index dcb365f..313bc8e 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -763,6 +763,44 @@ the same agent-store interop test. **Fix:** an empty contiguous dataset gets the undefined address (all `0xff`), which is what libhdf5 itself writes. +## Range reads (`File::open_storage`) limits + +**Status:** open (added 2026-09-26, milestone M2 of +`docs/design/range-reads.md`). `File::open_storage` reads any +`clawhdf5_format::storage::Storage` through the whole read API, and every +format-crate read path works through `Storage::read_at`/`read_ranges`, but: + +- **No remote backend and no block cache yet** (milestone M3). A `Storage` + is asked for each structure as the parsers need it, several times over + for some (an object header is re-read by each lookup through it): one + pass over the conformance corpus — open, list, every attribute, every + dataset once — is 176 092 `read_at` calls for 621 files, 92 489 of them + for the 35 001-group `h5stat_newgrat.h5` (2026-09-26, tank, + `crates/clawhdf5/tests/storage_equivalence.rs` with + `CLAWHDF5_STORAGE_CORPUS`). A backend over a network needs a cache in + front of it. `Storage::read_ranges` defaults to one `read_at` per range; + coalescing is the backend's job. +- A group lookup by name in a version-1 (symbol-table) group lists the whole + group (dense groups use their name index). Over a range backend that is + one read per symbol-table node and name, per lookup. +- External virtual-dataset source files are loaded whole through the + resolver (`File::set_vds_resolver`), as bytes; they are not read through + a `Storage`. +- The zero-copy methods (`Dataset::read_raw_ref`, `read_as_slice`, + `read_*_zerocopy`) need the file in memory and answer + `FormatError::ContiguousStorageRequired` otherwise; `File::as_bytes()` + panics for such a file (`File::contiguous_bytes()` is the fallible form). + `LazyFile`, `MmapFile`, the Python and wasm bindings and `h5rs` still read + a whole file. +- The file's length is read once, at open: a growing file (SWMR) is not + followed (milestone M5). +- Not new, but visible through the equivalence tests: a full read through + the file's chunk cache (`read_raw_data_cached`, `read_raw_data_indexed`, + and so `Dataset::read_*`) lists a damaged dataset's chunks in hash-map + order, so which failing chunk it reports can differ from one `File` to + the next (`cve-2025-2310.h5`); the values of a dataset that reads are + not affected. + ## `clawhdf5-wasm` (browser) limits **Status:** open (by design for now; added 2026-09-26). From 7d629f49e3fa6864dc6fcc527a3bc4a0087918d0 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:30:30 -0500 Subject: [PATCH 05/10] format: bound and batch every chunk fetch over Storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the full read split its chunk fetches into 64 MiB batches. The selection path, the indexed read and the parallel_read decoders fetched every chunk's stored bytes in one read_ranges call, each extent bounded only by the file length, so a crafted chunk index pointing many chunks at one large extent made File::open_storage hold chunks x extent bytes (3.3 GB from a 16.8 MB file) before the first decode error. - storage::for_each_extent_batch is now the one way raw-data reads fetch chunk bytes: batches of at most RAW_BATCH_BYTES (now pub), each decoded before the next is fetched. Used by the full, cached, indexed, selection and parallel_read paths; the sweep read uses read_extent per chunk. - ExtentReq carries each chunk's claimed extent (bounds-checked as before, same errors) and the prefix actually fetched: filters::stored_chunk_limit — the chunk size if unfiltered, else each applied filter's worst-case growth (n + n/4 + 4096 per codec; unbounded only for an application-registered codec). The in-memory path cuts the slice it decodes the same way, so both paths still agree. - tests/raw_fetch_bounds.rs: a crafted chunked_large.h5 (ten chunks all claiming 20 MiB at one padding blob) read through every path over a storage that records the largest single fetch; and 160 MiB of legitimate unfiltered chunks fetched batch by batch. Before: one 80 MiB fetch (selection) and one 160 MiB fetch; after: within the budget. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +- crates/clawhdf5-format/src/chunked_read.rs | 203 ++++++----- crates/clawhdf5-format/src/filter_registry.rs | 16 + crates/clawhdf5-format/src/filters.rs | 38 ++ crates/clawhdf5-format/src/parallel_read.rs | 238 +++++++------ crates/clawhdf5-format/src/partial_read.rs | 90 ++--- crates/clawhdf5-format/src/storage.rs | 162 ++++++--- .../clawhdf5-format/tests/raw_fetch_bounds.rs | 332 ++++++++++++++++++ crates/clawhdf5/src/reader.rs | 8 +- docs/design/range-reads.md | 8 +- 10 files changed, 825 insertions(+), 285 deletions(-) create mode 100644 crates/clawhdf5-format/tests/raw_fetch_bounds.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 661a2eb..21e857d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,9 +47,18 @@ `read_vl_strings_in`, `read_vl_bytes_in`), `AttributeMessage::read_vl_strings_in`, `provenance::verify_dataset_in`. - A chunked read first lists the chunks it needs, then fetches all their - stored bytes with **one `read_ranges` call** (per 64 MiB of stored - data), so a remote backend can coalesce and parallelise them, then - decodes as before (in parallel with the `parallel` feature). Chunks the + stored bytes with **one `read_ranges` call** per batch of at most 64 MiB + (`storage::RAW_BATCH_BYTES`), so a remote backend can coalesce and + parallelise them, then decodes each batch as before (in parallel with + the `parallel` feature) before fetching the next. Every path that reads + chunks — full, cached, indexed, sweep, selection and the + `parallel_read` decoders — goes through the same batching, and no chunk + fetches more of its stored bytes than its decoded size can need (the + chunk size if unfiltered; else each applied filter's worst-case growth, + generously: `n + n/4 + 4096` per codec, unbounded only for a codec the + application registered). A crafted chunk index that points every chunk + at one huge extent therefore costs a bounded fetch, not + `chunks x extent` bytes (`tests/raw_fetch_bounds.rs`). Chunks the file's chunk cache already holds are not fetched. A selection fetches only the chunks its bounding box overlaps; a contiguous selection only its runs (adjacent ones merged). A global-heap collection is read once diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 140cf37..fc35c38 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -19,7 +19,7 @@ use crate::filters::{DecodeScratch, decompress_chunk_exact_with}; #[cfg(feature = "std")] use crate::filters::{all_filters_skipped, decompress_chunk_exact}; use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks_in}; -use crate::storage::{ExtentBytes, Storage, Window, raw_batches, read_exact_at}; +use crate::storage::{ExtentBytes, ExtentReq, Storage, Window, for_each_extent_batch, read_extent}; #[cfg(feature = "std")] use std::sync::Arc; @@ -256,52 +256,17 @@ fn fill_from_chunks( cache: CacheUse<'_>, output: &mut [u8], ) -> Result<(), FormatError> { - let contiguous = file_data.as_contiguous().is_some(); let out = OutBuf::new(output); - let batches = raw_batches(chunks.len(), contiguous, |i| chunks[i].chunk_size as usize); - for batch in batches { - fill_batch( - file_data, - &chunks[batch], - pipeline, - placer, - chunk_total_bytes, - cache, - &out, - )?; - } - Ok(()) -} - -/// Whether a full read looks chunk `c` up in the cache (see -/// [`fill_from_chunks`]): a filtered chunk of a dataset the cache keeps. -#[cfg(feature = "std")] -fn uses_cache(cache: CacheUse<'_>, pipeline: Option<&FilterPipeline>, c: &ChunkInfo) -> bool { - matches!(cache, Some((_, _, true))) - && pipeline.is_some_and(|pl| !all_filters_skipped(pl, c.filter_mask)) -} - -/// [`fill_from_chunks`] for one batch of chunks. -#[allow(clippy::too_many_arguments)] -fn fill_batch( - file_data: &S, - chunks: &[ChunkInfo], - pipeline: Option<&FilterPipeline>, - placer: &ChunkPlacer, - chunk_total_bytes: usize, - cache: CacheUse<'_>, - out: &OutBuf<'_>, -) -> Result<(), FormatError> { - let rank = placer.rank; - let elem_size = placer.elem_size as u32; #[cfg(not(feature = "std"))] let _ = cache; + #[cfg(feature = "std")] + let rank = placer.rank; // Over a backend without the whole file in memory, the decoded chunks // the cache holds are taken now (so a chunk evicted before it is placed // is not left without bytes), and only the others are fetched. #[cfg(feature = "std")] - let hits: Vec>> = match cache { + let hits: Vec = match cache { Some((cache, key, true)) if file_data.as_contiguous().is_none() => chunks .iter() .map(|c| { @@ -314,25 +279,90 @@ fn fill_batch( .collect(), _ => Vec::new(), }; - let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { - Vec::new() - } else { - chunks - .iter() - .enumerate() - .map(|(i, c)| { - #[cfg(feature = "std")] - let wanted = hits.get(i).is_none_or(Option::is_none); - #[cfg(not(feature = "std"))] - let wanted = { - let _ = i; - true - }; - (c.address, c.chunk_size as usize, wanted) - }) - .collect() - }; - let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + #[cfg(not(feature = "std"))] + let hits: Vec = Vec::new(); + let reqs: Vec = chunks + .iter() + .enumerate() + .map(|(i, c)| { + let wanted = hits.get(i).is_none_or(Option::is_none); + chunk_req(c, pipeline, chunk_total_bytes, wanted) + }) + .collect(); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + fill_batch( + &chunks[batch.clone()], + &reqs[batch.clone()], + hits.get(batch.clone()).unwrap_or_default(), + batch.start, + raw_bytes, + pipeline, + placer, + chunk_total_bytes, + cache, + &out, + ) + }) +} + +/// The extent of chunk `c`'s stored bytes to fetch (`wanted`) or only +/// bounds-check: its whole stored size is checked against the file, and at +/// most [`crate::filters::stored_chunk_limit`] of it is read and decoded. +pub(crate) fn chunk_req( + c: &ChunkInfo, + pipeline: Option<&FilterPipeline>, + chunk_bytes: usize, + wanted: bool, +) -> ExtentReq { + let len = c.chunk_size as usize; + ExtentReq { + addr: c.address, + len, + fetch: wanted.then(|| { + len.min(crate::filters::stored_chunk_limit( + pipeline, + c.filter_mask, + chunk_bytes, + )) + }), + } +} + +/// A decoded chunk a full read took from the cache. +#[cfg(feature = "std")] +type CacheHit = Option>; +#[cfg(not(feature = "std"))] +type CacheHit = Option; + +/// Whether a full read looks chunk `c` up in the cache (see +/// [`fill_from_chunks`]): a filtered chunk of a dataset the cache keeps. +#[cfg(feature = "std")] +fn uses_cache(cache: CacheUse<'_>, pipeline: Option<&FilterPipeline>, c: &ChunkInfo) -> bool { + matches!(cache, Some((_, _, true))) + && pipeline.is_some_and(|pl| !all_filters_skipped(pl, c.filter_mask)) +} + +/// [`fill_from_chunks`] for one batch of chunks: `chunks` (with their +/// extents `reqs` and, over a backend without the file in memory, the cache +/// hits taken for them) are chunks `first..` of the read, whose stored bytes +/// `raw_bytes` holds. +#[allow(clippy::too_many_arguments)] +fn fill_batch( + chunks: &[ChunkInfo], + reqs: &[ExtentReq], + hits: &[CacheHit], + first: usize, + raw_bytes: &ExtentBytes<'_>, + pipeline: Option<&FilterPipeline>, + placer: &ChunkPlacer, + chunk_total_bytes: usize, + cache: CacheUse<'_>, + out: &OutBuf<'_>, +) -> Result<(), FormatError> { + let rank = placer.rank; + let elem_size = placer.elem_size as u32; + #[cfg(not(feature = "std"))] + let _ = cache; // Decode chunk `i` and place it. Its callers below run it either on one // thread, or on several for chunks whose regions are pairwise disjoint, @@ -346,15 +376,14 @@ fn fill_batch( ))); } let offsets = &c.offsets[..rank]; - let size = c.chunk_size as usize; #[cfg(feature = "std")] if let Some(Some(hit)) = hits.get(i) { - raw_bytes.check(i, c.address, size)?; + raw_bytes.check(first + i, &reqs[i])?; // SAFETY: see above. unsafe { placer.place(hit, offsets, out) }; return Ok(()); } - let raw = raw_bytes.get(i, c.address, size)?; + let raw = raw_bytes.get(first + i, &reqs[i])?; let Some(pl) = pipeline else { // SAFETY: see above. unsafe { placer.place(raw, offsets, out) }; @@ -1913,9 +1942,9 @@ pub fn read_chunked_data_sweep_in( cached } else { // Decompress from file - let c_addr = to_usize(chunk_info.address)?; - let size = chunk_info.chunk_size as usize; - let raw_chunk = read_exact_at(file_data, c_addr as u64, size)?; + to_usize(chunk_info.address)?; + let req = chunk_req(chunk_info, pipeline, chunk_total_bytes, true); + let raw_chunk = read_extent(file_data, &req)?; let raw_chunk = &*raw_chunk; let dec = if let Some(pl) = pipeline { decompress_chunk_exact( @@ -2052,46 +2081,60 @@ pub fn read_chunked_data_indexed_in( let chunk_total_bytes = plan.chunk_total_bytes; // The decoded chunks the cache holds, and the stored bytes of the - // others: fetched in one batch when the file is not in memory. + // others: fetched batch by batch when the file is not in memory. let hits: Vec>> = plan .mappings .iter() .map(|m| cache.get_decompressed_in(addr, &m.coord)) .collect(); - let extents: Vec<(u64, usize, bool)> = plan + let reqs: Vec = plan .mappings .iter() .zip(&hits) - .map(|(m, hit)| (m.file_offset, m.file_size as usize, hit.is_none())) + .map(|(m, hit)| { + let len = m.file_size as usize; + ExtentReq { + addr: m.file_offset, + len, + fetch: hit.is_none().then(|| { + len.min(crate::filters::stored_chunk_limit( + pipeline, + m.filter_mask, + chunk_total_bytes, + )) + }), + } + }) .collect(); - let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; // Decompress chunks (using LRU cache where possible) let mut chunk_buffers: Vec> = Vec::with_capacity(plan.mappings.len()); - for (i, (m, hit)) in plan.mappings.iter().zip(hits).enumerate() { - let (coord, file_offset, file_size, filter_mask) = - (&m.coord, &m.file_offset, &m.file_size, &m.filter_mask); - if let Some(cached) = hit { - chunk_buffers.push(cached); - } else { - let raw_chunk = raw_bytes.get(i, *file_offset, *file_size as usize)?; + let mut hits = hits.into_iter(); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + for i in batch { + let m = &plan.mappings[i]; + if let Some(cached) = hits.next().flatten() { + chunk_buffers.push(cached); + continue; + } + let raw_chunk = raw_bytes.get(i, &reqs[i])?; let decompressed = if let Some(pl) = pipeline { decompress_chunk_exact( raw_chunk, pl, chunk_total_bytes, elem_size as u32, - *filter_mask, - coord, + m.filter_mask, + &m.coord, )? } else { raw_chunk.to_vec() }; let aligned = CacheAlignedBuffer::from_vec(decompressed); - let arc = cache.put_decompressed_aligned_in(addr, coord.clone(), aligned); - chunk_buffers.push(arc); + chunk_buffers.push(cache.put_decompressed_aligned_in(addr, m.coord.clone(), aligned)); } - } + Ok(()) + })?; // Assemble using pre-computed layout let mut output = vec![0u8; plan.output_bytes]; diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 7670a8b..9855c24 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -188,6 +188,22 @@ pub fn is_filter_available(id: u16) -> bool { } } +/// Whether chunks filtered with `id` may be decoded by a codec the +/// application registered (whose stored sizes this crate cannot bound). +pub(crate) fn may_be_registered(id: u16) -> bool { + if builtin_filter(id).is_some_and(|b| !b.is_shared()) { + return false; + } + #[cfg(feature = "std")] + { + registered(id).is_some() + } + #[cfg(not(feature = "std"))] + { + false + } +} + #[cfg(feature = "std")] mod custom { use super::FilterCodec; diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 6390aee..e247983 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -63,6 +63,44 @@ fn filter_output_bound(filter_id: u16, input: usize) -> usize { } } +/// Most stored bytes a chunk whose decoded size is `chunk_bytes` can need: +/// what a raw-data read fetches of (and decodes from) a chunk, whatever size +/// its index entry claims. A crafted index that points many chunks at huge +/// extents then costs no more than legitimate chunks would. +/// +/// An unfiltered chunk (no pipeline, or every filter skipped by +/// `filter_mask`) is the chunk itself: `chunk_bytes`, and bytes past them +/// were never used. A filtered chunk is bounded by each applied filter's +/// worst-case growth in the write direction: shuffle keeps the size, +/// Fletcher32 adds 4 bytes, and any other codec gets `n + n/4 + 4096` — a +/// deliberately generous bound (bzip2 grows 1000 random bytes by 252, more +/// than the decoders' own `n + n/8 + 64` output bound), since a legitimate +/// chunk cut short here would fail to read. A filter handled by a codec the +/// application registered is not ours to bound: such a chunk is limited +/// only by the file. +pub(crate) fn stored_chunk_limit( + pipeline: Option<&FilterPipeline>, + filter_mask: u32, + chunk_bytes: usize, +) -> usize { + let Some(pipeline) = pipeline else { + return chunk_bytes; + }; + let mut size = chunk_bytes; + for (i, filter) in pipeline.filters.iter().enumerate() { + if filter_skipped(filter_mask, i) { + continue; + } + size = match filter.filter_id { + FILTER_SHUFFLE => size, + FILTER_FLETCHER32 => size.saturating_add(4), + id if filter_registry::may_be_registered(id) => return usize::MAX, + _ => size.saturating_add(size / 4).saturating_add(4096), + }; + } + size +} + /// Whether bit `index` of a chunk's filter mask says filter `index` was /// skipped when the chunk was written. fn filter_skipped(filter_mask: u32, index: usize) -> bool { diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 75a4bd3..606173c 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -12,24 +12,21 @@ use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::decompress_chunk_exact; use crate::lane_partition::{self, LaneStats, PartitionStats}; -use crate::storage::{ExtentBytes, Storage}; +use crate::storage::{ExtentReq, Storage, for_each_extent_batch}; -/// The stored bytes of every chunk in `chunks`, fetched in one -/// [`Storage::read_ranges`] call when the file is not in memory (each -/// chunk's bounds error is reported when that chunk is decoded, as before). -fn fetch_all<'a, S: Storage + ?Sized>( - file_data: &'a S, +/// The extents of `chunks`' stored bytes (see +/// [`crate::chunked_read::chunk_req`]), fetched batch by batch with +/// [`for_each_extent_batch`] when the file is not in memory (each chunk's +/// bounds error is reported when that chunk is decoded, as before). +fn chunk_reqs( chunks: &[ChunkInfo], -) -> Result, FormatError> { - let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { - Vec::new() - } else { - chunks - .iter() - .map(|c| (c.address, c.chunk_size as usize, true)) - .collect() - }; - ExtentBytes::fetch(file_data, &extents) + pipeline: Option<&FilterPipeline>, + chunk_total_bytes: usize, +) -> Vec { + chunks + .iter() + .map(|c| crate::chunked_read::chunk_req(c, pipeline, chunk_total_bytes, true)) + .collect() } /// Threshold: only use parallel decompression when chunk count exceeds this. @@ -238,62 +235,75 @@ pub fn decompress_chunks_lane_partitioned_in( .unwrap_or(1) }); - let raw_bytes = fetch_all(file_data, chunks)?; - let assignments = lane_partition::partition_chunks(chunks.len(), lanes, seed); - let num_lanes = assignments.len(); - - // Each lane processes its assigned chunks and returns results + stats. - let lane_results: Result, LaneStats)>, FormatError> = assignments - .into_par_iter() - .map(|indices| { - let mut results = Vec::with_capacity(indices.len()); - let mut stats = LaneStats::default(); - - for &index in &indices { - let chunk_info = &chunks[index]; - let size = chunk_info.chunk_size as usize; - let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; - - let decompressed = decompress_chunk_exact( - raw_chunk, - pipeline, - chunk_total_bytes, - element_size, - chunk_info.filter_mask, - &chunk_info.offsets, - )?; - - stats.chunks_processed += 1; - stats.compressed_bytes += size as u64; - stats.decompressed_bytes += decompressed.len() as u64; - - results.push(DecompressedChunk { - index, - data: decompressed, - }); - } - - Ok((results, stats)) - }) - .collect(); - - let lane_results = lane_results?; - - // Aggregate stats - let mut partition_stats = PartitionStats::new(num_lanes); + let reqs = chunk_reqs(chunks, Some(pipeline), chunk_total_bytes); + let mut ordered: Vec> = Vec::with_capacity(chunks.len()); + let mut partition_stats = PartitionStats::new(0); partition_stats.total_chunks = chunks.len(); - for (lane_idx, (_, stats)) in lane_results.iter().enumerate() { - partition_stats.per_lane[lane_idx] = stats.clone(); - } + // Each batch of fetched chunks is partitioned into lanes and decoded + // before the next batch is fetched (with the file in memory there is + // one batch: all the chunks). + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + let assignments = lane_partition::partition_chunks(batch.len(), lanes, seed); + // Each lane processes its assigned chunks and returns results + stats. + let lane_results: Result, LaneStats)>, FormatError> = + assignments + .into_par_iter() + .map(|indices| { + let mut results = Vec::with_capacity(indices.len()); + let mut stats = LaneStats::default(); - // Flatten and sort by original index to restore order - let mut all_chunks: Vec = lane_results - .into_iter() - .flat_map(|(chunks, _)| chunks) - .collect(); - all_chunks.sort_by_key(|dc| dc.index); + for &local in &indices { + let index = batch.start + local; + let chunk_info = &chunks[index]; + let size = chunk_info.chunk_size as usize; + let raw_chunk = raw_bytes.get(index, &reqs[index])?; - let ordered = all_chunks.into_iter().map(|dc| dc.data).collect(); + let decompressed = decompress_chunk_exact( + raw_chunk, + pipeline, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + &chunk_info.offsets, + )?; + + stats.chunks_processed += 1; + stats.compressed_bytes += size as u64; + stats.decompressed_bytes += decompressed.len() as u64; + + results.push(DecompressedChunk { + index, + data: decompressed, + }); + } + + Ok((results, stats)) + }) + .collect(); + let lane_results = lane_results?; + + // Aggregate stats + if partition_stats.per_lane.len() < lane_results.len() { + partition_stats + .per_lane + .resize_with(lane_results.len(), LaneStats::default); + partition_stats.num_lanes = lane_results.len(); + } + for (lane, (_, stats)) in partition_stats.per_lane.iter_mut().zip(&lane_results) { + lane.chunks_processed += stats.chunks_processed; + lane.compressed_bytes += stats.compressed_bytes; + lane.decompressed_bytes += stats.decompressed_bytes; + } + + // Flatten and sort by original index to restore order + let mut all_chunks: Vec = lane_results + .into_iter() + .flat_map(|(chunks, _)| chunks) + .collect(); + all_chunks.sort_by_key(|dc| dc.index); + ordered.extend(all_chunks.into_iter().map(|dc| dc.data)); + Ok(()) + })?; Ok((ordered, partition_stats)) } @@ -325,33 +335,38 @@ pub fn decompress_chunks_parallel_in( ) -> Result>, FormatError> { use rayon::prelude::*; - let raw_bytes = fetch_all(file_data, chunks)?; - let results: Result, FormatError> = chunks - .par_iter() - .enumerate() - .map(|(index, chunk_info)| { - let size = chunk_info.chunk_size as usize; - let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; + let reqs = chunk_reqs(chunks, Some(pipeline), chunk_total_bytes); + let mut ordered: Vec> = Vec::with_capacity(chunks.len()); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + let results: Result, FormatError> = batch + .clone() + .into_par_iter() + .map(|index| { + let chunk_info = &chunks[index]; + let raw_chunk = raw_bytes.get(index, &reqs[index])?; - let decompressed = decompress_chunk_exact( - raw_chunk, - pipeline, - chunk_total_bytes, - element_size, - chunk_info.filter_mask, - &chunk_info.offsets, - )?; + let decompressed = decompress_chunk_exact( + raw_chunk, + pipeline, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + &chunk_info.offsets, + )?; - Ok(DecompressedChunk { - index, - data: decompressed, + Ok(DecompressedChunk { + index, + data: decompressed, + }) }) - }) - .collect(); + .collect(); - let mut result_vec = results?; - result_vec.sort_by_key(|dc| dc.index); - Ok(result_vec.into_iter().map(|dc| dc.data).collect()) + let mut result_vec = results?; + result_vec.sort_by_key(|dc| dc.index); + ordered.extend(result_vec.into_iter().map(|dc| dc.data)); + Ok(()) + })?; + Ok(ordered) } /// Decompress chunks sequentially (fallback when parallel is not warranted). @@ -373,26 +388,29 @@ pub fn decompress_chunks_sequential_in( chunk_total_bytes: usize, element_size: u32, ) -> Result>, FormatError> { - let raw_bytes = fetch_all(file_data, chunks)?; + let reqs = chunk_reqs(chunks, pipeline, chunk_total_bytes); let mut result = Vec::with_capacity(chunks.len()); - for (i, chunk_info) in chunks.iter().enumerate() { - let size = chunk_info.chunk_size as usize; - let raw_chunk = raw_bytes.get(i, chunk_info.address, size)?; + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + for i in batch { + let chunk_info = &chunks[i]; + let raw_chunk = raw_bytes.get(i, &reqs[i])?; - let decompressed = if let Some(pl) = pipeline { - decompress_chunk_exact( - raw_chunk, - pl, - chunk_total_bytes, - element_size, - chunk_info.filter_mask, - &chunk_info.offsets, - )? - } else { - raw_chunk.to_vec() - }; - result.push(decompressed); - } + let decompressed = if let Some(pl) = pipeline { + decompress_chunk_exact( + raw_chunk, + pl, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + &chunk_info.offsets, + )? + } else { + raw_chunk.to_vec() + }; + result.push(decompressed); + } + Ok(()) + })?; Ok(result) } diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index a9f630c..3361485 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -26,7 +26,7 @@ use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::{all_filters_skipped, decompress_chunk_exact_with}; use crate::selection::Selection; -use crate::storage::{ExtentBytes, Storage}; +use crate::storage::{ExtentReq, Storage, for_each_extent_batch}; /// The smallest axis-aligned box containing every selected element, as /// `(start, extent)` per dimension. `None` when there is nothing to gain or @@ -373,50 +373,50 @@ pub fn read_selection_in( }) }) .collect(); - // Their stored bytes, in one batch when the file is not in memory. - let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { - Vec::new() - } else { - wanted - .iter() - .map(|c| (c.address, c.chunk_size as usize, true)) - .collect() - }; - let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; - // Chunks are decoded into this thread's reusable buffers. - crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { - for (i, chunk) in wanted.iter().enumerate() { - let origin = &chunk.offsets[..rank]; - usize::try_from(chunk.address) - .map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?; - let raw = raw_bytes.get(i, chunk.address, chunk.chunk_size as usize)?; - // Mirrors the full-read path: filter-mask bit i set means - // filter i was not applied to this chunk. - let data: &[u8] = match pipeline { - Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { - decompress_chunk_exact_with( - raw, - pl, - chunk_bytes, - elem_size as u32, - chunk.filter_mask, - &chunk.offsets[..rank], - scratch, - )? - } - _ => raw, - }; - copy_overlap( - data, - origin, - &chunk_shape, - &mut boxed, - &box_start, - &box_extent, - elem_size, - ); - } - Ok(()) + // Their stored bytes, batch by batch when the file is not in + // memory; each batch's chunks are decoded into this thread's + // reusable buffers before the next batch is fetched. + let reqs: Vec = wanted + .iter() + .map(|c| crate::chunked_read::chunk_req(c, pipeline, chunk_bytes, true)) + .collect(); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { + for i in batch { + let chunk = wanted[i]; + let origin = &chunk.offsets[..rank]; + usize::try_from(chunk.address).map_err(|_| { + FormatError::Overflow("chunk address exceeds usize".into()) + })?; + let raw = raw_bytes.get(i, &reqs[i])?; + // Mirrors the full-read path: filter-mask bit i set + // means filter i was not applied to this chunk. + let data: &[u8] = match pipeline { + Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { + decompress_chunk_exact_with( + raw, + pl, + chunk_bytes, + elem_size as u32, + chunk.filter_mask, + &chunk.offsets[..rank], + scratch, + )? + } + _ => raw, + }; + copy_overlap( + data, + origin, + &chunk_shape, + &mut boxed, + &box_start, + &box_extent, + elem_size, + ); + } + Ok(()) + }) })?; } _ => return Ok(None), diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 0dd0540..2b4b9f1 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -337,14 +337,57 @@ pub fn read_upto( } /// Most stored bytes fetched by one [`Storage::read_ranges`] call when a -/// read gathers many extents (a chunked dataset's chunks): a larger read is -/// fetched and decoded batch by batch, so a remote backend never holds more -/// than this much undecoded data per read. -pub(crate) const RAW_BATCH_BYTES: usize = 64 << 20; +/// read gathers many extents (a chunked dataset's chunks, a selection's +/// runs): a larger read is fetched and decoded batch by batch, so a backend +/// without the file in memory never holds more than this much undecoded +/// data per read (or one extent, when a single one is larger — and every +/// chunk's extent is bounded by what the chunk can need, see +/// [`crate::filters::stored_chunk_limit`]). +pub const RAW_BATCH_BYTES: usize = 64 << 20; -/// The stored bytes of a list of extents (chunks, contiguous runs), fetched -/// together: [`Storage::read_ranges`] is called once for all of them, so a -/// remote backend can coalesce and parallelise the requests. +/// One extent of a raw-data read: `len` bytes stored at `addr`, whose +/// bounds are checked against the file, of which the first `fetch` bytes +/// are read (`None`: only checked, not read — its bytes are not needed). +/// +/// `fetch` below `len` bounds what a crafted size field can make a read +/// fetch: a chunk never needs more of its stored bytes than its decoded +/// size allows, however large its index entry says it is. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ExtentReq { + pub addr: u64, + pub len: usize, + pub fetch: Option, +} + +impl ExtentReq { + /// How many bytes are read for this extent. + #[inline] + fn fetch_len(&self) -> usize { + self.fetch.map_or(0, |f| f.min(self.len)) + } +} + +/// One extent's bytes on their own (see [`ExtentReq`]): the whole extent's +/// bounds checked as [`read_exact_at`] checks them, and its first +/// `req.fetch` bytes read (none when `fetch` is `None`). +pub(crate) fn read_extent<'a, S: Storage + ?Sized>( + file: &'a S, + req: &ExtentReq, +) -> Result, FormatError> { + let start = usize::try_from(req.addr).unwrap_or(usize::MAX); + match start.checked_add(req.len) { + Some(end) if end <= len_usize(file) => read_exact_at(file, req.addr, req.fetch_len()), + _ => Err(FormatError::UnexpectedEof { + expected: start.saturating_add(req.len), + available: len_usize(file), + }), + } +} + +/// The stored bytes of one batch of extents (chunks, contiguous runs), +/// fetched together: [`Storage::read_ranges`] is called once for the batch, +/// so a remote backend can coalesce and parallelise the requests. See +/// [`for_each_extent_batch`], which is how every raw-data read gets them. /// /// With the whole file in memory nothing is fetched: [`Self::get`] slices /// it, as the slice readers did. Either way an extent that does not lie in @@ -356,8 +399,12 @@ pub(crate) const RAW_BATCH_BYTES: usize = 64 << 20; pub(crate) enum ExtentBytes<'a> { /// The whole file. Contiguous(&'a [u8]), - /// Each extent's bytes, or its bounds error. - Fetched(Vec>), + /// Each extent's bytes, or its bounds error; the first is extent + /// `base` of the read. + Fetched { + base: usize, + extents: Vec>, + }, } /// One extent of [`ExtentBytes::Fetched`]. @@ -371,33 +418,35 @@ pub(crate) enum Extent<'a> { } impl<'a> ExtentBytes<'a> { - /// Fetch `extents` (`(address, length, wanted)`): the bytes of those - /// `wanted`, and the bounds check of all of them. - pub(crate) fn fetch( + /// Fetch `reqs`, extents `base..base + reqs.len()` of the read: the + /// bytes of those wanted, and the bounds check of all of them. + fn fetch( file: &'a S, - extents: &[(u64, usize, bool)], + reqs: &[ExtentReq], + base: usize, ) -> Result { if let Some(all) = file.as_contiguous() { return Ok(ExtentBytes::Contiguous(all)); } let file_len = len_usize(file); let mut ranges = Vec::new(); - let mut out = Vec::with_capacity(extents.len()); + let mut out = Vec::with_capacity(reqs.len()); // Positions in `out` of the extents being read, in `ranges` order. let mut slots = Vec::new(); - for &(addr, len, wanted) in extents { - let checked = - crate::addr::to_usize(addr).and_then(|start| match start.checked_add(len) { + for req in reqs { + let checked = crate::addr::to_usize(req.addr).and_then(|start| { + match start.checked_add(req.len) { Some(end) if end <= file_len => Ok(()), _ => Err(FormatError::UnexpectedEof { - expected: start.saturating_add(len), + expected: start.saturating_add(req.len), available: file_len, }), - }); + } + }); match checked { - Ok(()) if wanted => { + Ok(()) if req.fetch.is_some() => { slots.push(out.len()); - ranges.push(addr..addr + len as u64); + ranges.push(req.addr..req.addr + req.fetch_len() as u64); out.push(Extent::NotFetched); } Ok(()) => out.push(Extent::NotFetched), @@ -418,41 +467,46 @@ impl<'a> ExtentBytes<'a> { out[slot] = Extent::Bytes(bytes); } } - Ok(ExtentBytes::Fetched(out)) + Ok(ExtentBytes::Fetched { base, extents: out }) } - /// Whether extent `i` (at `addr`, `len` bytes, as passed to - /// [`Self::fetch`]) lies in the file: its bounds error if not. - pub(crate) fn check(&self, i: usize, addr: u64, len: usize) -> Result<(), FormatError> { + /// Whether extent `i` of the read (`req`) lies in the file: its bounds + /// error if not. + pub(crate) fn check(&self, i: usize, req: &ExtentReq) -> Result<(), FormatError> { match self { - ExtentBytes::Contiguous(_) => self.get(i, addr, len).map(|_| ()), - ExtentBytes::Fetched(v) => match v.get(i) { - Some(Extent::Err(e)) => Err(e.clone()), - Some(_) => Ok(()), - None => Err(not_fetched()), - }, + ExtentBytes::Contiguous(_) => self.get(i, req).map(|_| ()), + ExtentBytes::Fetched { base, extents } => { + match i.checked_sub(*base).and_then(|j| extents.get(j)) { + Some(Extent::Err(e)) => Err(e.clone()), + Some(_) => Ok(()), + None => Err(not_fetched()), + } + } } } - /// Extent `i`'s bytes (at `addr`, `len` bytes, as passed to - /// [`Self::fetch`]). - pub(crate) fn get(&self, i: usize, addr: u64, len: usize) -> Result<&[u8], FormatError> { + /// Extent `i` of the read (`req`): its first `req.fetch` bytes, the + /// same whether the file is in memory or not. + pub(crate) fn get(&self, i: usize, req: &ExtentReq) -> Result<&[u8], FormatError> { match self { ExtentBytes::Contiguous(all) => { - let start = crate::addr::to_usize(addr)?; + let start = crate::addr::to_usize(req.addr)?; start - .checked_add(len) + .checked_add(req.len) .and_then(|end| all.get(start..end)) + .map(|b| &b[..req.fetch_len()]) .ok_or(FormatError::UnexpectedEof { - expected: start.saturating_add(len), + expected: start.saturating_add(req.len), available: <[u8]>::len(all), }) } - ExtentBytes::Fetched(v) => match v.get(i) { - Some(Extent::Bytes(b)) => Ok(b), - Some(Extent::Err(e)) => Err(e.clone()), - _ => Err(not_fetched()), - }, + ExtentBytes::Fetched { base, extents } => { + match i.checked_sub(*base).and_then(|j| extents.get(j)) { + Some(Extent::Bytes(b)) => Ok(b), + Some(Extent::Err(e)) => Err(e.clone()), + _ => Err(not_fetched()), + } + } } } } @@ -462,6 +516,30 @@ fn not_fetched() -> FormatError { FormatError::Storage("an extent that was not fetched was asked for".into()) } +/// The one way raw-data reads fetch stored bytes: `reqs` are split into +/// consecutive batches of at most [`RAW_BATCH_BYTES`] of fetched bytes (at +/// least one extent each — and no extent fetches more than its +/// [`ExtentReq::fetch`]), and for each batch in turn its bytes are fetched +/// with one [`Storage::read_ranges`] call and `f(batch, &bytes)` is called, +/// with `bytes` indexed by the extent's position in `reqs`. A batch's bytes +/// are dropped before the next batch is fetched, and an error from `f` +/// stops the read before anything more is fetched. +/// +/// With the whole file in memory there is nothing to fetch: one call, over +/// all of `reqs`, that slices the file. +pub(crate) fn for_each_extent_batch<'a, S: Storage + ?Sized>( + file: &'a S, + reqs: &[ExtentReq], + mut f: impl FnMut(Range, &ExtentBytes<'a>) -> Result<(), FormatError>, +) -> Result<(), FormatError> { + let contiguous = file.as_contiguous().is_some(); + for batch in raw_batches(reqs.len(), contiguous, |i| reqs[i].fetch_len()) { + let bytes = ExtentBytes::fetch(file, &reqs[batch.clone()], batch.start)?; + f(batch, &bytes)?; + } + Ok(()) +} + /// Split `n` extents, whose sizes `size(i)` gives, into consecutive batches /// of at most [`RAW_BATCH_BYTES`] (at least one extent each): the ranges of /// `0..n` to fetch together. With the whole file in memory (`contiguous`) diff --git a/crates/clawhdf5-format/tests/raw_fetch_bounds.rs b/crates/clawhdf5-format/tests/raw_fetch_bounds.rs new file mode 100644 index 0000000..8217323 --- /dev/null +++ b/crates/clawhdf5-format/tests/raw_fetch_bounds.rs @@ -0,0 +1,332 @@ +//! What a raw-data read fetches from a [`Storage`] without the file in +//! memory is bounded: by batch (at most `RAW_BATCH_BYTES` per +//! `read_ranges` call) and by chunk (never more of a chunk's stored bytes +//! than its decoded size can need), on every path that reads chunks — full, +//! cached, indexed, sweep, selection and the `parallel_read` decoders. +//! +//! A crafted chunk index can point every chunk at one huge extent. Slicing +//! an in-memory file costs nothing there, but a backend that fetches would +//! hold `chunks x extent` bytes before the first chunk failed to decode. + +use std::borrow::Cow; +use std::ops::Range; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +use clawhdf5_format::chunk_cache::ChunkCache; +use clawhdf5_format::chunked_read::{ + ChunkInfo, SweepContext, list_chunks, read_chunked_data_sweep_in, +}; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read::{ + read_raw_data_cached_in, read_raw_data_full_in, read_raw_data_indexed_in, + read_raw_data_selection_in, +}; +use clawhdf5_format::dataspace::Dataspace; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::filter_pipeline::FilterPipeline; +use clawhdf5_format::group_v2; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::selection::Selection; +use clawhdf5_format::storage::{RAW_BATCH_BYTES, Storage}; +use clawhdf5_format::superblock::Superblock; + +/// A read_at-only storage that records the most bytes one call fetched +/// (a `read_ranges` call counts all its ranges together) and the total. +struct PeakStorage { + data: Vec, + peak: AtomicU64, + total: AtomicU64, +} + +impl PeakStorage { + fn new(data: Vec) -> Self { + PeakStorage { + data, + peak: AtomicU64::new(0), + total: AtomicU64::new(0), + } + } + + fn reset(&self) { + self.peak.store(0, Relaxed); + self.total.store(0, Relaxed); + } + + fn served(&self, offset: u64, len: usize) -> Vec { + self.data + .as_slice() + .read_at(offset, len) + .unwrap() + .into_owned() + } +} + +impl Storage for PeakStorage { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let got = self.served(offset, len); + self.peak.fetch_max(got.len() as u64, Relaxed); + self.total.fetch_add(got.len() as u64, Relaxed); + Ok(Cow::Owned(got)) + } + + fn len(&self) -> u64 { + self.data.len() as u64 + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + let got: Vec> = ranges + .iter() + .map(|r| self.served(r.start, (r.end - r.start) as usize)) + .collect(); + let bytes: u64 = got.iter().map(|g| g.len() as u64).sum(); + self.peak.fetch_max(bytes, Relaxed); + self.total.fetch_add(bytes, Relaxed); + Ok(got.into_iter().map(Cow::Owned).collect()) + } +} + +struct Chunked { + layout: DataLayout, + dataspace: Dataspace, + datatype: Datatype, + pipeline: Option, + os: u8, + ls: u8, +} + +/// The one chunked dataset of fixture `name`. +fn chunked(bytes: &[u8]) -> Chunked { + let sb = Superblock::parse(bytes, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + for child in group_v2::resolve_group_children(bytes, &sb, sb.root_group_address).unwrap() { + let header = + ObjectHeader::parse(bytes, child.object_header_address as usize, os, ls).unwrap(); + let msg = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| m.data.clone()) + }; + let Some(dl) = msg(MessageType::DataLayout) else { + continue; + }; + let layout = DataLayout::parse(&dl, os, ls).unwrap(); + if !matches!(layout, DataLayout::Chunked { .. }) { + continue; + } + return Chunked { + layout, + datatype: Datatype::parse(&msg(MessageType::Datatype).unwrap()) + .unwrap() + .0, + dataspace: Dataspace::parse(&msg(MessageType::Dataspace).unwrap(), ls).unwrap(), + pipeline: msg(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p).unwrap()), + os, + ls, + }; + } + panic!("no chunked dataset"); +} + +/// Claimed stored size of every chunk in the crafted file. +const HUGE: u32 = 20 << 20; + +/// `chunked_large.h5` (1000 `i32` in ten gzip chunks, a v1 B-tree index) +/// with `HUGE` bytes of padding appended and every chunk's index entry +/// rewritten to claim `HUGE` stored bytes at the padding: ten chunks, 200 +/// MiB of extents, in a 20 MiB file. +fn crafted() -> (Vec, Chunked, Vec) { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut bytes = std::fs::read(dir.join("chunked_large.h5")).unwrap(); + let ds = chunked(&bytes); + let es = ds.datatype.type_size() as usize; + let (chunks, _) = list_chunks(&bytes, &ds.layout, &ds.dataspace, es, ds.os, ds.ls).unwrap(); + assert_eq!(chunks.len(), 10); + let blob = bytes.len() as u64; + for c in &chunks { + // v1 B-tree key (size, filter mask, offsets + 0) then the child + // address. + let mut pat = Vec::new(); + pat.extend_from_slice(&c.chunk_size.to_le_bytes()); + pat.extend_from_slice(&c.filter_mask.to_le_bytes()); + // The key holds one offset per dimension plus the element offset + // (0); `offsets` may or may not list that last one. + for d in 0..=ds.dataspace.dimensions.len() { + pat.extend_from_slice(&c.offsets.get(d).copied().unwrap_or(0).to_le_bytes()); + } + pat.extend_from_slice(&c.address.to_le_bytes()); + let at = bytes + .windows(pat.len()) + .position(|w| w == pat.as_slice()) + .expect("chunk key"); + bytes[at..at + 4].copy_from_slice(&HUGE.to_le_bytes()); + let a = at + pat.len() - 8; + bytes[a..a + 8].copy_from_slice(&blob.to_le_bytes()); + } + bytes.resize(bytes.len() + HUGE as usize, 0x5a); + let ds = chunked(&bytes); + let (chunks, _) = list_chunks(&bytes, &ds.layout, &ds.dataspace, es, ds.os, ds.ls).unwrap(); + assert!( + chunks + .iter() + .all(|c| c.chunk_size == HUGE && c.address == blob) + ); + (bytes, ds, chunks) +} + +/// Most a crafted chunk of the fixture may fetch: its decoded size (400 +/// bytes) grown by one codec, generously. +const CHUNK_LIMIT: u64 = 400 + 100 + 4096; + +#[test] +fn crafted_chunk_index_cannot_amplify_fetches() { + let (bytes, ds, chunks) = crafted(); + let st = PeakStorage::new(bytes.clone()); + let pl = ds.pipeline.as_ref(); + let (dl, sp, dt, os, ls) = (&ds.layout, &ds.dataspace, &ds.datatype, ds.os, ds.ls); + let check = + |what: &str, got: Result, FormatError>, want: Result, FormatError>| { + // Same outcome as slicing the whole file. + assert_eq!(got, want, "{what}"); + let (peak, total) = (st.peak.load(Relaxed), st.total.load(Relaxed)); + assert!( + peak <= RAW_BATCH_BYTES as u64, + "{what}: one fetch of {peak} bytes" + ); + // Every chunk's fetch is bounded by what it can need, whatever its + // index entry claims (plus the index and header reads). + assert!( + total <= chunks.len() as u64 * CHUNK_LIMIT + 64 * 1024, + "{what}: fetched {total} bytes" + ); + st.reset(); + }; + let slice: &[u8] = &bytes; + + let sel = Selection::Hyperslab { + start: vec![100], + stride: vec![1], + count: vec![400], + block: vec![1], + }; + check( + "selection", + read_raw_data_selection_in(&st, dl, sp, dt, pl, os, ls, &sel), + read_raw_data_selection_in(slice, dl, sp, dt, pl, os, ls, &sel), + ); + check( + "full", + read_raw_data_full_in(&st, dl, sp, dt, pl, os, ls), + read_raw_data_full_in(slice, dl, sp, dt, pl, os, ls), + ); + check( + "cached", + read_raw_data_cached_in(&st, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + read_raw_data_cached_in(slice, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + ); + check( + "indexed", + read_raw_data_indexed_in(&st, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + read_raw_data_indexed_in(slice, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + ); + check( + "sweep", + read_chunked_data_sweep_in( + &st, + dl, + sp, + dt, + pl, + os, + ls, + &ChunkCache::new(), + &mut SweepContext::new(4, 2), + ), + read_chunked_data_sweep_in( + slice, + dl, + sp, + dt, + pl, + os, + ls, + &ChunkCache::new(), + &mut SweepContext::new(4, 2), + ), + ); + #[cfg(feature = "parallel")] + { + use clawhdf5_format::parallel_read::{ + decompress_chunks_lane_partitioned_in, decompress_chunks_parallel_in, + decompress_chunks_sequential_in, + }; + let pl = pl.unwrap(); + let flat = |r: Result>, FormatError>| r.map(|v| v.concat()); + check( + "parallel", + flat(decompress_chunks_parallel_in(&st, &chunks, pl, 400, 4)), + flat(decompress_chunks_parallel_in(slice, &chunks, pl, 400, 4)), + ); + check( + "sequential", + flat(decompress_chunks_sequential_in( + &st, + &chunks, + Some(pl), + 400, + 4, + )), + flat(decompress_chunks_sequential_in( + slice, + &chunks, + Some(pl), + 400, + 4, + )), + ); + check( + "lane partitioned", + flat( + decompress_chunks_lane_partitioned_in(&st, &chunks, pl, 400, 4, 7, Some(3)) + .map(|(v, _)| v), + ), + flat( + decompress_chunks_lane_partitioned_in(slice, &chunks, pl, 400, 4, 7, Some(3)) + .map(|(v, _)| v), + ), + ); + } +} + +/// Legitimately large chunks (unfiltered, 4 MiB each, 160 MiB in all) are +/// fetched batch by batch: no call holds more than the batch budget, and +/// the data is right. +#[cfg(feature = "parallel")] +#[test] +fn large_reads_are_fetched_in_batches() { + use clawhdf5_format::parallel_read::decompress_chunks_sequential_in; + const CHUNK: usize = 4 << 20; + let data: Vec = (0..2 * CHUNK).map(|i| (i % 251) as u8).collect(); + let chunks: Vec = (0..40u64) + .map(|i| ChunkInfo { + chunk_size: CHUNK as u32, + filter_mask: 0, + offsets: vec![i * CHUNK as u64], + address: (i % 2) * CHUNK as u64, + }) + .collect(); + let st = PeakStorage::new(data.clone()); + let got = decompress_chunks_sequential_in(&st, &chunks, None, CHUNK, 1).unwrap(); + assert_eq!(got.len(), 40); + for (i, c) in got.iter().enumerate() { + let at = (i % 2) * CHUNK; + assert!(c == &data[at..at + CHUNK], "chunk {i}"); + } + let peak = st.peak.load(Relaxed); + assert!(peak <= RAW_BATCH_BYTES as u64, "one fetch of {peak} bytes"); + assert_eq!(st.total.load(Relaxed), 40 * CHUNK as u64); +} diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 6bd7b35..0b1ac71 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -349,9 +349,11 @@ impl File { /// selections, variable-length data, virtual datasets). /// /// Every read goes through the storage's [`Storage::read_at`] and - /// [`Storage::read_ranges`] (a chunked read fetches all the chunks it - /// needs with one `read_ranges` call per 64 MiB), so nothing is read that - /// the operation does not need. A storage that has the whole file in + /// [`Storage::read_ranges`] (a chunked read fetches the chunks it needs + /// with one `read_ranges` call per batch of at most 64 MiB, decoding + /// each batch before the next, and never more of a chunk than its + /// decoded size can need), so nothing is read that the operation does + /// not need. A storage that has the whole file in /// memory ([`Storage::as_contiguous`]) is read as [`File::from_bytes`] /// reads its buffer. The storage holds the whole file: a user block is /// found and skipped, and bytes past the end of file the superblock diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1098922..c42c5d3 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -435,8 +435,12 @@ fast path within benchmark noise. (checked: identical conformance results; the bench gate below still has to be run on an idle machine). - Chunked reads fetch in batches of at most 64 MiB of stored bytes, one - `read_ranges` call each, so a remote read never holds more than that - undecoded; chunks already in the chunk cache are not fetched. + `read_ranges` call each, decoding each batch before fetching the next, + on every chunk-reading path (one helper, + `storage::for_each_extent_batch`); and no chunk fetches more than its + decoded size can need (`filters::stored_chunk_limit`), so a remote read + holds at most one batch undecoded even over a crafted chunk index; + chunks already in the chunk cache are not fetched. - The typed readers' zero-copy fast path became "read the contiguous bytes once": over a range storage a contiguous `read_f64` is one read, and a native contiguous selection reads only its runs. From b086dc3c2bba49fbfa562c8f4ce0ded537a454a5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:34:31 -0500 Subject: [PATCH 06/10] format: read a contiguous selection's runs merged across small gaps gather_storage merged only runs that touch, so a strided selection of a contiguous dataset over a Storage became one range (and one owned Vec) per element: a stride-2 read of 32M f32 through File::open_storage made 16,777,232 read_at calls, took 2.0 s and peaked at 2.09 GB. The selection is now walked twice. The first walk checks the runs and plans spans: runs in increasing order at most 4 KiB apart (GATHER_GAP_BYTES) are read as one span up to 8 MiB (GATHER_SPAN_BYTES; a longer run is split), so nothing is stored per run. The spans are fetched in RAW_BATCH_BYTES batches while the second walk copies each run out of its span. Same checks and errors as before. The same read is now 32 reads and 0.31 s (File::open: 0.08 s). contiguous_read_interop: every h5py-checked selection is also read through File::open_storage and must give libhdf5's bytes; a new test bounds the range reads of strided, blocked, column and point selections (stride 2: at most 1 data read; 563,200 before). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +- crates/clawhdf5-format/src/gather.rs | 283 +++++++++++++----- crates/clawhdf5-format/src/storage.rs | 2 +- .../clawhdf5/tests/contiguous_read_interop.rs | 85 ++++++ 4 files changed, 294 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21e857d..3c6a7b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,10 @@ `chunks x extent` bytes (`tests/raw_fetch_bounds.rs`). Chunks the file's chunk cache already holds are not fetched. A selection fetches only the chunks its bounding box overlaps; a contiguous selection only - its runs (adjacent ones merged). A global-heap collection is read once + its runs, merged into reads of up to 8 MiB across gaps of up to 4 KiB + (a stride-2 selection of 32M `f32` is 32 reads and 0.3 s over a + `CountingStorage`, where one read per element was 16.8M reads, 2.0 s + and 2.1 GB peak). A global-heap collection is read once per resolver and kept (within the resolver's 32 MiB budget). - Each extent's bounds error is the one the slice readers gave, reported when the read reaches that extent, so a damaged file fails with the diff --git a/crates/clawhdf5-format/src/gather.rs b/crates/clawhdf5-format/src/gather.rs index 2b8a4ab..08480f1 100644 --- a/crates/clawhdf5-format/src/gather.rs +++ b/crates/clawhdf5-format/src/gather.rs @@ -13,7 +13,7 @@ use alloc::{vec, vec::Vec}; use crate::data_read::NativeElement; use crate::error::FormatError; use crate::selection::Selection; -use crate::storage::Storage; +use crate::storage::{ExtentBytes, ExtentReq, Storage, raw_batches}; /// Row-major element strides of `dims` (the last dimension has stride 1). fn strides(dims: &[u64]) -> Vec { @@ -262,12 +262,88 @@ pub(crate) fn gather( Ok(out) } +/// Largest gap between two of a selection's runs that [`gather_storage`] +/// reads through rather than asking for the runs separately: skipping a +/// few KiB costs a remote backend far less than another request (and a +/// local one less than another call and allocation). +pub(crate) const GATHER_GAP_BYTES: usize = 4 << 10; + +/// Largest single read [`gather_storage`] makes of a selection's runs: runs +/// are merged into reads up to this size, and a longer run is split. +pub(crate) const GATHER_SPAN_BYTES: usize = 8 << 20; + +/// Call `emit(first_element, element_count)` for each run of a validated +/// hyperslab or point selection (in output order; see [`hyperslab_runs`]), +/// or the error for a hyperslab of the wrong rank or a point outside `dims` +/// (runs before that point have been emitted). +fn selection_runs( + dims: &[u64], + selection: &Selection, + emit: &mut dyn FnMut(u64, u64), +) -> Result<(), FormatError> { + match selection { + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let rank = dims.len(); + if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { + return Err(FormatError::SelectionOutOfBounds( + "hyperslab rank does not match dataset rank".into(), + )); + } + hyperslab_runs(dims, start, stride, count, block, emit); + } + Selection::Points(points) => { + let strides = strides(dims); + let mut coalesce = Coalesce { + start: 0, + len: 0, + emit, + }; + for p in points { + if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) { + return Err(FormatError::SelectionOutOfBounds( + "selection addresses elements outside the dataset".into(), + )); + } + let at = p + .iter() + .zip(&strides) + .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s))); + coalesce.push(at, 1); + } + coalesce.flush(); + } + Selection::None | Selection::All => {} + } + Ok(()) +} + +/// One read of [`gather_storage`]: bytes `[start, end)` of the dataset, +/// which hold the output's bytes up to `out_end` (from where the previous +/// span's end left off). +#[derive(Clone, Copy)] +struct Span { + start: usize, + end: usize, + out_end: usize, +} + /// [`gather`] of bytes (`T = u8`) from a dataset that is not in memory: the /// dataset's `src_len` bytes start at `base` in `file`, which must hold all -/// of them (the caller checks). The selection's runs are collected first, -/// adjacent ones merged, and fetched with one [`Storage::read_ranges`] call, -/// so only the selected bytes are read. Same checks and errors as -/// [`gather`]. +/// of them (the caller checks). Same checks and errors as [`gather`]. +/// +/// The selection's runs are walked twice. The first walk checks them and +/// plans the reads: runs in increasing order with at most +/// [`GATHER_GAP_BYTES`] between them are read as one span (the gap is read +/// and dropped), up to [`GATHER_SPAN_BYTES`] per span. So a strided +/// selection is a few large reads, not one per element, and nothing is +/// allocated per run. The spans are fetched batch by batch (one +/// [`Storage::read_ranges`] call per [`crate::storage::RAW_BATCH_BYTES`]) +/// while the second walk copies each run out of its span. pub(crate) fn gather_storage( file: &S, base: u64, @@ -297,11 +373,15 @@ pub(crate) fn gather_storage( } }; let out_bytes = crate::chunked_read::checked_byte_len(n_elements, elem_size)?; - // Runs as (byte offset in the dataset, byte length), in output order. - let mut runs: Vec<(usize, usize)> = Vec::new(); + let outside = || { + FormatError::SelectionOutOfBounds("selection addresses elements outside the dataset".into()) + }; + + // First walk: check every run and plan the spans. + let mut spans: Vec = Vec::new(); let mut total = 0usize; let mut failed = false; - let mut collect = |first: u64, n: u64| { + selection_runs(dims, selection, &mut |first: u64, n: u64| { if failed { return; } @@ -314,78 +394,123 @@ pub(crate) fn gather_storage( .and_then(|n| n.checked_mul(elem_size)), ) .and_then(|(at, len)| Some((at, len, at.checked_add(len)?))); - match range { - Some((at, len, end)) if end <= src_len && total + len <= out_bytes => { - match runs.last_mut() { - Some((a, l)) if *a + *l == at => *l += len, - _ => runs.push((at, len)), - } - total += len; - } - _ => failed = true, - } - }; - let mut bad_point = false; - match selection { - Selection::Hyperslab { - start, - stride, - count, - block, - } => { - let rank = dims.len(); - if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { - return Err(FormatError::SelectionOutOfBounds( - "hyperslab rank does not match dataset rank".into(), - )); - } - hyperslab_runs(dims, start, stride, count, block, &mut collect); - } - Selection::Points(points) => { - let strides = strides(dims); - let mut coalesce = Coalesce { - start: 0, - len: 0, - emit: &mut collect, - }; - for p in points { - if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) { - bad_point = true; - break; - } - let at = p - .iter() - .zip(&strides) - .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s))); - coalesce.push(at, 1); - } - coalesce.flush(); - } - Selection::None | Selection::All => {} - } - if failed || bad_point || total != out_bytes { - return Err(FormatError::SelectionOutOfBounds( - "selection addresses elements outside the dataset".into(), - )); - } - let ranges: Vec> = runs - .iter() - .map(|&(at, len)| base + at as u64..base + (at + len) as u64) - .collect(); - let fetched = file.read_ranges(&ranges)?; - if fetched.len() != ranges.len() { - return Err(FormatError::Storage( - "read_ranges returned the wrong number of ranges".into(), - )); - } - let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes); - for (bytes, &(_, len)) in fetched.iter().zip(&runs) { - let Some(b) = bytes.get(..len) else { - return Err(FormatError::Storage( - "short read inside the file (the storage shrank or the backend failed)".into(), - )); + let Some((mut at, mut len)) = range + .filter(|&(_, len, end)| end <= src_len && len <= out_bytes - total) + .map(|(at, len, _)| (at, len)) + else { + failed = true; + return; }; - out.extend_from_slice(b); + while len > 0 { + let room = match spans.last_mut() { + Some(s) + if at >= s.end + && at - s.end <= GATHER_GAP_BYTES + && at - s.start < GATHER_SPAN_BYTES => + { + let take = len.min(GATHER_SPAN_BYTES - (at - s.start)); + s.end = at + take; + s.out_end += take; + take + } + _ => { + let take = len.min(GATHER_SPAN_BYTES); + spans.push(Span { + start: at, + end: at + take, + out_end: total + take, + }); + take + } + }; + total += room; + at += room; + len -= room; + } + })?; + if failed || total != out_bytes { + return Err(outside()); + } + + // The spans' reads, and the batches they are fetched in. + let reqs: Vec = spans + .iter() + .map(|s| ExtentReq { + addr: base + s.start as u64, + len: s.end - s.start, + fetch: Some(s.end - s.start), + }) + .collect(); + let batches = raw_batches(reqs.len(), false, |i| reqs[i].len); + + // Second walk: copy each run out of its span, fetching each batch of + // spans when the walk reaches it (and dropping the previous one). + let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes); + let mut span = 0usize; + let mut batch = 0usize; + let mut fetched: Option> = None; + let mut error: Option = None; + selection_runs(dims, selection, &mut |first: u64, n: u64| { + if error.is_some() { + return; + } + // Checked by the first walk. + let mut at = first as usize * elem_size; + let mut len = n as usize * elem_size; + while len > 0 { + while spans.get(span).is_some_and(|s| s.out_end <= out.len()) { + span += 1; + } + if fetched.is_none() || span >= batches[batch].end { + fetched = None; + while batches.get(batch).is_some_and(|b| span >= b.end) { + batch += 1; + } + let (Some(b), Some(_)) = (batches.get(batch).cloned(), spans.get(span)) else { + // The second walk emitted more than the first. + error = Some(outside()); + return; + }; + match ExtentBytes::fetch(file, &reqs[b.clone()], b.start) { + Ok(f) => fetched = Some(f), + Err(e) => { + error = Some(e); + return; + } + } + } + let s = spans[span]; + let take = len.min(s.out_end - out.len()); + let bytes = match fetched + .as_ref() + .map(|f| f.get(span, &reqs[span])) + .unwrap_or_else(|| Err(outside())) + { + Ok(b) => b, + Err(e) => { + error = Some(e); + return; + } + }; + match at + .checked_sub(s.start) + .and_then(|o| bytes.get(o..o.checked_add(take)?)) + { + Some(b) => out.extend_from_slice(b), + None => { + error = Some(outside()); + return; + } + } + at += take; + len -= take; + } + })?; + if let Some(e) = error { + return Err(e); + } + if out.len() != out_bytes { + return Err(outside()); } Ok(out) } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 2b4b9f1..7425296 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -420,7 +420,7 @@ pub(crate) enum Extent<'a> { impl<'a> ExtentBytes<'a> { /// Fetch `reqs`, extents `base..base + reqs.len()` of the read: the /// bytes of those wanted, and the bounds check of all of them. - fn fetch( + pub(crate) fn fetch( file: &'a S, reqs: &[ExtentReq], base: usize, diff --git a/crates/clawhdf5/tests/contiguous_read_interop.rs b/crates/clawhdf5/tests/contiguous_read_interop.rs index 188856e..ed4d4fa 100644 --- a/crates/clawhdf5/tests/contiguous_read_interop.rs +++ b/crates/clawhdf5/tests/contiguous_read_interop.rs @@ -12,9 +12,11 @@ use std::path::Path; use std::process::Command; +use std::sync::Arc; use clawhdf5::File; use clawhdf5_format::selection::Selection; +use clawhdf5_format::storage::CountingStorage; fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) @@ -364,9 +366,31 @@ with h5py.File("{path}", "r") as f: )); let file = File::open(&path).unwrap(); + // The same file over a storage that serves only range reads: the + // selections read only their runs (see `strided_selections_over_storage_ + // read_few_ranges`) and must give the same bytes. + let remote = File::open_storage(Arc::new(CountingStorage::new( + std::fs::read(&path).unwrap(), + ))) + .unwrap(); for (k, (name, code, sel)) in cases.iter().enumerate() { let ds = file.dataset(name).unwrap(); let want_bytes = std::fs::read(dir.path().join(format!("sel_{k}.bin"))).unwrap(); + let rds = remote.dataset(name).unwrap(); + assert!( + rds.read_selection(sel).unwrap() == want_bytes, + "{name} {sel:?}: bytes over a range storage differ from libhdf5's" + ); + assert_eq!( + rds.read_f64_selection(sel).unwrap(), + ds.read_f64_selection(sel).unwrap(), + "{name} {sel:?}: f64 over a range storage" + ); + assert_eq!( + rds.read_i32_selection(sel).unwrap(), + ds.read_i32_selection(sel).unwrap(), + "{name} {sel:?}: i32 over a range storage" + ); let got_bytes = ds.read_selection(sel).unwrap(); assert!( got_bytes == want_bytes, @@ -401,3 +425,64 @@ with h5py.File("{path}", "r") as f: ); } } + +/// Over a storage without the file in memory, a selection of a contiguous +/// dataset reads its runs merged across small gaps: a strided selection is +/// a few large reads, not one per element (563 200 for the stride-2 case +/// before), and the values are libhdf5's. +#[test] +fn strided_selections_over_storage_read_few_ranges() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("contig.h5"); + write_file(&path); + let storage = Arc::new(CountingStorage::new(std::fs::read(&path).unwrap())); + let remote = File::open_storage(storage.clone()).unwrap(); + let local = File::open(&path).unwrap(); + // f4le_big is 1100 x 1024 f32: 4 KiB rows, 4.4 MB in all. + let name = "f4le_big"; + let every_100th: Vec> = (0..1100u64 * 1024) + .step_by(100) + .map(|i| vec![i / 1024, i % 1024]) + .collect(); + let backwards: Vec> = every_100th.iter().rev().take(50).cloned().collect(); + // (selection, most range reads it may take) + let cases: Vec<(Selection, u64)> = vec![ + // Every other element of every row: one read of the whole dataset. + (slab(&[(0, 1, 1100, 1), (0, 2, 512, 1)]), 1), + // Blocks of 3 every 7 on every other row: rows are 4 KiB apart, so + // one read per selected row at most. + (slab(&[(0, 2, 550, 1), (1, 7, 146, 3)]), 550), + // Every 100th element, in order: 400-byte gaps, one read. + (Selection::Points(every_100th), 1), + // Points going backwards are not merged. + (Selection::Points(backwards), 50), + // A column: 4 KiB apart, merged. + (slab(&[(0, 1, 1100, 1), (5, 1, 1, 1)]), 1), + ]; + let ds = remote.dataset(name).unwrap(); + for (sel, most) in cases { + storage.reset(); + let got = ds.read_f32_selection(&sel).unwrap(); + let reads = storage.reads(); + assert_eq!( + got, + local + .dataset(name) + .unwrap() + .read_f32_selection(&sel) + .unwrap(), + "{sel:?}" + ); + // A few reads of metadata besides the data. + assert!(reads <= most + 8, "{sel:?}: {reads} range reads"); + storage.reset(); + let bytes = ds.read_selection(&sel).unwrap(); + assert!( + storage.reads() <= most + 8, + "{sel:?}: {} reads", + storage.reads() + ); + assert_eq!(bytes.len(), got.len() * 4); + } +} From 89e7977943776598945b80f50cc0de8f3c236892 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:41:10 -0500 Subject: [PATCH 07/10] clawhdf5: parse local files through the slice, and cache the contiguous view Since M2 the facade handed the metadata parsers its FileData view, so a local file ran the parsers monomorphised for FileData, whose Storage impl worked out contiguous() (patched and overlay checks, two range conversions) on every structure read. A metadata walk of h5stat_newgrat.h5 (35,001 groups: open, entries and attrs of each) was about 4.5% slower than at 8f59b2e. - FileData works out its contiguous slice once at open (a borrow of its own heap/mapped buffer, kept as a pointer; see the SAFETY notes). - with_bytes! hands the in-memory slice to the format parsers when the file has one (header parsing, attributes, group listings and lookups, path resolution, shared messages, VL decoding), so local files run the [u8] parsers as before; storage-backed files still get FileData. Provisional A/B on tank (load 5-11), best of 30, 5 alternating rounds: walk 26.07-26.37 ms at 8f59b2e, 27.24-28.03 ms before this commit, 26.52-26.98 ms after. Caching alone did not move it (27.02-27.47 ms); the dispatch did. File::open read_f32 on 32M f32 (contiguous, chunked, gzip) stays within noise of 8f59b2e. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/src/reader.rs | 188 ++++++++++++++++++++++++---------- 1 file changed, 135 insertions(+), 53 deletions(-) diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 0b1ac71..191c614 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -64,6 +64,24 @@ impl Backing { } } +/// Evaluate `$body` with `$d` bound to the bytes the `clawhdf5_format` +/// parsers read: the in-memory slice when the file has one (a `Vec`, an +/// mmap), so local files run the parsers monomorphised for `[u8]` — the +/// slice code, as before the range-read migration — and otherwise the +/// [`FileData`] itself, whose reads go to the storage. +macro_rules! with_bytes { + ($data:expr, |$d:ident| $body:expr) => {{ + let data: &FileData = $data; + match data.contiguous() { + Some($d) => $body, + None => { + let $d = data; + $body + } + } + }}; +} + /// The file's bytes, viewed from the superblock on and up to the end of /// file the superblock records. A file may start with a user block (the /// superblock at 512, 1024, …); every HDF5 address is relative to the @@ -93,8 +111,28 @@ struct FileData { /// then); every object lookup here fails with this error, and no /// metadata is read from the file's own, possibly stale, bytes. image_error: Option, + /// [`Self::contiguous`], worked out once at open: every structure a + /// parser reads asks for it, and the patched/overlay checks and range + /// conversions behind it cost a local metadata walk a few percent. + contiguous: Option, } +/// A borrow of the HDF5 data held by a [`FileData`]'s own `backing` or +/// `patched` buffer (see [`FileData::contiguous`]). +#[derive(Clone, Copy)] +struct WholeView { + ptr: *const u8, + len: usize, +} + +// SAFETY: a `WholeView` is only a borrow of bytes owned (through `backing` +// or `patched`) by the `FileData` holding it, which is `Send + Sync`: the +// bytes are never written after open, so sharing the pointer across +// threads is sharing a `&[u8]`. +unsafe impl Send for WholeView {} +// SAFETY: as above. +unsafe impl Sync for WholeView {} + impl FileData { /// Locate the superblock and parse it. A truncated file is refused, and /// bytes past the recorded end of file are not read, as in libhdf5. @@ -127,17 +165,17 @@ impl FileData { ImageView::Patched(p) => (Some(p), None), ImageView::Unloadable(e) => (None, Some(e)), }; - Ok(( - Self { - backing, - base: base as u64, - end: end as u64, - patched, - overlay: Vec::new(), - image_error, - }, - superblock, - )) + let mut data = Self { + backing, + base: base as u64, + end: end as u64, + patched, + overlay: Vec::new(), + image_error, + contiguous: None, + }; + data.contiguous = data.find_contiguous(); + Ok((data, superblock)) } /// [`Self::new`] for a [`Storage`] backend: the same checks, through @@ -152,7 +190,11 @@ impl FileData { patched: None, overlay: Vec::new(), image_error: None, + contiguous: None, }; + // Worked out again below, once the end of file and any cache image + // are known. + data.contiguous = data.find_contiguous(); let superblock = Superblock::parse_in(&data, 0)?; data.end = base + superblock.data_end(base, file_len)?; match superblock_ext::cache_image_state_in(&data, &superblock)? { @@ -167,13 +209,37 @@ impl FileData { .collect(); } } + data.contiguous = data.find_contiguous(); Ok((data, superblock)) } /// The HDF5 data as one slice, when the file is in memory (a `Vec`, an /// mmap, or a storage that holds it all and has no cache image to lay /// over it). + #[inline] fn contiguous(&self) -> Option<&[u8]> { + // SAFETY: `find_contiguous` borrowed these bytes from `backing` or + // `patched`, which this `FileData` owns and never changes after + // open. They live on the heap or in a mapping (a `Vec`'s buffer, an + // mmap, a private copy, or a buffer inside the `Arc`'d storage), so + // they stay put when the `FileData` moves, and they live as long as + // `self`. + self.contiguous + .map(|v| unsafe { core::slice::from_raw_parts(v.ptr, v.len) }) + } + + /// [`Self::contiguous`], worked out from `backing` and `patched`. + fn find_contiguous(&self) -> Option { + let bytes = self.compute_contiguous()?; + Some(WholeView { + ptr: bytes.as_ptr(), + len: bytes.len(), + }) + } + + /// The HDF5 data as one slice, from `backing` and `patched` (see + /// [`Self::contiguous`]). + fn compute_contiguous(&self) -> Option<&[u8]> { if let Some(p) = &self.patched { return p.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?); } @@ -224,6 +290,7 @@ impl FileData { } impl Storage for FileData { + #[inline] fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { if let Some(all) = self.contiguous() { return all.read_at(offset, len); @@ -261,6 +328,7 @@ impl Storage for FileData { .collect()) } + #[inline] fn as_contiguous(&self) -> Option<&[u8]> { self.contiguous() } @@ -400,8 +468,11 @@ impl File { /// /// The path uses `/` separators (e.g., `"group1/values"`). pub fn dataset(&self, path: &str) -> Result, Error> { - let data = self.data.meta()?; - let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; + let addr = with_bytes!(self.data.meta()?, |d| group_v2::resolve_path_any_in( + d, + &self.superblock, + path + ))?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); @@ -447,8 +518,11 @@ impl File { /// The path uses `/` separators (e.g., `"sensors"`). /// Use `"/"` or `""` for the root group. pub fn group(&self, path: &str) -> Result, Error> { - let data = self.data.meta()?; - let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; + let addr = with_bytes!(self.data.meta()?, |d| group_v2::resolve_path_any_in( + d, + &self.superblock, + path + ))?; Ok(Group { file: self, address: addr, @@ -560,13 +634,13 @@ impl File { /// [`AttrValue::Raw`] attribute. Variable-length strings are resolved in /// this file's global heap; see [`Dataset::read_string`] for the values. pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result, Error> { - crate::vlen::decode_strings( - &self.data, + with_bytes!(&self.data, |d| crate::vlen::decode_strings( + d, datatype, raw, self.offset_size(), self.length_size(), - ) + )) } /// Like [`decode_strings`](Self::decode_strings) for variable-length @@ -577,13 +651,13 @@ impl File { datatype: &Datatype, raw: &[u8], ) -> Result>, Error> { - crate::vlen::decode_string_bytes( - self.data.meta()?, + with_bytes!(self.data.meta()?, |d| crate::vlen::decode_string_bytes( + d, datatype, raw, self.offset_size(), self.length_size(), - ) + )) } /// Decode the variable-length sequences in `raw`, a buffer of elements @@ -595,22 +669,22 @@ impl File { datatype: &Datatype, raw: &[u8], ) -> Result>, Error> { - crate::vlen::decode_vlen( - self.data.meta()?, + with_bytes!(self.data.meta()?, |d| crate::vlen::decode_vlen( + d, datatype, raw, self.offset_size(), self.length_size(), - ) + )) } fn parse_header(&self, address: u64) -> Result { - ObjectHeader::parse_in( - self.data.meta()?, + with_bytes!(self.data.meta()?, |d| ObjectHeader::parse_in( + d, address, self.superblock.offset_size, self.superblock.length_size, - ) + )) } fn offset_size(&self) -> u8 { @@ -688,8 +762,12 @@ impl<'f> Group<'f> { &self, ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.parse_header(self.address)?; - let data = &self.file.data; - read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) + with_bytes!(&self.file.data, |d| read_attrs( + d, + &hdr, + self.file.offset_size(), + self.file.length_size() + )) } /// Get a dataset within this group by name. @@ -719,14 +797,13 @@ impl<'f> Group<'f> { /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { let hdr = self.file.parse_header(self.address)?; - let data = &self.file.data; - read_attr( - data, + with_bytes!(&self.file.data, |d| read_attr( + d, &hdr, name, self.file.offset_size(), self.file.length_size(), - ) + )) } /// The object header address of the child called `name`: the entry of @@ -734,9 +811,13 @@ impl<'f> Group<'f> { /// name index rather than by listing the group (see /// [`group_v2::resolve_child`]). fn child_address(&self, name: &str) -> Result { - let data = self.file.data.meta()?; - group_v2::resolve_child_in(data, &self.file.superblock, self.address, name) - .map_err(Error::Format) + with_bytes!(self.file.data.meta()?, |d| group_v2::resolve_child_in( + d, + &self.file.superblock, + self.address, + name + )) + .map_err(Error::Format) } /// This group's children that can be opened, as `(name, object header @@ -757,9 +838,10 @@ impl<'f> Group<'f> { /// [`group_v2::resolve_group_children`]); dangling, external and /// user-defined links are left out. fn children(&self) -> Result, Error> { - let data = self.file.data.meta()?; - group_v2::resolve_group_children_in(data, &self.file.superblock, self.address) - .map_err(Error::Format) + with_bytes!(self.file.data.meta()?, |d| { + group_v2::resolve_group_children_in(d, &self.file.superblock, self.address) + }) + .map_err(Error::Format) } } @@ -1332,13 +1414,12 @@ impl<'f> Dataset<'f> { pub fn attrs_with_errors( &self, ) -> Result<(HashMap, Vec), Error> { - let data = &self.file.data; - read_attrs( - data, + with_bytes!(&self.file.data, |d| read_attrs( + d, &self.header, self.file.offset_size(), self.file.length_size(), - ) + )) } /// The attribute called `name`, or `None` if it has none by that name @@ -1346,14 +1427,13 @@ impl<'f> Dataset<'f> { /// that name, found without reading the other attributes when they are /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { - let data = &self.file.data; - read_attr( - data, + with_bytes!(&self.file.data, |d| read_attr( + d, &self.header, name, self.file.offset_size(), self.file.length_size(), - ) + )) } /// Verify this dataset's content against its stored provenance hash @@ -1393,12 +1473,14 @@ impl<'f> Dataset<'f> { .iter() .find(|m| m.msg_type == msg_type) .map(|msg| { - clawhdf5_format::shared_message::message_data_in( - &self.file.data, - msg, - self.file.offset_size(), - self.file.length_size(), - ) + with_bytes!(&self.file.data, |d| { + clawhdf5_format::shared_message::message_data_in( + d, + msg, + self.file.offset_size(), + self.file.length_size(), + ) + }) .map_err(Error::Format) }) .transpose() From 6185874f9caaeb7edf61688d003e8a93cc517cc7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:43:00 -0500 Subject: [PATCH 08/10] format, clawhdf5: cut every Storage read to the range asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ExtentBytes and read_exact_at/read_upto rejected short results but passed longer-than-asked ones through, and FileData forwarded them too, so a Storage that broke read_at's contract by returning extra bytes had them decoded or returned as data (a contiguous dataset read gained 37 junk bytes). gather_storage alone trimmed. - storage::exact_len (new, pub): a read of len bytes as exactly len — cut when longer, an error when short. read_exact_at, read_upto and ExtentBytes (so chunk fetches and selection gathers) go through it. - FileData cuts a backend's answer to what it asked for before laying the cache image over it. - Tests: over a storage that appends 37 junk bytes to every read, every format-crate fixture reads exactly as from the slice (overlong_reads_are_cut_to_the_range_asked_for), and every facade fixture opens and reads through File::open_storage as through File::open (overlong_storage_reads_identically). Both failed before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 + crates/clawhdf5-format/src/storage.rs | 43 ++-- .../tests/storage_equivalence.rs | 195 +++++++++++------- crates/clawhdf5/src/reader.rs | 26 ++- crates/clawhdf5/tests/storage_equivalence.rs | 47 +++++ 5 files changed, 225 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c6a7b6..bbd2eb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,10 @@ - Each extent's bounds error is the one the slice readers gave, reported when the read reaches that extent, so a damaged file fails with the same error, in the same order, through either path. + - A backend that answers a read with more bytes than asked (breaking + `read_at`'s contract) never has the extra bytes used: every read is + cut to the range asked for (`storage::exact_len`), and a short answer + inside the file is an error. - **No behaviour change for in-memory and mapped files:** with `as_contiguous()` every path slices the file as before (checked below). - Tests (2026-09-26, tank): diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 7425296..482cfad 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -43,6 +43,9 @@ pub trait Storage { /// end of the storage (and empty when `offset` is at or past the end); /// a backend that cannot serve a range returns an error instead of a /// short read. + /// It is never longer than `len`; the parsers cut a longer result to + /// `len` (see [`exact_len`]) rather than read bytes from outside the + /// range. fn read_at(&self, offset: u64, len: usize) -> Result, FormatError>; /// Current length of the storage in bytes. @@ -218,13 +221,30 @@ pub fn read_exact_at( Some(end) if end <= file.len() => {} _ => return Err(eof()), } - let bytes = file.read_at(offset, len)?; - if bytes.len() < len { - // The storage shrank or the backend served a short read inside the - // file: never parse a partial structure. - return Err(short_read()); + // A short read (the storage shrank, or the backend served less inside + // the file) is an error: never parse a partial structure. + exact_len(file.read_at(offset, len)?, len) +} + +/// `bytes`, the result of asking a [`Storage`] for `len` bytes, as exactly +/// `len` bytes: a longer result (a backend that broke +/// [`Storage::read_at`]'s contract) is cut to `len`, so bytes from outside +/// the range asked for are never parsed or returned; a shorter one is an +/// error (the storage shrank, or the backend failed), never a partial +/// structure. +#[inline] +pub fn exact_len(bytes: Cow<'_, [u8]>, len: usize) -> Result, FormatError> { + match bytes.len().cmp(&len) { + core::cmp::Ordering::Equal => Ok(bytes), + core::cmp::Ordering::Less => Err(short_read()), + core::cmp::Ordering::Greater => Ok(match bytes { + Cow::Borrowed(b) => Cow::Borrowed(&b[..len]), + Cow::Owned(mut v) => { + v.truncate(len); + Cow::Owned(v) + } + }), } - Ok(bytes) } #[cold] @@ -329,11 +349,7 @@ pub fn read_upto( } let avail = file.len().saturating_sub(offset); let len = usize::try_from(avail).map_or(max, |a| a.min(max)); - let bytes = file.read_at(offset, len)?; - if bytes.len() < len { - return Err(short_read()); - } - Ok(bytes) + exact_len(file.read_at(offset, len)?, len) } /// Most stored bytes fetched by one [`Storage::read_ranges`] call when a @@ -461,10 +477,7 @@ impl<'a> ExtentBytes<'a> { )); } for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) { - if (bytes.len() as u64) < r.end - r.start { - return Err(short_read()); - } - out[slot] = Extent::Bytes(bytes); + out[slot] = Extent::Bytes(exact_len(bytes, (r.end - r.start) as usize)?); } } Ok(ExtentBytes::Fetched { base, extents: out }) diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index 71b0dc0..a25a79a 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -753,6 +753,73 @@ fn corpus_parses_identically_through_storage() { assert!(tally.files > 0); } +/// The whole read of every object reachable from the root group of `sb`'s +/// file — each dataset whole (fill-aware) and half of it through a +/// selection, and each group's listing — as one result to compare. +fn read_everything(file: &dyn Storage, sb: &Superblock) -> Result { + let (os, ls) = (sb.offset_size, sb.length_size); + let mut out = String::new(); + let mut queue = VecDeque::from([sb.root_group_address]); + let mut seen = HashSet::new(); + while let Some(addr) = queue.pop_front() { + if seen.len() > 200 || !seen.insert(addr) { + continue; + } + let header = ObjectHeader::parse_in(file, addr, os, ls)?; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| message_data_with_sohm_in(file, m, os, ls)) + .transpose() + }; + if let (Some(dt), Some(ds), Some(dl)) = ( + find(MessageType::Datatype)?, + find(MessageType::Dataspace)?, + find(MessageType::DataLayout)?, + ) { + let dt = Datatype::parse(&dt)?.0; + let ds = Dataspace::parse(&ds, ls)?; + let dl = DataLayout::parse(&dl, os, ls)?; + let pl = find(MessageType::FilterPipeline)? + .map(|p| FilterPipeline::parse(&p)) + .transpose()?; + let data = read_full_with_fill_in( + &header.messages, + file, + &dl, + &ds, + dt.type_size() as usize, + os, + ls, + || read_raw_data_full_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls), + ); + out.push_str(&format!("{addr}: {data:?}\n")); + if let Some(&d0) = ds.dimensions.first() { + let rank = ds.dimensions.len(); + let sel = Selection::Hyperslab { + start: vec![0; rank], + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(2)) + .chain(ds.dimensions[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + let part = + read_raw_data_selection_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel); + out.push_str(&format!("{addr} half: {part:?}\n")); + } + } + let children = group_v2::resolve_group_children_in(file, sb, addr); + out.push_str(&format!("{addr} children: {children:?}\n")); + if let Ok(c) = children { + queue.extend(c.iter().map(|c| c.object_header_address)); + } + } + Ok(out) +} + /// A storage that misbehaves: fails its `fail_at`-th read (1-based, `0` /// never), and, with `short`, serves one byte less than asked for inside /// the file (a truncated response). @@ -808,78 +875,7 @@ fn misbehaving_storage_never_returns_wrong_data() { let Ok(sb) = Superblock::parse(hdf5, 0) else { continue; }; - // The whole read of every object, as one result to compare. - let everything = |file: &dyn Storage| -> Result { - let (os, ls) = (sb.offset_size, sb.length_size); - let mut out = String::new(); - let mut queue = VecDeque::from([sb.root_group_address]); - let mut seen = HashSet::new(); - while let Some(addr) = queue.pop_front() { - if seen.len() > 200 || !seen.insert(addr) { - continue; - } - let header = ObjectHeader::parse_in(file, addr, os, ls)?; - let find = |t: MessageType| { - header - .messages - .iter() - .find(|m| m.msg_type == t) - .map(|m| message_data_with_sohm_in(file, m, os, ls)) - .transpose() - }; - if let (Some(dt), Some(ds), Some(dl)) = ( - find(MessageType::Datatype)?, - find(MessageType::Dataspace)?, - find(MessageType::DataLayout)?, - ) { - let dt = Datatype::parse(&dt)?.0; - let ds = Dataspace::parse(&ds, ls)?; - let dl = DataLayout::parse(&dl, os, ls)?; - let pl = find(MessageType::FilterPipeline)? - .map(|p| FilterPipeline::parse(&p)) - .transpose()?; - let data = read_full_with_fill_in( - &header.messages, - file, - &dl, - &ds, - dt.type_size() as usize, - os, - ls, - || read_raw_data_full_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls), - ); - out.push_str(&format!("{addr}: {data:?}\n")); - if let Some(&d0) = ds.dimensions.first() { - let rank = ds.dimensions.len(); - let sel = Selection::Hyperslab { - start: vec![0; rank], - stride: vec![1; rank], - count: std::iter::once(d0.div_ceil(2)) - .chain(ds.dimensions[1..].iter().copied()) - .collect(), - block: vec![1; rank], - }; - let part = read_raw_data_selection_in( - file, - &dl, - &ds, - &dt, - pl.as_ref(), - os, - ls, - &sel, - ); - out.push_str(&format!("{addr} half: {part:?}\n")); - } - } - let children = group_v2::resolve_group_children_in(file, &sb, addr); - out.push_str(&format!("{addr} children: {children:?}\n")); - if let Ok(c) = children { - queue.extend(c.iter().map(|c| c.object_header_address)); - } - } - Ok(out) - }; + let everything = |file: &dyn Storage| read_everything(file, &sb); let want = everything(&hdf5); let counting = CountingStorage::new(hdf5.to_vec()); assert_eq!( @@ -947,6 +943,61 @@ fn misbehaving_storage_never_returns_wrong_data() { ); } +/// A storage that breaks `read_at`'s contract the other way: every read +/// comes back with 37 bytes more than asked for (junk past the range). +struct Overlong { + data: Vec, +} + +impl Storage for Overlong { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let mut v = self.data.as_slice().read_at(offset, len)?.into_owned(); + v.extend(std::iter::repeat_n(0xa5, 37)); + Ok(std::borrow::Cow::Owned(v)) + } + + fn len(&self) -> u64 { + self.data.len() as u64 + } +} + +/// Bytes past the range asked for are never used: every read through a +/// storage that returns more than asked gives exactly the in-memory result. +#[test] +fn overlong_reads_are_cut_to_the_range_asked_for() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut files = Vec::new(); + hdf5_files(&dir, &mut files); + files.sort(); + let mut compared = 0; + for path in &files { + let Ok(bytes) = std::fs::read(path) else { + continue; + }; + let Ok((_, hdf5)) = split_user_block(&bytes) else { + continue; + }; + let Ok(sb) = Superblock::parse(hdf5, 0) else { + continue; + }; + let want = read_everything(&hdf5, &sb); + let got = read_everything( + &Overlong { + data: hdf5.to_vec(), + }, + &sb, + ); + assert_eq!( + format!("{got:?}"), + format!("{want:?}"), + "{}", + path.display() + ); + compared += 1; + } + assert!(compared > 40, "{compared}"); +} + /// A read_at-only storage that also counts `read_ranges` calls and ranges. struct BatchCounting { inner: CountingStorage, diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 191c614..a99373f 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -300,7 +300,7 @@ impl Storage for FileData { if len == 0 { return Ok(Cow::Borrowed(&[])); } - let bytes = self.remote()?.read_at(self.base + offset, len)?; + let bytes = cut_to(self.remote()?.read_at(self.base + offset, len)?, len); Ok(self.with_overlay(offset, bytes)) } @@ -323,8 +323,11 @@ impl Storage for FileData { let got = self.remote()?.read_ranges(&shifted)?; Ok(got .into_iter() - .zip(ranges) - .map(|(bytes, r)| self.with_overlay(r.start, bytes)) + .zip(ranges.iter().zip(&shifted)) + .map(|(bytes, (r, asked))| { + let bytes = cut_to(bytes, (asked.end - asked.start) as usize); + self.with_overlay(r.start, bytes) + }) .collect()) } @@ -334,6 +337,23 @@ impl Storage for FileData { } } +/// `bytes`, a backend's answer to a read of `len` bytes, without anything +/// past those `len` (a backend that returns more than asked breaks +/// [`Storage::read_at`]'s contract; the extra bytes are not the file's). A +/// short answer is passed on: the parsers' own checks refuse it. +fn cut_to(bytes: Cow<'_, [u8]>, len: usize) -> Cow<'_, [u8]> { + if bytes.len() <= len { + return bytes; + } + match bytes { + Cow::Borrowed(b) => Cow::Borrowed(&b[..len]), + Cow::Owned(mut v) => { + v.truncate(len); + Cow::Owned(v) + } + } +} + // --------------------------------------------------------------------------- // File // --------------------------------------------------------------------------- diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs index 7a2749b..71768d2 100644 --- a/crates/clawhdf5/tests/storage_equivalence.rs +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -432,3 +432,50 @@ fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() { "as_bytes over a range storage must not answer" ); } + +/// A storage that returns 37 junk bytes more than every read asked for. +struct Overlong(Vec); + +impl clawhdf5::Storage for Overlong { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let mut v = clawhdf5::Storage::read_at(self.0.as_slice(), offset, len)?.into_owned(); + v.extend(std::iter::repeat_n(0xa5, 37)); + Ok(std::borrow::Cow::Owned(v)) + } + + fn len(&self) -> u64 { + self.0.len() as u64 + } +} + +/// Bytes a misbehaving storage returns past the range asked for are never +/// read as the file's: every fixture reads through it as through +/// `File::open`. +#[test] +fn overlong_storage_reads_identically() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + files.sort(); + let mut compared = 0; + for path in &files { + let Ok(bytes) = std::fs::read(path) else { + continue; + }; + let Ok(local) = File::open(path) else { + continue; + }; + let mut remote = File::open_storage(Arc::new(Overlong(bytes))) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + remote.set_vds_resolver(sibling_resolver(path.parent().map(Path::to_path_buf))); + assert_eq!( + transcript(&local), + transcript(&remote), + "{}", + path.display() + ); + compared += 1; + } + assert!(compared >= 40, "{compared}"); +} From 75444950f3cd0dd30ac313df0ed9a06e4d4c4e91 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 19:04:56 -0500 Subject: [PATCH 09/10] clawhdf5: storage harness compares errors, not just failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The facade equivalence harness turned every data-read error into "Err", so it could not see File::open_storage failing differently from File::open (a Storage or ContiguousStorageRequired error where the mmap path gives a decode error, say). - value() keeps the whole error. The only allowance is for a line on which File::open itself varies between opens — the chunk cache lists a damaged dataset's chunks in hash-map order, so which failing chunk a full read reports varies (cve-2025-2310.h5, the one corpus file where this shows): both sides must fail there, and a fresh File::open (up to 64) must reproduce the storage's exact error. Open errors were already compared in full; they still agree. - The storage transcript may not contain ContiguousStorageRequired. - More selections: a strided hyperslab (every third row) through read_f64_selection, and out-of-order points through read_selection and read_i64_selection. - harness_compares_errors_not_just_failures checks the harness itself: two different errors are different values, and a difference File::open does not produce is reported. With full errors the harness passes on the 61 fixtures and on the corpus (701 files, 621 open). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 +- crates/clawhdf5/tests/storage_equivalence.rs | 136 ++++++++++++++++--- 2 files changed, 126 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbd2eb0..c74077f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,7 +98,15 @@ that open) through `File::open` and through `File::open_storage` over `CountingStorage`: the tree, every attribute (all, and each by name), every dataset's shape, types and values (all bytes, `f64`, `f32`, - `i64`, a hyperslab, strings, VL sequences) must be identical, and are. + `i64`, a box hyperslab, a strided one, out-of-order points, strings, VL + sequences) must be identical, and are — errors included, in full (open + errors and every read's). The one allowance is a line on which + `File::open` itself varies between two opens (the chunk cache lists a + damaged dataset's chunks in hash-map order, so which failing chunk a + full read reports varies: `cve-2025-2310.h5`), and then only if a fresh + `File::open` reproduces the storage's error. No storage read may answer + `ContiguousStorageRequired`. A storage that returns more bytes than + asked reads every fixture identically too. It also counts what one pass — open, list, read every attribute and every dataset once — asks of a storage with no cache: 176 092 `read_at` calls and 208 MB for the 621 corpus files (254 MB of files); the most diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs index 71768d2..9a5ace8 100644 --- a/crates/clawhdf5/tests/storage_equivalence.rs +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -47,15 +47,15 @@ fn digest(v: &T) -> String { format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len()) } -/// A data read's result: its value, or just `Err` — a full read goes +/// A data read's result: its value, or its error in full (the two paths +/// must fail the same way, not just both fail). One case is known to vary +/// between two `File`s and is allowed for in [`check`]: a full read goes /// through the file's chunk cache, which lists a damaged dataset's chunks -/// in hash-map order, so which failing chunk it reports varies from one -/// `File` to the next (the format crate's harness compares these errors on -/// the uncached path). -fn value(r: &Result) -> String { +/// in hash-map order, so which failing chunk it reports varies. +fn value(r: &Result) -> String { match r { Ok(v) => digest(v), - Err(_) => "Err".into(), + Err(e) => format!("Err({e:?})"), } } @@ -173,6 +173,43 @@ fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { value(&ds.read_selection(&sel)) ) .unwrap(); + // Every third row (a strided hyperslab), and a few points out + // of order: the last element, the first, one in the middle. + let strided = Selection::Hyperslab { + start: vec![0; rank], + stride: std::iter::once(3) + .chain(std::iter::repeat_n(1, rank - 1)) + .collect(), + count: std::iter::once(d0.div_ceil(3)) + .chain(shape[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + writeln!( + out, + "{path} f64 strided {}", + value(&ds.read_f64_selection(&strided)) + ) + .unwrap(); + if shape.iter().all(|&d| d > 0) { + let points = Selection::Points(vec![ + shape.iter().map(|&d| d - 1).collect(), + vec![0; rank], + shape.iter().map(|&d| d / 2).collect(), + ]); + writeln!( + out, + "{path} bytes points {}", + value(&ds.read_selection(&points)) + ) + .unwrap(); + writeln!( + out, + "{path} i64 points {}", + value(&ds.read_i64_selection(&points)) + ) + .unwrap(); + } } } match &raw_dt { @@ -296,20 +333,16 @@ fn check(path: &Path, totals: &mut Totals) { assert_eq!(local.user_block_size(), remote.user_block_size(), "{name}"); let want = transcript(&local); let got = transcript(&remote); + // Every read path works over a storage without the file in memory. + assert!( + !got.contains("ContiguousStorageRequired"), + "{name}: a read needed the file in memory" + ); if want != got { - let first = want - .lines() - .zip(got.lines()) - .find(|(w, g)| w != g) - .map(|(w, g)| format!("\n local: {w}\n storage: {g}")) - .unwrap_or_else(|| { - format!( - "\n {} vs {} lines", - want.lines().count(), - got.lines().count() - ) - }); - panic!("{name}: File::open_storage differs from File::open{first}"); + let first = unexplained_difference(path, &want, &got); + if let Some(first) = first { + panic!("{name}: File::open_storage differs from File::open{first}"); + } } totals.reads += storage.reads(); totals.bytes += storage.bytes_read(); @@ -334,6 +367,40 @@ fn check(path: &Path, totals: &mut Totals) { .push((reads, read_bytes, bytes.len() as u64, name)); } +/// Why the storage transcript `got` differs from the `File::open` one +/// `want`, or `None` when every line that differs is one `File::open` can +/// give too: a line that varies between two `File`s (the chunk cache's +/// hash-map order picks which failing chunk a damaged dataset's full read +/// reports) and whose storage value some fresh `File::open` reproduces. +/// Nothing else is allowed to differ. +fn unexplained_difference(path: &Path, want: &str, got: &str) -> Option { + let (want, got): (Vec<&str>, Vec<&str>) = (want.lines().collect(), got.lines().collect()); + if want.len() != got.len() { + return Some(format!("\n {} vs {} lines", want.len(), got.len())); + } + let mut open: Vec = (0..want.len()).filter(|&i| want[i] != got[i]).collect(); + // Only reads that fail on both sides may vary. + if let Some(&i) = open + .iter() + .find(|&&i| !(want[i].contains(" Err(") && got[i].contains(" Err("))) + { + return Some(format!("\n local: {}\n storage: {}", want[i], got[i])); + } + for _ in 0..64 { + let again = transcript(&File::open(path).unwrap()); + let again: Vec<&str> = again.lines().collect(); + open.retain(|&i| again.get(i) != Some(&got[i])); + if open.is_empty() { + return None; + } + } + let i = open[0]; + Some(format!( + "\n local: {}\n storage: {}\n (no File::open of 64 gave the storage's result)", + want[i], got[i] + )) +} + fn report(what: &str, totals: &mut Totals) { eprintln!( "{what}: {} files ({} open, {} bytes); comparison: {} read_at calls, {} bytes; \ @@ -479,3 +546,34 @@ fn overlong_storage_reads_identically() { } assert!(compared >= 40, "{compared}"); } + +/// The harness tells failures apart: two different errors are two +/// different transcripts, and only a difference `File::open` itself +/// produces between two opens is let through. +#[test] +fn harness_compares_errors_not_just_failures() { + let a: Result<(), FormatError> = Err(FormatError::ContiguousStorageRequired("x")); + let b: Result<(), FormatError> = Err(FormatError::Storage("x".into())); + assert_ne!(value(&a), value(&b)); + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../clawhdf5-format/tests/fixtures/chunked_2d.h5"); + let want = transcript(&File::open(&path).unwrap()); + assert_eq!(unexplained_difference(&path, &want, &want), None); + // A read that fails differently through the storage. + let line = want.lines().position(|l| l.contains(" all ")).unwrap(); + let (mut w, mut g): (Vec, Vec) = ( + want.lines().map(String::from).collect(), + want.lines().map(String::from).collect(), + ); + w[line] = format!( + "{} Err(Format(DataSizeMismatch))", + &w[line][..w[line].find(" all ").unwrap() + 4] + ); + g[line] = format!( + "{} Err(Format(Storage(\"injected\")))", + &g[line][..g[line].find(" all ").unwrap() + 4] + ); + assert!(unexplained_difference(&path, &w.join("\n"), &g.join("\n")).is_some()); + // A value against an error is never let through. + assert!(unexplained_difference(&path, &want, &g.join("\n")).is_some()); +} From ea0508aaa5a6f20a8261ffbb056a46767549cb58 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 19:05:45 -0500 Subject: [PATCH 10/10] format: no truncating u64 -> usize casts in gather_storage and ExtentBytes check-32bit-casts.sh flagged two casts added by the previous commits; both values are bounded (checked by gather_storage's first walk, and built from a usize fetch length), so they go through addr::saturating_usize. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/gather.rs | 6 +++--- crates/clawhdf5-format/src/storage.rs | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/src/gather.rs b/crates/clawhdf5-format/src/gather.rs index 08480f1..5f73dbd 100644 --- a/crates/clawhdf5-format/src/gather.rs +++ b/crates/clawhdf5-format/src/gather.rs @@ -454,9 +454,9 @@ pub(crate) fn gather_storage( if error.is_some() { return; } - // Checked by the first walk. - let mut at = first as usize * elem_size; - let mut len = n as usize * elem_size; + // Checked by the first walk (these cannot saturate or wrap). + let mut at = crate::addr::saturating_usize(first).wrapping_mul(elem_size); + let mut len = crate::addr::saturating_usize(n).wrapping_mul(elem_size); while len > 0 { while spans.get(span).is_some_and(|s| s.out_end <= out.len()) { span += 1; diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 482cfad..08720ab 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -477,7 +477,8 @@ impl<'a> ExtentBytes<'a> { )); } for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) { - out[slot] = Extent::Bytes(exact_len(bytes, (r.end - r.start) as usize)?); + let len = crate::addr::saturating_usize(r.end - r.start); + out[slot] = Extent::Bytes(exact_len(bytes, len)?); } } Ok(ExtentBytes::Fetched { base, extents: out })