From 42894bf93b4685945bc3401f5e13811ab231facc Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:13:01 -0500 Subject: [PATCH 01/49] format: v2 B-trees, dense groups and group listings over Storage BTreeV2Header::parse_in, collect_btree_v2_records_in and find_btree_v2_records_in read one bounded window per node (its size is known from the parent before the node is read; a count stretched past node_size is checked against the end of the file first), with the whole-file bounds errors unchanged. With them, dense attributes, a SOHM B-tree index and huge fractal-heap objects no longer answer ContiguousStorageRequired, and group_v1/group_v2 listings, lookups and path resolution get *_in cores (resolve_group_children_in, resolve_child_in, resolve_path_any_in, ...). The &[u8] functions are thin wrappers, as in M1. The equivalence harness now fails on any ContiguousStorageRequired and compares v2 B-tree headers, records and descents, group listings, child lookups and paths; a unit test compares a two-level tree through a read_at-only storage truncated at every length and with every node byte flipped. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/addr.rs | 9 + crates/clawhdf5-format/src/attribute.rs | 55 ++-- crates/clawhdf5-format/src/btree_v2.rs | 306 ++++++++++++------ crates/clawhdf5-format/src/btree_v2_write.rs | 51 +++ crates/clawhdf5-format/src/fractal_heap.rs | 36 +-- crates/clawhdf5-format/src/group_v1.rs | 85 +++-- crates/clawhdf5-format/src/group_v2.rs | 138 +++++--- crates/clawhdf5-format/src/shared_message.rs | 23 +- .../tests/storage_equivalence.rs | 138 ++++---- 9 files changed, 544 insertions(+), 297 deletions(-) diff --git a/crates/clawhdf5-format/src/addr.rs b/crates/clawhdf5-format/src/addr.rs index 3198f1f..e6471d0 100644 --- a/crates/clawhdf5-format/src/addr.rs +++ b/crates/clawhdf5-format/src/addr.rs @@ -23,6 +23,15 @@ pub fn to_usize(value: u64) -> Result { to_index::(value) } +/// A file address for a [`crate::storage::Storage`] read, checked as +/// [`to_usize`] checks it: the parsers read through 64-bit offsets, but an +/// address that could not index an in-memory file on this platform is the +/// same [`FormatError::Overflow`] the slice parsers gave for it. +#[inline] +pub fn checked_addr(value: u64) -> Result { + to_usize(value).map(|_| value) +} + /// [`to_usize`] for an index type of any width. `usize` is 64 bits wide on /// the hosts CI tests on, where the error path cannot be reached through /// `usize`; tests run the same code with `u32` in its place, as on a 32-bit diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 78aaf12..4b30fac 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -7,7 +7,7 @@ use std::borrow::Cow; use crate::addr::to_usize; use crate::attribute_info::AttributeInfoMessage; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in}; use crate::checksum::jenkins_lookup3; use crate::data_read; use crate::dataspace::Dataspace; @@ -17,7 +17,7 @@ use crate::fractal_heap::FractalHeapHeader; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; use crate::shared_message; -use crate::storage::{Storage, require_contiguous}; +use crate::storage::Storage; use crate::vl_data; /// A parsed HDF5 attribute message. @@ -578,9 +578,12 @@ pub fn find_attribute_in( .find(|a| a.name == name), ); }; - let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?; - let btree_hdr = - BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + to_usize(btree_addr)? as u64, + offset_size, + length_size, + )?; let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?; if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 { return Ok( @@ -610,7 +613,7 @@ pub fn find_attribute_in( // hash is the last field. let hash = jenkins_lookup3(name.as_bytes()); let hash_at = usize::from(btree_hdr.record_size) - 4; - let records = find_btree_v2_records(contiguous, &btree_hdr, offset_size, &mut |r| match r + let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| match r .get(hash_at..hash_at + 4) { Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), @@ -722,10 +725,13 @@ fn extract_dense_attributes( expected: 1, available: 0, })?; - let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?; - let btree_hdr = - BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?; - let records = collect_btree_v2_records(contiguous, &btree_hdr, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + to_usize(btree_addr)? as u64, + offset_size, + length_size, + )?; + let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { // Per HDF5 spec, both type 8 and type 9 records start with heap_id: @@ -1156,11 +1162,9 @@ mod tests { } /// Every object's attributes in h5py-written files read identically - /// through a read_at-only CountingStorage — compact ones, shared ones - /// and those behind an Attribute Info message — except dense storage, - /// whose v2 B-tree index is not read over Storage yet: that is the clean - /// ContiguousStorageRequired error, never a partial list. Through a - /// slice as Storage every object matches. + /// through a read_at-only CountingStorage — compact ones, shared ones, + /// those behind an Attribute Info message and dense storage (its v2 + /// B-tree name index included) — and through a slice as Storage. #[test] fn storage_reads_match_slice_reads() { use crate::storage::CountingStorage; @@ -1206,18 +1210,17 @@ mod tests { .unwrap() .is_some_and(|i| i.fractal_heap_address.is_some()); if is_dense { - let e = FormatError::ContiguousStorageRequired( - "dense attribute storage (a v2 B-tree)", - ); - assert_eq!(got.unwrap_err(), e, "{name}"); - assert_eq!(got_t.unwrap_err(), e, "{name}"); dense += 1; - } else { - attrs += want.as_ref().map_or(0, Vec::len); - assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}"); - let want_t = extract_attributes_tolerant(file, &header, os, ls); - assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}"); - same += 1; + } + attrs += want.as_ref().map_or(0, Vec::len); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}"); + let want_t = extract_attributes_tolerant(file, &header, os, ls); + assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}"); + same += 1; + for a in want.iter().flatten() { + let one = find_attribute_in(&storage, &header, &a.name, os, ls); + let want_one = find_attribute_in_file(file, &header, &a.name, os, ls); + assert_eq!(format!("{one:?}"), format!("{want_one:?}"), "{name}"); } } } diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index d3f6250..5641eb7 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -9,6 +9,7 @@ use byteorder::{ByteOrder, LittleEndian}; use crate::addr::to_usize; use crate::error::FormatError; +use crate::storage::{Storage, Window, len_usize}; /// Parsed B-tree v2 header (signature "BTHD"). #[derive(Debug, Clone)] @@ -99,38 +100,52 @@ impl BTreeV2Header { offset_size: u8, length_size: u8, ) -> Result { - ensure_len(file_data, offset, 4)?; - if &file_data[offset..offset + 4] != b"BTHD" { + Self::parse_in(file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one bounded read of the + /// header. + pub fn parse_in( + file: &S, + offset: u64, + offset_size: u8, + length_size: u8, + ) -> Result { + // Every field and the checksum; the window holds all of it or ends + // at the end of the file, so its bounds checks are the whole-file + // ones. + let full = 16 + usize::from(offset_size) + 2 + usize::from(length_size) + 4; + let w = Window::read(file, offset, full)?; + let d = &w.bytes; + w.ensure(0, 4)?; + if &d[..4] != b"BTHD" { return Err(FormatError::InvalidBTreeV2Signature); } - ensure_len(file_data, offset, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?; - let version = file_data[offset + 4]; + w.ensure(0, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?; + let version = d[4]; if version != 0 { return Err(FormatError::InvalidBTreeV2Version(version)); } - let tree_type = file_data[offset + 5]; - let node_size = u32::from_le_bytes([ - file_data[offset + 6], - file_data[offset + 7], - file_data[offset + 8], - file_data[offset + 9], - ]); - let record_size = u16::from_le_bytes([file_data[offset + 10], file_data[offset + 11]]); - let depth = u16::from_le_bytes([file_data[offset + 12], file_data[offset + 13]]); - let _split_percent = file_data[offset + 14]; - let _merge_percent = file_data[offset + 15]; + let tree_type = d[5]; + let node_size = u32::from_le_bytes([d[6], d[7], d[8], d[9]]); + let record_size = u16::from_le_bytes([d[10], d[11]]); + let depth = u16::from_le_bytes([d[12], d[13]]); + let _split_percent = d[14]; + let _merge_percent = d[15]; - let mut pos = offset + 16; - let root_node_address = read_offset(file_data, pos, offset_size)?; + let mut pos = 16; + w.ensure(pos, usize::from(offset_size))?; + let root_node_address = read_offset(d, pos, offset_size)?; pos += offset_size as usize; - ensure_len(file_data, pos, 2)?; - let num_records_in_root = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); + w.ensure(pos, 2)?; + let num_records_in_root = u16::from_le_bytes([d[pos], d[pos + 1]]); pos += 2; - let total_records = read_offset(file_data, pos, length_size)?; + w.ensure(pos, usize::from(length_size))?; + let total_records = read_offset(d, pos, length_size)?; #[allow(unused_assignments)] { pos += length_size as usize; @@ -139,9 +154,9 @@ impl BTreeV2Header { // Validate header checksum #[cfg(feature = "checksum")] { - ensure_len(file_data, pos, 4)?; - let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]); - let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]); + w.ensure(pos, 4)?; + let stored = LittleEndian::read_u32(&d[pos..pos + 4]); + let computed = crate::checksum::jenkins_lookup3(&d[..pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, @@ -191,6 +206,17 @@ pub fn collect_btree_v2_records( header: &BTreeV2Header, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + collect_btree_v2_records_in(file_data, header, offset_size, length_size) +} + +/// [`collect_btree_v2_records`] over any [`Storage`]: one bounded read per +/// node. +pub fn collect_btree_v2_records_in( + file: &S, + header: &BTreeV2Header, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { if header.total_records == 0 || header.num_records_in_root == 0 { return Ok(Vec::new()); @@ -210,23 +236,24 @@ pub fn collect_btree_v2_records( // millions of records from a few kilobytes. Counting against what the // file could physically contain bounds that without trusting the // header's own `total_records`. - let mut budget = file_data.len() / usize::from(header.record_size.max(1)); + let mut budget = len_usize(file) / usize::from(header.record_size.max(1)); let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size); if header.depth == 0 { // Root is a leaf parse_leaf_records( - file_data, + file, to_usize(header.root_node_address)?, header.num_records_in_root, header.record_size, + header.node_size, ) } else { // Root is internal; traverse recursively let mut records = Vec::new(); collect_internal_records( - file_data, + file, to_usize(header.root_node_address)?, header.num_records_in_root, header.depth, @@ -242,36 +269,72 @@ pub fn collect_btree_v2_records( } } +/// A node's bytes: `want` bytes at `offset` (fewer only at the end of the +/// file), after checking its 4-byte signature. A node is read in one piece +/// when it fits in `node_size` (every valid node does); a larger claimed +/// extent — record counts from a damaged parent — is first checked against +/// the end of the file, so it costs a read only of bytes the file has. +/// Bounds errors are the whole-file ones: the signature check needs the +/// first 6 bytes, then `checks` — `(position, length)` pairs relative to +/// the node, in the order the parser checks them — must lie in the file. +fn read_node<'a, S: Storage + ?Sized>( + file: &'a S, + offset: usize, + want: usize, + node_size: u32, + signature: &[u8; 4], + checks: &[(usize, usize)], +) -> Result, FormatError> { + let one_read = usize::try_from(node_size).unwrap_or(usize::MAX).max(6); + let w = Window::read(file, offset as u64, want.min(one_read))?; + w.ensure(0, 6)?; + if &w.bytes[..4] != signature { + return Err(FormatError::InvalidBTreeV2Signature); + } + if want <= one_read { + return Ok(w); + } + for &(rel, len) in checks { + Window::check_extent(file, offset as u64, rel, len)?; + } + Window::read(file, offset as u64, want) +} + /// Parse records from a leaf node (signature "BTLF"). -fn parse_leaf_records( - file_data: &[u8], +fn parse_leaf_records( + file: &S, offset: usize, num_records: u16, record_size: u16, + node_size: u32, ) -> Result, FormatError> { // signature(4) + version(1) + type(1) = 6 bytes header - ensure_len(file_data, offset, 6)?; - if &file_data[offset..offset + 4] != b"BTLF" { - return Err(FormatError::InvalidBTreeV2Signature); - } - - let pos = offset + 6; + let pos = 6; let rs = record_size as usize; let total = (num_records as usize) .checked_mul(rs) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: len_usize(file), })?; - ensure_len(file_data, pos, total)?; + let w = read_node( + file, + offset, + pos + total + 4, + node_size, + b"BTLF", + &[(pos, total)], + )?; + let d = &w.bytes; + w.ensure(pos, total)?; // Validate checksum: 4 bytes after records + padding #[cfg(feature = "checksum")] { let checksum_pos = pos + total; - if file_data.len() >= checksum_pos + 4 { - let stored = LittleEndian::read_u32(&file_data[checksum_pos..checksum_pos + 4]); - let computed = crate::checksum::jenkins_lookup3(&file_data[offset..checksum_pos]); + if d.len() >= checksum_pos + 4 { + let stored = LittleEndian::read_u32(&d[checksum_pos..checksum_pos + 4]); + let computed = crate::checksum::jenkins_lookup3(&d[..checksum_pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, @@ -285,17 +348,41 @@ fn parse_leaf_records( for i in 0..num_records as usize { let start = pos + i * rs; records.push(BTreeV2Record { - data: file_data[start..start + rs].to_vec(), + data: d[start..start + rs].to_vec(), }); } Ok(records) } +/// An internal node read from the file: its bytes (from the signature on), +/// where its records start, and its children as `(address, record count)`. +struct InternalNode<'a> { + node: Window<'a>, + records_start: usize, + children: Vec<(u64, u16)>, +} + +impl InternalNode<'_> { + /// Record `i`, `rs` bytes long. + fn record(&self, i: usize, rs: usize) -> Result<&[u8], FormatError> { + let overflow = || FormatError::UnexpectedEof { + expected: usize::MAX, + available: usize::MAX, + }; + let rec_start = i + .checked_mul(rs) + .and_then(|o| self.records_start.checked_add(o)) + .ok_or_else(overflow)?; + self.node.ensure(rec_start, rs)?; + Ok(&self.node.bytes[rec_start..rec_start + rs]) + } +} + /// An internal node's layout: where its records start, and its children as /// `(address, record count)`. #[allow(clippy::too_many_arguments)] -fn read_internal_node( - file_data: &[u8], +fn read_internal_node( + file: &S, offset: usize, num_records: u16, depth: u16, @@ -303,25 +390,15 @@ fn read_internal_node( node_size: u32, offset_size: u8, max_leaf_nrec: u64, -) -> Result<(usize, Vec<(u64, u16)>), FormatError> { - // signature(4) + version(1) + type(1) = 6 - ensure_len(file_data, offset, 6)?; - if &file_data[offset..offset + 4] != b"BTIN" { - return Err(FormatError::InvalidBTreeV2Signature); - } - +) -> Result, FormatError> { let nr = num_records as usize; let rs = record_size as usize; - let mut pos = offset + 6; // Records first let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: len_usize(file), })?; - ensure_len(file_data, pos, records_total)?; - let records_start = pos; - pos += records_total; // Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the // child's record count is always encoded in the width needed for a @@ -344,13 +421,30 @@ fn read_internal_node( let num_children = nr + 1; let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width; - ensure_len(file_data, pos, num_children * child_ptr_size)?; + let pointers = num_children * child_ptr_size; + + // signature(4) + version(1) + type(1) = 6, records, pointers, checksum. + let w = read_node( + file, + offset, + 6 + records_total + pointers + 4, + node_size, + b"BTIN", + &[(6, records_total), (6 + records_total, pointers)], + )?; + let d = &w.bytes; + let mut pos = 6; + w.ensure(pos, records_total)?; + let records_start = pos; + pos += records_total; + + w.ensure(pos, pointers)?; let mut children = Vec::with_capacity(num_children); for _ in 0..num_children { - let addr = read_offset(file_data, pos, offset_size)?; + let addr = read_offset(d, pos, offset_size)?; pos += offset_size as usize; - let child_nrec = read_var_uint(file_data, pos, nrec_width)? as u16; + let child_nrec = read_var_uint(d, pos, nrec_width)? as u16; pos += nrec_width; pos += total_nrec_width; // skip total records in subtree children.push((addr, child_nrec)); @@ -362,9 +456,9 @@ fn read_internal_node( // a mismatch here, and so does this. #[cfg(feature = "checksum")] { - ensure_len(file_data, pos, 4)?; - let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]); - let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]); + w.ensure(pos, 4)?; + let stored = LittleEndian::read_u32(&d[pos..pos + 4]); + let computed = crate::checksum::jenkins_lookup3(&d[..pos]); if computed != stored { return Err(FormatError::ChecksumMismatch { expected: stored, @@ -372,37 +466,17 @@ fn read_internal_node( }); } } - Ok((records_start, children)) -} - -/// Record `i` of an internal node whose records start at `records_start`. -fn internal_record( - file_data: &[u8], - records_start: usize, - i: usize, - rs: usize, -) -> Result<&[u8], FormatError> { - let overflow = || FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - }; - let rec_start = i - .checked_mul(rs) - .and_then(|o| records_start.checked_add(o)) - .ok_or_else(overflow)?; - let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?; - file_data - .get(rec_start..rec_end) - .ok_or(FormatError::UnexpectedEof { - expected: rec_end, - available: file_data.len(), - }) + Ok(InternalNode { + node: w, + records_start, + children, + }) } /// Recursively collect records from an internal node. #[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)] -fn collect_internal_records( - file_data: &[u8], +fn collect_internal_records( + file: &S, offset: usize, num_records: u16, depth: u16, @@ -416,8 +490,8 @@ fn collect_internal_records( ) -> Result<(), FormatError> { let nr = num_records as usize; let rs = record_size as usize; - let (records_start, children) = read_internal_node( - file_data, + let node = read_internal_node( + file, offset, num_records, depth, @@ -430,16 +504,21 @@ fn collect_internal_records( // Interleave: child[0], record[0], child[1], record[1], ..., child[nr] // We collect child[0] records, then record[0], then child[1], etc. - for (i, &(child_addr, child_nrec)) in children.iter().enumerate() { + for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() { if child_depth == 0 { // Before parsing, so a refused tree is not also a large allocation. spend(budget, usize::from(child_nrec))?; - let leaf_recs = - parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?; + let leaf_recs = parse_leaf_records( + file, + to_usize(child_addr)?, + child_nrec, + record_size, + node_size, + )?; out.extend(leaf_recs); } else { collect_internal_records( - file_data, + file, to_usize(child_addr)?, child_nrec, child_depth, @@ -455,7 +534,7 @@ fn collect_internal_records( // Add record[i] (except after the last child) if i < nr { - let data = internal_record(file_data, records_start, i, rs)?; + let data = node.record(i, rs)?; spend(budget, 1)?; out.push(BTreeV2Record { data: data.to_vec(), @@ -481,6 +560,17 @@ pub fn find_btree_v2_records( header: &BTreeV2Header, offset_size: u8, cmp: &mut dyn FnMut(&[u8]) -> Ordering, +) -> Result, FormatError> { + find_btree_v2_records_in(file_data, header, offset_size, cmp) +} + +/// [`find_btree_v2_records`] over any [`Storage`]: one bounded read per +/// node visited. +pub fn find_btree_v2_records_in( + file: &S, + header: &BTreeV2Header, + offset_size: u8, + cmp: &mut dyn FnMut(&[u8]) -> Ordering, ) -> Result, FormatError> { if header.total_records == 0 || header.num_records_in_root == 0 { return Ok(Vec::new()); @@ -490,11 +580,11 @@ pub fn find_btree_v2_records( } // As in `collect_btree_v2_records`: a valid tree cannot hold more // records than the file has room for, however its children are shared. - let mut budget = file_data.len() / usize::from(header.record_size.max(1)); + let mut budget = len_usize(file) / usize::from(header.record_size.max(1)); let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size); let mut out = Vec::new(); find_in_node( - file_data, + file, header, to_usize(header.root_node_address)?, header.num_records_in_root, @@ -509,8 +599,8 @@ pub fn find_btree_v2_records( } #[allow(clippy::too_many_arguments)] -fn find_in_node( - file_data: &[u8], +fn find_in_node( + file: &S, header: &BTreeV2Header, offset: usize, num_records: u16, @@ -523,7 +613,13 @@ fn find_in_node( ) -> Result<(), FormatError> { spend(budget, usize::from(num_records))?; if depth == 0 { - let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?; + let records = parse_leaf_records( + file, + offset, + num_records, + header.record_size, + header.node_size, + )?; out.extend( records .into_iter() @@ -532,8 +628,8 @@ fn find_in_node( return Ok(()); } let rs = usize::from(header.record_size); - let (records_start, children) = read_internal_node( - file_data, + let node = read_internal_node( + file, offset, num_records, depth, @@ -545,17 +641,17 @@ fn find_in_node( let nr = usize::from(num_records); let mut order = Vec::with_capacity(nr); for i in 0..nr { - order.push(cmp(internal_record(file_data, records_start, i, rs)?)); + order.push(cmp(node.record(i, rs)?)); } // Child `i` holds the keys between record `i - 1` and record `i`: it can // hold a match unless the record before it is already past the range or // the record after it is still before it. - for (i, &(child_addr, child_nrec)) in children.iter().enumerate() { + for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() { let after_left = i == 0 || order[i - 1] != Ordering::Greater; let before_right = i == nr || order[i] != Ordering::Less; if after_left && before_right { find_in_node( - file_data, + file, header, to_usize(child_addr)?, child_nrec, @@ -569,7 +665,7 @@ fn find_in_node( } if i < nr && order[i] == Ordering::Equal { out.push(BTreeV2Record { - data: internal_record(file_data, records_start, i, rs)?.to_vec(), + data: node.record(i, rs)?.to_vec(), }); } } diff --git a/crates/clawhdf5-format/src/btree_v2_write.rs b/crates/clawhdf5-format/src/btree_v2_write.rs index 38fa7c1..dd8e9cf 100644 --- a/crates/clawhdf5-format/src/btree_v2_write.rs +++ b/crates/clawhdf5-format/src/btree_v2_write.rs @@ -441,6 +441,57 @@ mod tests { } } + /// A two-level tree read through a `read_at`-only storage gives what + /// the slice gives — records, descents and errors — whole, truncated + /// at every length, and with each byte of its nodes flipped, and each + /// node costs one read. + #[test] + fn storage_reads_match_slice_reads() { + use crate::btree_v2::{ + collect_btree_v2_records_in, find_btree_v2_records, find_btree_v2_records_in, + }; + use crate::storage::CountingStorage; + let (rs, n, base) = (11usize, 120usize, 64usize); + let recs = records(n, rs); + let tree = build_btree_v2(params(128, 11), &recs, base as u64, 8, 8).unwrap(); + let mut whole = vec![0u8; base]; + whole.extend_from_slice(&tree); + let hdr = BTreeV2Header::parse(&whole, base, 8, 8).unwrap(); + assert!(hdr.depth >= 1, "{hdr:?}"); + let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap()); + let mut files = Vec::new(); + for cut in base..=whole.len() { + files.push(whole[..cut].to_vec()); + } + for at in base..whole.len() { + let mut bad = whole.clone(); + bad[at] ^= 0x5a; + files.push(bad); + } + let mut ok = 0; + for f in &files { + let st = CountingStorage::new(f.clone()); + let want_h = BTreeV2Header::parse(f, base, 8, 8); + let got_h = BTreeV2Header::parse_in(&st, base as u64, 8, 8); + assert_eq!(format!("{got_h:?}"), format!("{want_h:?}")); + // The nodes of the intact header, over each damaged file. + let want = collect_btree_v2_records(f, &hdr, 8, 8); + st.reset(); + let got = collect_btree_v2_records_in(&st, &hdr, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + if want.is_ok() { + ok += 1; + assert!(st.reads() <= 1 + n as u64 / 3, "{} reads", st.reads()); + } + for k in [0u64, 7, 60, 119, 500] { + let want = find_btree_v2_records(f, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)); + let got = find_btree_v2_records_in(&st, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + } + } + assert!(ok > 1); + } + #[test] fn a_node_too_small_or_too_big_is_an_error() { assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err()); diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index 7bdce6f..83cfd33 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -7,10 +7,10 @@ use alloc::{format, vec::Vec}; use byteorder::{ByteOrder, LittleEndian}; use crate::addr::to_usize; -use crate::btree_v2::{BTreeV2Header, find_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, find_btree_v2_records_in}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; -use crate::storage::{Storage, Window, len_usize, read_exact_at, require_contiguous}; +use crate::storage::{Storage, Window, len_usize, read_exact_at}; /// Parsed fractal heap header (signature "FRHP"). #[derive(Debug, Clone)] @@ -386,9 +386,7 @@ impl FractalHeapHeader { self.read_managed_object_in(file_data, id_bytes, offset_size) } - /// [`Self::read_managed_object`] over any [`Storage`]. A huge object - /// found through the huge-object v2 B-tree still needs the whole file - /// in memory ([`FormatError::ContiguousStorageRequired`] otherwise). + /// [`Self::read_managed_object`] over any [`Storage`]. pub fn read_managed_object_in( &self, file_data: &S, @@ -491,11 +489,9 @@ impl FractalHeapHeader { "huge object ID but the heap has no huge-object index", )); } - // The v2 B-tree is read from a slice until it is converted. - let file_data = require_contiguous(file, "a huge fractal-heap object's B-tree")?; - let hdr = BTreeV2Header::parse( - file_data, - to_usize(self.huge_btree_address)?, + let hdr = BTreeV2Header::parse_in( + file, + self.huge_btree_address, self.offset_size, self.length_size, )?; @@ -513,7 +509,7 @@ impl FractalHeapHeader { // Records are ordered by ID (the last field): descend to the ones // equal to `key` instead of reading the whole index. let id_at = rec_len - ls; - let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| { + let records = find_btree_v2_records_in(file, &hdr, self.offset_size, &mut |r| { le_uint(&r[id_at..id_at + ls]).cmp(&key) })?; for rec in &records { @@ -1317,22 +1313,20 @@ mod tests { assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want); } - /// A huge object found through the huge-object B-tree needs the whole - /// file in memory until the B-tree reader is converted: a clean error - /// on other storage. + /// A huge object found through the huge-object B-tree reads the + /// B-tree through Storage: the same result (here an error, there is no + /// B-tree at that address) as from the slice. #[test] - fn huge_object_btree_needs_contiguous_storage() { + fn huge_object_btree_reads_through_storage() { use crate::storage::CountingStorage; let (file, _) = build_simple_heap(8, 8); let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap(); hdr.huge_btree_address = 700; + let id = [0x10, 1, 0, 0, 0, 0, 0]; + let want = hdr.read_managed_object(&file, &id, 8); + assert!(want.is_err()); let storage = CountingStorage::new(file); - assert_eq!( - hdr.read_managed_object_in(&storage, &[0x10, 1, 0, 0, 0, 0, 0], 8), - Err(FormatError::ContiguousStorageRequired( - "a huge fractal-heap object's B-tree" - )) - ); + assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want); } /// A header with an I/O filter pipeline (read in a second, longer diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index 3dff9cd..ebd4d32 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -3,12 +3,13 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; -use crate::addr::to_usize; -use crate::btree_v1::collect_symbol_table_nodes; +use crate::addr::checked_addr; +use crate::btree_v1::collect_symbol_table_nodes_in; use crate::error::FormatError; use crate::local_heap::LocalHeap; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; +use crate::storage::Storage; use crate::symbol_table::{SymbolTableMessage, SymbolTableNode}; /// A resolved group entry (child name + object header address). @@ -36,6 +37,16 @@ pub fn resolve_v1_group_entries( sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + resolve_v1_group_entries_in(file_data, sym_table_msg, offset_size, length_size) +} + +/// [`resolve_v1_group_entries`] over any [`Storage`]. +pub fn resolve_v1_group_entries_in( + file_data: &S, + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size)?; if entries.iter().any(|e| e.name.is_empty()) { @@ -46,22 +57,22 @@ pub fn resolve_v1_group_entries( /// Every entry of a v1 group, empty names included — for looking a name up, /// which never matches an empty name. -pub(crate) fn v1_group_entries( - file_data: &[u8], +pub(crate) fn v1_group_entries( + file_data: &S, sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, ) -> Result, FormatError> { // Parse local heap - let heap = LocalHeap::parse( + let heap = LocalHeap::parse_in( file_data, - to_usize(sym_table_msg.local_heap_address)?, + checked_addr(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; // Collect all SNOD addresses from B-tree - let snod_addrs = collect_symbol_table_nodes( + let snod_addrs = collect_symbol_table_nodes_in( file_data, sym_table_msg.btree_address, offset_size, @@ -71,15 +82,15 @@ pub(crate) fn v1_group_entries( let mut entries = Vec::new(); let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; + let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?; for entry in &snod.entries { // Like libhdf5, look at the heap's free list only once a name is // needed: an empty group with a damaged heap still lists. if !heap_checked { - heap.validate_free_list(file_data, length_size)?; + heap.validate_free_list_in(file_data, length_size)?; heap_checked = true; } - let name = heap.read_string(file_data, entry.link_name_offset)?; + let name = heap.read_string_in(file_data, entry.link_name_offset)?; entries.push(GroupEntry { name, object_header_address: entry.object_header_address, @@ -103,6 +114,17 @@ pub fn find_v1_soft_link( name: &str, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + find_v1_soft_link_in(file_data, sym_table_msg, name, offset_size, length_size) +} + +/// [`find_v1_soft_link`] over any [`Storage`]. +pub fn find_v1_soft_link_in( + file_data: &S, + sym_table_msg: &SymbolTableMessage, + name: &str, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let mut found = None; for_each_v1_soft_link( @@ -125,6 +147,16 @@ pub fn v1_soft_links( sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + v1_soft_links_in(file_data, sym_table_msg, offset_size, length_size) +} + +/// [`v1_soft_links`] over any [`Storage`]. +pub fn v1_soft_links_in( + file_data: &S, + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let mut links = Vec::new(); for_each_v1_soft_link( @@ -143,21 +175,21 @@ pub fn v1_soft_links( /// Visit the soft links of a v1 group whose name passes `wanted`, with their /// target paths, until `visit` returns false. -fn for_each_v1_soft_link( - file_data: &[u8], +fn for_each_v1_soft_link( + file_data: &S, sym_table_msg: &SymbolTableMessage, offset_size: u8, length_size: u8, wanted: impl Fn(&str) -> bool, mut visit: impl FnMut(&str, String) -> bool, ) -> Result<(), FormatError> { - let heap = LocalHeap::parse( + let heap = LocalHeap::parse_in( file_data, - to_usize(sym_table_msg.local_heap_address)?, + checked_addr(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; - let snod_addrs = collect_symbol_table_nodes( + let snod_addrs = collect_symbol_table_nodes_in( file_data, sym_table_msg.btree_address, offset_size, @@ -165,16 +197,16 @@ fn for_each_v1_soft_link( )?; let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; + let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?; for entry in &snod.entries { if entry.cache_type != CACHE_TYPE_SOFT_LINK { continue; } if !heap_checked { - heap.validate_free_list(file_data, length_size)?; + heap.validate_free_list_in(file_data, length_size)?; heap_checked = true; } - let name = heap.read_string(file_data, entry.link_name_offset)?; + let name = heap.read_string_in(file_data, entry.link_name_offset)?; if !wanted(&name) { continue; } @@ -184,7 +216,7 @@ fn for_each_v1_soft_link( entry.scratch_pad[2], entry.scratch_pad[3], ]); - let target = heap.read_string(file_data, u64::from(value_offset))?; + let target = heap.read_string_in(file_data, u64::from(value_offset))?; if !visit(&name, target) { return Ok(()); } @@ -222,6 +254,17 @@ pub fn resolve_path( path: &str, offset_size: u8, length_size: u8, +) -> Result { + resolve_path_in(file_data, root_sym_table, path, offset_size, length_size) +} + +/// [`resolve_path`] over any [`Storage`]. +pub fn resolve_path_in( + file_data: &S, + root_sym_table: &SymbolTableMessage, + path: &str, + offset_size: u8, + length_size: u8, ) -> Result { let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); if components.is_empty() { @@ -241,9 +284,9 @@ pub fn resolve_path( return Ok(entry.object_header_address); } // Not last — must be a group, parse its object header to get symbol table - let obj_header = ObjectHeader::parse( + let obj_header = ObjectHeader::parse_in( file_data, - to_usize(entry.object_header_address)?, + checked_addr(entry.object_header_address)?, offset_size, length_size, )?; diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index 0edf972..9dda843 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -11,8 +11,8 @@ use alloc::collections::BTreeSet; #[cfg(feature = "std")] use std::collections::BTreeSet; -use crate::addr::to_usize; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::addr::checked_addr; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in}; use crate::checksum::jenkins_lookup3; use crate::error::FormatError; use crate::fractal_heap::FractalHeapHeader; @@ -21,6 +21,7 @@ use crate::link_info::LinkInfoMessage; use crate::link_message::{LinkMessage, LinkTarget}; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; +use crate::storage::Storage; use crate::superblock::Superblock; use crate::symbol_table::SymbolTableMessage; @@ -32,6 +33,16 @@ pub fn resolve_v2_group_entries( object_header: &ObjectHeader, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size) +} + +/// [`resolve_v2_group_entries`] over any [`Storage`]. +pub fn resolve_v2_group_entries_in( + file_data: &S, + object_header: &ObjectHeader, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { // Look for Link Info message to determine storage type let link_info = find_link_info(object_header, offset_size)?; @@ -91,8 +102,8 @@ fn resolve_compact_entries( } /// Visit every link in dense storage (fractal heap + B-tree v2 name index). -fn for_each_dense_link( - file_data: &[u8], +fn for_each_dense_link( + file_data: &S, link_info: &LinkInfoMessage, fh_addr: u64, offset_size: u8, @@ -100,15 +111,20 @@ fn for_each_dense_link( mut visit: impl FnMut(LinkMessage), ) -> Result<(), FormatError> { // Parse fractal heap - let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + let fh = + FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?; // Parse B-tree v2 for name index let btree_addr = link_info .btree_name_index_address .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; - let btree_hdr = - BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; - let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + checked_addr(btree_addr)?, + offset_size, + length_size, + )?; + let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { // For type 5 (name index): hash(4) + heap_id(heap_id_length) @@ -125,7 +141,7 @@ fn for_each_dense_link( let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize]; // Read managed object from fractal heap - let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; + let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?; if let Some(link) = parse_link(&link_data, offset_size)? { visit(link); } @@ -134,8 +150,8 @@ fn for_each_dense_link( } /// Resolve entries from dense storage (fractal heap + B-tree v2). -fn resolve_dense_entries( - file_data: &[u8], +fn resolve_dense_entries( + file_data: &S, link_info: &LinkInfoMessage, fh_addr: u64, offset_size: u8, @@ -167,8 +183,8 @@ fn resolve_dense_entries( /// The soft link called `name` in a v1 (symbol table) group, if there is /// one. Hard links are what `resolve_group_entries` returns; this is /// consulted only when a path component isn't among them. -fn find_v1_symbolic_link( - file_data: &[u8], +fn find_v1_symbolic_link( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -182,7 +198,7 @@ fn find_v1_symbolic_link( return Ok(None); }; let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::find_v1_soft_link(file_data, &stm, name, offset_size, length_size) + group_v1::find_v1_soft_link_in(file_data, &stm, name, offset_size, length_size) .map(|target| target.map(|target_path| LinkTarget::Soft { target_path })) } @@ -199,8 +215,8 @@ const LINK_NAME_INDEX: u8 = 5; /// link. libhdf5 orders records with equal hashes by name; all of them are /// read and compared here, so that order does not matter. An index of /// another type is scanned in full. -fn links_named( - file_data: &[u8], +fn links_named( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -220,12 +236,17 @@ fn links_named( return Ok(found); }; - let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + let fh = + FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?; let btree_addr = link_info .btree_name_index_address .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; - let btree_hdr = - BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; + let btree_hdr = BTreeV2Header::parse_in( + file_data, + checked_addr(btree_addr)?, + offset_size, + length_size, + )?; if btree_hdr.tree_type != LINK_NAME_INDEX { for_each_dense_link( file_data, @@ -244,7 +265,7 @@ fn links_named( // Record: hash(4) + heap ID. let hash = jenkins_lookup3(name.as_bytes()); - let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| { + let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| { match r.get(..4) { Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), // Too short to hold a hash (a corrupt record size): never a match. @@ -256,7 +277,7 @@ fn links_named( let Some(id_bytes) = record.data.get(4..4 + id_len) else { continue; }; - let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; + let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?; if let Some(link) = parse_link(&link_data, offset_size)? && link.name == name { @@ -278,8 +299,8 @@ fn links_named( /// and may land on another of several exact duplicates. The listing /// ([`resolve_group_children`]), [`resolve_child`] and path resolution all /// apply this rule, so they agree. -fn first_link_named( - file_data: &[u8], +fn first_link_named( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -296,8 +317,8 @@ fn first_link_named( /// the group with header `object_header`: a hard link (as `Hard`), else a /// soft or external link of that name, else `None`. Fails with /// `PathNotFound` if the object is not a group. -fn lookup_link( - file_data: &[u8], +fn lookup_link( + file_data: &S, object_header: &ObjectHeader, name: &str, offset_size: u8, @@ -346,13 +367,23 @@ pub fn resolve_child( superblock: &Superblock, group_address: u64, name: &str, +) -> Result { + resolve_child_in(file_data, superblock, group_address, name) +} + +/// [`resolve_child`] over any [`Storage`]. +pub fn resolve_child_in( + file_data: &S, + superblock: &Superblock, + group_address: u64, + name: &str, ) -> Result { let os = superblock.offset_size; let ls = superblock.length_size; let not_found = || FormatError::PathNotFound(String::from(name)); - let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; + let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?; if !is_v2_group(&header) || is_v1_group(&header) { - return resolve_group_children(file_data, superblock, group_address)? + return resolve_group_children_in(file_data, superblock, group_address)? .into_iter() .find(|e| e.name == name) .map(|e| e.object_header_address) @@ -365,7 +396,7 @@ pub fn resolve_child( object_header_address, }) => Ok(object_header_address), Some(LinkTarget::Soft { target_path }) => { - match resolve_path_from(file_data, superblock, group_address, &target_path) { + match resolve_path_from_in(file_data, superblock, group_address, &target_path) { // Left out of the listing: dangling, cyclic, or in another file. Err( FormatError::PathNotFound(_) @@ -421,6 +452,15 @@ pub fn resolve_path_any( file_data: &[u8], superblock: &Superblock, path: &str, +) -> Result { + resolve_path_any_in(file_data, superblock, path) +} + +/// [`resolve_path_any`] over any [`Storage`]. +pub fn resolve_path_any_in( + file_data: &S, + superblock: &Superblock, + path: &str, ) -> Result { resolve_path_following_links( file_data, @@ -439,6 +479,16 @@ pub fn resolve_path_from( superblock: &Superblock, group_address: u64, path: &str, +) -> Result { + resolve_path_from_in(file_data, superblock, group_address, path) +} + +/// [`resolve_path_from`] over any [`Storage`]. +pub fn resolve_path_from_in( + file_data: &S, + superblock: &Superblock, + group_address: u64, + path: &str, ) -> Result { let start = if path.starts_with('/') { superblock.root_group_address @@ -462,10 +512,19 @@ pub fn resolve_group_children( file_data: &[u8], superblock: &Superblock, group_address: u64, +) -> Result, FormatError> { + resolve_group_children_in(file_data, superblock, group_address) +} + +/// [`resolve_group_children`] over any [`Storage`]. +pub fn resolve_group_children_in( + file_data: &S, + superblock: &Superblock, + group_address: u64, ) -> Result, FormatError> { let os = superblock.offset_size; let ls = superblock.length_size; - let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; + let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?; let mut entries = Vec::new(); let mut soft = Vec::new(); @@ -476,9 +535,9 @@ pub fn resolve_group_children( .find(|m| m.msg_type == MessageType::SymbolTable) .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?; let stm = SymbolTableMessage::parse(&sym_msg.data, os)?; - let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?; + let all = group_v1::resolve_v1_group_entries_in(file_data, &stm, os, ls)?; if all.iter().any(group_v1::is_v1_soft_link) { - soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?; + soft = group_v1::v1_soft_links_in(file_data, &stm, os, ls)?; } entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e))); } else if is_v2_group(&header) { @@ -515,7 +574,7 @@ pub fn resolve_group_children( } for (name, target) in soft { - match resolve_path_from(file_data, superblock, group_address, &target) { + match resolve_path_from_in(file_data, superblock, group_address, &target) { Ok(object_header_address) => entries.push(GroupEntry { name, object_header_address, @@ -538,8 +597,8 @@ pub fn resolve_group_children( const MAX_SOFT_LINK_DEPTH: u8 = 16; /// Walk `path` from the group at `start`, following soft links. -fn resolve_path_following_links( - file_data: &[u8], +fn resolve_path_following_links( + file_data: &S, superblock: &Superblock, start: u64, path: &str, @@ -557,7 +616,7 @@ fn resolve_path_following_links( let ls = superblock.length_size; let mut current_addr = start; - let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?; + let mut current_header = ObjectHeader::parse_in(file_data, checked_addr(start)?, os, ls)?; for (i, component) in components.iter().enumerate() { match lookup_link(file_data, ¤t_header, component, os, ls)? { @@ -568,7 +627,8 @@ fn resolve_path_following_links( return Ok(object_header_address); } current_addr = object_header_address; - current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?; + current_header = + ObjectHeader::parse_in(file_data, checked_addr(current_addr)?, os, ls)?; } found => { return match found { @@ -607,8 +667,8 @@ fn resolve_path_following_links( } /// Resolve group entries from an object header, auto-detecting v1 vs v2. -fn resolve_group_entries( - file_data: &[u8], +fn resolve_group_entries( + file_data: &S, object_header: &ObjectHeader, offset_size: u8, length_size: u8, @@ -625,7 +685,7 @@ fn resolve_group_entries( // skipped by the name comparison, as in libhdf5. group_v1::v1_group_entries(file_data, &stm, offset_size, length_size) } else if is_v2_group(object_header) { - resolve_v2_group_entries(file_data, object_header, offset_size, length_size) + resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size) } else { Err(FormatError::PathNotFound(String::from( "object header is not a group", diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 05ee649..5f0fc2e 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -23,12 +23,12 @@ use alloc::vec::Vec; #[cfg(feature = "std")] use std::borrow::Cow; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in}; use crate::error::FormatError; use crate::fractal_heap::FractalHeapHeader; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; -use crate::storage::{Storage, Window, read_exact_at, require_contiguous}; +use crate::storage::{Storage, Window, read_exact_at}; /// Fractal heap ID length for SOHM entries (fixed at 8 bytes). const FHEAP_ID_LEN: usize = 8; @@ -423,19 +423,15 @@ pub fn parse_sohm_btree_entries( parse_sohm_btree_entries_in(file_data, btree_addr as u64, offset_size, length_size) } -/// [`parse_sohm_btree_entries`] over any [`Storage`]. The v2 B-tree is not -/// read over [`Storage`] yet, so this needs the whole file in memory -/// ([`FormatError::ContiguousStorageRequired`] otherwise). +/// [`parse_sohm_btree_entries`] over any [`Storage`]. pub fn parse_sohm_btree_entries_in( file: &S, btree_addr: u64, offset_size: u8, length_size: u8, ) -> Result, FormatError> { - let file_data = require_contiguous(file, "a shared-message B-tree index")?; - let btree_addr = usize::try_from(btree_addr).unwrap_or(usize::MAX); - let header = BTreeV2Header::parse(file_data, btree_addr, offset_size, length_size)?; - let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?; + let header = BTreeV2Header::parse_in(file, btree_addr, offset_size, length_size)?; + let records = collect_btree_v2_records_in(file, &header, offset_size, length_size)?; let mut entries = Vec::with_capacity(records.len()); for rec in &records { let entry = parse_sohm_entry(&rec.data, offset_size)?; @@ -1243,11 +1239,12 @@ mod tests { } } assert!(compared > 200); - // The B-tree index is not read over Storage yet: a clean error. - let st = CountingStorage::new(vec![0u8; 64]); + // The B-tree index reads through Storage too, errors included. + let junk = vec![0u8; 64]; + let st = CountingStorage::new(junk.clone()); assert_eq!( - parse_sohm_btree_entries_in(&st, 0, 8, 8).unwrap_err(), - FormatError::ContiguousStorageRequired("a shared-message B-tree index") + format!("{:?}", parse_sohm_btree_entries_in(&st, 0, 8, 8)), + format!("{:?}", parse_sohm_btree_entries(&junk, 0, 8, 8)) ); } } diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index 5cc71b9..02a4f17 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -1,24 +1,19 @@ //! Equivalence harness for the range-read migration -//! (`docs/design/range-reads.md`, milestone M1). +//! (`docs/design/range-reads.md`, milestones M1 and M2). //! -//! Every metadata parser converted to [`Storage`] must give exactly what its -//! `&[u8]` form gives. This walks real files — the fixtures, files h5py -//! writes to exercise the less common structures, and optionally the -//! conformance corpus — and, for every object, runs each converted parser -//! twice: over the file as a slice, and over a [`CountingStorage`] that -//! serves the same bytes through `read_at` only (`as_contiguous()` is -//! `None`, so no parser can fall back to the whole slice). The results must -//! be identical, value for value and error for error. +//! Every parser converted to [`Storage`] must give exactly what its `&[u8]` +//! form gives. This walks real files — the fixtures, files h5py writes to +//! exercise the less common structures, and optionally the conformance +//! corpus — and, for every object, runs each converted parser twice: over +//! the file as a slice, and over a [`CountingStorage`] that serves the same +//! bytes through `read_at` only (`as_contiguous()` is `None`, so no parser +//! can fall back to the whole slice). The results must be identical, value +//! for value and error for error. //! -//! The one allowed difference is [`FormatError::ContiguousStorageRequired`] -//! from the storage path, and only from the structures still indexed by a v2 -//! B-tree (dense attributes, a SOHM B-tree index, huge fractal-heap objects; -//! see `CONTIGUOUS_REQUIRED`), which fail cleanly instead of reading the -//! whole file. Those are counted; the error from any other site or check -//! fails the harness. -//! -//! Milestones M2/M3 extend `check_object` with the raw-data and group -//! parsers as they are converted. +//! Nothing may answer [`FormatError::ContiguousStorageRequired`] any more: +//! since milestone M2 every read path works over `read_at` alone, the v2 +//! B-tree structures (dense groups and attributes, a SOHM B-tree index, +//! huge fractal-heap objects) included. //! //! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/ //! `.nc`/`.h5ad` file under those directories (the conformance corpus is @@ -38,7 +33,10 @@ use clawhdf5_format::attribute::{ }; use clawhdf5_format::attribute_info::AttributeInfoMessage; use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in}; -use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use clawhdf5_format::btree_v2::{ + BTreeV2Header, collect_btree_v2_records, collect_btree_v2_records_in, find_btree_v2_records, + find_btree_v2_records_in, +}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; @@ -51,6 +49,7 @@ use clawhdf5_format::fixed_array::{ FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in, }; use clawhdf5_format::fractal_heap::FractalHeapHeader; +use clawhdf5_format::group_v2; use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::local_heap::LocalHeap; use clawhdf5_format::message_type::MessageType; @@ -73,44 +72,11 @@ use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode}; const MAX_OBJECTS: usize = 1500; const MAX_HEAP_IDS: usize = 200; -/// The structures that still need the whole file in memory, because they -/// are found through a version-2 B-tree (not converted yet), and the checks -/// that can reach each of them. Anything else answering -/// [`FormatError::ContiguousStorageRequired`] is a converted parser falling -/// back to the whole file, and fails the harness. -const CONTIGUOUS_REQUIRED: &[(&str, &[&str])] = &[ - ( - "dense attribute storage (a v2 B-tree)", - &["attributes", "attributes (tolerant)"], - ), - ( - "a shared-message B-tree index", - &[ - "SOHM B-tree", - "shared message", - "fill value", - "attributes", - "attributes (tolerant)", - ], - ), - ( - "a huge fractal-heap object's B-tree", - &["heap object", "attributes", "attributes (tolerant)"], - ), -]; - -fn may_require_contiguous(check: &str, site: &str) -> bool { - CONTIGUOUS_REQUIRED - .iter() - .any(|(s, checks)| *s == site && checks.contains(&check)) -} - #[derive(Default, Debug)] struct Tally { files: usize, objects: usize, checks: usize, - contiguous_required: usize, reads: u64, bytes: u64, /// Chunk indexes (fixed and extensible arrays) read, and the most bytes @@ -127,8 +93,8 @@ struct Walk<'a> { } impl Walk<'_> { - /// The storage result must equal the slice result, or be the clean - /// "needs the whole file" error. + /// The storage result must equal the slice result; no parser may ask + /// for the whole file. fn same( &mut self, what: &str, @@ -137,14 +103,11 @@ impl Walk<'_> { ) { self.tally.checks += 1; if let Err(FormatError::ContiguousStorageRequired(site)) = got { - assert!( - may_require_contiguous(what, site), - "{}: {what} fell back to the whole file ({site}), which only the \ - v2-B-tree-indexed structures may do", + panic!( + "{}: {what} fell back to the whole file ({site}); every read path \ + must work through read_at", self.name ); - self.tally.contiguous_required += 1; - return; } let (w, g) = (format!("{want:?}"), format!("{got:?}")); assert!( @@ -206,13 +169,34 @@ impl Walk<'_> { } self.tally.objects += 1; self.check_object(&sb, addr); - // Traversal only (group lookups are milestone M0/M3 work). - if let Ok(children) = - clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr) - { + let children = group_v2::resolve_group_children(slice, &sb, addr); + let got = group_v2::resolve_group_children_in(self.st(), &sb, addr); + self.same("group listing", &children, &got); + if let Ok(children) = children { + for c in children.iter().take(MAX_HEAP_IDS) { + let want = group_v2::resolve_child(slice, &sb, addr, &c.name); + let got = group_v2::resolve_child_in(self.st(), &sb, addr, &c.name); + self.same("child lookup", &want, &got); + } + // A name no group has: the lookup's not-found path. + let want = group_v2::resolve_child(slice, &sb, addr, "no such child"); + let got = group_v2::resolve_child_in(self.st(), &sb, addr, "no such child"); + self.same("child lookup (missing)", &want, &got); queue.extend(children.iter().map(|c| c.object_header_address)); } } + // Paths: every listed name from the root, and one that is missing. + if let Ok(children) = group_v2::resolve_group_children(slice, &sb, sb.root_group_address) { + for c in children.iter().take(MAX_HEAP_IDS) { + let path = format!("/{}", c.name); + let want = group_v2::resolve_path_any(slice, &sb, &path); + let got = group_v2::resolve_path_any_in(self.st(), &sb, &path); + self.same("path", &want, &got); + } + } + let want = group_v2::resolve_path_any(slice, &sb, "/no/such/path"); + let got = group_v2::resolve_path_any_in(self.st(), &sb, "/no/such/path"); + self.same("path (missing)", &want, &got); } fn check_object(&mut self, sb: &Superblock, addr: u64) { @@ -336,12 +320,24 @@ impl Walk<'_> { let (Ok(fh), Some(index)) = (fh, index) else { return; }; - let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else { - return; - }; - let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else { - return; - }; + let bt = BTreeV2Header::parse(slice, index as usize, os, ls); + self.same( + "v2 B-tree header", + &bt, + &BTreeV2Header::parse_in(self.st(), index, os, ls), + ); + let Ok(bt) = bt else { return }; + let records = collect_btree_v2_records(slice, &bt, os, ls); + let got = collect_btree_v2_records_in(self.st(), &bt, os, ls); + self.same("v2 B-tree records", &records, &got); + let Ok(records) = records else { return }; + // Descents to single records (by their bytes), as name lookups do. + for rec in records.iter().take(8) { + let key = rec.data.clone(); + let want = find_btree_v2_records(slice, &bt, os, &mut |r| r.cmp(&key[..])); + let got = find_btree_v2_records_in(self.st(), &bt, os, &mut |r| r.cmp(&key[..])); + self.same("v2 B-tree descent", &want, &got); + } let id_len = fh.heap_id_length as usize; for rec in records.iter().take(MAX_HEAP_IDS) { let Some(id) = rec.data.get(id_at..id_at + id_len) else { @@ -696,6 +692,4 @@ fn h5py_files_parse_identically_through_storage() { } eprintln!("h5py files: {tally:?}"); assert!(tally.objects >= 700, "{tally:?}"); - // Dense attributes and the SOHM B-tree are the known clean errors. - assert!(tally.contiguous_required > 0, "{tally:?}"); } -- 2.54.0 From 3fa5ed1dda7088c6a90757e399142bc789859545 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:28:01 -0500 Subject: [PATCH 02/49] format: raw data, VDS and VL data over Storage Every raw-data path has a generic *_in core, with the &[u8] functions as thin wrappers: data_read (read_raw_data*, read_raw_data_selection, read_chunked_native), chunked_read (the v1 B-tree chunk index, list_chunks, the full, cached, sweep and indexed reads), parallel_read, partial_read, fill_value (read_full_with_fill, apply_to_unallocated_chunks; and dataset_fill_value_from_storage is now generic), vds (the virtual file through Storage, external sources still through the resolver), vl_data (VlResolver<'a, S = [u8]>, read_vl_strings_in, read_vl_bytes_in), AttributeMessage::read_vl_strings_in and provenance::verify_dataset_in. With the whole file in memory nothing changes: chunks and contiguous data are sliced from it as before. Otherwise a chunked read lists its chunks, fetches their stored bytes with one Storage::read_ranges call per 64 MiB batch (chunks the cache already holds are not fetched), then decodes as today; a selection fetches only the chunks it overlaps, and a contiguous selection only its runs. Each extent's bounds error is the one the slice code gave, reported when that extent is reached, so errors keep their order. Tests: the equivalence harness now reads every dataset's values (whole, fill-aware, cached, indexed, three selections, VDS, VL strings and sequences) through the read_at-only storage and requires the slice results (all 653 corpus files agree); a misbehaving storage (a failing Nth read, short reads) only ever yields errors or the right values; and chunked reads are checked to use one read_ranges call. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/attribute.rs | 12 +- crates/clawhdf5-format/src/chunked_read.rs | 437 ++++++++++++--- crates/clawhdf5-format/src/data_read.rs | 215 +++++++- crates/clawhdf5-format/src/fill_value.rs | 65 ++- crates/clawhdf5-format/src/gather.rs | 129 +++++ crates/clawhdf5-format/src/global_heap.rs | 2 +- crates/clawhdf5-format/src/parallel_read.rs | 105 ++-- crates/clawhdf5-format/src/partial_read.rs | 106 +++- crates/clawhdf5-format/src/provenance.rs | 17 +- crates/clawhdf5-format/src/storage.rs | 153 ++++++ crates/clawhdf5-format/src/vds.rs | 123 ++++- crates/clawhdf5-format/src/vl_data.rs | 204 ++++--- .../tests/storage_equivalence.rs | 514 +++++++++++++++++- 13 files changed, 1803 insertions(+), 279 deletions(-) diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 4b30fac..c8f69b5 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -336,9 +336,19 @@ impl AttributeMessage { file_data: &[u8], offset_size: u8, length_size: u8, + ) -> Result, FormatError> { + self.read_vl_strings_in(file_data, offset_size, length_size) + } + + /// [`Self::read_vl_strings`] over any [`Storage`]. + pub fn read_vl_strings_in( + &self, + file_data: &S, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let num_elements = self.dataspace.num_elements(); - vl_data::read_vl_strings( + vl_data::read_vl_strings_in( file_data, &self.raw_data, num_elements, diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index e249168..140cf37 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -6,19 +6,20 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; -use crate::addr::to_usize; +use crate::addr::{checked_addr, to_usize}; #[cfg(feature = "std")] use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::datatype::Datatype; use crate::error::FormatError; -use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks}; +use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks_in}; use crate::filter_pipeline::FilterPipeline; use crate::filters::{DecodeScratch, decompress_chunk_exact_with}; #[cfg(feature = "std")] use crate::filters::{all_filters_skipped, decompress_chunk_exact}; -use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks}; +use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks_in}; +use crate::storage::{ExtentBytes, Storage, Window, raw_batches, read_exact_at}; #[cfg(feature = "std")] use std::sync::Arc; @@ -238,22 +239,101 @@ type CacheUse<'a> = Option<&'a core::convert::Infallible>; /// ([`parallel_read::run_with_helpers`]), so the caller never waits on a /// busy pool. The error returned is the first failing chunk's, in `chunks` /// order. +/// +/// With the whole file in memory the chunks are sliced from it. Otherwise +/// their stored bytes are fetched first, with one +/// [`Storage::read_ranges`] call per batch of up to +/// [`crate::storage::RAW_BATCH_BYTES`] (all of them, for most datasets), +/// and then decoded as above; chunks the cache already holds are not +/// fetched. #[allow(clippy::too_many_arguments)] -fn fill_from_chunks( - file_data: &[u8], +fn fill_from_chunks( + file_data: &S, chunks: &[ChunkInfo], pipeline: Option<&FilterPipeline>, placer: &ChunkPlacer, chunk_total_bytes: usize, cache: CacheUse<'_>, output: &mut [u8], +) -> Result<(), FormatError> { + let contiguous = file_data.as_contiguous().is_some(); + let out = OutBuf::new(output); + let batches = raw_batches(chunks.len(), contiguous, |i| chunks[i].chunk_size as usize); + for batch in batches { + fill_batch( + file_data, + &chunks[batch], + pipeline, + placer, + chunk_total_bytes, + cache, + &out, + )?; + } + Ok(()) +} + +/// Whether a full read looks chunk `c` up in the cache (see +/// [`fill_from_chunks`]): a filtered chunk of a dataset the cache keeps. +#[cfg(feature = "std")] +fn uses_cache(cache: CacheUse<'_>, pipeline: Option<&FilterPipeline>, c: &ChunkInfo) -> bool { + matches!(cache, Some((_, _, true))) + && pipeline.is_some_and(|pl| !all_filters_skipped(pl, c.filter_mask)) +} + +/// [`fill_from_chunks`] for one batch of chunks. +#[allow(clippy::too_many_arguments)] +fn fill_batch( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: Option<&FilterPipeline>, + placer: &ChunkPlacer, + chunk_total_bytes: usize, + cache: CacheUse<'_>, + out: &OutBuf<'_>, ) -> Result<(), FormatError> { let rank = placer.rank; let elem_size = placer.elem_size as u32; - let out = OutBuf::new(output); #[cfg(not(feature = "std"))] let _ = cache; + // Over a backend without the whole file in memory, the decoded chunks + // the cache holds are taken now (so a chunk evicted before it is placed + // is not left without bytes), and only the others are fetched. + #[cfg(feature = "std")] + let hits: Vec>> = match cache { + Some((cache, key, true)) if file_data.as_contiguous().is_none() => chunks + .iter() + .map(|c| { + if c.offsets.len() >= rank && uses_cache(Some((cache, key, true)), pipeline, c) { + cache.get_decompressed_in(key, &c.offsets[..rank]) + } else { + None + } + }) + .collect(), + _ => Vec::new(), + }; + let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { + Vec::new() + } else { + chunks + .iter() + .enumerate() + .map(|(i, c)| { + #[cfg(feature = "std")] + let wanted = hits.get(i).is_none_or(Option::is_none); + #[cfg(not(feature = "std"))] + let wanted = { + let _ = i; + true + }; + (c.address, c.chunk_size as usize, wanted) + }) + .collect() + }; + let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + // Decode chunk `i` and place it. Its callers below run it either on one // thread, or on several for chunks whose regions are pairwise disjoint, // each chunk once: no two threads ever write the same bytes. @@ -266,13 +346,18 @@ fn fill_from_chunks( ))); } let offsets = &c.offsets[..rank]; - let c_addr = to_usize(c.address)?; let size = c.chunk_size as usize; - ensure_len(file_data, c_addr, size)?; - let raw = &file_data[c_addr..c_addr + size]; + #[cfg(feature = "std")] + if let Some(Some(hit)) = hits.get(i) { + raw_bytes.check(i, c.address, size)?; + // SAFETY: see above. + unsafe { placer.place(hit, offsets, out) }; + return Ok(()); + } + let raw = raw_bytes.get(i, c.address, size)?; let Some(pl) = pipeline else { // SAFETY: see above. - unsafe { placer.place(raw, offsets, &out) }; + unsafe { placer.place(raw, offsets, out) }; return Ok(()); }; // A chunk stored as-is (every filter skipped) is checked and placed @@ -296,7 +381,7 @@ fn fill_from_chunks( } }; // SAFETY: see above. - unsafe { placer.place(&cached, offsets, &out) }; + unsafe { placer.place(&cached, offsets, out) }; return Ok(()); } let data = decompress_chunk_exact_with( @@ -309,7 +394,7 @@ fn fill_from_chunks( scratch, )?; // SAFETY: see above. - unsafe { placer.place(data, offsets, &out) }; + unsafe { placer.place(data, offsets, out) }; Ok(()) }; @@ -370,7 +455,29 @@ pub fn decompress_all_chunks_with_stats( seed: u64, num_lanes: Option, ) -> Result<(Vec>, PartitionStats), FormatError> { - parallel_read::decompress_chunks_lane_partitioned( + decompress_all_chunks_with_stats_in( + file_data, + chunks, + pipeline, + chunk_total_bytes, + element_size, + seed, + num_lanes, + ) +} + +/// [`decompress_all_chunks_with_stats`] over any [`Storage`]. +#[cfg(feature = "parallel")] +pub fn decompress_all_chunks_with_stats_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: &FilterPipeline, + chunk_total_bytes: usize, + element_size: u32, + seed: u64, + num_lanes: Option, +) -> Result<(Vec>, PartitionStats), FormatError> { + parallel_read::decompress_chunks_lane_partitioned_in( file_data, chunks, pipeline, @@ -394,21 +501,6 @@ pub struct ChunkInfo { pub address: u64, } -/// Checks that `[offset, offset + needed)` fits within `data`, guarding the -/// addition against `usize` overflow from a crafted near-`usize::MAX` offset. -fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { - if offset - .checked_add(needed) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: offset.saturating_add(needed), - available: data.len(), - }); - } - Ok(()) -} - /// `elements * elem_size` for sizes that come from the file. Dataspace and /// chunk dimensions are untrusted 64-bit fields, so a crafted file can make /// the plain product wrap to a small number (or to something enormous). @@ -588,6 +680,17 @@ pub fn collect_chunk_info( ndims: usize, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + collect_chunk_info_in(file_data, btree_address, ndims, offset_size, length_size) +} + +/// [`collect_chunk_info`] over any [`Storage`]. +pub fn collect_chunk_info_in( + file_data: &S, + btree_address: u64, + ndims: usize, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let _ = length_size; let mut chunks = Vec::new(); @@ -630,6 +733,23 @@ pub fn collect_chunk_info_checked( chunk_dimensions: &[u32], offset_size: u8, length_size: u8, +) -> Result, FormatError> { + collect_chunk_info_checked_in( + file_data, + btree_address, + chunk_dimensions, + offset_size, + length_size, + ) +} + +/// [`collect_chunk_info_checked`] over any [`Storage`]. +pub fn collect_chunk_info_checked_in( + file_data: &S, + btree_address: u64, + chunk_dimensions: &[u32], + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let _ = length_size; let ndims = chunk_dimensions.len(); @@ -813,8 +933,8 @@ const MAX_CHUNK_BTREE_DEPTH: usize = 64; /// Parse the v1 B-tree chunk index node at `btree_address` and its /// subtree, appending its chunks to `stored` in tree order. -fn parse_chunk_node( - file_data: &[u8], +fn parse_chunk_node( + file_data: &S, btree_address: u64, ndims: usize, chunk_dimensions: Option<&[u32]>, @@ -831,21 +951,24 @@ fn parse_chunk_node( // Parse B-tree v1 header let header_size = 8 + os * 2; - ensure_len(file_data, offset, header_size)?; + let head = Window::read(file_data, offset as u64, header_size)?; + head.ensure(0, header_size)?; + let h = &head.bytes; - if &file_data[offset..offset + 4] != b"TREE" { + if &h[..4] != b"TREE" { return Err(FormatError::InvalidBTreeSignature); } - let node_type = file_data[offset + 4]; + let node_type = h[4]; if node_type != 1 { return Err(FormatError::InvalidBTreeNodeType(node_type)); } - let node_level = file_data[offset + 5]; - let entries_used = u16::from_le_bytes([file_data[offset + 6], file_data[offset + 7]]) as usize; + let node_level = h[5]; + let entries_used = u16::from_le_bytes([h[6], h[7]]) as usize; - let mut pos = offset + 8 + os * 2; // skip left/right sibling + // Positions below are relative to the node's start. + let mut pos = header_size; // skip left/right sibling // Key: chunk_size(4) + filter_mask(4) + one offset per dimension. The // offsets are always 8 bytes each — they are dataset coordinates, not file @@ -858,28 +981,20 @@ fn parse_chunk_node( // key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N] let needed = entries_used * (key_size + os) + key_size; - ensure_len(file_data, pos, needed)?; + let node = Window::read(file_data, offset as u64, header_size + needed)?; + node.ensure(pos, needed)?; + let d = &node.bytes; let mut keys = Vec::with_capacity((entries_used + 1) * ndims); let mut chunks = Vec::new(); let mut child_addrs = Vec::new(); for _ in 0..entries_used { - let chunk_size = u32::from_le_bytes([ - file_data[pos], - file_data[pos + 1], - file_data[pos + 2], - file_data[pos + 3], - ]); - let filter_mask = u32::from_le_bytes([ - file_data[pos + 4], - file_data[pos + 5], - file_data[pos + 6], - file_data[pos + 7], - ]); + let chunk_size = u32::from_le_bytes([d[pos], d[pos + 1], d[pos + 2], d[pos + 3]]); + let filter_mask = u32::from_le_bytes([d[pos + 4], d[pos + 5], d[pos + 6], d[pos + 7]]); let k = keys.len(); - read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?; + read_key_offsets(d, pos, ndims, chunk_dimensions, &mut keys)?; pos += key_size; - let address = read_offset(file_data, pos, offset_size)?; + let address = read_offset(d, pos, offset_size)?; pos += os; if node_level == 0 { chunks.push(stored.len()); @@ -894,7 +1009,7 @@ fn parse_chunk_node( } } // The final key only bounds the node; libhdf5 still checks it. - read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?; + read_key_offsets(d, pos, ndims, chunk_dimensions, &mut keys)?; let children = if node_level == 0 { ChunkChildren::Chunks(chunks) @@ -984,18 +1099,18 @@ const BT2_CHUNK_FILTERED: u8 = 11; /// The width of the stored-size field depends on the largest possible chunk; /// rather than re-derive the library's formula it is taken from the record /// size the tree header declares, which is what actually governs the bytes. -fn read_btree_v2_chunks( - file_data: &[u8], +fn read_btree_v2_chunks( + file_data: &S, addr: u64, chunk_dims: &[usize], elem_size: usize, offset_size: u8, length_size: u8, ) -> Result, FormatError> { - use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; + use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in}; let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}")); - let header = BTreeV2Header::parse(file_data, to_usize(addr)?, offset_size, length_size)?; + let header = BTreeV2Header::parse_in(file_data, checked_addr(addr)?, offset_size, length_size)?; let rank = chunk_dims.len(); let os = offset_size as usize; let record_size = header.record_size as usize; @@ -1022,7 +1137,7 @@ fn read_btree_v2_chunks( let unfiltered_bytes = u32::try_from(unfiltered_bytes).map_err(|_| bad("chunk larger than 4 GiB"))?; - let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?; + let records = collect_btree_v2_records_in(file_data, &header, offset_size, length_size)?; let mut chunks = Vec::with_capacity(records.len()); for record in &records { let data = record.data.as_slice(); @@ -1085,6 +1200,25 @@ pub fn list_chunks( elem_size: usize, offset_size: u8, length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + list_chunks_in( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + ) +} + +/// [`list_chunks`] over any [`Storage`]. +pub fn list_chunks_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + offset_size: u8, + length_size: u8, ) -> Result<(Vec, Vec), FormatError> { let ( chunk_dimensions, @@ -1132,9 +1266,13 @@ pub fn list_chunks( // Collect chunks based on version and index type let mut chunks = match (version, chunk_index_type) { - (3, _) => { - collect_chunk_info_checked(file_data, addr, chunk_dimensions, offset_size, length_size)? - } + (3, _) => collect_chunk_info_checked_in( + file_data, + addr, + chunk_dimensions, + offset_size, + length_size, + )?, (4, Some(1)) => { // Single chunk — one chunk covering the entire dataset let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?; @@ -1163,9 +1301,13 @@ pub fn list_chunks( (4, Some(3)) => { // Fixed Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = - FixedArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; - read_fixed_array_chunks( + let header = FixedArrayHeader::parse_in( + file_data, + checked_addr(addr)?, + offset_size, + length_size, + )?; + read_fixed_array_chunks_in( file_data, &header, &dataspace.dimensions, @@ -1179,9 +1321,13 @@ pub fn list_chunks( (4, Some(4)) => { // Extensible Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = - ExtensibleArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; - read_extensible_array_chunks( + let header = ExtensibleArrayHeader::parse_in( + file_data, + checked_addr(addr)?, + offset_size, + length_size, + )?; + read_extensible_array_chunks_in( file_data, &header, &dataspace.dimensions, @@ -1248,7 +1394,28 @@ pub fn list_chunks_for_read( offset_size: u8, length_size: u8, ) -> Result<(Vec, Vec), FormatError> { - let (chunks, chunk_dims) = list_chunks( + list_chunks_for_read_in( + file_data, + layout, + dataspace, + elem_size, + pipeline, + offset_size, + length_size, + ) +} + +/// [`list_chunks_for_read`] over any [`Storage`]. +pub fn list_chunks_for_read_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + let (chunks, chunk_dims) = list_chunks_in( file_data, layout, dataspace, @@ -1284,8 +1451,8 @@ pub(crate) type CacheRef<'a> = Option<&'a core::convert::Infallible>; /// output with `alloc` (zeroed, `total_bytes` long, as bytes through /// `bytes`), and decode every chunk straight into it. #[allow(clippy::too_many_arguments)] -pub(crate) fn read_chunked_full( - file_data: &[u8], +pub(crate) fn read_chunked_full( + file_data: &S, layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, @@ -1299,7 +1466,7 @@ pub(crate) fn read_chunked_full( check_chunk_element_size(layout, datatype, offset_size)?; let elem_size = datatype.type_size() as usize; let list = || { - list_chunks_for_read( + list_chunks_for_read_in( file_data, layout, dataspace, @@ -1390,6 +1557,27 @@ pub fn read_chunked_data( pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + read_chunked_data_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + ) +} + +/// [`read_chunked_data`] over any [`Storage`]. +pub fn read_chunked_data_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { read_chunked_full( file_data, @@ -1422,6 +1610,31 @@ pub fn read_chunked_data_cached( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_chunked_data_cached_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_chunked_data_cached`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_data_cached_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { read_chunked_full( file_data, @@ -1592,6 +1805,33 @@ pub fn read_chunked_data_sweep( length_size: u8, cache: &ChunkCache, sweep: &mut SweepContext, +) -> Result, FormatError> { + read_chunked_data_sweep_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + sweep, + ) +} + +/// [`read_chunked_data_sweep`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_data_sweep_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, + sweep: &mut SweepContext, ) -> Result, FormatError> { let (chunk_dimensions, version, addr_opt) = match layout { DataLayout::Chunked { @@ -1623,7 +1863,7 @@ pub fn read_chunked_data_sweep( // lookup is keyed by this dataset's chunk-index address, so another // dataset's index or chunks are never used for this read. let chunks = cache.chunks_for(addr, rank, || { - list_chunks_for_read( + list_chunks_for_read_in( file_data, layout, dataspace, @@ -1675,8 +1915,8 @@ pub fn read_chunked_data_sweep( // Decompress from file let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; - ensure_len(file_data, c_addr, size)?; - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = read_exact_at(file_data, c_addr as u64, size)?; + let raw_chunk = &*raw_chunk; let dec = if let Some(pl) = pipeline { decompress_chunk_exact( raw_chunk, @@ -1736,6 +1976,31 @@ pub fn read_chunked_data_indexed( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_chunked_data_indexed_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_chunked_data_indexed`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_data_indexed_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { let (chunk_dimensions, version, addr_opt) = match layout { DataLayout::Chunked { @@ -1769,7 +2034,7 @@ pub fn read_chunked_data_indexed( addr, rank, || { - list_chunks_for_read( + list_chunks_for_read_in( file_data, layout, dataspace, @@ -1786,18 +2051,30 @@ pub fn read_chunked_data_indexed( )?; let chunk_total_bytes = plan.chunk_total_bytes; + // The decoded chunks the cache holds, and the stored bytes of the + // others: fetched in one batch when the file is not in memory. + let hits: Vec>> = plan + .mappings + .iter() + .map(|m| cache.get_decompressed_in(addr, &m.coord)) + .collect(); + let extents: Vec<(u64, usize, bool)> = plan + .mappings + .iter() + .zip(&hits) + .map(|(m, hit)| (m.file_offset, m.file_size as usize, hit.is_none())) + .collect(); + let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + // Decompress chunks (using LRU cache where possible) let mut chunk_buffers: Vec> = Vec::with_capacity(plan.mappings.len()); - for m in &plan.mappings { + for (i, (m, hit)) in plan.mappings.iter().zip(hits).enumerate() { let (coord, file_offset, file_size, filter_mask) = (&m.coord, &m.file_offset, &m.file_size, &m.filter_mask); - if let Some(cached) = cache.get_decompressed_in(addr, coord) { + if let Some(cached) = hit { chunk_buffers.push(cached); } else { - let c_addr = to_usize(*file_offset)?; - let size = *file_size as usize; - ensure_len(file_data, c_addr, size)?; - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(i, *file_offset, *file_size as usize)?; let decompressed = if let Some(pl) = pipeline { decompress_chunk_exact( raw_chunk, diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index bcf1a18..28bc4f2 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -1,7 +1,9 @@ //! Raw data reading and typed conversion for HDF5 datasets. #[cfg(not(feature = "std"))] -use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; +use alloc::{borrow::Cow, collections::BTreeMap, format, string::String, vec, vec::Vec}; +#[cfg(feature = "std")] +use std::borrow::Cow; #[cfg(feature = "std")] use std::collections::BTreeMap; @@ -9,14 +11,15 @@ use std::collections::BTreeMap; use crate::addr::to_usize; #[cfg(feature = "std")] use crate::chunk_cache::ChunkCache; -use crate::chunked_read::read_chunked_data; +use crate::chunked_read::read_chunked_data_in; #[cfg(feature = "std")] -use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed}; +use crate::chunked_read::{read_chunked_data_cached_in, read_chunked_data_indexed_in}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::datatype::{Datatype, DatatypeByteOrder}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; +use crate::storage::{Storage, read_exact_at}; /// Checks that `[offset, offset + needed)` fits within `data`, guarding the /// addition against `usize` overflow from a crafted near-`usize::MAX` offset. @@ -149,7 +152,17 @@ pub fn read_raw_data( dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { - read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8) + read_raw_data_in(file_data, layout, dataspace, datatype) +} + +/// [`read_raw_data`] over any [`Storage`]. +pub fn read_raw_data_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, +) -> Result, FormatError> { + read_raw_data_full_in(file_data, layout, dataspace, datatype, None, 8, 8) } /// Resolves a Virtual Dataset source **file name** (as stored in the mapping, @@ -171,6 +184,27 @@ pub fn read_raw_data_full( pipeline: Option<&FilterPipeline>, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + read_raw_data_full_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + ) +} + +/// [`read_raw_data_full`] over any [`Storage`]. +pub fn read_raw_data_full_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, @@ -196,6 +230,30 @@ pub fn read_raw_data_full_with_resolver( offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, +) -> Result, FormatError> { + read_raw_data_full_with_resolver_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + resolver, + ) +} + +/// [`read_raw_data_full_with_resolver`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_full_with_resolver_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { read_raw_data_full_impl( file_data, @@ -210,8 +268,8 @@ pub fn read_raw_data_full_with_resolver( } #[allow(clippy::too_many_arguments)] -fn read_raw_data_full_impl( - file_data: &[u8], +fn read_raw_data_full_impl( + file_data: &S, layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, @@ -242,12 +300,17 @@ fn read_raw_data_full_impl( let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = to_usize(addr)?; let sz = contiguous_read_len(*size, expected_size)?; - ensure_len(file_data, addr, sz)?; - let mut out = crate::bulk_alloc::vec_for_bulk(sz); - out.extend_from_slice(&file_data[addr..addr + sz]); - Ok(out) + match read_exact_at(file_data, addr as u64, sz)? { + Cow::Borrowed(bytes) => { + let mut out = crate::bulk_alloc::vec_for_bulk(sz); + out.extend_from_slice(bytes); + Ok(out) + } + // Fetched for this read: already the caller's copy. + Cow::Owned(out) => Ok(out), + } } - DataLayout::Chunked { .. } => read_chunked_data( + DataLayout::Chunked { .. } => read_chunked_data_in( file_data, layout, dataspace, @@ -284,9 +347,34 @@ pub fn read_raw_data_cached( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_raw_data_cached_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_raw_data_cached`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_cached_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { match layout { - DataLayout::Chunked { .. } => read_chunked_data_cached( + DataLayout::Chunked { .. } => read_chunked_data_cached_in( file_data, layout, dataspace, @@ -296,7 +384,7 @@ pub fn read_raw_data_cached( length_size, cache, ), - _ => read_raw_data_full( + _ => read_raw_data_full_in( file_data, layout, dataspace, @@ -325,9 +413,34 @@ pub fn read_raw_data_indexed( offset_size: u8, length_size: u8, cache: &ChunkCache, +) -> Result, FormatError> { + read_raw_data_indexed_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_raw_data_indexed`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_indexed_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: &ChunkCache, ) -> Result, FormatError> { match layout { - DataLayout::Chunked { .. } => read_chunked_data_indexed( + DataLayout::Chunked { .. } => read_chunked_data_indexed_in( file_data, layout, dataspace, @@ -337,7 +450,7 @@ pub fn read_raw_data_indexed( length_size, cache, ), - _ => read_raw_data_full( + _ => read_raw_data_full_in( file_data, layout, dataspace, @@ -366,6 +479,30 @@ pub fn read_raw_data_selection( offset_size: u8, length_size: u8, selection: &crate::selection::Selection, +) -> Result, FormatError> { + read_raw_data_selection_in( + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + selection, + ) +} + +/// [`read_raw_data_selection`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_raw_data_selection_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + selection: &crate::selection::Selection, ) -> Result, FormatError> { use crate::selection::Selection; @@ -375,7 +512,7 @@ pub fn read_raw_data_selection( // Read only what the selection's bounding box touches when that is // possible; everything below is the decode-everything-then-pick path, // kept for the cases `partial_read` declines. - if let Some(selected) = crate::partial_read::read_selection( + if let Some(selected) = crate::partial_read::read_selection_in( file_data, layout, dataspace, @@ -390,7 +527,7 @@ pub fn read_raw_data_selection( match selection { Selection::All => { - return read_raw_data_full( + return read_raw_data_full_in( file_data, layout, dataspace, @@ -410,7 +547,7 @@ pub fn read_raw_data_selection( match layout { DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => { // Read all data, then extract the selection - let full_data = read_raw_data_full( + let full_data = read_raw_data_full_in( file_data, layout, dataspace, @@ -434,7 +571,7 @@ pub fn read_raw_data_selection( // implicit-index generator, which then indexed past the rank and // panicked — only to decode the full dataset anyway. crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?; - let full_data = read_raw_data_full( + let full_data = read_raw_data_full_in( file_data, layout, dataspace, @@ -447,7 +584,7 @@ pub fn read_raw_data_selection( } DataLayout::Virtual { .. } => { // Assemble the full virtual dataset, then apply the read selection. - let full_data = read_raw_data_full( + let full_data = read_raw_data_full_in( file_data, layout, dataspace, @@ -471,8 +608,8 @@ pub fn read_raw_data_selection( /// would report differently from the stored dataspace (unlimited mappings). /// Use [`crate::vds::read_virtual_dataset`] to read those. #[allow(clippy::too_many_arguments)] -fn read_virtual_data( - file_data: &[u8], +fn read_virtual_data( + file_data: &S, layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, @@ -483,7 +620,7 @@ fn read_virtual_data( let wrapped = resolver.map(|r| move |name: &str| -> Result>, FormatError> { Ok(r(name)) }); let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver); - let v = crate::vds::read_virtual_dataset( + let v = crate::vds::read_virtual_dataset_in( file_data, layout, dataspace, @@ -885,6 +1022,33 @@ pub fn read_chunked_native( offset_size: u8, length_size: u8, cache: Option<&ChunkCache>, +) -> Result>, FormatError> { + read_chunked_native_in( + messages, + file_data, + layout, + dataspace, + datatype, + pipeline, + offset_size, + length_size, + cache, + ) +} + +/// [`read_chunked_native`] over any [`Storage`]. +#[cfg(feature = "std")] +#[allow(clippy::too_many_arguments)] +pub fn read_chunked_native_in( + messages: &[crate::object_header::HeaderMessage], + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + cache: Option<&ChunkCache>, ) -> Result>, FormatError> { use crate::fill_value; use crate::message_type::MessageType; @@ -919,8 +1083,9 @@ pub fn read_chunked_native( }, |values| bytes_of_mut(values), )?; - let fill = fill_value::dataset_fill_value_in(file_data, messages, offset_size, length_size)?; - fill_value::apply_to_unallocated_chunks( + let fill = + fill_value::dataset_fill_value_from_storage(file_data, messages, offset_size, length_size)?; + fill_value::apply_to_unallocated_chunks_in( bytes_of_mut(&mut values), file_data, layout, diff --git a/crates/clawhdf5-format/src/fill_value.rs b/crates/clawhdf5-format/src/fill_value.rs index f0f442e..ac23044 100644 --- a/crates/clawhdf5-format/src/fill_value.rs +++ b/crates/clawhdf5-format/src/fill_value.rs @@ -13,7 +13,7 @@ use alloc::{format, vec, vec::Vec}; use crate::addr::to_usize; -use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks}; +use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_in}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; use crate::error::FormatError; @@ -122,10 +122,11 @@ pub fn dataset_fill_value_in( } /// [`dataset_fill_value_in`] with the file behind any -/// [`Storage`](crate::storage::Storage). (The trait is not imported here: -/// its `len` would shadow the slice method in this module.) -pub fn dataset_fill_value_from_storage( - file: &dyn crate::storage::Storage, +/// [`Storage`](crate::storage::Storage) (a `&dyn Storage` too). (The trait +/// is not imported here: its `len` would shadow the slice method in this +/// module.) +pub fn dataset_fill_value_from_storage( + file: &S, messages: &[HeaderMessage], offset_size: u8, length_size: u8, @@ -212,6 +213,30 @@ pub fn read_full_with_fill>( offset_size: u8, length_size: u8, read: impl FnOnce() -> Result, E>, +) -> Result, E> { + read_full_with_fill_in( + messages, + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + read, + ) +} + +/// [`read_full_with_fill`] over any [`Storage`](crate::storage::Storage). +#[allow(clippy::too_many_arguments)] +pub fn read_full_with_fill_in, S: crate::storage::Storage + ?Sized>( + messages: &[HeaderMessage], + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + offset_size: u8, + length_size: u8, + read: impl FnOnce() -> Result, E>, ) -> Result, E> { // A dataset with external raw data also has no data address in this // file. It is NOT unallocated — its values live elsewhere — so it must @@ -222,12 +247,12 @@ pub fn read_full_with_fill>( { return Err(FormatError::ExternalDataFilesUnsupported.into()); } - let fill = dataset_fill_value_in(file_data, messages, offset_size, length_size)?; + let fill = dataset_fill_value_from_storage(file_data, messages, offset_size, length_size)?; if !has_storage(layout) { return Ok(filled_dataset(dataspace, elem_size, fill.as_deref())?); } let mut output = read()?; - apply_to_unallocated_chunks( + apply_to_unallocated_chunks_in( &mut output, file_data, layout, @@ -253,6 +278,30 @@ pub fn apply_to_unallocated_chunks( fill: Option<&[u8]>, offset_size: u8, length_size: u8, +) -> Result<(), FormatError> { + apply_to_unallocated_chunks_in( + output, + file_data, + layout, + dataspace, + elem_size, + fill, + offset_size, + length_size, + ) +} + +/// [`apply_to_unallocated_chunks`] over any [`Storage`](crate::storage::Storage). +#[allow(clippy::too_many_arguments)] +pub fn apply_to_unallocated_chunks_in( + output: &mut [u8], + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + fill: Option<&[u8]>, + offset_size: u8, + length_size: u8, ) -> Result<(), FormatError> { let Some(fill) = fill.filter(|f| f.len() == elem_size && !is_default(Some(f))) else { return Ok(()); @@ -260,7 +309,7 @@ pub fn apply_to_unallocated_chunks( if !matches!(layout, DataLayout::Chunked { .. }) || elem_size == 0 { return Ok(()); } - let (chunks, chunk_dims) = list_chunks( + let (chunks, chunk_dims) = list_chunks_in( file_data, layout, dataspace, diff --git a/crates/clawhdf5-format/src/gather.rs b/crates/clawhdf5-format/src/gather.rs index 22d2ae7..2b8a4ab 100644 --- a/crates/clawhdf5-format/src/gather.rs +++ b/crates/clawhdf5-format/src/gather.rs @@ -13,6 +13,7 @@ use alloc::{vec, vec::Vec}; use crate::data_read::NativeElement; use crate::error::FormatError; use crate::selection::Selection; +use crate::storage::Storage; /// Row-major element strides of `dims` (the last dimension has stride 1). fn strides(dims: &[u64]) -> Vec { @@ -261,6 +262,134 @@ pub(crate) fn gather( Ok(out) } +/// [`gather`] of bytes (`T = u8`) from a dataset that is not in memory: the +/// dataset's `src_len` bytes start at `base` in `file`, which must hold all +/// of them (the caller checks). The selection's runs are collected first, +/// adjacent ones merged, and fetched with one [`Storage::read_ranges`] call, +/// so only the selected bytes are read. Same checks and errors as +/// [`gather`]. +pub(crate) fn gather_storage( + file: &S, + base: u64, + src_len: usize, + dims: &[u64], + elem_size: usize, + selection: &Selection, +) -> Result, FormatError> { + if elem_size == 0 { + return Err(FormatError::DataSizeMismatch { + expected: 1, + actual: elem_size, + }); + } + let n_elements = match selection { + Selection::None => 0, + Selection::Hyperslab { count, block, .. } => count + .iter() + .zip(block) + .try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?)) + .ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?, + Selection::Points(points) => points.len() as u64, + Selection::All => { + return Err(FormatError::SelectionOutOfBounds( + "gather does not take Selection::All".into(), + )); + } + }; + let out_bytes = crate::chunked_read::checked_byte_len(n_elements, elem_size)?; + // Runs as (byte offset in the dataset, byte length), in output order. + let mut runs: Vec<(usize, usize)> = Vec::new(); + let mut total = 0usize; + let mut failed = false; + let mut collect = |first: u64, n: u64| { + if failed { + return; + } + let range = usize::try_from(first) + .ok() + .and_then(|f| f.checked_mul(elem_size)) + .zip( + usize::try_from(n) + .ok() + .and_then(|n| n.checked_mul(elem_size)), + ) + .and_then(|(at, len)| Some((at, len, at.checked_add(len)?))); + match range { + Some((at, len, end)) if end <= src_len && total + len <= out_bytes => { + match runs.last_mut() { + Some((a, l)) if *a + *l == at => *l += len, + _ => runs.push((at, len)), + } + total += len; + } + _ => failed = true, + } + }; + let mut bad_point = false; + match selection { + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let rank = dims.len(); + if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { + return Err(FormatError::SelectionOutOfBounds( + "hyperslab rank does not match dataset rank".into(), + )); + } + hyperslab_runs(dims, start, stride, count, block, &mut collect); + } + Selection::Points(points) => { + let strides = strides(dims); + let mut coalesce = Coalesce { + start: 0, + len: 0, + emit: &mut collect, + }; + for p in points { + if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) { + bad_point = true; + break; + } + let at = p + .iter() + .zip(&strides) + .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s))); + coalesce.push(at, 1); + } + coalesce.flush(); + } + Selection::None | Selection::All => {} + } + if failed || bad_point || total != out_bytes { + return Err(FormatError::SelectionOutOfBounds( + "selection addresses elements outside the dataset".into(), + )); + } + let ranges: Vec> = runs + .iter() + .map(|&(at, len)| base + at as u64..base + (at + len) as u64) + .collect(); + let fetched = file.read_ranges(&ranges)?; + if fetched.len() != ranges.len() { + return Err(FormatError::Storage( + "read_ranges returned the wrong number of ranges".into(), + )); + } + let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes); + for (bytes, &(_, len)) in fetched.iter().zip(&runs) { + let Some(b) = bytes.get(..len) else { + return Err(FormatError::Storage( + "short read inside the file (the storage shrank or the backend failed)".into(), + )); + }; + out.extend_from_slice(b); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index 9861e8f..be69189 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -152,7 +152,7 @@ impl GlobalHeapCollection { /// Read the collection at `offset` and index its objects: the /// collection's bytes, its offset as a `usize`, and the index (with /// file offsets). - fn read_collection( + pub(crate) fn read_collection( file: &S, offset: u64, length_size: u8, diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 92fb1fa..75a4bd3 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -7,12 +7,30 @@ //! The lane assignment is seeded by dataset metadata so repeated reads of //! the same region produce identical partitions (cache-friendly, reproducible). -use crate::addr::to_usize; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::decompress_chunk_exact; use crate::lane_partition::{self, LaneStats, PartitionStats}; +use crate::storage::{ExtentBytes, Storage}; + +/// The stored bytes of every chunk in `chunks`, fetched in one +/// [`Storage::read_ranges`] call when the file is not in memory (each +/// chunk's bounds error is reported when that chunk is decoded, as before). +fn fetch_all<'a, S: Storage + ?Sized>( + file_data: &'a S, + chunks: &[ChunkInfo], +) -> Result, FormatError> { + let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { + Vec::new() + } else { + chunks + .iter() + .map(|c| (c.address, c.chunk_size as usize, true)) + .collect() + }; + ExtentBytes::fetch(file_data, &extents) +} /// Threshold: only use parallel decompression when chunk count exceeds this. const PARALLEL_THRESHOLD: usize = 4; @@ -190,6 +208,27 @@ pub fn decompress_chunks_lane_partitioned( element_size: u32, seed: u64, num_lanes: Option, +) -> Result<(Vec>, PartitionStats), FormatError> { + decompress_chunks_lane_partitioned_in( + file_data, + chunks, + pipeline, + chunk_total_bytes, + element_size, + seed, + num_lanes, + ) +} + +/// [`decompress_chunks_lane_partitioned`] over any [`Storage`]. +pub fn decompress_chunks_lane_partitioned_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: &FilterPipeline, + chunk_total_bytes: usize, + element_size: u32, + seed: u64, + num_lanes: Option, ) -> Result<(Vec>, PartitionStats), FormatError> { use rayon::prelude::*; @@ -199,6 +238,7 @@ pub fn decompress_chunks_lane_partitioned( .unwrap_or(1) }); + let raw_bytes = fetch_all(file_data, chunks)?; let assignments = lane_partition::partition_chunks(chunks.len(), lanes, seed); let num_lanes = assignments.len(); @@ -211,19 +251,8 @@ pub fn decompress_chunks_lane_partitioned( for &index in &indices { let chunk_info = &chunks[index]; - let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; - - if c_addr - .checked_add(size) - .is_none_or(|end| end > file_data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: c_addr.saturating_add(size), - available: file_data.len(), - }); - } - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; let decompressed = decompress_chunk_exact( raw_chunk, @@ -282,25 +311,27 @@ pub fn decompress_chunks_parallel( pipeline: &FilterPipeline, chunk_total_bytes: usize, element_size: u32, +) -> Result>, FormatError> { + decompress_chunks_parallel_in(file_data, chunks, pipeline, chunk_total_bytes, element_size) +} + +/// [`decompress_chunks_parallel`] over any [`Storage`]. +pub fn decompress_chunks_parallel_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: &FilterPipeline, + chunk_total_bytes: usize, + element_size: u32, ) -> Result>, FormatError> { use rayon::prelude::*; + let raw_bytes = fetch_all(file_data, chunks)?; let results: Result, FormatError> = chunks .par_iter() .enumerate() .map(|(index, chunk_info)| { - let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; - if c_addr - .checked_add(size) - .is_none_or(|end| end > file_data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: c_addr.saturating_add(size), - available: file_data.len(), - }); - } - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; let decompressed = decompress_chunk_exact( raw_chunk, @@ -331,20 +362,22 @@ pub fn decompress_chunks_sequential( chunk_total_bytes: usize, element_size: u32, ) -> Result>, FormatError> { + decompress_chunks_sequential_in(file_data, chunks, pipeline, chunk_total_bytes, element_size) +} + +/// [`decompress_chunks_sequential`] over any [`Storage`]. +pub fn decompress_chunks_sequential_in( + file_data: &S, + chunks: &[ChunkInfo], + pipeline: Option<&FilterPipeline>, + chunk_total_bytes: usize, + element_size: u32, +) -> Result>, FormatError> { + let raw_bytes = fetch_all(file_data, chunks)?; let mut result = Vec::with_capacity(chunks.len()); - for chunk_info in chunks { - let c_addr = to_usize(chunk_info.address)?; + for (i, chunk_info) in chunks.iter().enumerate() { let size = chunk_info.chunk_size as usize; - if c_addr - .checked_add(size) - .is_none_or(|end| end > file_data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: c_addr.saturating_add(size), - available: file_data.len(), - }); - } - let raw_chunk = &file_data[c_addr..c_addr + size]; + let raw_chunk = raw_bytes.get(i, chunk_info.address, size)?; let decompressed = if let Some(pl) = pipeline { decompress_chunk_exact( diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index ad5dc8e..a9f630c 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -18,7 +18,7 @@ use alloc::{format, vec, vec::Vec}; #[cfg(feature = "std")] use std::string as alloc_or_std; -use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read}; +use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read_in}; use crate::data_layout::DataLayout; use crate::data_read::extract_selection_from_buffer; use crate::dataspace::Dataspace; @@ -26,6 +26,7 @@ use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::{all_filters_skipped, decompress_chunk_exact_with}; use crate::selection::Selection; +use crate::storage::{ExtentBytes, Storage}; /// The smallest axis-aligned box containing every selected element, as /// `(start, extent)` per dimension. `None` when there is nothing to gain or @@ -256,6 +257,30 @@ pub fn read_selection( offset_size: u8, length_size: u8, selection: &Selection, +) -> Result>, FormatError> { + read_selection_in( + file_data, + layout, + dataspace, + elem_size, + pipeline, + offset_size, + length_size, + selection, + ) +} + +/// [`read_selection`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_selection_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + elem_size: usize, + pipeline: Option<&FilterPipeline>, + offset_size: u8, + length_size: u8, + selection: &Selection, ) -> Result>, FormatError> { let dims = &dataspace.dimensions; if dims.is_empty() || elem_size == 0 { @@ -276,14 +301,33 @@ pub fn read_selection( validate(selection, dims)?; let base = usize::try_from(*address) .map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?; - let data = file_data - .get(base..) - .and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?)) - .ok_or(FormatError::UnexpectedEof { - expected: base, - available: file_data.len(), - })?; - return crate::gather::gather::(data, dims, elem_size, selection).map(Some); + let file_len = crate::storage::len_usize(file_data); + let eof = FormatError::UnexpectedEof { + expected: base, + available: file_len, + }; + if let Some(all) = file_data.as_contiguous() { + let data = all + .get(base..) + .and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?)) + .ok_or(eof)?; + return crate::gather::gather::(data, dims, elem_size, selection).map(Some); + } + // Not in memory: the same bounds check, then only the selected runs + // are read. + let len = checked_byte_len(total, elem_size) + .ok() + .filter(|&len| base <= file_len && len <= file_len - base) + .ok_or(eof)?; + return crate::gather::gather_storage( + file_data, + base as u64, + len, + dims, + elem_size, + selection, + ) + .map(Some); } let Some((box_start, box_extent)) = bounding_box(selection, dims) else { return Ok(None); @@ -303,7 +347,7 @@ pub fn read_selection( btree_address: Some(_), .. } => { - let (chunks, chunk_dims) = list_chunks_for_read( + let (chunks, chunk_dims) = list_chunks_for_read_in( file_data, layout, dataspace, @@ -315,29 +359,37 @@ pub fn read_selection( let rank = dims.len(); let chunk_shape: Vec = chunk_dims.iter().map(|&d| d as u64).collect(); let chunk_bytes = crate::chunked_read::checked_chunk_byte_len(&chunk_dims, elem_size)?; - // Chunks are decoded into this thread's reusable buffers. - crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { - for chunk in &chunks { + // The chunks overlapping the box, in index order. + let wanted: Vec<&crate::chunked_read::ChunkInfo> = chunks + .iter() + .filter(|chunk| { if chunk.offsets.len() < rank || chunk.address == u64::MAX { - continue; + return false; } let origin = &chunk.offsets[..rank]; - let overlaps = (0..rank).all(|d| { + (0..rank).all(|d| { origin[d] < box_start[d] + box_extent[d] && origin[d].saturating_add(chunk_shape[d]) > box_start[d] - }); - if !overlaps { - continue; - } - let at = usize::try_from(chunk.address) + }) + }) + .collect(); + // Their stored bytes, in one batch when the file is not in memory. + let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { + Vec::new() + } else { + wanted + .iter() + .map(|c| (c.address, c.chunk_size as usize, true)) + .collect() + }; + let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + // Chunks are decoded into this thread's reusable buffers. + crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { + for (i, chunk) in wanted.iter().enumerate() { + let origin = &chunk.offsets[..rank]; + usize::try_from(chunk.address) .map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?; - let raw = at - .checked_add(chunk.chunk_size as usize) - .and_then(|end| file_data.get(at..end)) - .ok_or(FormatError::UnexpectedEof { - expected: at.saturating_add(chunk.chunk_size as usize), - available: file_data.len(), - })?; + let raw = raw_bytes.get(i, chunk.address, chunk.chunk_size as usize)?; // Mirrors the full-read path: filter-mask bit i set means // filter i was not applied to this chunk. let data: &[u8] = match pipeline { diff --git a/crates/clawhdf5-format/src/provenance.rs b/crates/clawhdf5-format/src/provenance.rs index ee74994..0bdccab 100644 --- a/crates/clawhdf5-format/src/provenance.rs +++ b/crates/clawhdf5-format/src/provenance.rs @@ -13,7 +13,6 @@ use sha2::{Digest, Sha256}; use crate::attribute::AttributeMessage; use crate::data_layout::DataLayout; -use crate::data_read::read_raw_data; use crate::dataspace::Dataspace; use crate::datatype::Datatype; use crate::error::FormatError; @@ -128,10 +127,20 @@ pub fn verify_dataset( header: &ObjectHeader, offset_size: u8, length_size: u8, +) -> Result { + verify_dataset_in(file_data, header, offset_size, length_size) +} + +/// [`verify_dataset`] over any [`Storage`](crate::storage::Storage). +pub fn verify_dataset_in( + file_data: &S, + header: &ObjectHeader, + offset_size: u8, + length_size: u8, ) -> Result { // 1. Extract all attributes (compact + dense). let attrs = - crate::attribute::extract_attributes_full(file_data, header, offset_size, length_size)?; + crate::attribute::extract_attributes_full_in(file_data, header, offset_size, length_size)?; // 2. Find the stored hash. let stored_hash = attrs @@ -174,7 +183,7 @@ pub fn verify_dataset( .transpose()?; let raw = match &dl { - DataLayout::Chunked { .. } => crate::chunked_read::read_chunked_data( + DataLayout::Chunked { .. } => crate::chunked_read::read_chunked_data_in( file_data, &dl, &ds, @@ -183,7 +192,7 @@ pub fn verify_dataset( offset_size, length_size, )?, - _ => read_raw_data(file_data, &dl, &ds, &dt)?, + _ => crate::data_read::read_raw_data_in(file_data, &dl, &ds, &dt)?, }; // 4. Compare. diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 555ac78..0dd0540 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -336,6 +336,159 @@ pub fn read_upto( Ok(bytes) } +/// Most stored bytes fetched by one [`Storage::read_ranges`] call when a +/// read gathers many extents (a chunked dataset's chunks): a larger read is +/// fetched and decoded batch by batch, so a remote backend never holds more +/// than this much undecoded data per read. +pub(crate) const RAW_BATCH_BYTES: usize = 64 << 20; + +/// The stored bytes of a list of extents (chunks, contiguous runs), fetched +/// together: [`Storage::read_ranges`] is called once for all of them, so a +/// remote backend can coalesce and parallelise the requests. +/// +/// With the whole file in memory nothing is fetched: [`Self::get`] slices +/// it, as the slice readers did. Either way an extent that does not lie in +/// the file is the error the slice readers gave for it +/// ([`FormatError::UnexpectedEof`] with its end and the file length, or +/// [`FormatError::Overflow`] for an address past this platform's `usize`), +/// reported when that extent is asked for — so a read reports the first +/// failing extent in its own order, whatever fails after it. +pub(crate) enum ExtentBytes<'a> { + /// The whole file. + Contiguous(&'a [u8]), + /// Each extent's bytes, or its bounds error. + Fetched(Vec>), +} + +/// One extent of [`ExtentBytes::Fetched`]. +pub(crate) enum Extent<'a> { + /// Its bytes. + Bytes(Cow<'a, [u8]>), + /// In the file, but not fetched (the caller did not want its bytes). + NotFetched, + /// The error reading it gives. + Err(FormatError), +} + +impl<'a> ExtentBytes<'a> { + /// Fetch `extents` (`(address, length, wanted)`): the bytes of those + /// `wanted`, and the bounds check of all of them. + pub(crate) fn fetch( + file: &'a S, + extents: &[(u64, usize, bool)], + ) -> Result { + if let Some(all) = file.as_contiguous() { + return Ok(ExtentBytes::Contiguous(all)); + } + let file_len = len_usize(file); + let mut ranges = Vec::new(); + let mut out = Vec::with_capacity(extents.len()); + // Positions in `out` of the extents being read, in `ranges` order. + let mut slots = Vec::new(); + for &(addr, len, wanted) in extents { + let checked = + crate::addr::to_usize(addr).and_then(|start| match start.checked_add(len) { + Some(end) if end <= file_len => Ok(()), + _ => Err(FormatError::UnexpectedEof { + expected: start.saturating_add(len), + available: file_len, + }), + }); + match checked { + Ok(()) if wanted => { + slots.push(out.len()); + ranges.push(addr..addr + len as u64); + out.push(Extent::NotFetched); + } + Ok(()) => out.push(Extent::NotFetched), + Err(e) => out.push(Extent::Err(e)), + } + } + if !ranges.is_empty() { + let got = file.read_ranges(&ranges)?; + if got.len() != ranges.len() { + return Err(FormatError::Storage( + "read_ranges returned the wrong number of ranges".into(), + )); + } + for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) { + if (bytes.len() as u64) < r.end - r.start { + return Err(short_read()); + } + out[slot] = Extent::Bytes(bytes); + } + } + Ok(ExtentBytes::Fetched(out)) + } + + /// Whether extent `i` (at `addr`, `len` bytes, as passed to + /// [`Self::fetch`]) lies in the file: its bounds error if not. + pub(crate) fn check(&self, i: usize, addr: u64, len: usize) -> Result<(), FormatError> { + match self { + ExtentBytes::Contiguous(_) => self.get(i, addr, len).map(|_| ()), + ExtentBytes::Fetched(v) => match v.get(i) { + Some(Extent::Err(e)) => Err(e.clone()), + Some(_) => Ok(()), + None => Err(not_fetched()), + }, + } + } + + /// Extent `i`'s bytes (at `addr`, `len` bytes, as passed to + /// [`Self::fetch`]). + pub(crate) fn get(&self, i: usize, addr: u64, len: usize) -> Result<&[u8], FormatError> { + match self { + ExtentBytes::Contiguous(all) => { + let start = crate::addr::to_usize(addr)?; + start + .checked_add(len) + .and_then(|end| all.get(start..end)) + .ok_or(FormatError::UnexpectedEof { + expected: start.saturating_add(len), + available: <[u8]>::len(all), + }) + } + ExtentBytes::Fetched(v) => match v.get(i) { + Some(Extent::Bytes(b)) => Ok(b), + Some(Extent::Err(e)) => Err(e.clone()), + _ => Err(not_fetched()), + }, + } + } +} + +#[cold] +fn not_fetched() -> FormatError { + FormatError::Storage("an extent that was not fetched was asked for".into()) +} + +/// Split `n` extents, whose sizes `size(i)` gives, into consecutive batches +/// of at most [`RAW_BATCH_BYTES`] (at least one extent each): the ranges of +/// `0..n` to fetch together. With the whole file in memory (`contiguous`) +/// there is nothing to fetch, and one batch. +pub(crate) fn raw_batches( + n: usize, + contiguous: bool, + size: impl Fn(usize) -> usize, +) -> Vec> { + if contiguous || n == 0 { + return core::iter::once(0..n).collect(); + } + let mut out = Vec::new(); + let (mut start, mut bytes) = (0, 0usize); + for i in 0..n { + let s = size(i); + if i > start && bytes.saturating_add(s) > RAW_BATCH_BYTES { + out.push(start..i); + start = i; + bytes = 0; + } + bytes = bytes.saturating_add(s); + } + out.push(start..n); + out +} + /// Borrow the whole file for a code path that has not been converted to /// [`Storage`] yet. On a backend without a contiguous view this is the /// clean [`FormatError::ContiguousStorageRequired`] error, never a guess. diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs index df1b068..92ad4e7 100644 --- a/crates/clawhdf5-format/src/vds.rs +++ b/crates/clawhdf5-format/src/vds.rs @@ -15,12 +15,13 @@ #[cfg(not(feature = "std"))] use alloc::{format, string::String, vec, vec::Vec}; -use crate::addr::to_usize; +use crate::addr::{checked_addr, to_usize}; use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings}; use crate::dataspace::Dataspace; use crate::datatype::Datatype; use crate::error::FormatError; use crate::selection::{SerializedSelection, UNLIMITED}; +use crate::storage::Storage; /// Resolves the name of an external VDS source file, as stored in the /// mapping, to that file's bytes. @@ -192,8 +193,8 @@ fn non_unlimited_elements(sel: &SerializedSelection, skip: usize) -> Option } /// Load and decode the mapping list of a virtual layout. -fn load_mappings( - file_data: &[u8], +fn load_mappings( + file_data: &S, layout: &DataLayout, length_size: u8, ) -> Result, FormatError> { @@ -208,8 +209,11 @@ fn load_mappings( let Some(addr) = *global_heap_address else { return Ok(Vec::new()); }; - let coll = - crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?; + let coll = crate::global_heap::GlobalHeapCollection::parse_in( + file_data, + checked_addr(addr)?, + length_size, + )?; let index = u16::try_from(*global_heap_index) .map_err(|_| vds_err("VDS mapping heap index out of range"))?; let obj = coll @@ -330,7 +334,11 @@ enum Step { /// Work out the extent libhdf5 gives the virtual dataset /// (`H5D__virtual_set_extent_unlim`, default view `H5D_VDS_LAST_AVAILABLE` /// with a printf gap of 0) and how much of each unlimited mapping is read. -fn plan(mappings: &[Mapping], stored: &[u64], sources: &mut Sources) -> Result { +fn plan( + mappings: &[Mapping], + stored: &[u64], + sources: &mut Sources<'_, '_, S>, +) -> Result { let overflow = || FormatError::Overflow("VDS extent overflow".into()); let rank = stored.len(); let mut new_dims: Vec> = vec![None; rank]; @@ -472,6 +480,25 @@ pub fn virtual_dataset_extent( _offset_size: u8, length_size: u8, resolver: Option<&VdsFileResolver>, +) -> Result, FormatError> { + virtual_dataset_extent_in( + file_data, + layout, + dataspace, + _offset_size, + length_size, + resolver, + ) +} + +/// [`virtual_dataset_extent`] over any [`Storage`]. +pub fn virtual_dataset_extent_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + _offset_size: u8, + length_size: u8, + resolver: Option<&VdsFileResolver>, ) -> Result, FormatError> { let mappings = load_mappings(file_data, layout, length_size)?; if mappings.iter().all(|m| m.kind == Kind::Fixed) { @@ -498,6 +525,30 @@ pub fn read_virtual_dataset( _offset_size: u8, length_size: u8, resolver: Option<&VdsFileResolver>, +) -> Result { + read_virtual_dataset_in( + file_data, + layout, + dataspace, + datatype, + fill, + _offset_size, + length_size, + resolver, + ) +} + +/// [`read_virtual_dataset`] over any [`Storage`]. +#[allow(clippy::too_many_arguments)] +pub fn read_virtual_dataset_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + fill: Option<&[u8]>, + _offset_size: u8, + length_size: u8, + resolver: Option<&VdsFileResolver>, ) -> Result { let mappings = load_mappings(file_data, layout, length_size)?; let mut sources = Sources::new(file_data, resolver); @@ -511,7 +562,7 @@ pub fn read_virtual_dataset( let mut data = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len( total, elem_size, )?)?; - if let Some(fill) = fill.filter(|f| f.len() == elem_size && f.iter().any(|&b| b != 0)) { + if let Some(fill) = fill.filter(|f| <[u8]>::len(f) == elem_size && f.iter().any(|&b| b != 0)) { for element in data.chunks_exact_mut(elem_size) { element.copy_from_slice(fill); } @@ -780,14 +831,17 @@ struct SourceData { /// Source files and datasets, fetched on demand. The most recently used /// external file is kept, since consecutive mappings usually share one. -struct Sources<'a, 'r> { - file_data: &'a [u8], +/// +/// The virtual dataset's own file (`"."`) is read through its [`Storage`]; +/// an external source file is loaded whole, through the resolver. +struct Sources<'a, 'r, S: Storage + ?Sized> { + file_data: &'a S, resolver: Option<&'r VdsFileResolver<'r>>, cached_file: Option<(String, Option>)>, } -impl<'a, 'r> Sources<'a, 'r> { - fn new(file_data: &'a [u8], resolver: Option<&'r VdsFileResolver<'r>>) -> Self { +impl<'a, 'r, S: Storage + ?Sized> Sources<'a, 'r, S> { + fn new(file_data: &'a S, resolver: Option<&'r VdsFileResolver<'r>>) -> Self { Sources { file_data, resolver, @@ -795,11 +849,9 @@ impl<'a, 'r> Sources<'a, 'r> { } } - /// The bytes of source file `name`, or `None` if it does not exist. - fn file(&mut self, name: &str) -> Result, FormatError> { - if name == "." { - return Ok(Some(self.file_data)); - } + /// The bytes of external source file `name` (not `"."`), or `None` if it + /// does not exist. + fn external(&mut self, name: &str) -> Result, FormatError> { if self.cached_file.as_ref().is_none_or(|(n, _)| n != name) { let resolver = self.resolver.ok_or_else(|| { vds_err("external-file virtual dataset sources require a file resolver") @@ -821,7 +873,10 @@ impl<'a, 'r> Sources<'a, 'r> { /// The extent of source dataset `path` in file `file`, or `None` when /// either does not exist. fn dims(&mut self, file: &str, path: &str) -> Result>, FormatError> { - let Some(bytes) = self.file(file)? else { + if file == "." { + return Ok(open_source(self.file_data, path)?.map(|s| s.dataspace.dimensions)); + } + let Some(bytes) = self.external(file)? else { return Ok(None); }; Ok(open_source(bytes, path)?.map(|s| s.dataspace.dimensions)) @@ -846,7 +901,13 @@ impl<'a, 'r> Sources<'a, 'r> { from another file is not supported" ))); } - let Some(bytes) = self.file(file)? else { + if file == "." { + let Some(src) = open_source(self.file_data, path)? else { + return Ok(None); + }; + return read_source(self.file_data, src, path, datatype).map(Some); + } + let Some(bytes) = self.external(file)? else { return Ok(None); }; let Some(src) = open_source(bytes, path)? else { @@ -910,19 +971,23 @@ fn source_message<'h>( /// Open source dataset `path` of the file in `file_data`, or `None` if there /// is no such object (libhdf5 reads a missing source as fill). -fn open_source(file_data: &[u8], path: &str) -> Result, FormatError> { +fn open_source( + file_data: &S, + path: &str, +) -> Result, FormatError> { use crate::message_type::MessageType; - use crate::shared_message::message_data_with_sohm; + use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm; - // `file_data` starts at the superblock (see `Sources::file`). - let sb = crate::superblock::Superblock::parse(file_data, 0)?; + // `file_data` starts at the superblock (see `Sources::external`). + let sb = crate::superblock::Superblock::parse_in(file_data, 0)?; let (os, ls) = (sb.offset_size, sb.length_size); - let addr = match crate::group_v2::resolve_path_any(file_data, &sb, path) { + let addr = match crate::group_v2::resolve_path_any_in(file_data, &sb, path) { Ok(a) => a, Err(FormatError::PathNotFound(_)) => return Ok(None), Err(e) => return Err(e), }; - let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?; + let header = + crate::object_header::ObjectHeader::parse_in(file_data, checked_addr(addr)?, os, ls)?; let mut src = OpenSource { offset_size: os, length_size: ls, @@ -941,15 +1006,15 @@ fn open_source(file_data: &[u8], path: &str) -> Result, Forma /// Read an opened source dataset in full (its own fill value applied to /// unallocated chunks). -fn read_source( - file_data: &[u8], +fn read_source( + file_data: &S, src: OpenSource, path: &str, datatype: &Datatype, ) -> Result { use crate::filter_pipeline::FilterPipeline; use crate::message_type::MessageType; - use crate::shared_message::message_data_with_sohm; + use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm; let (os, ls) = (src.offset_size, src.length_size); let dt_msg = source_message(&src, path, MessageType::Datatype)?; @@ -987,7 +1052,7 @@ fn read_source( message_data_with_sohm(file_data, m, os, ls).and_then(|d| FilterPipeline::parse(&d)) }) .transpose()?; - let raw = crate::fill_value::read_full_with_fill( + let raw = crate::fill_value::read_full_with_fill_in( &src.header.messages, file_data, &layout, @@ -996,7 +1061,7 @@ fn read_source( os, ls, || { - crate::data_read::read_raw_data_full( + crate::data_read::read_raw_data_full_in( file_data, &layout, &src.dataspace, diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index a998a62..f37638b 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -5,9 +5,9 @@ //! `sequence_length(4 LE) + collection_address(offset_size LE) + object_index(4 LE)`. #[cfg(not(feature = "std"))] -use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; +use alloc::{borrow::Cow, collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] -use std::collections::BTreeMap; +use std::{borrow::Cow, collections::BTreeMap}; use crate::addr::to_usize; use crate::error::FormatError; @@ -137,13 +137,15 @@ pub fn check_element_size(stored_size: u32, offset_size: u8) -> Result<(), Forma /// A collection's objects, located in the file data but not copied: /// `(index, offset, size)` of the first object with each index, sorted by -/// index. -struct CachedCollection { +/// index. Over a storage without the whole file in memory, also the +/// collection's bytes (`(offset, bytes)`), read once when it is indexed. +struct CachedCollection<'a> { objects: Vec<(u16, usize, usize)>, + bytes: Option<(usize, Cow<'a, [u8]>)>, } -impl CachedCollection { - fn new(index: GlobalHeapIndex) -> Self { +impl<'a> CachedCollection<'a> { + fn new(index: GlobalHeapIndex, bytes: Option<(usize, Cow<'a, [u8]>)>) -> Self { let mut objects: Vec<(u16, usize, usize)> = index .objects .iter() @@ -152,12 +154,16 @@ impl CachedCollection { // Stable, so the first object with a repeated index is kept. objects.sort_by_key(|o| o.0); objects.dedup_by_key(|o| o.0); - Self { objects } + Self { objects, bytes } } /// What this entry costs to keep, in bytes (roughly). fn cost(&self) -> usize { - 64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>() + let held = match &self.bytes { + Some((_, Cow::Owned(b))) => b.len(), + _ => 0, + }; + 64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>() + held } fn get(&self, index: u32) -> Option<(usize, usize)> { @@ -170,6 +176,8 @@ impl CachedCollection { /// How many bytes of collection indexes a [`VlResolver`] keeps before it /// drops them and starts again. Values are never copied into the cache, so /// this bounds what a read retains however many collections it visits. +/// (Over a storage without the whole file in memory the collections' bytes +/// are kept too, and count against this.) const CACHE_BUDGET: usize = 32 << 20; /// Resolves variable-length elements against a file's global heap, parsing @@ -185,11 +193,18 @@ const CACHE_BUDGET: usize = 32 << 20; /// that overlap one another are refused (libhdf5 never writes them), so a /// file cannot make the resolver parse the same bytes as the objects of /// many collections. -pub struct VlResolver<'a> { - file_data: &'a [u8], +/// +/// The file is any [`Storage`](crate::storage::Storage) (`S`, a slice by default). Over one without +/// the whole file in memory each collection is read once, when first used, +/// and kept (within the budget above); [`Self::strings`], +/// [`Self::string_bytes`] and [`Self::sequences`] work over any storage, +/// [`Self::element`] and [`Self::string_element`], which borrow from the +/// file, over a slice. +pub struct VlResolver<'a, S: crate::storage::Storage + ?Sized = [u8]> { + file_data: &'a S, offset_size: u8, length_size: u8, - cache: BTreeMap, + cache: BTreeMap>, cached_bytes: usize, budget: usize, /// Start → end of every collection parsed so far (kept when the cache @@ -201,6 +216,56 @@ impl<'a> VlResolver<'a> { /// A resolver over `file_data` (the file from its superblock on), with /// the superblock's offset and length sizes. pub fn new(file_data: &'a [u8], offset_size: u8, length_size: u8) -> Self { + Self::new_in(file_data, offset_size, length_size) + } + + /// One element (the first [`element_size`](Self::element_size) bytes of + /// `elem`) of a variable-length sequence whose base type is `base_size` + /// bytes: its `length × base_size` bytes, or `None` for a null element + /// (heap address 0). + pub fn element( + &mut self, + elem: &[u8], + base_size: usize, + ) -> Result, FormatError> { + let vl = parse_vl_references(elem, 1, self.offset_size)?; + let vl = &vl[0]; + if vl.collection_address == 0 { + return Ok(None); + } + let (start, size) = self.locate(vl)?; + let data = &self.file_data[start..start + size]; + check_object_size(vl, data.len(), base_size)?; + Ok(Some(data)) + } + + /// One variable-length string element: its bytes up to the first NUL, + /// or `None` for a null element (h5dump prints it as `NULL`, h5py + /// returns it as empty). + pub fn string_element(&mut self, elem: &[u8]) -> Result, FormatError> { + Ok(self.element(elem, 1)?.map(cut_at_nul)) + } +} + +/// `data_len`, the size of `vl`'s heap object, against the `length × +/// base_size` bytes the element says it holds. +fn check_object_size(vl: &VlElement, data_len: usize, base_size: usize) -> Result<(), FormatError> { + let expected = (vl.length as usize) + .checked_mul(base_size) + .ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?; + if data_len != expected { + return Err(FormatError::VlDataError(format!( + "global heap object {} in the collection at {} holds {data_len} bytes; the element \ + says {} × {base_size}", + vl.object_index, vl.collection_address, vl.length + ))); + } + Ok(()) +} + +impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> { + /// [`VlResolver::new`] over any [`Storage`](crate::storage::Storage). + pub fn new_in(file_data: &'a S, offset_size: u8, length_size: u8) -> Self { Self { file_data, offset_size, @@ -231,51 +296,15 @@ impl<'a> VlResolver<'a> { /// The bytes of one element: `length × base_size` bytes from the heap, /// or `None` for a null element. - fn resolve( - &mut self, - vl: &VlElement, - base_size: usize, - ) -> Result, FormatError> { - let addr = vl.collection_address; - if addr == 0 { + fn resolve(&mut self, vl: &VlElement, base_size: usize) -> Result, FormatError> { + if vl.collection_address == 0 { return Ok(None); } let data = self.object(vl)?; - let expected = (vl.length as usize) - .checked_mul(base_size) - .ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?; - if data.len() != expected { - return Err(FormatError::VlDataError(format!( - "global heap object {} in the collection at {addr} holds {} bytes; the element \ - says {} × {base_size}", - vl.object_index, - data.len(), - vl.length - ))); - } + check_object_size(vl, data.len(), base_size)?; Ok(Some(data)) } - /// One element (the first [`element_size`](Self::element_size) bytes of - /// `elem`) of a variable-length sequence whose base type is `base_size` - /// bytes: its `length × base_size` bytes, or `None` for a null element - /// (heap address 0). - pub fn element( - &mut self, - elem: &[u8], - base_size: usize, - ) -> Result, FormatError> { - let vl = parse_vl_references(elem, 1, self.offset_size)?; - self.resolve(&vl[0], base_size) - } - - /// One variable-length string element: its bytes up to the first NUL, - /// or `None` for a null element (h5dump prints it as `NULL`, h5py - /// returns it as empty). - pub fn string_element(&mut self, elem: &[u8]) -> Result, FormatError> { - Ok(self.element(elem, 1)?.map(cut_at_nul)) - } - /// The strings of the variable-length string elements in `raw`, as /// bytes. A string ends at its first NUL, as libhdf5 returns it (it /// converts each to a C string); a null element is empty. @@ -330,9 +359,20 @@ pub fn read_vl_strings( num_elements: u64, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + read_vl_strings_in(file_data, raw_data, num_elements, offset_size, length_size) +} + +/// [`read_vl_strings`] over any [`Storage`](crate::storage::Storage). +pub fn read_vl_strings_in( + file_data: &S, + raw_data: &[u8], + num_elements: u64, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let raw = first_elements(raw_data, num_elements, offset_size)?; - VlResolver::new(file_data, offset_size, length_size).strings(raw) + VlResolver::new_in(file_data, offset_size, length_size).strings(raw) } /// The first `num_elements` elements of `raw`, or an error if it is shorter. @@ -364,9 +404,20 @@ pub fn read_vl_bytes( num_elements: u64, offset_size: u8, length_size: u8, +) -> Result>, FormatError> { + read_vl_bytes_in(file_data, raw_data, num_elements, offset_size, length_size) +} + +/// [`read_vl_bytes`] over any [`Storage`](crate::storage::Storage). +pub fn read_vl_bytes_in( + file_data: &S, + raw_data: &[u8], + num_elements: u64, + offset_size: u8, + length_size: u8, ) -> Result>, FormatError> { let refs = parse_vl_references(raw_data, num_elements, offset_size)?; - let mut resolver = VlResolver::new(file_data, offset_size, length_size); + let mut resolver = VlResolver::new_in(file_data, offset_size, length_size); let mut result = Vec::with_capacity(refs.len()); for vl in &refs { @@ -385,10 +436,10 @@ pub fn read_vl_bytes( Ok(result) } -impl<'a> VlResolver<'a> { - /// The heap object `vl` points to, whatever its size; its collection is - /// parsed on first use. - fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> { +impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> { + /// Where the heap object `vl` points to lies in the file, whatever its + /// size (`(offset, size)`); its collection is parsed on first use. + fn locate(&mut self, vl: &VlElement) -> Result<(usize, usize), FormatError> { let addr = vl.collection_address; // libhdf5 writes a null element with address 0, never the undefined // address, and fails to read one ("addr undefined") even when its @@ -402,14 +453,20 @@ impl<'a> VlResolver<'a> { if !self.cache.contains_key(&addr) { let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof { expected: usize::MAX, - available: self.file_data.len(), + available: crate::storage::len_usize(self.file_data), })?; - let index = - GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?; - // parse_index checked that the collection lies in the file. + let (bytes, base, index) = + GlobalHeapCollection::read_collection(self.file_data, addr, self.length_size)?; + // read_collection checked that the collection lies in the file. let end = offset + to_usize(index.collection_size)?; self.check_overlap(offset, end)?; - let coll = CachedCollection::new(index); + // With the whole file in memory the objects are sliced from it; + // otherwise the collection's bytes are kept. + let bytes = match self.file_data.as_contiguous() { + Some(_) => None, + None => Some((base, bytes)), + }; + let coll = CachedCollection::new(index, bytes); if self.cached_bytes.saturating_add(coll.cost()) > self.budget { self.cache.clear(); self.cached_bytes = 0; @@ -417,13 +474,27 @@ impl<'a> VlResolver<'a> { self.cached_bytes += coll.cost(); self.cache.insert(addr, coll); } - let (start, size) = self.cache[&addr].get(vl.object_index).ok_or( - FormatError::GlobalHeapObjectNotFound { + self.cache[&addr] + .get(vl.object_index) + .ok_or(FormatError::GlobalHeapObjectNotFound { collection_address: addr, index: vl.object_index as u16, - }, - )?; - Ok(&self.file_data[start..start + size]) + }) + } + + /// The heap object `vl` points to, whatever its size; its collection is + /// parsed on first use. + fn object(&mut self, vl: &VlElement) -> Result<&[u8], FormatError> { + let (start, size) = self.locate(vl)?; + if let Some(all) = self.file_data.as_contiguous() { + return Ok(&all[start..start + size]); + } + match &self.cache[&vl.collection_address].bytes { + Some((base, bytes)) => Ok(&bytes[start - base..start - base + size]), + None => Err(FormatError::Storage( + "global heap collection bytes were not kept".into(), + )), + } } /// Record the collection at `start..end`, refusing one that overlaps a @@ -709,6 +780,7 @@ mod tests { let mut r = VlResolver::new(&file_data, 8, 8); let one = CachedCollection { objects: vec![(0, 0, 0); 3], + bytes: None, } .cost(); r.budget = 2 * one + 1; diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index 02a4f17..71b0dc0 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -37,14 +37,25 @@ use clawhdf5_format::btree_v2::{ BTreeV2Header, collect_btree_v2_records, collect_btree_v2_records_in, find_btree_v2_records, find_btree_v2_records_in, }; +use clawhdf5_format::chunk_cache::ChunkCache; +use clawhdf5_format::chunked_read::{list_chunks, list_chunks_in}; use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read::{ + read_raw_data_cached, read_raw_data_cached_in, read_raw_data_full, read_raw_data_full_in, + read_raw_data_indexed, read_raw_data_indexed_in, read_raw_data_selection, + read_raw_data_selection_in, +}; use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; use clawhdf5_format::extensible_array::{ ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in, }; -use clawhdf5_format::fill_value::{dataset_fill_value_from_storage, dataset_fill_value_in}; +use clawhdf5_format::fill_value::{ + dataset_fill_value_from_storage, dataset_fill_value_in, read_full_with_fill, + read_full_with_fill_in, +}; +use clawhdf5_format::filter_pipeline::FilterPipeline; use clawhdf5_format::fixed_array::{ FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in, }; @@ -54,6 +65,7 @@ use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::local_heap::LocalHeap; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::selection::Selection; use clawhdf5_format::shared_message::{ self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in, parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in, @@ -66,11 +78,18 @@ use clawhdf5_format::superblock_ext::{ read_superblock_extension_in, }; use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode}; +use clawhdf5_format::vds::{ + read_virtual_dataset, read_virtual_dataset_in, virtual_dataset_extent, + virtual_dataset_extent_in, +}; +use clawhdf5_format::vl_data::{VlResolver, read_vl_bytes, read_vl_bytes_in}; /// Objects visited per file, heap objects read per heap: enough to cover /// every structure kind while keeping a 35 000-group file fast. const MAX_OBJECTS: usize = 1500; const MAX_HEAP_IDS: usize = 200; +/// Datasets larger than this are not read (their chunk indexes still are). +const MAX_DATA_BYTES: u64 = 16 << 20; #[derive(Default, Debug)] struct Tally { @@ -89,6 +108,8 @@ struct Walk<'a> { slice: &'a [u8], storage: &'a CountingStorage, name: String, + /// The file's directory, for external VDS sources. + dir: Option, tally: &'a mut Tally, } @@ -264,6 +285,7 @@ impl Walk<'_> { } } self.check_layout(&header, os, ls); + self.check_data(&header, os, ls); } /// A symbol-table group: its local heap, B-tree, nodes and names. @@ -352,6 +374,179 @@ impl Walk<'_> { } } + /// A dataset's values through every raw-data path: whole reads (plain, + /// cached, indexed, fill-aware, virtual), chunk listings, selections + /// (a box, a strided hyperslab, points), VL strings and sequences. + fn check_data(&mut self, header: &ObjectHeader, os: u8, ls: u8) { + let slice = self.slice; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok()) + }; + let (Some(dt), Some(ds), Some(dl)) = ( + find(MessageType::Datatype), + find(MessageType::Dataspace), + find(MessageType::DataLayout), + ) else { + return; + }; + let (Ok((dt, _)), Ok(ds), Ok(dl)) = ( + Datatype::parse(&dt), + Dataspace::parse(&ds, ls), + DataLayout::parse(&dl, os, ls), + ) else { + return; + }; + let pipeline = match find(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p)) { + Some(Ok(p)) => Some(p), + Some(Err(_)) => return, + None => None, + }; + let pl = pipeline.as_ref(); + let elem = dt.type_size() as u64; + let bytes = ds + .dimensions + .iter() + .try_fold(elem, |a, &d| a.checked_mul(d)); + if bytes.is_none_or(|b| b > MAX_DATA_BYTES) { + return; + } + + if matches!(dl, DataLayout::Virtual { .. }) { + let resolver = self.resolver(); + let r: &clawhdf5_format::vds::VdsFileResolver = &resolver; + let want = virtual_dataset_extent(slice, &dl, &ds, os, ls, Some(r)); + let got = virtual_dataset_extent_in(self.st(), &dl, &ds, os, ls, Some(r)); + self.same("VDS extent", &want, &got); + let want = read_virtual_dataset(slice, &dl, &ds, &dt, None, os, ls, Some(r)); + let got = read_virtual_dataset_in(self.st(), &dl, &ds, &dt, None, os, ls, Some(r)); + self.same("VDS read", &want, &got); + return; + } + + let want = read_raw_data_full(slice, &dl, &ds, &dt, pl, os, ls); + let got = read_raw_data_full_in(self.st(), &dl, &ds, &dt, pl, os, ls); + self.same("raw data", &want, &got); + let want_fill = read_full_with_fill( + &header.messages, + slice, + &dl, + &ds, + elem as usize, + os, + ls, + || read_raw_data_full(slice, &dl, &ds, &dt, pl, os, ls), + ); + let got_fill = read_full_with_fill_in( + &header.messages, + self.st(), + &dl, + &ds, + elem as usize, + os, + ls, + || read_raw_data_full_in(self.st(), &dl, &ds, &dt, pl, os, ls), + ); + self.same("raw data with fill", &want_fill, &got_fill); + + if matches!(dl, DataLayout::Chunked { .. }) { + let want = list_chunks(slice, &dl, &ds, elem as usize, os, ls); + let got = list_chunks_in(self.st(), &dl, &ds, elem as usize, os, ls); + self.same("chunk list", &want, &got); + // Through a chunk cache, twice (the second read is served from + // it), and through the indexed path. + let (c1, c2) = (ChunkCache::new(), ChunkCache::new()); + for _ in 0..2 { + let want = read_raw_data_cached(slice, &dl, &ds, &dt, pl, os, ls, &c1); + let got = read_raw_data_cached_in(self.st(), &dl, &ds, &dt, pl, os, ls, &c2); + // A cache lists the chunks in hash-map order (see below). + if want.is_err() && got.is_err() { + self.tally.checks += 1; + } else { + self.same("raw data (cached)", &want, &got); + } + } + let (c1, c2) = (ChunkCache::new(), ChunkCache::new()); + let want = read_raw_data_indexed(slice, &dl, &ds, &dt, pl, os, ls, &c1); + let got = read_raw_data_indexed_in(self.st(), &dl, &ds, &dt, pl, os, ls, &c2); + // The indexed path decodes chunks in hash-map order, so which + // failing chunk it reports varies between two caches (with the + // slice alone, too): only whether it fails must agree. + if want.is_err() && got.is_err() { + self.tally.checks += 1; + } else { + self.same("raw data (indexed)", &want, &got); + } + } + + let dims = &ds.dimensions; + if !dims.is_empty() && dims.iter().all(|&d| d > 0) { + let rank = dims.len(); + let ones = vec![1u64; rank]; + let quarter = Selection::Hyperslab { + start: dims.iter().map(|&d| d / 4).collect(), + stride: ones.clone(), + count: dims.iter().map(|&d| (d / 3).max(1)).collect(), + block: ones.clone(), + }; + let mut stride = ones.clone(); + stride[rank - 1] = 2; + let mut count = dims.clone(); + count[rank - 1] = dims[rank - 1].div_ceil(2); + let strided = Selection::Hyperslab { + start: vec![0; rank], + stride, + count, + block: ones.clone(), + }; + let points = Selection::Points(vec![ + dims.iter().map(|&d| d - 1).collect(), + vec![0; rank], + dims.iter().map(|&d| d / 2).collect(), + ]); + for (what, sel) in [ + ("selection (box)", &quarter), + ("selection (strided)", &strided), + ("selection (points)", &points), + ] { + let want = read_raw_data_selection(slice, &dl, &ds, &dt, pl, os, ls, sel); + let got = read_raw_data_selection_in(self.st(), &dl, &ds, &dt, pl, os, ls, sel); + self.same(what, &want, &got); + } + } + + // Variable-length strings and sequences, resolved in the global heap. + if let (Datatype::VariableLength { base_type, .. }, Ok(raw)) = (&dt, &want) { + let n = raw.len() / clawhdf5_format::vl_data::element_size(os).max(1); + let raw = &raw[..n * clawhdf5_format::vl_data::element_size(os)]; + let want = VlResolver::new(slice, os, ls).string_bytes(raw); + let got = VlResolver::new_in(self.st(), os, ls).string_bytes(raw); + self.same("VL strings", &want, &got); + let base = base_type.type_size() as usize; + let want = VlResolver::new(slice, os, ls).sequences(raw, base); + let got = VlResolver::new_in(self.st(), os, ls).sequences(raw, base); + self.same("VL sequences", &want, &got); + let want = read_vl_bytes(slice, raw, n as u64, os, ls); + let got = read_vl_bytes_in(self.st(), raw, n as u64, os, ls); + self.same("VL bytes", &want, &got); + } + } + + /// External VDS source files: siblings of the file being walked. + fn resolver(&self) -> impl Fn(&str) -> Result>, FormatError> + use<> { + let dir = self.dir.clone(); + move |name: &str| { + let (Some(dir), false) = (dir.as_ref(), name.contains("..") || name.starts_with('/')) + else { + return Ok(None); + }; + Ok(std::fs::read(dir.join(name)).ok()) + } + } + /// A dataset's layout: VDS mappings, and fixed/extensible array chunk /// indexes. fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) { @@ -461,10 +656,19 @@ fn check_file(path: &Path, tally: &mut Tally) { let Ok(bytes) = std::fs::read(path) else { return; }; - check_bytes(&path.display().to_string(), &bytes, tally); + check_bytes_in( + &path.display().to_string(), + &bytes, + path.parent().map(Path::to_path_buf), + tally, + ); } fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) { + check_bytes_in(name, bytes, None, tally); +} + +fn check_bytes_in(name: &str, bytes: &[u8], dir: Option, tally: &mut Tally) { let Ok((_, hdf5)) = split_user_block(bytes) else { return; }; @@ -474,6 +678,7 @@ fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) { slice: hdf5, storage: &storage, name: name.to_string(), + dir, tally, }; walk.run(); @@ -548,6 +753,311 @@ fn corpus_parses_identically_through_storage() { assert!(tally.files > 0); } +/// A storage that misbehaves: fails its `fail_at`-th read (1-based, `0` +/// never), and, with `short`, serves one byte less than asked for inside +/// the file (a truncated response). +struct Adversary { + data: Vec, + reads: std::sync::atomic::AtomicUsize, + fail_at: usize, + short: bool, +} + +impl Storage for Adversary { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let n = self + .reads + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1; + if n == self.fail_at { + return Err(FormatError::Storage(format!( + "injected failure of read {n}" + ))); + } + let got = self.data.as_slice().read_at(offset, len)?; + let mut v = got.into_owned(); + if self.short && v.len() > 1 { + v.pop(); + } + Ok(std::borrow::Cow::Owned(v)) + } + + fn len(&self) -> u64 { + self.data.len() as u64 + } +} + +/// Every group listing and every dataset's values (whole, fill-aware and +/// through a selection) read through a storage that fails or serves short +/// reads: each result is an error or exactly the in-memory result, never +/// other data; and a failing read is reported as that failure. +#[test] +fn misbehaving_storage_never_returns_wrong_data() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut files = Vec::new(); + hdf5_files(&dir, &mut files); + files.sort(); + let (mut compared, mut failures_seen) = (0usize, 0usize); + for path in &files { + let Ok(bytes) = std::fs::read(path) else { + continue; + }; + let Ok((_, hdf5)) = split_user_block(&bytes) else { + continue; + }; + let Ok(sb) = Superblock::parse(hdf5, 0) else { + continue; + }; + // The whole read of every object, as one result to compare. + let everything = |file: &dyn Storage| -> Result { + let (os, ls) = (sb.offset_size, sb.length_size); + let mut out = String::new(); + let mut queue = VecDeque::from([sb.root_group_address]); + let mut seen = HashSet::new(); + while let Some(addr) = queue.pop_front() { + if seen.len() > 200 || !seen.insert(addr) { + continue; + } + let header = ObjectHeader::parse_in(file, addr, os, ls)?; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| message_data_with_sohm_in(file, m, os, ls)) + .transpose() + }; + if let (Some(dt), Some(ds), Some(dl)) = ( + find(MessageType::Datatype)?, + find(MessageType::Dataspace)?, + find(MessageType::DataLayout)?, + ) { + let dt = Datatype::parse(&dt)?.0; + let ds = Dataspace::parse(&ds, ls)?; + let dl = DataLayout::parse(&dl, os, ls)?; + let pl = find(MessageType::FilterPipeline)? + .map(|p| FilterPipeline::parse(&p)) + .transpose()?; + let data = read_full_with_fill_in( + &header.messages, + file, + &dl, + &ds, + dt.type_size() as usize, + os, + ls, + || read_raw_data_full_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls), + ); + out.push_str(&format!("{addr}: {data:?}\n")); + if let Some(&d0) = ds.dimensions.first() { + let rank = ds.dimensions.len(); + let sel = Selection::Hyperslab { + start: vec![0; rank], + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(2)) + .chain(ds.dimensions[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + let part = read_raw_data_selection_in( + file, + &dl, + &ds, + &dt, + pl.as_ref(), + os, + ls, + &sel, + ); + out.push_str(&format!("{addr} half: {part:?}\n")); + } + } + let children = group_v2::resolve_group_children_in(file, &sb, addr); + out.push_str(&format!("{addr} children: {children:?}\n")); + if let Ok(c) = children { + queue.extend(c.iter().map(|c| c.object_header_address)); + } + } + Ok(out) + }; + let want = everything(&hdf5); + let counting = CountingStorage::new(hdf5.to_vec()); + assert_eq!( + format!("{:?}", everything(&counting)), + format!("{want:?}"), + "{}", + path.display() + ); + let total = counting.reads() as usize; + let step = (total / 25).max(1); + for fail_at in (1..=total).step_by(step) { + for short in [false, true] { + if short && fail_at != 1 { + continue; + } + let adv = Adversary { + data: hdf5.to_vec(), + reads: Default::default(), + fail_at: if short { 0 } else { fail_at }, + short, + }; + let got = everything(&adv); + compared += 1; + match (&got, &want) { + (Ok(g), Ok(w)) => { + // Per-object results inside may be errors; values + // that were read must be the right ones. + for (gl, wl) in g.lines().zip(w.lines()) { + if gl != wl { + assert!( + gl.contains("Err("), + "{}: fail_at {fail_at} short {short}:\n got {gl}\n want {wl}", + path.display() + ); + failures_seen += 1; + // A listing that failed ends the walk + // differently from here on. + if gl.contains("children: Err(") { + break; + } + } + } + } + (Err(FormatError::Storage(_)), _) => failures_seen += 1, + (Err(e), Ok(_)) => panic!( + "{}: fail_at {fail_at} short {short}: {e:?} instead of a storage error", + path.display() + ), + (Err(_), Err(_)) => {} + // A listing failed, so the walk never reached the + // object that fails in memory. + (Ok(g), Err(e)) => assert!( + g.contains("children: Err(Storage"), + "{}: fail_at {fail_at} short {short}: read where memory fails ({e:?})", + path.display() + ), + } + } + } + } + eprintln!("misbehaving storage: {compared} runs, {failures_seen} failures reported"); + assert!( + compared > 500 && failures_seen > 100, + "{compared} {failures_seen}" + ); +} + +/// A read_at-only storage that also counts `read_ranges` calls and ranges. +struct BatchCounting { + inner: CountingStorage, + batches: std::sync::atomic::AtomicUsize, + ranges: std::sync::atomic::AtomicUsize, +} + +impl Storage for BatchCounting { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + self.inner.read_at(offset, len) + } + + fn len(&self) -> u64 { + self.inner.len() + } + + fn read_ranges( + &self, + ranges: &[std::ops::Range], + ) -> Result>, FormatError> { + use std::sync::atomic::Ordering::Relaxed; + self.batches.fetch_add(1, Relaxed); + self.ranges.fetch_add(ranges.len(), Relaxed); + ranges + .iter() + .map(|r| self.inner.read_at(r.start, (r.end - r.start) as usize)) + .collect() + } +} + +/// A chunked read lists its chunks, then fetches all their bytes with one +/// `read_ranges` call (a remote backend coalesces and parallelises it), and +/// a selection fetches only the chunks it overlaps, in one call too. +#[test] +fn chunked_reads_fetch_their_chunks_in_one_batch() { + use std::sync::atomic::Ordering::Relaxed; + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut datasets = 0; + for name in [ + "chunked_large.h5", + "chunked_deflate.h5", + "chunked_2d.h5", + "v4_fixed_array.h5", + ] { + let bytes = std::fs::read(dir.join(name)).unwrap(); + let sb = Superblock::parse(&bytes, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + let st = BatchCounting { + inner: CountingStorage::new(bytes.clone()), + batches: Default::default(), + ranges: Default::default(), + }; + for child in group_v2::resolve_group_children(&bytes, &sb, sb.root_group_address).unwrap() { + let header = + ObjectHeader::parse(&bytes, child.object_header_address as usize, os, ls).unwrap(); + let msg = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| m.data.clone()) + }; + let Some(dl) = msg(MessageType::DataLayout) else { + continue; + }; + let dl = DataLayout::parse(&dl, os, ls).unwrap(); + if !matches!(dl, DataLayout::Chunked { .. }) { + continue; + } + let dt = Datatype::parse(&msg(MessageType::Datatype).unwrap()) + .unwrap() + .0; + let ds = Dataspace::parse(&msg(MessageType::Dataspace).unwrap(), ls).unwrap(); + let pl = msg(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p).unwrap()); + let es = dt.type_size() as usize; + let (chunks, _) = list_chunks(&bytes, &dl, &ds, es, os, ls).unwrap(); + let want = read_raw_data_full(&bytes, &dl, &ds, &dt, pl.as_ref(), os, ls).unwrap(); + st.batches.store(0, Relaxed); + st.ranges.store(0, Relaxed); + let got = read_raw_data_full_in(&st, &dl, &ds, &dt, pl.as_ref(), os, ls).unwrap(); + assert_eq!(got, want, "{name} {}", child.name); + assert_eq!(st.batches.load(Relaxed), 1, "{name} {}", child.name); + assert_eq!( + st.ranges.load(Relaxed), + chunks.len(), + "{name} {}", + child.name + ); + // The first chunk only. + let rank = ds.dimensions.len(); + let sel = Selection::Hyperslab { + start: vec![0; rank], + stride: vec![1; rank], + count: vec![1; rank], + block: vec![1; rank], + }; + let want = read_raw_data_selection(&bytes, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel); + st.batches.store(0, Relaxed); + st.ranges.store(0, Relaxed); + let got = read_raw_data_selection_in(&st, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel); + assert_eq!(got, want, "{name} {}", child.name); + if chunks.len() > 2 { + assert_eq!(st.batches.load(Relaxed), 1, "{name} {}", child.name); + assert_eq!(st.ranges.load(Relaxed), 1, "{name} {}", child.name); + } + datasets += 1; + } + } + assert!(datasets >= 4, "{datasets}"); +} + fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) } -- 2.54.0 From 17201e279dec2b4211a2ef809a96213c06f276ad Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:35:06 -0500 Subject: [PATCH 03/49] format: implicit chunk index addresses over the maximum chunk grid libhdf5 allocates an implicit index's chunks for the whole maximum extent and places chunk `scaled` at its row-major position in the maximum chunk grid (H5D__none_idx_get_addr, max_down_chunks). The reader used the current grid, so a dataset below its maximum shape with more chunk columns at its maximum read other chunks' values from the second chunk row on (h5py early allocation, fixed maxshape). generate_implicit_chunks_in_grid takes the maximum dimensions; generate_implicit_chunks keeps its signature (grid = current extent). Regression: implicit_chunks_use_the_maximum_grid here, and implicit_index_below_its_maximum_reads_like_libhdf5 (h5py file) with the editor tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 51 ++++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index e249168..f428285 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -933,16 +933,40 @@ pub fn generate_implicit_chunks( dataset_dims: &[u64], chunk_dimensions: &[u32], element_size: u32, +) -> Vec { + generate_implicit_chunks_in_grid( + base_address, + dataset_dims, + dataset_dims, + chunk_dimensions, + element_size, + ) +} + +/// [`generate_implicit_chunks`] for a dataset whose maximum dimensions +/// (`max_dims`) exceed its current ones: libhdf5 allocates the chunks of +/// the whole maximum extent and places chunk `scaled` at its row-major +/// position in the *maximum* chunk grid (`H5D__none_idx_get_addr`, +/// `max_down_chunks`), so the current extent's chunks are not contiguous. +/// Only the chunks of the current extent are listed. +pub fn generate_implicit_chunks_in_grid( + base_address: u64, + dataset_dims: &[u64], + max_dims: &[u64], + chunk_dimensions: &[u32], + element_size: u32, ) -> Vec { let rank = chunk_dimensions.len(); let chunk_byte_size: u64 = chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; let mut num_chunks_per_dim = Vec::with_capacity(rank); + let mut grid_per_dim = Vec::with_capacity(rank); for d in 0..rank { - let ds = dataset_dims[d]; let ch = chunk_dimensions[d] as u64; - num_chunks_per_dim.push(ds.div_ceil(ch)); + let n = dataset_dims[d].div_ceil(ch); + num_chunks_per_dim.push(n); + grid_per_dim.push(max_dims.get(d).map_or(n, |m| m.div_ceil(ch)).max(n)); } let total_chunks: u64 = num_chunks_per_dim.iter().product(); @@ -951,18 +975,22 @@ pub fn generate_implicit_chunks( for linear_idx in 0..total_chunks { let mut offsets = vec![0u64; rank]; let mut remaining = linear_idx; + let mut grid_idx = 0u64; + let mut down = 1u64; for d in (0..rank).rev() { let nchunks = num_chunks_per_dim[d]; let chunk_idx = remaining % nchunks; remaining /= nchunks; offsets[d] = chunk_idx * chunk_dimensions[d] as u64; + grid_idx = grid_idx.saturating_add(chunk_idx.saturating_mul(down)); + down = down.saturating_mul(grid_per_dim[d]); } chunks.push(ChunkInfo { chunk_size: chunk_byte_size as u32, filter_mask: 0, offsets, - address: base_address + linear_idx * chunk_byte_size, + address: base_address.saturating_add(grid_idx.saturating_mul(chunk_byte_size)), }); } @@ -1153,9 +1181,13 @@ pub fn list_chunks( (4, Some(2)) => { // Implicit index — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - generate_implicit_chunks( + generate_implicit_chunks_in_grid( addr, &dataspace.dimensions, + dataspace + .max_dimensions + .as_deref() + .unwrap_or(&dataspace.dimensions), spatial_chunk_dims, elem_size as u32, ) @@ -3015,6 +3047,17 @@ mod tests { } } + /// A dataset below its maximum extent: libhdf5 lays chunks out over the + /// maximum chunk grid, so row 1 starts after a whole maximum row (here + /// 4 chunks), not after the current row of 3. + #[test] + fn implicit_chunks_use_the_maximum_grid() { + let chunks = generate_implicit_chunks_in_grid(0x100, &[2, 3], &[4, 4], &[1, 1], 4); + let addrs: Vec = chunks.iter().map(|c| (c.address - 0x100) / 4).collect(); + assert_eq!(addrs, vec![0, 1, 2, 4, 5, 6]); + assert_eq!(chunks[3].offsets, vec![1, 0]); + } + #[test] fn implicit_chunks_partial_last() { // 25 elements, chunk size 10 => 3 chunks (last partial) -- 2.54.0 From e9c71e5d2e7c237b7105caf9125d508be75d3aa2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:35:26 -0500 Subject: [PATCH 04/49] edit: version-2 B-tree chunk indexes, shrinking, early allocation FileEditor can now: - add, move and resize chunks of datasets with two or more unlimited dimensions (version-2 B-tree chunk index, record types 10/11). The new edit/btree2.rs follows libhdf5's H5B2 code: H5B2_update (modify, or insert into a leaf with room, or fall back to H5B2__insert), the preemptive split/redistribute loop with its two retries, split1, split_root (depth growth, node geometry per depth), redistribute2/3, cumulative record counts and the pointer widths H5B2__hdr_init derives, a checksum per node. Removal (H5B2_remove: merge2/3, redistribution, root collapse, the internal-record swap with a leaf) is there too. A dataset without an index yet gets one from the layout message's node size and split/merge percentages. - shrink a chunked dataset along any dimension (resize to a smaller shape), as H5D__set_extent / H5D__chunk_prune_by_extent do: the same chunks visited in the same order; chunks wholly outside the new extent are removed from the index (version-1 B-tree: H5B_remove with its sibling key and link fix-ups and the empty-root case; version-2 B-tree; Fixed/Extensible Array elements reset to the fill element; an implicit index keeps its chunks, as libhdf5 does) and their space noted as free; the part of each partial edge chunk outside the extent is overwritten with the fill value, so elements that come back after a later growth read as fill. - under early allocation, allocate and fill the chunks a growth brings in (H5D__chunk_allocate), which an implicit index needs: libhdf5 refills them, and they may hold the data of chunks pruned earlier. Tests (crates/clawhdf5-tools/tests/edit_coverage_interop.rs): growth in both dimensions of v110/latest files, unfiltered and deflated, gives node-for-node the version-2 B-tree libhdf5 builds (h5py with its chunk cache off, so chunks enter the index in the editor's order), through a depth increase; random chunk order; 60 random shrink/grow/write steps on Extensible Array, version-2 B-tree, Fixed Array, 1-D and implicit datasets (earliest/v110/latest, with and without gzip+shuffle) give the values h5py gets doing the same and the same index shape (version-1 and version-2 B-tree node shapes, Extensible Array statistics); h5py r+ continues on every result; h5dump and h5rs check accept them. edit_interop's version-2 B-tree case now appends instead of expecting a refusal; shrinking is no longer an error in edit_tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 832 ++++++++++++ crates/clawhdf5-tools/tests/edit_interop.rs | 25 +- crates/clawhdf5/src/edit/btree1.rs | 167 ++- crates/clawhdf5/src/edit/btree2.rs | 1128 +++++++++++++++++ crates/clawhdf5/src/edit/earray.rs | 29 +- crates/clawhdf5/src/edit/farray.rs | 15 +- crates/clawhdf5/src/edit/image.rs | 17 + crates/clawhdf5/src/edit/mod.rs | 582 ++++++++- crates/clawhdf5/tests/edit_tests.rs | 32 +- 9 files changed, 2738 insertions(+), 89 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/edit_coverage_interop.rs create mode 100644 crates/clawhdf5/src/edit/btree2.rs diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs new file mode 100644 index 0000000..f2c27fd --- /dev/null +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -0,0 +1,832 @@ +//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends, +//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or +//! more unlimited dimensions), datasets shrunk, attributes in dense storage +//! (and moved there), and space reused within an editing session. Files are +//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by +//! clawhdf5; after each edit h5py must read exactly what a model says, +//! h5dump must read the file, `h5rs check --data` must find nothing, and our +//! reader must agree; h5py then opens the file `r+` and goes on. Where the +//! same operations can be done by libhdf5, the structures it builds (B-tree +//! shapes, heap and free-space bookkeeping) are compared with the editor's. +//! +//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips +//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::{Command, Output}; + +use clawhdf5::{Error, File, FileEditor, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn available(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the tools are here; false (skip) or a panic when interop is +/// required. +fn tools_ok() -> bool { + let ok = + available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]); + if !ok { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available" + ); + eprintln!("SKIP: h5py/numpy or h5dump not available"); + } + ok +} + +fn py(script: &str) -> String { + let o = Command::new(python()) + .args(["-c", script]) + .output() + .expect("run python"); + assert!( + o.status.success(), + "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + String::from_utf8_lossy(&o.stdout).trim().to_string() +} + +fn text(o: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ) +} + +/// The file is valid for libhdf5's tools and for `h5rs check --data`. +/// HDF5 2.0 `latest` files are beyond h5dump 1.14. +fn check_tools(path: &Path, h5dump: bool) { + let p = path.to_str().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + if h5dump { + let o = Command::new("h5dump") + .args(["-o", "/dev/null", p]) + .output() + .unwrap(); + assert!( + o.status.success() && o.stderr.is_empty(), + "h5dump {p}:\n{}", + text(&o) + ); + } +} + +/// A dataset's expected contents. +#[derive(Clone, Debug)] +struct Model { + shape: Vec, + /// Row-major values. + data: Vec, +} + +impl Model { + fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self { + let n: u64 = shape.iter().product(); + Self { + shape: shape.to_vec(), + data: (0..n).map(f).collect(), + } + } + + fn index(&self, c: &[u64]) -> usize { + c.iter() + .zip(&self.shape) + .fold(0u64, |a, (&x, &d)| a * d + x) as usize + } + + /// Change the extent to `shape`: elements inside both keep their + /// values, new ones are `fill`. + fn resize(&mut self, shape: &[u64], fill: i32) { + let old = self.clone(); + *self = Self::new(shape, |_| fill); + let n: u64 = old.shape.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; old.shape.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % old.shape[d]; + r /= old.shape[d]; + } + if c.iter().zip(shape).all(|(x, s)| x < s) { + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } + } + } + + /// Apply a hyperslab write of `vals` (row-major over the block). + fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) { + let n: u64 = count.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; count.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = start[d] + r % count[d]; + r /= count[d]; + } + let i = self.index(&c); + self.data[i] = vals[flat as usize]; + } + } +} + +fn block(start: &[u64], count: &[u64]) -> Selection { + Selection::Hyperslab { + start: start.to_vec(), + stride: vec![1; start.len()], + count: count.to_vec(), + block: vec![1; start.len()], + } +} + +/// h5py and our reader both read `m` from dataset `name`. +fn verify(path: &Path, name: &str, m: &Model) { + let f = File::open(path).unwrap(); + let ds = f.dataset(name).unwrap(); + assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}"); + let got = ds.read_i32().unwrap(); + if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) { + panic!( + "our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}", + path.display(), + m.shape, + got[i], + m.data[i] + ); + } + let exp = path.with_extension("expect"); + let bytes: Vec = m.data.iter().flat_map(|v| v.to_le_bytes()).collect(); + std::fs::write(&exp, bytes).unwrap(); + let shape: Vec = m.shape.iter().map(|d| d.to_string()).collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + a = f[{name:?}][()]\n\ + e = np.fromfile({e:?}, dtype=' u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +fn tmpdir() -> tempfile::TempDir { + let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + tempfile::TempDir::new_in(base).unwrap() +} + +#[allow(dead_code)] +fn unsupported(r: Result) { + match r { + Err(Error::Unsupported(_)) => {} + other => panic!("expected Error::Unsupported, got {other:?}"), + } +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} + +fn enc_size(v: u64) -> usize { + ((63 - v.max(1).leading_zeros()) / 8 + 1) as usize +} + +/// The shape of the version-2 B-tree whose header is at `hdr` in `b` +/// (8-byte addresses and lengths): its depth, record count, and every +/// node's (depth, records) breadth-first. +fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) { + let h = &b[hdr..]; + assert_eq!(&h[0..4], b"BTHD"); + let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap())); + let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap())); + let depth = u16::from_le_bytes(h[12..14].try_into().unwrap()); + let root = le(&h[16..24]); + let root_n = le(&h[24..26]); + let total = le(&h[26..34]); + // Node geometry, as H5B2__hdr_init computes it. + let leaf_max = (node_size - 10) / rs; + let nrec_w = enc_size(leaf_max); + let mut cum = vec![(leaf_max, 0usize)]; + for d in 1..=u64::from(depth) { + let (below, below_w) = cum[d as usize - 1]; + let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 }; + let max = (node_size - 10 - ptr) / (rs + ptr); + let c = (max + 1) * below + max; + cum.push((c, enc_size(c))); + } + let mut out = Vec::new(); + let mut level = vec![(root, root_n)]; + let mut dep = depth; + loop { + let mut next = Vec::new(); + for &(addr, n) in &level { + out.push((dep, n)); + if dep == 0 { + assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF"); + continue; + } + let node = &b[addr as usize..]; + assert_eq!(&node[0..4], b"BTIN"); + let mut q = 6 + (n * rs) as usize; + let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 }; + for _ in 0..=n { + let a = le(&node[q..q + 8]); + let c = le(&node[q + 8..q + 8 + nrec_w]); + q += 8 + nrec_w + all_w; + next.push((a, c)); + } + } + if dep == 0 { + break; + } + dep -= 1; + level = next; + } + (depth, total, out) +} + +/// The shape of the file's only chunk B-tree (record type 10 or 11). +fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) { + let b = std::fs::read(path).unwrap(); + let hdrs: Vec = b + .windows(6) + .enumerate() + .filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11)) + .map(|(i, _)| i) + .collect(); + assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display()); + bt2_shape_at(&b, hdrs[0]) +} + +/// Grow a dataset with two unlimited dimensions in both directions many +/// times, writing the new parts, with libhdf5 (its chunk cache off, so each +/// chunk enters the index as it is written, in the same order as the +/// editor's) and with the editor: the same values, and the same B-tree — +/// depth, record count, and every node's record count — through leaf and +/// internal splits, redistributions and a depth increase. +fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) { + let dir = tmpdir(); + let a = dir.path().join(format!("bt2_{tag}_h5py.h5")); + let b = dir.path().join(format!("bt2_{tag}_edit.h5")); + // Extents: both dimensions grow; new columns fall between existing + // chunks in the index's key order (row first). + let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect(); + let val = |r: u64, c: u64| (r * 1000 + c) as i32; + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\ + \x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \ + dtype=' c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\ + \x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\ + \x20 r0, c0 = r, c\n", + p = p.to_str().unwrap(), + write = if write { "True" } else { "False" }, + )) + }; + create(&a, true); + create(&b, false); + let mut m = Model::new(&[0, 0], |_| 0); + let mut ed = FileEditor::open(&b).unwrap(); + let (mut r0, mut c0) = (0u64, 0u64); + for &(r, c) in &steps { + ed.resize("x", &[r, c]).unwrap(); + m.resize(&[r, c], 0); + if c > c0 && r0 > 0 { + let vals: Vec = (0..r0) + .flat_map(|i| (c0..c).map(move |j| val(i, j))) + .collect(); + ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals) + .unwrap(); + m.write_block(&[0, c0], &[r0, c - c0], &vals); + } + if r > r0 { + let vals: Vec = (r0..r) + .flat_map(|i| (0..c).map(move |j| val(i, j))) + .collect(); + ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals) + .unwrap(); + m.write_block(&[r0, 0], &[r - r0, c], &vals); + } + (r0, c0) = (r, c); + } + drop(ed); + verify(&b, "x", &m); + verify(&a, "x", &m); + check_tools(&b, h5dump); + let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b)); + assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}"); + assert_eq!(sb, sa, "B-tree shape differs from libhdf5's"); + // libhdf5 goes on growing what the editor built. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 r, c = d.shape\n\ + \x20 d.resize((r + 3, c + 2))\n\ + \x20 d[:, c:] = -1\n\ + \x20 d[r:, :] = -2\n", + p = b.to_str().unwrap() + )); + let (r, c) = (m.shape[0], m.shape[1]); + m.resize(&[r + 3, c + 2], 0); + m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]); + m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]); + verify(&b, "x", &m); + check_tools(&b, h5dump); +} + +/// Dataset `name`'s layout: its chunk index type (0 for a version-1 +/// B-tree), index address and chunk rank (element-size dimension +/// included). +fn chunk_index(path: &Path, name: &str) -> (u8, Option, usize) { + use clawhdf5_format::data_layout::DataLayout; + use clawhdf5_format::message_type::MessageType; + use clawhdf5_format::object_header::ObjectHeader; + let f = File::open(path).unwrap(); + let sb = f.superblock(); + let (os, ls) = (sb.offset_size, sb.length_size); + let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap(); + let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap(); + let m = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + .unwrap(); + match DataLayout::parse(&m.data, os, ls).unwrap() { + DataLayout::Chunked { + chunk_dimensions, + btree_address, + chunk_index_type, + .. + } => ( + chunk_index_type.unwrap_or(0), + btree_address, + chunk_dimensions.len(), + ), + other => panic!("{other:?}"), + } +} + +/// A version-1 chunk B-tree's shape: every node's (level, entries), +/// breadth-first from the root. +fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> { + let ks = 8 + 8 * ndims; + let mut out = Vec::new(); + let mut level = vec![root]; + while !level.is_empty() { + let mut next = Vec::new(); + for a in level { + let n = &b[a as usize..]; + assert_eq!(&n[0..5], b"TREE\x01", "node at {a}"); + let lv = n[5]; + let e = le(&n[6..8]); + out.push((lv, e)); + if lv > 0 { + for i in 0..e as usize { + next.push(le( + &n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8] + )); + } + } + } + level = next; + } + out +} + +/// The shape of dataset `name`'s chunk index, for comparing with the one +/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or +/// nothing to compare (Fixed Array, implicit). +fn index_shape(path: &Path, name: &str) -> String { + let b = std::fs::read(path).unwrap(); + match chunk_index(path, name) { + (_, None, _) => "none".into(), + (0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)), + (5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)), + (4, Some(a), _) => { + let s: Vec = (0..6) + .map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k])) + .collect(); + format!("ea {s:?}") + } + (t, Some(_), _) => format!("type {t}"), + } +} + +/// One step of a resize workload. +#[derive(Debug, Clone)] +enum Op { + Resize(Vec), + /// A block write; values from `vals`. + Write(Vec, Vec, u64), +} + +fn op_vals(seed: u64, n: u64) -> Vec { + (0..n) + .map(|i| ((i * 31 + seed * 7919) % 100_000) as i32) + .collect() +} + +/// Random resizes (shrinking and growing any dimension within `max`) +/// and block writes. +fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec { + let mut shape = start.to_vec(); + let mut ops = Vec::new(); + for k in 0..n { + if k % 3 == 0 { + shape = (0..shape.len()) + .map(|d| { + let cap = max[d].min(shape[d] * 2 + 6); + rng.below(cap + 1) + }) + .collect(); + ops.push(Op::Resize(shape.clone())); + } else if shape.iter().all(|&s| s > 0) { + let st: Vec = shape.iter().map(|&s| rng.below(s)).collect(); + let cnt: Vec = (0..shape.len()) + .map(|d| 1 + rng.below((shape[d] - st[d]).min(6))) + .collect(); + ops.push(Op::Write(st, cnt, rng.next() % 1000)); + } + } + ops +} + +/// The python statements applying `ops` to dataset `d`. +fn py_ops(ops: &[Op]) -> String { + let mut s = String::new(); + for op in ops { + match op { + Op::Resize(shape) => { + s += &format!("\x20 d.resize({shape:?})\n") + .replace('[', "(") + .replace(']', ",)"); + } + Op::Write(st, cnt, seed) => { + let n: u64 = cnt.iter().product(); + let sl: Vec = st + .iter() + .zip(cnt) + .map(|(a, c)| format!("{a}:{}", a + c)) + .collect(); + s += &format!( + "\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\ + .astype(' Vec { + s.trim_matches(|c| c == '[' || c == ']') + .split(',') + .filter(|t| !t.trim().is_empty()) + .map(|t| t.trim().parse().unwrap()) + .collect() + }; + (p(x), p(y)) + }) + .unwrap(); + let initial = py(&format!( + "import h5py\n\ + d = h5py.File({p:?}, 'r')['x']\n\ + print(' '.join(str(v) for v in d[()].ravel()))\n", + p = b.to_str().unwrap() + )); + let mut m = Model { + shape: start.clone(), + data: initial + .split_whitespace() + .map(|v| v.parse().unwrap()) + .collect(), + }; + let mut rng = Rng(seed); + let ops = random_resize_ops(&mut rng, &start, &max, 60); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\ + \x20 d = f['x']\n{o}", + p = a.to_str().unwrap(), + o = py_ops(&ops) + )); + let mut ed = FileEditor::open(&b).unwrap(); + for (k, op) in ops.iter().enumerate() { + match op { + Op::Resize(s) => { + ed.resize("x", s) + .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}")); + m.resize(s, fill); + } + Op::Write(st, cnt, seed) => { + let vals = op_vals(*seed, cnt.iter().product()); + ed.write_values("x", &block(st, cnt), &vals) + .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}")); + m.write_block(st, cnt, &vals); + } + } + } + drop(ed); + if tag == "implicit" { + // An implicit index cannot drop chunks: libhdf5 leaves the data of + // chunks wholly outside a shrunk extent in place and does not + // refill those that come back unless they lie past the extent it + // grows from, so they can show old values. The editor must match + // libhdf5, not the model. + let got = py(&format!( + "import h5py\n\ + d = h5py.File({p:?}, 'r')['x']\n\ + print(' '.join(str(v) for v in d[()].ravel()))\n", + p = a.to_str().unwrap() + )); + m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect(); + } + verify(&a, "x", &m); + verify(&b, "x", &m); + check_tools(&b, h5dump); + assert_eq!( + index_shape(&b, "x"), + index_shape(&a, "x"), + "{tag}: chunk index differs from libhdf5's" + ); + // libhdf5 goes on: grow back to the start shape and write everything. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 d.resize({s:?})\n\ + \x20 d[...] = 3\n", + p = b.to_str().unwrap(), + s = start + )); + let n: u64 = start.iter().product(); + verify( + &b, + "x", + &Model { + shape: start.clone(), + data: vec![3; n as usize], + }, + ); + check_tools(&b, h5dump); +} + +/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every +/// dimension, mixed with growth and writes, on every chunk index: the +/// version-1 B-tree (`earliest`), Extensible Array (one unlimited +/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape) +/// and implicit (early allocation), unfiltered and deflated. +#[test] +fn shrink_matches_libhdf5() { + if !tools_ok() { + return; + } + let kinds: &[(&str, &str, i32)] = &[ + ( + "ea", + "\x20 d = f.create_dataset('x', data=np.arange(40, dtype=' = (0..cnt[0] * cnt[1]) + .map(|_| (rng.next() % 1_000_000) as i32) + .collect(); + ed.write_values("x", &block(&[r0, c0], &cnt), &vals) + .unwrap(); + m.write_block(&[r0, c0], &cnt, &vals); + if step % 100 == 99 { + drop(ed); + verify(&path, "x", &m); + check_tools(&path, false); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "x", &m); + let shape = chunk_bt2_shape(&path); + assert!(shape.0 >= 1, "{shape:?}"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['x'][...] = 5\n", + p = path.to_str().unwrap() + )); + let n = m.data.len(); + m.data = vec![5; n]; + verify(&path, "x", &m); + check_tools(&path, false); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 85d4d0c..6a2dbf6 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -752,19 +752,20 @@ fn overwrite_every_layout() { } check_tools(&path, *dump); } - // A version-2 B-tree index can take new chunks only from libhdf5 for - // now: growing works, writing the new chunks is refused and changes - // nothing. + // A version-2 B-tree index takes new chunks too. if *lv != "'earliest'" { let mut ed = FileEditor::open(&path).unwrap(); - ed.resize("bt2", &[8, 6]).unwrap(); - let before = std::fs::read(&path).unwrap(); - unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12])); - assert!( - std::fs::read(&path).unwrap() == before, - "a refused edit changed the file" - ); - models[12].resize(&[8, 6], 0); + ed.resize("bt2", &[8, 7]).unwrap(); + models[12].resize(&[8, 7], 0); + let vals: Vec = (0..23).collect(); + ed.write_values("bt2", &block(&[6, 0], &[2, 7]), &vals[..14]) + .unwrap(); + models[12].write_block(&[6, 0], &[2, 7], &vals[..14]); + ed.write_values("bt2", &block(&[0, 6], &[6, 1]), &vals[14..20]) + .unwrap(); + models[12].write_block(&[0, 6], &[6, 1], &vals[14..20]); + drop(ed); + verify(&path, "bt2", &models[12]); } // libhdf5 goes on modifying what we wrote. py(&format!( @@ -1027,7 +1028,7 @@ fn refused_edits_change_nothing() { let before = std::fs::read(&path).unwrap(); let mut ed = FileEditor::open(&path).unwrap(); unsupported(ed.write_all("s", &[0u8; 32])); - unsupported(ed.resize("x", &[4])); + unsupported(ed.resize("c", &[4])); unsupported( ed.resize("c", &[6, 1]) .map_err(|_| Error::Unsupported(String::new())), diff --git a/crates/clawhdf5/src/edit/btree1.rs b/crates/clawhdf5/src/edit/btree1.rs index 7289c3f..25c626a 100644 --- a/crates/clawhdf5/src/edit/btree1.rs +++ b/crates/clawhdf5/src/edit/btree1.rs @@ -56,6 +56,15 @@ fn bad(why: &str) -> Error { )) } +/// What a removal did below a node (`H5B_ins_t`), with the removed chunk's +/// address and size. +enum Rm { + NotFound, + Noop((u64, u32)), + /// The child is gone: the parent must drop it. + Remove((u64, u32)), +} + enum Ins { Done, /// The node split; the new right sibling and its first key. @@ -144,7 +153,14 @@ impl BTree1 { put_uint(&mut d[8 + osz..], node.right, os); let ks = self.key_size(); let mut p = 8 + 2 * osz; - for (i, k) in node.keys.iter().enumerate() { + // An empty node (a root whose last chunk was removed) stores no + // keys, as libhdf5 writes it. + let nkeys = if node.children.is_empty() { + 0 + } else { + node.keys.len() + }; + for (i, k) in node.keys.iter().take(nkeys).enumerate() { d[p..p + 4].copy_from_slice(&k.size.to_le_bytes()); d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes()); for (j, o) in k.offs.iter().enumerate() { @@ -210,6 +226,18 @@ impl BTree1 { return Err(bad("bad chunk key")); } let root = self.read(img, self.root)?; + if root.children.is_empty() { + // Every chunk was removed (H5B__insert_helper's first + // insertion): the root, a leaf again, takes it. + let right = self.right_key_after(&key); + let node = Node { + level: 0, + keys: vec![key, right], + children: vec![addr], + ..root + }; + return self.write(img, &node); + } if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? { // The root split: move its (left) half to a new node so the root // keeps its address, then make the root the parent of both. @@ -372,6 +400,143 @@ impl BTree1 { cmp(&key.offs, &right.keys[0].offs) != Ordering::Less } + /// Remove the chunk at offsets `offs` (element-size coordinate 0), as + /// `H5B_remove` does for the chunk index (whose critical key is the + /// left one): no rebalancing; a node left without children is deleted + /// and its siblings relinked (the left one takes over its right key), + /// a root left empty becomes an empty leaf. Returns the chunk's address + /// and stored size, or `None` when the tree has no such chunk (nothing + /// changes then). Deleted nodes are freed in `img`. + pub(crate) fn remove( + &mut self, + img: &mut Image<'_>, + offs: &[u64], + ) -> Result, Error> { + if offs.len() != self.ndims { + return Err(bad("bad chunk key")); + } + let mut lt = None; + match self.remove_at(img, self.root, 0, offs, &mut lt, 64)? { + Rm::NotFound => Ok(None), + Rm::Noop(c) | Rm::Remove(c) => Ok(Some(c)), + } + } + + fn remove_at( + &self, + img: &mut Image<'_>, + addr: u64, + level: usize, + offs: &[u64], + lt_out: &mut Option, + depth: u8, + ) -> Result { + if depth == 0 { + return Err(bad("tree too deep")); + } + let mut node = self.read(img, addr)?; + let n = node.children.len(); + // H5D__btree_cmp3 over (keys[i], keys[i + 1]), binary search. + let (mut lo, mut hi, mut idx) = (0usize, n, 0usize); + let mut c = 1i32; + while lo < hi && c != 0 { + idx = (lo + hi) / 2; + c = if cmp(offs, &node.keys[idx + 1].offs) != Ordering::Less { + 1 + } else if cmp(offs, &node.keys[idx].offs) == Ordering::Less { + -1 + } else { + 0 + }; + if c < 0 { + hi = idx; + } else { + lo = idx + 1; + } + } + if c != 0 { + return Ok(Rm::NotFound); + } + let mut lt_changed = None; + let res = if node.level > 0 { + let child = self.read(img, node.children[idx])?; + if usize::from(child.level) + 1 != usize::from(node.level) { + return Err(bad("inconsistent node levels")); + } + self.remove_at( + img, + node.children[idx], + level + 1, + offs, + &mut lt_changed, + depth - 1, + )? + } else { + if node.keys[idx].offs != offs { + return Ok(Rm::NotFound); + } + Rm::Remove((node.children[idx], node.keys[idx].size)) + }; + let chunk = match res { + Rm::NotFound => return Ok(Rm::NotFound), + Rm::Noop(c) | Rm::Remove(c) => c, + }; + let mut dirty = false; + if let Some(k) = lt_changed { + node.keys[idx] = k; + dirty = true; + if idx == 0 { + *lt_out = Some(node.keys[0].clone()); + } + } + let out = Rm::Noop(chunk); + if let Rm::Remove(_) = res { + let undefined = undef(img.os); + if n == 1 { + if level > 0 { + if node.left != undefined { + let mut sib = self.read(img, node.left)?; + let last = sib.children.len(); + sib.keys[last] = node.keys[1].clone(); + sib.right = node.right; + self.write(img, &sib)?; + } + if node.right != undefined { + let mut sib = self.read(img, node.right)?; + sib.left = node.left; + self.write(img, &sib)?; + } + img.free(addr, self.node_size(img.os) as u64); + return Ok(Rm::Remove(chunk)); + } + node.children.clear(); + node.keys.truncate(1); + node.level = 0; + } else if idx == 0 { + node.keys.remove(0); + node.children.remove(0); + *lt_out = Some(node.keys[0].clone()); + } else { + // Right-most or middle child: its left key goes, the next + // key becomes the following child's left key. + node.keys.remove(idx); + node.children.remove(idx); + } + dirty = true; + } + if dirty { + self.write(img, &node)?; + } + // The left sibling's right key follows a changed left key. + if lt_out.is_some() && node.left != undef(img.os) && level > 0 { + let mut sib = self.read(img, node.left)?; + let last = sib.children.len(); + sib.keys[last] = node.keys[0].clone(); + self.write(img, &sib)?; + } + Ok(out) + } + fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) { let n = node.children.len(); if node.level == 0 { diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs new file mode 100644 index 0000000..8ff7205 --- /dev/null +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -0,0 +1,1128 @@ +//! Changing a version-2 B-tree (`BTHD`/`BTIN`/`BTLF`) in place, as libhdf5's +//! `H5B2` code does: the same insertion (`H5B2_update`, falling back to +//! `H5B2__insert` when the leaf is full), node splits and redistributions +//! (`H5B2__split1`, `H5B2__split_root`, `H5B2__redistribute2/3`), and removal +//! with merges and redistributions (`H5B2_remove`, `H5B2__merge2/3`, root +//! collapse, the internal-record swap with a leaf) — so a tree the editor +//! changes has the nodes libhdf5 would have made for the same operations. +//! +//! Split and merge thresholds come from the header's split and merge +//! percentages; each node pointer carries the child's record count and, +//! below the first internal level, the child subtree's total, in the widths +//! libhdf5 derives from the node size (`H5B2__hdr_init`). Every node written +//! gets its checksum; the header's depth, root and counts are updated by +//! [`Bt2::finish`]. +//! +//! Nodes are loaded on first use, changed in memory and written back by +//! [`Bt2::finish`]; nodes a merge empties are dropped and their space handed +//! to the image's free list ([`Image::free`]). + +use std::cmp::Ordering; +use std::collections::{BTreeSet, HashMap}; + +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +/// A comparison of the record being looked for against a stored record +/// (`Less`: the target sorts before the stored record). +pub(crate) type Cmp<'c> = dyn FnMut(&Image<'_>, &[u8]) -> Result + 'c; + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("version-2 B-tree: {why}"), + )) +} + +/// A pointer to a node, as its parent (or the header, for the root) holds +/// it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Ptr { + addr: u64, + nrec: u16, + all: u64, +} + +#[derive(Debug, Clone)] +struct Node { + depth: u16, + recs: Vec>, + /// `recs.len() + 1` children (internal nodes only). + ptrs: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct Level { + max_nrec: u64, + split_nrec: u64, + merge_nrec: u64, + cum_max_nrec: u64, + cum_max_nrec_size: usize, +} + +/// `H5VM_limit_enc_size`: bytes to encode values up to `v`. +fn enc_size(v: u64) -> usize { + ((63 - v.max(1).leading_zeros()) / 8 + 1) as usize +} + +pub(crate) struct Bt2 { + addr: u64, + tree_type: u8, + node_size: u32, + rec_size: usize, + split_pct: u8, + merge_pct: u8, + depth: u16, + root: Ptr, + levels: Vec, + max_nrec_size: usize, + nodes: HashMap, + dirty: BTreeSet, + hdr_dirty: bool, +} + +/// What `update` did below a node. +#[derive(Debug, PartialEq, Eq)] +enum Status { + Modified, + Inserted, + ChildFull, +} + +const MAX_DEPTH: u16 = 32; + +impl Bt2 { + fn header_len(os: u8, ls: u8) -> usize { + 16 + os as usize + 2 + ls as usize + 4 + } + + /// Open the tree whose header is at `addr`. + pub(crate) fn open(img: &Image<'_>, addr: u64) -> Result { + let len = Self::header_len(img.os, img.ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"BTHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let os = img.os as usize; + let mut t = Self { + addr, + tree_type: d[5], + node_size: u32::from_le_bytes([d[6], d[7], d[8], d[9]]), + rec_size: usize::from(u16::from_le_bytes([d[10], d[11]])), + depth: u16::from_le_bytes([d[12], d[13]]), + split_pct: d[14], + merge_pct: d[15], + root: Ptr { + addr: get_uint(&d[16..], img.os), + nrec: u16::from_le_bytes([d[16 + os], d[17 + os]]), + all: get_uint(&d[18 + os..], img.ls), + }, + levels: Vec::new(), + max_nrec_size: 0, + nodes: HashMap::new(), + dirty: BTreeSet::new(), + hdr_dirty: false, + }; + if t.depth > MAX_DEPTH || t.rec_size == 0 || t.split_pct == 0 || t.split_pct > 100 { + return Err(bad("unsupported header parameters")); + } + t.compute_levels(img.os)?; + Ok(t) + } + + /// Create an empty tree (header only, undefined root), as + /// `H5B2_create` does. + pub(crate) fn create( + img: &mut Image<'_>, + tree_type: u8, + node_size: u32, + rec_size: usize, + split_pct: u8, + merge_pct: u8, + ) -> Result { + let addr = img.alloc(Self::header_len(img.os, img.ls) as u64)?; + let mut t = Self { + addr, + tree_type, + node_size, + rec_size, + split_pct, + merge_pct, + depth: 0, + root: Ptr { + addr: undef(img.os), + nrec: 0, + all: 0, + }, + levels: Vec::new(), + max_nrec_size: 0, + nodes: HashMap::new(), + dirty: BTreeSet::new(), + hdr_dirty: true, + }; + t.compute_levels(img.os)?; + t.write_header(img)?; + Ok(t) + } + + pub(crate) fn address(&self) -> u64 { + self.addr + } + + pub(crate) fn record_size(&self) -> usize { + self.rec_size + } + + pub(crate) fn tree_type(&self) -> u8 { + self.tree_type + } + + /// Records in the whole tree. + pub(crate) fn len(&self) -> u64 { + self.root.all + } + + /// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`, + /// `H5B2__split_root`). + fn compute_levels(&mut self, os: u8) -> Result<(), Error> { + let ns = u64::from(self.node_size); + let rs = self.rec_size as u64; + let leaf_max = ns.saturating_sub(10) / rs; + if leaf_max < 2 || leaf_max > u64::from(u16::MAX) { + return Err(bad("node size holds too few or too many records")); + } + self.max_nrec_size = enc_size(leaf_max); + let pct = |n: u64, p: u8| n * u64::from(p) / 100; + let mut levels = vec![Level { + max_nrec: leaf_max, + split_nrec: pct(leaf_max, self.split_pct), + merge_nrec: pct(leaf_max, self.merge_pct), + cum_max_nrec: leaf_max, + cum_max_nrec_size: 0, + }]; + for d in 1..=self.depth { + let below = levels[usize::from(d) - 1]; + let ptr = u64::from(os) + + self.max_nrec_size as u64 + + if d > 1 { + below.cum_max_nrec_size as u64 + } else { + 0 + }; + let max = ns.saturating_sub(10 + ptr) / (rs + ptr); + if max < 2 { + return Err(bad("internal nodes hold too few records")); + } + let cum = max + .saturating_add(1) + .saturating_mul(below.cum_max_nrec) + .saturating_add(max); + levels.push(Level { + max_nrec: max, + split_nrec: pct(max, self.split_pct), + merge_nrec: pct(max, self.merge_pct), + cum_max_nrec: cum, + cum_max_nrec_size: enc_size(cum), + }); + } + if levels.iter().any(|l| l.split_nrec == 0) { + return Err(bad("split percentage too small")); + } + self.levels = levels; + Ok(()) + } + + fn ptr_size(&self, os: u8, depth: u16) -> usize { + os as usize + + self.max_nrec_size + + if depth > 1 { + self.levels[usize::from(depth) - 1].cum_max_nrec_size + } else { + 0 + } + } + + /// Load the node `p` points at (a node of `depth`). + fn load(&mut self, img: &Image<'_>, p: Ptr, depth: u16) -> Result<(), Error> { + if let Some(n) = self.nodes.get(&p.addr) { + if n.depth != depth || n.recs.len() != usize::from(p.nrec) { + return Err(bad("node does not match its pointer")); + } + return Ok(()); + } + if p.addr == undef(img.os) { + return Err(bad("undefined node address")); + } + let nrec = usize::from(p.nrec); + if nrec as u64 > self.levels[usize::from(depth)].max_nrec { + return Err(bad("node holds more records than it can")); + } + let ps = if depth > 0 { + self.ptr_size(img.os, depth) + } else { + 0 + }; + let body = 6 + nrec * self.rec_size + if depth > 0 { (nrec + 1) * ps } else { 0 }; + if body + 4 > self.node_size as usize { + return Err(bad("node overflows its size")); + } + let d = img.read(p.addr, body + 4)?; + let sig: &[u8] = if depth == 0 { b"BTLF" } else { b"BTIN" }; + if &d[0..4] != sig || d[4] != 0 || d[5] != self.tree_type { + return Err(bad("bad node signature")); + } + let stored = u32::from_le_bytes(d[body..body + 4].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..body]) != stored { + return Err(bad("node checksum mismatch")); + } + let mut q = 6; + let mut recs = Vec::with_capacity(nrec); + for _ in 0..nrec { + recs.push(d[q..q + self.rec_size].to_vec()); + q += self.rec_size; + } + let mut ptrs = Vec::new(); + if depth > 0 { + let all_w = if depth > 1 { + self.levels[usize::from(depth) - 1].cum_max_nrec_size + } else { + 0 + }; + for _ in 0..=nrec { + let addr = get_uint(&d[q..], img.os); + q += img.os as usize; + let n = le(&d[q..q + self.max_nrec_size]); + q += self.max_nrec_size; + let all = if depth > 1 { + let a = le(&d[q..q + all_w]); + q += all_w; + a + } else { + n + }; + let n = u16::try_from(n).map_err(|_| bad("record count"))?; + ptrs.push(Ptr { addr, nrec: n, all }); + } + } + self.nodes.insert(p.addr, Node { depth, recs, ptrs }); + Ok(()) + } + + fn node(&mut self, addr: u64) -> &mut Node { + self.dirty.insert(addr); + self.nodes.get_mut(&addr).expect("node loaded") + } + + fn peek(&self, addr: u64) -> &Node { + self.nodes.get(&addr).expect("node loaded") + } + + fn new_node(&mut self, img: &mut Image<'_>, depth: u16) -> Result { + let addr = img.alloc_reusing(u64::from(self.node_size))?; + self.nodes.insert( + addr, + Node { + depth, + recs: Vec::new(), + ptrs: Vec::new(), + }, + ); + self.dirty.insert(addr); + Ok(addr) + } + + fn drop_node(&mut self, img: &mut Image<'_>, addr: u64) { + self.nodes.remove(&addr); + self.dirty.remove(&addr); + img.free(addr, u64::from(self.node_size)); + } + + /// `H5B2__locate_record`: binary search; the index and the comparison + /// there. + fn locate( + img: &Image<'_>, + recs: &[Vec], + cmp: &mut Cmp<'_>, + ) -> Result<(usize, Ordering), Error> { + let (mut lo, mut hi) = (0usize, recs.len()); + let mut idx = 0; + let mut c = Ordering::Less; + while lo < hi && c != Ordering::Equal { + idx = (lo + hi) / 2; + c = cmp(img, &recs[idx])?; + if c == Ordering::Less { + hi = idx; + } else { + lo = idx + 1; + } + } + Ok((idx, c)) + } + + /// The record matching `cmp`, if any. + pub(crate) fn find( + &mut self, + img: &Image<'_>, + cmp: &mut Cmp<'_>, + ) -> Result>, Error> { + if self.root.addr == undef(img.os) || self.root.nrec == 0 { + return Ok(None); + } + let mut p = self.root; + let mut depth = self.depth; + loop { + self.load(img, p, depth)?; + let n = self.peek(p.addr); + let (idx, c) = Self::locate(img, &n.recs, cmp)?; + if c == Ordering::Equal { + return Ok(Some(n.recs[idx].clone())); + } + if depth == 0 { + return Ok(None); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + p = n.ptrs[i]; + depth -= 1; + } + } + + /// Every record, in key order. + pub(crate) fn records(&mut self, img: &Image<'_>) -> Result>, Error> { + let mut out = Vec::new(); + if self.root.addr != undef(img.os) && self.root.nrec > 0 { + self.walk(img, self.root, self.depth, &mut out)?; + } + Ok(out) + } + + fn walk( + &mut self, + img: &Image<'_>, + p: Ptr, + depth: u16, + out: &mut Vec>, + ) -> Result<(), Error> { + self.load(img, p, depth)?; + let n = self.peek(p.addr).clone(); + for i in 0..=n.recs.len() { + if depth > 0 { + self.walk(img, n.ptrs[i], depth - 1, out)?; + } + if i < n.recs.len() { + out.push(n.recs[i].clone()); + } + } + Ok(()) + } + + /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). + pub(crate) fn update( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + if rec.len() != self.rec_size { + return Err(bad("record of the wrong size")); + } + if self.root.addr == undef(img.os) { + let a = self.new_node(img, 0)?; + self.root = Ptr { + addr: a, + nrec: 0, + all: 0, + }; + self.hdr_dirty = true; + } + let mut root = self.root; + let st = self.update_at(img, &mut root, self.depth, cmp, rec)?; + self.root = root; + self.hdr_dirty = true; + if st == Status::ChildFull { + self.insert(img, cmp, rec)?; + } + Ok(()) + } + + fn update_at( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result { + self.load(img, *p, depth)?; + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c == Ordering::Equal { + self.node(p.addr).recs[idx] = rec.to_vec(); + return Ok(Status::Modified); + } + if depth == 0 { + if u64::from(p.nrec) == self.levels[0].split_nrec { + return Ok(Status::ChildFull); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + self.node(p.addr).recs.insert(i, rec.to_vec()); + p.nrec += 1; + p.all += 1; + return Ok(Status::Inserted); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + let mut child = self.peek(p.addr).ptrs[i]; + let st = self.update_at(img, &mut child, depth - 1, cmp, rec)?; + match st { + Status::Modified => Ok(st), + Status::Inserted => { + self.node(p.addr).ptrs[i] = child; + p.all += 1; + Ok(st) + } + Status::ChildFull => { + // H5B2__update_internal: a full child is split or + // redistributed here, unless this node is full too and the + // child and a neighbour are close to full (the split could + // reach this node): then the caller deals with it. + let d = usize::from(depth); + let (nrec, ptrs) = { + let n = self.peek(p.addr); + (n.recs.len(), n.ptrs.clone()) + }; + if nrec as u64 == self.levels[d].split_nrec { + let near = (self.levels[d - 1].split_nrec * 2).saturating_sub(1); + let pair = |a: usize, b: usize| { + u64::from(ptrs[a].nrec) + u64::from(ptrs[b].nrec) >= near + }; + let could_split = if i == 0 { + pair(0, 1) + } else if i == nrec { + pair(i - 1, i) + } else { + pair(i - 1, i) || pair(i, i + 1) + }; + if could_split { + return Ok(Status::ChildFull); + } + } + self.insert_internal(img, p, depth, cmp, rec)?; + Ok(Status::Inserted) + } + } + } + + /// `H5B2__insert`: insert a record that is not in the tree, splitting + /// or redistributing full nodes on the way down. + pub(crate) fn insert( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + if rec.len() != self.rec_size { + return Err(bad("record of the wrong size")); + } + if self.root.addr == undef(img.os) { + let a = self.new_node(img, 0)?; + self.root = Ptr { + addr: a, + nrec: 0, + all: 0, + }; + } else if u64::from(self.root.nrec) == self.levels[usize::from(self.depth)].split_nrec { + self.split_root(img)?; + } + self.hdr_dirty = true; + let mut root = self.root; + if self.depth > 0 { + self.insert_internal(img, &mut root, self.depth, cmp, rec)?; + } else { + self.insert_leaf(img, &mut root, cmp, rec)?; + } + self.root = root; + Ok(()) + } + + fn insert_leaf( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + self.load(img, *p, 0)?; + let mut i = 0; + if p.nrec > 0 { + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c == Ordering::Equal { + return Err(bad("record is already in the tree")); + } + i = if c == Ordering::Greater { idx + 1 } else { idx }; + } + if u64::from(p.nrec) >= self.levels[0].max_nrec { + return Err(bad("leaf full")); + } + self.node(p.addr).recs.insert(i, rec.to_vec()); + p.nrec += 1; + p.all += 1; + Ok(()) + } + + fn insert_internal( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + self.load(img, *p, depth)?; + let child_split = self.levels[usize::from(depth) - 1].split_nrec; + let mut i = self.child_index(img, p.addr, cmp)?; + // Preemptively split or redistribute the child to enter; a + // redistribution can move the insertion point into another full + // child, so after two tries it splits. + let mut retries = 2u32; + while u64::from(self.peek(p.addr).ptrs[i].nrec) == child_split { + let nrec = self.peek(p.addr).recs.len(); + let ptrs = self.peek(p.addr).ptrs.clone(); + let room = |j: usize| retries > 0 && u64::from(ptrs[j].nrec) < child_split; + if i == 0 { + if room(1) { + self.redistribute2(img, p.addr, depth, 0)?; + } else { + self.split1(img, p, depth, 0)?; + } + } else if i == nrec { + if room(i - 1) { + self.redistribute2(img, p.addr, depth, i - 1)?; + } else { + self.split1(img, p, depth, i)?; + } + } else if room(i + 1) || room(i - 1) { + self.redistribute3(img, p.addr, depth, i)?; + } else { + self.split1(img, p, depth, i)?; + } + i = self.child_index(img, p.addr, cmp)?; + retries = retries.saturating_sub(1); + } + let mut child = self.peek(p.addr).ptrs[i]; + if depth > 1 { + self.insert_internal(img, &mut child, depth - 1, cmp, rec)?; + } else { + self.insert_leaf(img, &mut child, cmp, rec)?; + } + self.node(p.addr).ptrs[i] = child; + p.all += 1; + Ok(()) + } + + /// The child of internal node `addr` to descend into for an insertion; + /// refuses a record that is already there. + fn child_index( + &mut self, + img: &Image<'_>, + addr: u64, + cmp: &mut Cmp<'_>, + ) -> Result { + let n = self.peek(addr); + if n.recs.is_empty() { + return Ok(0); + } + let (idx, c) = Self::locate(img, &n.recs, cmp)?; + match c { + Ordering::Equal => Err(bad("record is already in the tree")), + Ordering::Greater => Ok(idx + 1), + Ordering::Less => Ok(idx), + } + } + + /// Load the children `lo..=hi` of internal node `parent` (at `depth`). + fn load_children( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + lo: usize, + hi: usize, + ) -> Result<(), Error> { + for i in lo..=hi { + let c = self.peek(parent).ptrs[i]; + self.load(img, c, depth - 1)?; + } + Ok(()) + } + + /// Total records under a node with these records and children. + fn subtree_total(nrec: usize, ptrs: &[Ptr]) -> u64 { + nrec as u64 + ptrs.iter().map(|p| p.all).sum::() + } + + /// Lay `recs`/`ptrs` (the concatenation of several siblings and the + /// parent records between them) out over the children `lo..` of + /// `parent`, with `counts[k]` records in the k-th; the records between + /// them go back into the parent. + fn relayout( + &mut self, + parent: u64, + depth: u16, + lo: usize, + addrs: &[u64], + mut recs: Vec>, + mut ptrs: Vec, + counts: &[usize], + ) { + let internal = depth > 1; + let mut new_ptrs = Vec::with_capacity(addrs.len()); + let mut seps = Vec::with_capacity(addrs.len().saturating_sub(1)); + for (k, &a) in addrs.iter().enumerate() { + let n = counts[k]; + let rest_r = recs.split_off(n); + let mine_r = std::mem::replace(&mut recs, rest_r); + let mine_p = if internal { + let rest_p = ptrs.split_off(n + 1); + std::mem::replace(&mut ptrs, rest_p) + } else { + Vec::new() + }; + if k + 1 < addrs.len() { + seps.push(recs.remove(0)); + } + let all = if internal { + Self::subtree_total(n, &mine_p) + } else { + n as u64 + }; + new_ptrs.push(Ptr { + addr: a, + nrec: n as u16, + all, + }); + let node = self.node(a); + node.recs = mine_r; + node.ptrs = mine_p; + } + let pn = self.node(parent); + for (k, s) in seps.into_iter().enumerate() { + pn.recs[lo + k] = s; + } + for (k, p) in new_ptrs.into_iter().enumerate() { + pn.ptrs[lo + k] = p; + } + } + + /// The children `lo..=hi` of `parent` and the parent records between + /// them, concatenated. + fn gather(&self, parent: u64, lo: usize, hi: usize) -> (Vec, Vec>, Vec) { + let pn = self.peek(parent); + let mut addrs = Vec::new(); + let mut recs = Vec::new(); + let mut ptrs = Vec::new(); + for i in lo..=hi { + let c = self.peek(pn.ptrs[i].addr); + addrs.push(pn.ptrs[i].addr); + recs.extend(c.recs.iter().cloned()); + ptrs.extend(c.ptrs.iter().copied()); + if i < hi { + recs.push(pn.recs[i].clone()); + } + } + (addrs, recs, ptrs) + } + + /// `H5B2__redistribute2`: even out children `idx` and `idx + 1`. + fn redistribute2( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + self.load_children(img, parent, depth, idx, idx + 1)?; + let (l, r) = { + let pn = self.peek(parent); + ( + usize::from(pn.ptrs[idx].nrec), + usize::from(pn.ptrs[idx + 1].nrec), + ) + }; + let (nl, nr) = if l < r { + let nr = (l + r) / 2; + (l + r - nr, nr) + } else { + let nl = (l + r) / 2; + (nl, l + r - nl) + }; + let (addrs, recs, ptrs) = self.gather(parent, idx, idx + 1); + self.relayout(parent, depth, idx, &addrs, recs, ptrs, &[nl, nr]); + Ok(()) + } + + /// `H5B2__redistribute3`: even out children `idx - 1`, `idx`, `idx + 1`. + fn redistribute3( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + self.load_children(img, parent, depth, idx - 1, idx + 1)?; + let total = { + let pn = self.peek(parent); + (idx - 1..=idx + 1) + .map(|i| usize::from(pn.ptrs[i].nrec)) + .sum::() + + 2 + }; + let nm = (total - 2) / 3; + let nl = ((total - 2) - nm) / 2; + let nr = (total - 2) - (nl + nm); + let (addrs, recs, ptrs) = self.gather(parent, idx - 1, idx + 1); + self.relayout(parent, depth, idx - 1, &addrs, recs, ptrs, &[nl, nm, nr]); + Ok(()) + } + + /// `H5B2__split1`: split child `idx` of the node `p` points at, which + /// gains a record and a child. + fn split1( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx, idx)?; + let right = self.new_node(img, depth - 1)?; + let old = usize::from(self.peek(parent).ptrs[idx].nrec); + let mid = old / 2; + { + let pn = self.node(parent); + pn.recs.insert(idx, Vec::new()); + pn.ptrs.insert( + idx + 1, + Ptr { + addr: right, + nrec: 0, + all: 0, + }, + ); + } + let left = self.peek(parent).ptrs[idx].addr; + let (recs, ptrs) = { + let l = self.peek(left); + (l.recs.clone(), l.ptrs.clone()) + }; + self.relayout( + parent, + depth, + idx, + &[left, right], + recs, + ptrs, + &[mid, old - mid - 1], + ); + p.nrec += 1; + Ok(()) + } + + /// `H5B2__split_root`: the tree grows a level. + fn split_root(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + if self.depth >= MAX_DEPTH { + return Err(bad("tree too deep")); + } + self.depth += 1; + self.compute_levels(img.os)?; + let old = self.root; + let new_root = self.new_node(img, self.depth)?; + self.node(new_root).ptrs.push(old); + let mut root = Ptr { + addr: new_root, + nrec: 0, + all: old.all, + }; + let depth = self.depth; + self.split1(img, &mut root, depth, 0)?; + self.root = root; + self.hdr_dirty = true; + Ok(()) + } + + /// Remove the record `cmp` matches (`H5B2_remove`); returns it, or + /// `None` when the tree has no such record (nothing changes then). + pub(crate) fn remove( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + ) -> Result>, Error> { + let Some(found) = self.find(img, cmp)? else { + return Ok(None); + }; + let mut root = self.root; + if self.depth > 0 { + let mut decreased = false; + self.remove_internal(img, &mut root, self.depth, true, None, cmp, &mut decreased)?; + if decreased { + self.depth -= 1; + self.compute_levels(img.os)?; + } + } else { + self.remove_leaf(img, &mut root, cmp)?; + } + root.all -= 1; + self.root = root; + self.hdr_dirty = true; + Ok(Some(found)) + } + + fn remove_leaf( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + cmp: &mut Cmp<'_>, + ) -> Result<(), Error> { + self.load(img, *p, 0)?; + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c != Ordering::Equal { + return Err(bad("record to remove not found")); + } + self.node(p.addr).recs.remove(idx); + if self.peek(p.addr).recs.is_empty() { + self.drop_node(img, p.addr); + p.addr = undef(img.os); + } + p.nrec -= 1; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn remove_internal( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + is_root: bool, + mut swap: Option<(u64, usize)>, + cmp: &mut Cmp<'_>, + decreased: &mut bool, + ) -> Result<(), Error> { + self.load(img, *p, depth)?; + let merge = self.levels[usize::from(depth) - 1].merge_nrec; + let addr = p.addr; + let pn = self.peek(addr); + // libhdf5 checks every level; only a root can hold a single record. + if is_root + && pn.recs.len() == 1 + && u64::from(pn.ptrs[0].nrec) + u64::from(pn.ptrs[1].nrec) <= merge * 2 + 1 + { + // Collapse the root: merge its two children into the left one, + // which becomes the root. + let mut dummy = *p; + self.merge2(img, &mut dummy, depth, 0)?; + let child = self.peek(addr).ptrs[0]; + self.drop_node(img, addr); + p.addr = child.addr; + p.nrec = child.nrec; + *decreased = true; + if depth > 1 { + return self.remove_internal(img, p, depth - 1, true, swap, cmp, decreased); + } + return self.remove_leaf(img, p, cmp); + } + let mut idx; + let mut c = Ordering::Less; + if swap.is_some() { + idx = 0; + } else { + let (i, cc) = Self::locate(img, &pn.recs, cmp)?; + c = cc; + idx = if cc != Ordering::Less { i + 1 } else { i }; + } + let mut retries = 2u32; + while u64::from(self.peek(addr).ptrs[idx].nrec) == merge { + let n = self.peek(addr).recs.len(); + if n == 0 { + return Err(bad("internal node without records")); + } + let ptrs = self.peek(addr).ptrs.clone(); + let spare = |j: usize| retries > 0 && u64::from(ptrs[j].nrec) > merge; + if idx == 0 { + if spare(1) { + self.redistribute2(img, addr, depth, 0)?; + } else { + self.merge2(img, p, depth, 0)?; + } + } else if idx == n { + if spare(idx - 1) { + self.redistribute2(img, addr, depth, idx - 1)?; + } else { + self.merge2(img, p, depth, idx - 1)?; + } + } else if spare(idx + 1) || spare(idx - 1) { + self.redistribute3(img, addr, depth, idx)?; + } else { + self.merge3(img, p, depth, idx)?; + } + if swap.is_some() { + idx = 0; + } else { + let (i, cc) = Self::locate(img, &self.peek(addr).recs, cmp)?; + c = cc; + idx = if cc != Ordering::Less { i + 1 } else { i }; + } + retries = retries.saturating_sub(1); + } + if swap.is_none() && c == Ordering::Equal { + swap = Some((addr, idx - 1)); + } + if let (Some((sa, si)), 1) = (swap, depth) { + // H5B2__swap_leaf: the record to delete trades places with the + // leaf's first record. + let child = self.peek(addr).ptrs[idx]; + self.load(img, child, 0)?; + let leaf_first = self.peek(child.addr).recs[0].clone(); + let deleted = std::mem::replace(&mut self.node(sa).recs[si], leaf_first); + self.node(child.addr).recs[0] = deleted; + } + let mut child = self.peek(addr).ptrs[idx]; + if depth > 1 { + self.remove_internal(img, &mut child, depth - 1, false, swap, cmp, decreased)?; + } else { + self.remove_leaf(img, &mut child, cmp)?; + } + child.all -= 1; + self.node(addr).ptrs[idx] = child; + Ok(()) + } + + /// `H5B2__merge2`: merge child `idx + 1` of the node `p` points at into + /// child `idx`. + fn merge2( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx, idx + 1)?; + let (addrs, recs, ptrs) = self.gather(parent, idx, idx + 1); + let n = recs.len(); + { + let pn = self.node(parent); + pn.recs.remove(idx); + pn.ptrs.remove(idx + 1); + } + self.relayout(parent, depth, idx, &addrs[..1], recs, ptrs, &[n]); + self.drop_node(img, addrs[1]); + p.nrec -= 1; + Ok(()) + } + + /// `H5B2__merge3`: merge children `idx - 1`, `idx`, `idx + 1` of the + /// node `p` points at into two. + fn merge3( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx - 1, idx + 1)?; + let (addrs, recs, ptrs) = self.gather(parent, idx - 1, idx + 1); + let total = recs.len(); + let nl = (total - 1) / 2; + { + let pn = self.node(parent); + pn.recs.remove(idx); + pn.ptrs.remove(idx + 1); + } + self.relayout( + parent, + depth, + idx - 1, + &addrs[..2], + recs, + ptrs, + &[nl, total - 1 - nl], + ); + self.drop_node(img, addrs[2]); + p.nrec -= 1; + Ok(()) + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let len = Self::header_len(img.os, img.ls); + let os = img.os as usize; + let mut d = vec![0u8; len]; + d[0..4].copy_from_slice(b"BTHD"); + d[5] = self.tree_type; + d[6..10].copy_from_slice(&self.node_size.to_le_bytes()); + d[10..12].copy_from_slice(&(self.rec_size as u16).to_le_bytes()); + d[12..14].copy_from_slice(&self.depth.to_le_bytes()); + d[14] = self.split_pct; + d[15] = self.merge_pct; + put_uint(&mut d[16..], self.root.addr, img.os); + d[16 + os..18 + os].copy_from_slice(&self.root.nrec.to_le_bytes()); + put_uint(&mut d[18 + os..], self.root.all, img.ls); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]); + d[len - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d)?; + self.hdr_dirty = false; + Ok(()) + } + + /// Write every changed node and the header. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for addr in std::mem::take(&mut self.dirty) { + let Some(n) = self.nodes.get(&addr) else { + continue; + }; + let mut d = Vec::with_capacity(self.node_size as usize); + d.extend_from_slice(if n.depth == 0 { b"BTLF" } else { b"BTIN" }); + d.push(0); + d.push(self.tree_type); + for r in &n.recs { + d.extend_from_slice(r); + } + if n.depth > 0 { + let all_w = if n.depth > 1 { + self.levels[usize::from(n.depth) - 1].cum_max_nrec_size + } else { + 0 + }; + for p in &n.ptrs { + let mut a = vec![0u8; img.os as usize]; + put_uint(&mut a, p.addr, img.os); + d.extend_from_slice(&a); + d.extend_from_slice(&u64::from(p.nrec).to_le_bytes()[..self.max_nrec_size]); + if n.depth > 1 { + d.extend_from_slice(&p.all.to_le_bytes()[..all_w]); + } + } + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + if d.len() > self.node_size as usize { + return Err(bad("node overflows its size")); + } + d.resize(self.node_size as usize, 0); + img.write(addr, &d)?; + } + if self.hdr_dirty { + self.write_header(img)?; + } + Ok(()) + } +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} diff --git a/crates/clawhdf5/src/edit/earray.rs b/crates/clawhdf5/src/edit/earray.rs index 49d9621..bae196f 100644 --- a/crates/clawhdf5/src/edit/earray.rs +++ b/crates/clawhdf5/src/edit/earray.rs @@ -383,12 +383,23 @@ impl Ea { } /// Set element `idx` to `e`. - pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + /// Set element `idx` to `e`, or back to the fill element (`None`: a + /// removed chunk, `H5D__earray_idx_remove`), which creates no block. + pub(crate) fn set( + &mut self, + img: &mut Image<'_>, + idx: u64, + e: Option, + ) -> Result<(), Error> { let os = img.os; let osz = u64::from(os); let es = self.slot_size(os) as u64; - let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let enc = encode_elem(e, self.filtered, self.elem_size, os)?; + let clear = e.is_none(); if self.iblock == undef(os) { + if clear { + return Ok(()); + } self.create_iblock(img)?; } let ib = self.iblock; @@ -420,6 +431,9 @@ impl Ea { let dblk_idx = l.start_dblk + local; let slot = dblks_at + dblk_idx * osz; let mut addr = get_uint(&img.read(slot, os as usize)?, os); + if addr == undef(os) && clear { + return Ok(()); + } if addr == undef(os) { // libhdf5 records start_idx + (global data block index) // * nelmts here (H5EA__lookup_elmt), not the block's @@ -449,6 +463,9 @@ impl Ea { let sb_prefix = self.dblk_prefix_len(os); let sb_len = sb_prefix + bitmap_len + l.ndblks * osz; let mut sb = get_uint(&img.read(sslot, os as usize)?, os); + if sb == undef(os) && clear { + return Ok(()); + } if sb == undef(os) { let mut d = self.block_prefix(b"EASB", l.start_idx, os); d.resize(d.len() + bitmap_len as usize, 0); @@ -471,6 +488,9 @@ impl Ea { let local = (rel - l.start_idx) / l.dblk_nelmts; let dslot = sb + sb_prefix + bitmap_len + local * osz; let mut addr = get_uint(&img.read(dslot, os as usize)?, os); + if addr == undef(os) && clear { + return Ok(()); + } if addr == undef(os) { let off = l.start_idx + local * l.dblk_nelmts; addr = self.create_dblock(img, l.dblk_nelmts, off)?; @@ -491,6 +511,9 @@ impl Ea { let bpos = sb + sb_prefix + bit / 8; let mut byte = img.read(bpos, 1)?[0]; let mask = 0x80u8 >> (bit % 8); + if byte & mask == 0 && clear { + return Ok(()); + } if byte & mask == 0 { let fill = self.fill_elems(page, os)?; img.write(page_at, &fill)?; @@ -503,7 +526,7 @@ impl Ea { } } } - if idx + 1 > self.stats[4] { + if !clear && idx + 1 > self.stats[4] { self.stats[4] = idx + 1; self.dirty_hdr = true; } diff --git a/crates/clawhdf5/src/edit/farray.rs b/crates/clawhdf5/src/edit/farray.rs index 0a25fe7..e7a1696 100644 --- a/crates/clawhdf5/src/edit/farray.rs +++ b/crates/clawhdf5/src/edit/farray.rs @@ -137,13 +137,19 @@ impl Fa { Ok((fa, hdr)) } - /// Set element `idx` to `e`. - pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + /// Set element `idx` to `e`, or back to the fill element (`None`: a + /// removed chunk, `H5D__farray_idx_remove`), which creates no page. + pub(crate) fn set( + &mut self, + img: &mut Image<'_>, + idx: u64, + e: Option, + ) -> Result<(), Error> { let os = img.os; if idx >= self.nelmts { return Err(bad("index beyond the array")); } - let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let enc = encode_elem(e, self.filtered, self.elem_size, os)?; let es = self.slot(os); let prefix = 6 + u64::from(os); let page = self.page(); @@ -162,6 +168,9 @@ impl Fa { let bpos = self.dblk + prefix + p / 8; let mut byte = img.read(bpos, 1)?[0]; let mask = 0x80u8 >> (p % 8); + if byte & mask == 0 && e.is_none() { + return Ok(()); + } if byte & mask == 0 { let fill = encode_elem(None, self.filtered, self.elem_size, os)?; img.write(page_at, &fill.repeat(count as usize))?; diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs index 254abcd..722534d 100644 --- a/crates/clawhdf5/src/edit/image.rs +++ b/crates/clawhdf5/src/edit/image.rs @@ -35,6 +35,8 @@ pub(crate) struct Image<'a> { /// Width of addresses and lengths in the file. pub(crate) os: u8, pub(crate) ls: u8, + /// Space the edit stopped using. + freed: Vec<(u64, u64)>, } impl<'a> Image<'a> { @@ -47,6 +49,7 @@ impl<'a> Image<'a> { old_eoa: eoa, os, ls, + freed: Vec::new(), } } @@ -77,6 +80,20 @@ impl<'a> Image<'a> { Ok(addr) } + /// Allocate `size` bytes for metadata or data, from space an earlier + /// edit of this session freed when there is a block that fits, + /// otherwise at the end of the file. + pub(crate) fn alloc_reusing(&mut self, size: u64) -> Result { + self.alloc(size) + } + + /// Note that the edit no longer uses `[addr, addr + len)`. + pub(crate) fn free(&mut self, addr: u64, len: u64) { + if len > 0 { + self.freed.push((addr, len)); + } + } + /// If `[addr, addr + old_len)` is the last allocated space, grow it to /// `new_len` bytes (a structure at the end of the file can grow where /// it is) and return true. diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 4903672..b14244a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -16,6 +16,7 @@ //! is alive while the file changes (see `image`). mod btree1; +mod btree2; mod earray; mod farray; mod image; @@ -39,6 +40,7 @@ use crate::error::Error; use crate::reader::File; use crate::types::AttrValue; use btree1::{BTree1, Key}; +use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; use image::{Image, get_uint, put_uint, undef}; @@ -249,6 +251,14 @@ impl Target { fn dims(&self) -> &[u64] { &self.ds.dimensions } + + /// The chunk index (or implicit chunks') address. + fn layout_address(&self) -> Option { + match &self.layout { + DataLayout::Chunked { btree_address, .. } => *btree_address, + _ => None, + } + } } /// Datatypes whose stored bytes point elsewhere in the file (variable-length @@ -331,7 +341,7 @@ enum IndexEdit { Implicit, Fixed(Option), Extensible(Option), - BTree2, + BTree2(Option), } struct ChunkedEdit<'t> { @@ -398,7 +408,9 @@ impl<'t> ChunkedEdit<'t> { (_, Some(4)) => { IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?) } - (_, Some(5)) => IndexEdit::BTree2, + (_, Some(5)) => { + IndexEdit::BTree2(btree_address.map(|a| Bt2::open(img, a)).transpose()?) + } (v, i) => { return Err(Error::Unsupported(format!( "chunked layout version {v}, index type {i:?}" @@ -520,7 +532,7 @@ impl<'t> ChunkedEdit<'t> { self.patch_layout_addr(img, hdr_addr)?; } if let IndexEdit::Fixed(Some(fa)) = &mut self.index { - fa.set(img, idx, e)?; + fa.set(img, idx, Some(e))?; } } IndexEdit::Extensible(ea) => { @@ -549,30 +561,165 @@ impl<'t> ChunkedEdit<'t> { self.patch_layout_addr(img, hdr_addr)?; } if let IndexEdit::Extensible(Some(ea)) = &mut self.index { - ea.set(img, idx, e)?; + ea.set(img, idx, Some(e))?; } } - IndexEdit::BTree2 => { - return Err(Error::Unsupported( - "adding, moving or resizing a chunk in a version-2 B-tree chunk index \ - (datasets with more than one unlimited dimension)" - .into(), - )); + IndexEdit::BTree2(tree) => { + if tree.is_none() { + let at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("B-tree v2 parameters".into()))?; + let d = self.hdr.data(img, self.layout_msg)?; + let node_size = u32::from_le_bytes([d[at], d[at + 1], d[at + 2], d[at + 3]]); + let size_len = if filtered { + earray::chunk_size_len(self.chunk_bytes as u64, self.lpos.version) + 4 + } else { + 0 + }; + let rec_size = img.os as usize + size_len + 8 * self.cd.len(); + let new = Bt2::create( + img, + if filtered { 11 } else { 10 }, + node_size, + rec_size, + d[at + 4], + d[at + 5], + )?; + let a = new.address(); + *tree = Some(new); + self.patch_layout_addr(img, a)?; + } + let IndexEdit::BTree2(Some(tree)) = &mut self.index else { + unreachable!("created above") + }; + let rec = bt2_chunk_record(tree, scaled, e, filtered, img.os)?; + let mut cmp = bt2_chunk_cmp(scaled, tree.record_size()); + tree.update(img, &mut cmp, &rec)?; } } Ok(()) } + /// Remove chunk `scaled` (stored at `info`) from the index, as the + /// index's `remove` operation does when libhdf5 prunes a dataset, and + /// free its space. An implicit index cannot drop chunks: libhdf5 leaves + /// them (and their data) where they are, and so does this. + fn remove( + &mut self, + img: &mut Image<'_>, + scaled: &[u64], + info: &ChunkInfo, + ) -> Result<(), Error> { + let t = self.t; + match &mut self.index { + IndexEdit::Implicit => return Ok(()), + IndexEdit::Single => { + return Err(Error::Unsupported( + "removing the chunk of a single-chunk index".into(), + )); + } + IndexEdit::BTree1(Some(tree)) => { + let mut offs: Vec = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect(); + offs.push(0); + if tree.remove(img, &offs)?.is_none() { + return Err(Error::Unsupported("chunk missing from its B-tree".into())); + } + } + IndexEdit::BTree2(Some(tree)) => { + let mut cmp = bt2_chunk_cmp(scaled, tree.record_size()); + if tree.remove(img, &mut cmp)?.is_none() { + return Err(Error::Unsupported("chunk missing from its B-tree".into())); + } + } + IndexEdit::Fixed(Some(fa)) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, false)?; + fa.set(img, idx, None)?; + } + IndexEdit::Extensible(Some(ea)) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, true)?; + ea.set(img, idx, None)?; + } + _ => return Err(Error::Unsupported("chunk index missing".into())), + } + img.free(info.address, u64::from(info.chunk_size)); + Ok(()) + } + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { match &mut self.index { IndexEdit::Extensible(Some(ea)) => ea.finish(img)?, IndexEdit::Fixed(Some(fa)) => fa.finish(img)?, + IndexEdit::BTree2(Some(t)) => t.finish(img)?, _ => {} } self.hdr.finish(img) } } +/// A version-2 B-tree chunk record (type 10, or 11 when filtered): the +/// chunk's address, [its stored size and filter mask,] and its scaled +/// offsets. +fn bt2_chunk_record( + tree: &Bt2, + scaled: &[u64], + e: Elem, + filtered: bool, + os: u8, +) -> Result, Error> { + let rs = tree.record_size(); + let want_type = if filtered { 11 } else { 10 }; + let osz = os as usize; + let size_len = rs + .checked_sub(osz + 8 * scaled.len()) + .and_then(|n| { + if filtered { + n.checked_sub(4).filter(|&w| (1..=8).contains(&w)) + } else { + (n == 0).then_some(0) + } + }) + .filter(|_| tree.tree_type() == want_type) + .ok_or_else(|| Error::Unsupported("version-2 B-tree chunk record layout".into()))?; + let mut r = vec![0u8; rs]; + put_uint(&mut r, e.addr, os); + let mut q = osz; + if filtered { + if size_len < 8 && e.size >> (8 * size_len) != 0 { + return Err(Error::Unsupported(format!( + "filtered chunk of {} bytes does not fit the index's {size_len}-byte size field", + e.size + ))); + } + r[q..q + size_len].copy_from_slice(&e.size.to_le_bytes()[..size_len]); + q += size_len; + r[q..q + 4].copy_from_slice(&e.mask.to_le_bytes()); + q += 4; + } + for &s in scaled { + r[q..q + 8].copy_from_slice(&s.to_le_bytes()); + q += 8; + } + Ok(r) +} + +/// Compare chunk `scaled` with a chunk record (`H5D__bt2_compare`: the +/// scaled offsets, the first dimension most significant). +fn bt2_chunk_cmp( + scaled: &[u64], + rec_size: usize, +) -> impl FnMut(&Image<'_>, &[u8]) -> Result + '_ { + move |_, r| { + let at = rec_size - 8 * scaled.len(); + let theirs = (0..scaled.len()).map(|d| { + u64::from_le_bytes(r[at + 8 * d..at + 8 * d + 8].try_into().unwrap_or([0; 8])) + }); + Ok(scaled.iter().copied().cmp(theirs)) + } +} + /// The chunk B-tree's K: the superblock's (version 1), the superblock /// extension's (versions 2 and 3), or libhdf5's default of 32. fn chunk_btree_k(f: &File) -> Result { @@ -694,10 +841,15 @@ impl FileEditor { }) } - /// Change a chunked dataset's current dimensions to `shape`, which may - /// only grow each dimension, up to the dataset's maximum dimensions - /// (h5py's `Dataset.resize`). New elements read as the fill value until - /// written. Shrinking is [`Error::Unsupported`]. + /// Change a chunked dataset's current dimensions to `shape`: each + /// dimension may grow up to the dataset's maximum or shrink (h5py's + /// `Dataset.resize`), as `H5Dset_extent` does. New elements read as the + /// fill value until written (with early allocation their chunks are + /// allocated and filled now). Shrinking removes the chunks wholly + /// outside the new extent from the chunk index and frees them, and + /// overwrites the part of each partial edge chunk outside it with the + /// fill value, so elements that come back after a later growth read as + /// the fill value (`H5D__chunk_prune_by_extent`). pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> { self.edit(|f, img| { let t = Target::load(f, path)?; @@ -720,18 +872,16 @@ impl FileEditor { shape[d], max[d] ))); } - if shape[d] < dims[d] { - return Err(Error::Unsupported("shrinking a dataset".into())); - } } if !matches!(t.layout, DataLayout::Chunked { .. }) { return Err(Error::Unsupported( "resizing a dataset that is not chunked".into(), )); } - // Only the dataspace changes: every chunk index is keyed - // independently of the current extent (the array indexes by the - // maximum dimensions), and new chunks come with the writes. + // The dataspace changes; every chunk index is keyed + // independently of the current extent (the arrays index by the + // maximum dimensions). Growth allocates chunks only under early + // allocation, shrinking prunes. let mut hdr = Header::load(img, t.addr)?; let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage( clawhdf5_format::message_type::MessageType::Dataspace, @@ -754,7 +904,15 @@ impl FileEditor { put_uint(&mut dims_bytes[d * ls..], n, img.ls); } hdr.patch(img, i, first, &dims_bytes)?; - hdr.finish(img) + let fill = fill_info(img, &hdr)?; + hdr.finish(img)?; + let expand = shape.iter().zip(&dims).any(|(n, o)| n > o); + let shrink = shape.iter().zip(&dims).any(|(n, o)| n < o); + let early = expand && fill.alloc_time == ALLOC_EARLY; + if early || shrink { + resize_chunks(f, img, &t, &dims, shape, &fill, early, shrink)?; + } + Ok(()) }) } @@ -1187,52 +1345,7 @@ fn write_selection( Ok(()) })?; for (scaled, buf) in bufs { - // As libhdf5 does: an optional filter that fails (LZF that - // does not shrink the chunk) is skipped and its mask bit set. - let (bytes, mask) = match &t.pipeline { - Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, es as u32)?, - None => (buf, 0u32), - }; - let len = bytes.len() as u64; - let placed = match existing.get(&scaled) { - Some(info) if t.pipeline.is_none() => { - if u64::from(info.chunk_size) != len { - return Err(Error::Unsupported( - "unfiltered chunk stored at an unexpected size".into(), - )); - } - img.write(info.address, &bytes)?; - None - } - // Rewritten where it is when it still fits, or when it - // is the last thing in the file (the chunk an append - // keeps rewriting usually is) and can grow there. - Some(info) - if len <= u64::from(info.chunk_size) - || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => - { - img.write(info.address, &bytes)?; - (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some( - Elem { - addr: info.address, - size: len, - mask, - }, - ) - } - _ => { - let a = img.alloc(len)?; - img.write(a, &bytes)?; - Some(Elem { - addr: a, - size: len, - mask, - }) - } - }; - if let Some(e) = placed { - ce.set(img, &scaled, e)?; - } + store_chunk(img, &mut ce, existing.get(&scaled), &scaled, buf)?; } ce.finish(img) } @@ -1240,6 +1353,341 @@ fn write_selection( } } +/// `H5D_ALLOC_TIME_EARLY`. +const ALLOC_EARLY: u8 = 1; + +/// When a dataset's storage is allocated and filled, from its fill value +/// message (`H5O__fill_new_decode`); a dataset without one uses the chunked +/// defaults (incremental allocation, fill if set). +struct FillInfo { + alloc_time: u8, + /// 0 on allocation, 1 never, 2 if set. + fill_time: u8, + /// The fill value is undefined (`H5D_FILL_VALUE_UNDEFINED`). + undefined: bool, +} + +fn fill_info(img: &Image<'_>, hdr: &Header) -> Result { + let mut fi = FillInfo { + alloc_time: 3, + fill_time: 2, + undefined: false, + }; + let Some(i) = hdr.find(0x05) else { + return Ok(fi); + }; + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared fill value message".into())); + } + let d = hdr.data(img, i)?; + let short = || Error::Unsupported("short fill value message".into()); + match d.first() { + Some(1) | Some(2) => { + fi.alloc_time = *d.get(1).ok_or_else(short)?; + fi.fill_time = *d.get(2).ok_or_else(short)?; + fi.undefined = d[0] == 2 && *d.get(3).ok_or_else(short)? == 0; + } + Some(3) => { + let flags = *d.get(1).ok_or_else(short)?; + fi.alloc_time = flags & 0x03; + fi.fill_time = (flags >> 2) & 0x03; + fi.undefined = flags & 0x10 != 0; + } + _ => return Err(Error::Unsupported("fill value message version".into())), + } + Ok(fi) +} + +/// Visit the chunks libhdf5 visits, in its order, when the extent changes +/// from `old` to `new`: for allocation (`H5D__chunk_allocate`), every +/// chunk of the new extent outside the old one; `f` gets each scaled +/// offset. +fn for_each_new_chunk( + old: &[u64], + new: &[u64], + cd: &[u64], + mut f: impl FnMut(&[u64]) -> Result<(), Error>, +) -> Result<(), Error> { + let rank = new.len(); + if rank == 0 || new.contains(&0) { + return Ok(()); + } + let min: Vec = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect(); + let mut max: Vec = (0..rank).map(|d| (new[d] - 1) / cd[d]).collect(); + for op in 0..rank { + if min[op] > max[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min[op]; + loop { + f(&scaled)?; + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max[i] { + scaled[i] = if i == op { min[i] } else { 0 }; + } else { + carry = false; + break; + } + } + if carry { + break; + } + } + if min[op] == 0 { + break; + } + max[op] = min[op] - 1; + } + Ok(()) +} + +/// What pruning does to one chunk. +enum Prune { + /// Overwrite the part outside the new extent with the fill value. + Fill, + /// Drop it from the index. + Remove, +} + +/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks +/// from `old` to `new`, in its order. +fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { + let rank = new.len(); + let mut out = Vec::new(); + if old.contains(&0) { + return out; + } + let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); + let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_fill: Vec = (0..rank) + .map(|d| { + if new[d] == 0 { + -1 + } else { + ((new[d].min(old[d]) - 1) / cd[d]) as i64 + } + }) + .collect(); + let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); + let fill_dim: Vec = (0..rank) + .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) + .collect(); + for op in 0..rank { + if !shrunk[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min_mod[op]; + let mut outside: Vec = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); + let mut n_out = outside.iter().filter(|&&o| o).count(); + loop { + if n_out == 0 { + out.push((scaled.clone(), Prune::Fill)); + } else { + out.push((scaled.clone(), Prune::Remove)); + } + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max_mod[i] { + if i == op { + scaled[i] = min_mod[i]; + if outside[i] && fill_dim[i] { + outside[i] = false; + n_out -= 1; + } + } else { + scaled[i] = 0; + if outside[i] && max_fill[i] >= 0 { + outside[i] = false; + n_out -= 1; + } + } + } else { + if !outside[i] && scaled[i] as i64 > max_fill[i] { + outside[i] = true; + n_out += 1; + } + carry = false; + break; + } + } + if carry { + break; + } + } + if min_mod[op] == 0 { + // Every chunk was visited (the dimension shrank to nothing). + break; + } + max_mod[op] = min_mod[op] - 1; + } + out +} + +/// The chunk work of a resize: under early allocation, allocate and fill +/// the chunks the growth brings in (`H5D__chunk_allocate`); when a +/// dimension shrank, prune (`H5D__chunk_prune_by_extent`). +#[allow(clippy::too_many_arguments)] +fn resize_chunks( + f: &File, + img: &mut Image<'_>, + t: &Target, + old: &[u64], + new: &[u64], + fill: &FillInfo, + early: bool, + shrink: bool, +) -> Result<(), Error> { + let mut ce = ChunkedEdit::new(f, img, t)?; + let cd = ce.cd.clone(); + let es = t.es; + let chunk_bytes = ce.chunk_bytes; + let existing: HashMap, ChunkInfo> = match &t.layout { + DataLayout::Chunked { + btree_address: Some(_), + .. + } => { + let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?; + chunks + .into_iter() + .map(|c| { + let s: Vec = c.offsets.iter().zip(&cd).map(|(o, c)| o / c).collect(); + (s, c) + }) + .collect() + } + _ => HashMap::new(), + }; + let fill_chunk = t.fill.repeat(chunk_bytes / es); + if early { + let should_fill = + fill.fill_time == 0 || (fill.fill_time == 2 && !fill.undefined) || t.pipeline.is_some(); + let implicit = matches!(ce.index, IndexEdit::Implicit); + let base = t.layout_address(); + for_each_new_chunk(old, new, &cd, |scaled| { + if implicit { + if should_fill { + let base = + base.ok_or_else(|| Error::Unsupported("implicit index address".into()))?; + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, new, max, &cd, false)?; + let at = idx + .checked_mul(chunk_bytes as u64) + .and_then(|o| o.checked_add(base)) + .ok_or_else(|| Error::Unsupported("chunk address overflows".into()))?; + img.write(at, &fill_chunk)?; + } + Ok(()) + } else if existing.contains_key(scaled) { + Ok(()) + } else { + store_chunk(img, &mut ce, None, scaled, fill_chunk.clone()) + } + })?; + } + if shrink && !existing.is_empty() { + for (scaled, what) in prune_plan(old, new, &cd) { + let Some(info) = existing.get(&scaled) else { + continue; + }; + match what { + Prune::Remove => ce.remove(img, &scaled, info)?, + Prune::Fill => { + let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?; + // Keep [0, count) in each dimension; fill the rest. + let count: Vec = (0..cd.len()) + .map(|d| cd[d].min(new[d] - scaled[d] * cd[d])) + .collect(); + let n: u64 = cd.iter().product(); + for flat in 0..n { + let mut r = flat; + let mut inside = true; + for d in (0..cd.len()).rev() { + if r % cd[d] >= count[d] { + inside = false; + } + r /= cd[d]; + } + if !inside { + let at = flat as usize * es; + buf[at..at + es].copy_from_slice(&t.fill); + } + } + store_chunk(img, &mut ce, Some(info), &scaled, buf)?; + } + } + } + } + ce.finish(img) +} + +/// Encode chunk `scaled` (decoded bytes `buf`) and store it: an existing +/// unfiltered chunk in place; a filtered one in place when it still fits +/// (or can grow at the end of the file), else in new space, the old space +/// freed; a new chunk in new space. The index is updated when the chunk's +/// address, size or filter mask changed. +fn store_chunk( + img: &mut Image<'_>, + ce: &mut ChunkedEdit<'_>, + existing: Option<&ChunkInfo>, + scaled: &[u64], + buf: Vec, +) -> Result<(), Error> { + let t = ce.t; + // As libhdf5 does: an optional filter that fails (LZF that does not + // shrink the chunk) is skipped and its mask bit set. + let (bytes, mask) = match &t.pipeline { + Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, t.es as u32)?, + None => (buf, 0u32), + }; + let len = bytes.len() as u64; + let placed = match existing { + Some(info) if t.pipeline.is_none() => { + if u64::from(info.chunk_size) != len { + return Err(Error::Unsupported( + "unfiltered chunk stored at an unexpected size".into(), + )); + } + img.write(info.address, &bytes)?; + None + } + // Rewritten where it is when it still fits, or when it is the last + // thing in the file (the chunk an append keeps rewriting usually is) + // and can grow there. + Some(info) + if len <= u64::from(info.chunk_size) + || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => + { + img.write(info.address, &bytes)?; + (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some(Elem { + addr: info.address, + size: len, + mask, + }) + } + _ => { + let a = img.alloc_reusing(len)?; + img.write(a, &bytes)?; + if let Some(info) = existing { + img.free(info.address, u64::from(info.chunk_size)); + } + Some(Elem { + addr: a, + size: len, + mask, + }) + } + }; + if let Some(e) = placed { + ce.set(img, scaled, e)?; + } + Ok(()) +} + fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> { let start = usize::try_from(info.address) .map_err(|_| Error::Unsupported("chunk address out of range".into()))?; diff --git a/crates/clawhdf5/tests/edit_tests.rs b/crates/clawhdf5/tests/edit_tests.rs index 2467256..9190a2c 100644 --- a/crates/clawhdf5/tests/edit_tests.rs +++ b/crates/clawhdf5/tests/edit_tests.rs @@ -98,8 +98,8 @@ fn errors_leave_the_file_untouched() { let mut ed = FileEditor::open(&path).unwrap(); assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); assert!(ed.write_all("missing", &[0; 4]).is_err()); - // Wrong length, wrong type, outside the extent, beyond maxshape, - // shrinking, a rank change. + // Wrong length, wrong type, outside the extent, beyond maxshape, a + // rank change, resizing a dataset that is not chunked. assert!(matches!( ed.write_all("flat", &[0; 7]), Err(Error::InvalidArgument(_)) @@ -116,7 +116,10 @@ fn errors_leave_the_file_untouched() { ed.resize("raw", &[3, 5]), Err(Error::InvalidArgument(_)) )); - assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_)))); + assert!(matches!( + ed.resize("flat", &[2]), + Err(Error::Unsupported(_)) + )); assert!(matches!( ed.resize("ext", &[4, 1]), Err(Error::InvalidArgument(_)) @@ -136,3 +139,26 @@ fn errors_leave_the_file_untouched() { drop(ed); assert!(std::fs::read(&path).unwrap() == before); } + +/// Shrinking and growing again on a file clawhdf5 wrote: elements that come +/// back read as the fill value, the ones kept keep their values. +#[test] +fn shrink_then_grow_reads_fill() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + { + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("ext", &[2]).unwrap(); + ed.resize("ext", &[9]).unwrap(); + ed.resize("raw", &[1, 4]).unwrap(); + ed.resize("raw", &[3, 4]).unwrap(); + } + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("ext").unwrap().read_i32().unwrap(), + [0, 1, 0, 0, 0, 0, 0, 0, 0] + ); + let mut raw = vec![0.0f64; 12]; + raw[..4].fill(0.5); + assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw); +} -- 2.54.0 From 1c3ef98828e5bcd104063f0a84c6eb66af41a3ad Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:41:47 -0500 Subject: [PATCH 05/49] clawhdf5: File::open_storage reads any Storage through the full API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File::open_storage(Arc) opens a file served by any backend: groups, datasets, attributes, read_*, selections, VL data and virtual datasets (external sources through the new File::set_vds_resolver) all read through Storage::read_at/read_ranges. The file's view (user block skipped, bounded by the recorded end of file) is itself a Storage; File::open and File::from_bytes keep their mmap and in-memory paths, now as that view's as_contiguous() fast path. A storage-backed file's metadata cache image is laid over each read it covers (new CacheImage::entries), as libhdf5 loads it. The typed readers keep their fast paths over any storage: a contiguous dataset is read in one piece and converted (read_f64 and friends), and a contiguous native selection reads only its runs (data_read::read_selection_native_in). Zero-copy methods answer ContiguousStorageRequired when the bytes are not in memory, and File::as_bytes panics there (File::contiguous_bytes is the fallible form). Test: tests/storage_equivalence.rs reads every fixture, and with CLAWHDF5_STORAGE_CORPUS every conformance-corpus file, through File::open and through open_storage over a read_at-only CountingStorage — tree, attributes, and every dataset's values several ways — and requires identical transcripts; it prints the read_at calls and bytes a one-pass read costs per file. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 72 +++ crates/clawhdf5-format/src/superblock_ext.rs | 17 + crates/clawhdf5/src/lib.rs | 3 +- crates/clawhdf5/src/reader.rs | 395 ++++++++++++++--- crates/clawhdf5/src/types.rs | 22 +- crates/clawhdf5/src/vlen.rs | 19 +- crates/clawhdf5/tests/storage_equivalence.rs | 434 +++++++++++++++++++ 7 files changed, 869 insertions(+), 93 deletions(-) create mode 100644 crates/clawhdf5/tests/storage_equivalence.rs diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 28bc4f2..40826c1 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -956,6 +956,78 @@ pub fn read_selection_native( crate::gather::gather::(raw, dims, elem_size, selection).map(Some) } +/// [`read_selection_native`] of a contiguous dataset in any [`Storage`], +/// reading only the selected elements' runs (adjacent ones merged, one +/// [`Storage::read_ranges`] call) instead of the whole dataset. +/// +/// `Ok(None)` wherever the in-memory fast path does not apply and the +/// caller converts through the byte readers instead: `datatype` is not +/// `T`'s native representation, the layout is not contiguous, or the +/// dataset's bytes cannot be located in the file (no address, storage too +/// small, past the end of file: the cases [`read_raw_data_zerocopy`] +/// fails). Otherwise the result and errors are [`read_selection_native`]'s +/// over those bytes. +pub fn read_selection_native_in( + file_data: &S, + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + selection: &crate::selection::Selection, +) -> Result>, FormatError> { + if !T::is_native(datatype) { + return Ok(None); + } + let DataLayout::Contiguous { + address: Some(address), + size, + } = layout + else { + return Ok(None); + }; + // Where the dataset's bytes are, as `read_raw_data_zerocopy` finds them. + let located = to_usize(dataspace.num_elements()) + .ok() + .and_then(|n| n.checked_mul(datatype.type_size() as usize)) + .filter(|&len| contiguous_read_len(*size, len).is_ok()) + .filter(|&len| { + address + .checked_add(len as u64) + .is_some_and(|end| end <= file_data.len()) + }); + let Some(len) = located else { + return Ok(None); + }; + if let Some(all) = file_data.as_contiguous() { + let start = to_usize(*address)?; + return read_selection_native( + &all[start..start + len], + &dataspace.dimensions, + datatype, + selection, + ); + } + let dims = &dataspace.dimensions; + let elem_size = core::mem::size_of::(); + let total = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?; + let expected = crate::chunked_read::checked_byte_len(total, elem_size)?; + if len != expected { + return Err(FormatError::DataSizeMismatch { + expected, + actual: len, + }); + } + let bytes = if let crate::selection::Selection::All = selection { + read_exact_at(file_data, *address, len)?.into_owned() + } else { + crate::partial_read::validate(selection, dims)?; + crate::gather::gather_storage(file_data, *address, len, dims, elem_size, selection)? + }; + Ok(Some(native_to_vec(&bytes, bytes.len() / elem_size))) +} + /// The bytes of a slice of [`NativeElement`]s. #[cfg(feature = "std")] fn bytes_of_mut(values: &mut [T]) -> &mut [u8] { diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index 4a66473..7edbda6 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -490,6 +490,23 @@ impl CacheImage { image_block_in(file, self.location) } + /// Every entry as `(file address, its bytes)`, taken from `block` (the + /// image block, see [`Self::block_in`]), in the order [`Self::apply`] + /// writes them: for a reader that cannot write the image over the + /// file's bytes and lays the entries over each read instead. + pub fn entries<'b>(&self, block: &'b [u8]) -> Result, FormatError> { + let short = || FormatError::InvalidCacheImage("image applied to the wrong file"); + self.entries + .iter() + .map(|e| { + let src = block + .get(e.image_offset..e.image_offset + e.len) + .ok_or_else(short)?; + Ok((e.address, src)) + }) + .collect() + } + /// Write every entry over `dst`, the file's bytes from the superblock /// on (as long as the `data` the image was decoded from), taking the /// entries from `block` (the image block, see [`Self::block`]). `block` diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 280f64b..5ceec87 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -56,7 +56,7 @@ pub use error::Error; pub use lazy::{LazyDataset, LazyFile, LazyGroup}; #[cfg(feature = "mmap")] pub use mmap_file::{MmapDataset, MmapFile, MmapGroup}; -pub use reader::{Dataset, File, Group}; +pub use reader::{Dataset, File, Group, SharedStorage, VdsResolver}; pub use types::{AttrValue, DType}; pub use vlen::VlenValue; pub use writer::FileBuilder; @@ -72,6 +72,7 @@ pub use clawhdf5_format::property_list::{ #[cfg(feature = "provenance")] pub use clawhdf5_format::provenance; pub use clawhdf5_format::selection::Selection; +pub use clawhdf5_format::storage::Storage; pub use clawhdf5_format::superblock::swmr_flags; pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime}; diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index fcd3b67..6bd7b35 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -5,7 +5,10 @@ //! the traditional read-into-`Vec` fallback. [`File::from_bytes`] remains //! available for in-memory usage (tests, etc.). +use std::borrow::Cow; use std::collections::HashMap; +use std::ops::Range; +use std::sync::Arc; use clawhdf5_format::chunk_cache::ChunkCache; use clawhdf5_format::data_layout::DataLayout; @@ -19,29 +22,44 @@ use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; +use clawhdf5_format::storage::Storage; use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext::{self, CacheImageState}; use crate::cache_image::{self, ImageView}; use crate::error::Error; use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; // --------------------------------------------------------------------------- -// FileData — internal storage for either owned bytes or an mmap +// FileData — internal storage for owned bytes, an mmap, or any Storage // --------------------------------------------------------------------------- -/// Internal storage: either an owned `Vec` or a memory-mapped region. +/// A [`Storage`] a [`File`] can be opened over: any backend that can be +/// shared between threads (see [`File::open_storage`]). +pub type SharedStorage = Arc; + +/// Resolves an external Virtual Dataset source file name to that file's +/// bytes (`Ok(None)`: the file does not exist, its mappings read as the +/// fill value). See [`File::set_vds_resolver`]. +pub type VdsResolver = Arc Result>, FormatError> + Send + Sync>; + +/// Internal storage: an owned `Vec`, a memory-mapped region, or a +/// [`Storage`] backend. enum Backing { Owned(Vec), #[cfg(feature = "mmap")] Mmap(clawhdf5_io::MmapReader), + Storage(SharedStorage), } impl Backing { - fn whole_file(&self) -> &[u8] { + /// The whole file, when it is in memory. + fn whole_file(&self) -> Option<&[u8]> { match self { - Backing::Owned(v) => v, + Backing::Owned(v) => Some(v), #[cfg(feature = "mmap")] - Backing::Mmap(r) => r.as_bytes(), + Backing::Mmap(r) => Some(r.as_bytes()), + Backing::Storage(s) => s.as_contiguous(), } } } @@ -49,13 +67,16 @@ impl Backing { /// The file's bytes, viewed from the superblock on and up to the end of /// file the superblock records. A file may start with a user block (the /// superblock at 512, 1024, …); every HDF5 address is relative to the -/// superblock, so all parsing goes through [`Self::as_bytes`]. +/// superblock, so all parsing goes through this view, which is a +/// [`Storage`]: in memory (a `Vec`, an mmap) its reads are slices of the +/// file, as before; over any other storage they are reads of that storage, +/// shifted by the user block and bounded by the end of file. struct FileData { backing: Backing, /// Offset of the superblock in the file (the user-block size). - base: usize, + base: u64, /// End of the HDF5 data in the file (`Superblock::data_end`, absolute). - end: usize, + end: u64, /// A mapped file that holds a metadata cache image, with the image /// written in: a private copy-on-write mapping of the whole file, so /// only the pages the image's entries land on are copied (see @@ -63,6 +84,10 @@ struct FileData { /// an image is read straight from the mapping, and an owned buffer has /// the image written into it in place. patched: Option, + /// A [`Storage`]-backed file's metadata cache image: its entries + /// (address relative to the superblock, bytes), laid over every read in + /// order, as libhdf5 reads them instead of the file's own bytes. + overlay: Vec<(u64, Vec)>, /// The file has a metadata cache image libhdf5 cannot load. libhdf5 /// opens such a file and fails its first metadata read (the image loads /// then); every object lookup here fails with this error, and no @@ -74,7 +99,10 @@ impl FileData { /// Locate the superblock and parse it. A truncated file is refused, and /// bytes past the recorded end of file are not read, as in libhdf5. fn new(mut backing: Backing) -> Result<(Self, Superblock), Error> { - let whole = backing.whole_file(); + if let Backing::Storage(storage) = backing { + return Self::new_storage(storage); + } + let whole = backing.whole_file().unwrap_or_default(); let (user_block, hdf5) = signature::split_user_block(whole)?; let base = user_block.len(); let superblock = Superblock::parse(hdf5, 0)?; @@ -92,6 +120,7 @@ impl FileData { clawhdf5_io::HDF5Read::private_copy(r) })? } + Backing::Storage(_) => unreachable!("handled above"), }; let (patched, image_error) = match view { ImageView::Plain => (None, None), @@ -101,34 +130,140 @@ impl FileData { Ok(( Self { backing, - base, - end, + base: base as u64, + end: end as u64, patched, + overlay: Vec::new(), image_error, }, superblock, )) } - fn as_bytes(&self) -> &[u8] { - match &self.patched { - Some(p) => &p[self.base..self.end], - None => &self.backing.whole_file()[self.base..self.end], + /// [`Self::new`] for a [`Storage`] backend: the same checks, through + /// reads of the storage. + fn new_storage(storage: SharedStorage) -> Result<(Self, Superblock), Error> { + let file_len = storage.len(); + let base = signature::find_signature_in(&*storage)?; + let mut data = Self { + backing: Backing::Storage(storage), + base, + end: file_len, + patched: None, + overlay: Vec::new(), + image_error: None, + }; + let superblock = Superblock::parse_in(&data, 0)?; + data.end = base + superblock.data_end(base, file_len)?; + match superblock_ext::cache_image_state_in(&data, &superblock)? { + CacheImageState::Absent => {} + CacheImageState::Unloadable(e) => data.image_error = Some(e), + CacheImageState::Loaded(image) => { + let block = image.block_in(&data)?.into_owned(); + data.overlay = image + .entries(&block)? + .into_iter() + .map(|(addr, bytes)| (addr, bytes.to_vec())) + .collect(); + } } + Ok((data, superblock)) } - fn len(&self) -> usize { - self.as_bytes().len() + /// The HDF5 data as one slice, when the file is in memory (a `Vec`, an + /// mmap, or a storage that holds it all and has no cache image to lay + /// over it). + fn contiguous(&self) -> Option<&[u8]> { + if let Some(p) = &self.patched { + return p.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?); + } + if !self.overlay.is_empty() { + return None; + } + self.backing + .whole_file()? + .get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?) } /// The bytes to read metadata from; fails for a file whose cache image /// cannot be loaded (see [`Self::image_error`]). - fn meta(&self) -> Result<&[u8], FormatError> { + fn meta(&self) -> Result<&Self, FormatError> { match &self.image_error { Some(e) => Err(e.clone()), - None => Ok(self.as_bytes()), + None => Ok(self), } } + + /// The backend of a file that is not in memory. + fn remote(&self) -> Result<&SharedStorage, FormatError> { + match &self.backing { + Backing::Storage(s) => Ok(s), + _ => Err(FormatError::Storage( + "in-memory file has no contiguous view".into(), + )), + } + } + + /// `bytes`, read at `offset`, with the cache image entries they overlap + /// written over them. + fn with_overlay<'a>(&self, offset: u64, mut bytes: Cow<'a, [u8]>) -> Cow<'a, [u8]> { + let end = offset + bytes.len() as u64; + for (addr, entry) in &self.overlay { + let entry_end = addr + entry.len() as u64; + if *addr >= end || entry_end <= offset { + continue; + } + let from = (*addr).max(offset); + let to = entry_end.min(end); + let dst = bytes.to_mut(); + dst[(from - offset) as usize..(to - offset) as usize] + .copy_from_slice(&entry[(from - addr) as usize..(to - addr) as usize]); + } + bytes + } +} + +impl Storage for FileData { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if let Some(all) = self.contiguous() { + return all.read_at(offset, len); + } + let size = self.end - self.base; + let len = usize::try_from(size.saturating_sub(offset)).map_or(len, |avail| avail.min(len)); + if len == 0 { + return Ok(Cow::Borrowed(&[])); + } + let bytes = self.remote()?.read_at(self.base + offset, len)?; + Ok(self.with_overlay(offset, bytes)) + } + + fn len(&self) -> u64 { + self.end - self.base + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if let Some(all) = self.contiguous() { + return all.read_ranges(ranges); + } + let size = self.end - self.base; + let shifted: Vec> = ranges + .iter() + .map(|r| { + let (s, e) = (r.start.min(size), r.end.min(size)); + self.base + s..self.base + e.max(s) + }) + .collect(); + let got = self.remote()?.read_ranges(&shifted)?; + Ok(got + .into_iter() + .zip(ranges) + .map(|(bytes, r)| self.with_overlay(r.start, bytes)) + .collect()) + } + + fn as_contiguous(&self) -> Option<&[u8]> { + self.contiguous() + } } // --------------------------------------------------------------------------- @@ -149,6 +284,9 @@ pub struct File { /// Directory the file was opened from, used to resolve external Virtual /// Dataset source files relative to this file. `None` for in-memory files. base_dir: Option, + /// Resolves external Virtual Dataset source files instead of + /// `base_dir` (see [`File::set_vds_resolver`]). + vds_resolver: Option, } impl File { @@ -167,6 +305,7 @@ impl File { superblock, chunk_cache: ChunkCache::new(), base_dir, + vds_resolver: None, }) } #[cfg(not(feature = "mmap"))] @@ -200,9 +339,53 @@ impl File { superblock, chunk_cache: ChunkCache::new(), base_dir: None, + vds_resolver: None, }) } + /// Open an HDF5 file served by any [`Storage`]: a range-reading remote + /// backend, a block cache, or an in-memory buffer — through the same + /// read API as [`File::open`] (groups, datasets, attributes, `read_*`, + /// selections, variable-length data, virtual datasets). + /// + /// Every read goes through the storage's [`Storage::read_at`] and + /// [`Storage::read_ranges`] (a chunked read fetches all the chunks it + /// needs with one `read_ranges` call per 64 MiB), so nothing is read that + /// the operation does not need. A storage that has the whole file in + /// memory ([`Storage::as_contiguous`]) is read as [`File::from_bytes`] + /// reads its buffer. The storage holds the whole file: a user block is + /// found and skipped, and bytes past the end of file the superblock + /// records are not read. A metadata cache image is laid over the reads + /// it covers, as libhdf5 loads it. + /// + /// The zero-copy methods ([`Dataset::read_raw_ref`], + /// [`Dataset::read_as_slice`], `read_*_zerocopy`) borrow the file's bytes + /// and return [`FormatError::ContiguousStorageRequired`] over a storage + /// that does not hold them; [`File::as_bytes`] panics there (use + /// [`File::contiguous_bytes`]). External virtual-dataset source files + /// are read through [`File::set_vds_resolver`]; without one they cannot + /// be resolved. + pub fn open_storage(storage: SharedStorage) -> Result { + let (data, superblock) = FileData::new(Backing::Storage(storage))?; + Ok(Self { + data, + superblock, + chunk_cache: ChunkCache::new(), + base_dir: None, + vds_resolver: None, + }) + } + + /// Resolve external Virtual Dataset source files (their names as the + /// mappings store them) with `resolver`, instead of reading them from + /// the directory of the file (for [`File::open`]) or refusing them (for + /// in-memory and [`Storage`]-backed files). `Ok(None)` means the source + /// file does not exist, and its mappings read as the fill value, as in + /// libhdf5; `Err` fails the read. + pub fn set_vds_resolver(&mut self, resolver: VdsResolver) { + self.vds_resolver = Some(resolver); + } + /// Returns a handle to the root group. pub fn root(&self) -> Group<'_> { Group { @@ -216,7 +399,7 @@ impl File { /// The path uses `/` separators (e.g., `"group1/values"`). pub fn dataset(&self, path: &str) -> Result, Error> { let data = self.data.meta()?; - let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; + let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); @@ -263,7 +446,7 @@ impl File { /// Use `"/"` or `""` for the root group. pub fn group(&self, path: &str) -> Result, Error> { let data = self.data.meta()?; - let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; + let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; Ok(Group { file: self, address: addr, @@ -322,8 +505,22 @@ impl File { /// with a metadata cache image these are the bytes with the image /// applied; when the image cannot be loaded they are the file's own /// bytes, whose metadata may be stale (every object lookup fails then). + /// + /// # Panics + /// + /// For a file opened with [`File::open_storage`] over a storage that + /// does not hold the whole file in memory; use + /// [`contiguous_bytes`](Self::contiguous_bytes) there. pub fn as_bytes(&self) -> &[u8] { - self.data.as_bytes() + self.data + .contiguous() + .expect("File::as_bytes: the file is not in memory (see File::contiguous_bytes)") + } + + /// [`as_bytes`](Self::as_bytes), or `None` for a file whose bytes are + /// not all in memory ([`File::open_storage`]). + pub fn contiguous_bytes(&self) -> Option<&[u8]> { + self.data.contiguous() } /// The error of a metadata cache image libhdf5 cannot load, when the @@ -338,7 +535,7 @@ impl File { /// Size of the user block before the superblock (0 for most files). /// Matches h5py's `File.userblock_size`. pub fn user_block_size(&self) -> u64 { - self.data.base as u64 + self.data.base } /// Returns a reference to the parsed superblock. @@ -349,7 +546,7 @@ impl File { /// Returns `true` when the file is backed by memory-mapped I/O. pub fn is_mmap(&self) -> bool { match &self.data.backing { - Backing::Owned(_) => false, + Backing::Owned(_) | Backing::Storage(_) => false, #[cfg(feature = "mmap")] Backing::Mmap(_) => true, } @@ -362,7 +559,7 @@ impl File { /// this file's global heap; see [`Dataset::read_string`] for the values. pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result, Error> { crate::vlen::decode_strings( - self.as_bytes(), + &self.data, datatype, raw, self.offset_size(), @@ -406,9 +603,9 @@ impl File { } fn parse_header(&self, address: u64) -> Result { - ObjectHeader::parse( + ObjectHeader::parse_in( self.data.meta()?, - address as usize, + address, self.superblock.offset_size, self.superblock.length_size, ) @@ -426,7 +623,7 @@ impl File { impl std::fmt::Debug for File { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("File") - .field("size", &self.data.len()) + .field("size", &Storage::len(&self.data)) .field("superblock_version", &self.superblock.version) .field("mmap", &self.is_mmap()) .finish() @@ -489,7 +686,7 @@ impl<'f> Group<'f> { &self, ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.parse_header(self.address)?; - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) } @@ -520,7 +717,7 @@ impl<'f> Group<'f> { /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { let hdr = self.file.parse_header(self.address)?; - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attr( data, &hdr, @@ -536,7 +733,7 @@ impl<'f> Group<'f> { /// [`group_v2::resolve_child`]). fn child_address(&self, name: &str) -> Result { let data = self.file.data.meta()?; - group_v2::resolve_child(data, &self.file.superblock, self.address, name) + group_v2::resolve_child_in(data, &self.file.superblock, self.address, name) .map_err(Error::Format) } @@ -559,7 +756,7 @@ impl<'f> Group<'f> { /// user-defined links are left out. fn children(&self) -> Result, Error> { let data = self.file.data.meta()?; - group_v2::resolve_group_children(data, &self.file.superblock, self.address) + group_v2::resolve_group_children_in(data, &self.file.superblock, self.address) .map_err(Error::Format) } } @@ -589,7 +786,7 @@ impl<'f> Dataset<'f> { Ok((self.datatype()?, ds, self.data_layout()?)) })(); if let Ok((dt, ds, dl)) = decoded { - data_read::check_dataset_storage(&dl, &ds, &dt, self.file.data.len() as u64)?; + data_read::check_dataset_storage(&dl, &ds, &dt, Storage::len(&self.file.data))?; } Ok(self) } @@ -629,8 +826,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_f64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_f64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -649,8 +846,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_f32(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_f32(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -664,8 +861,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_i32(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_i32(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -679,8 +876,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_i64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_i64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -694,8 +891,8 @@ impl<'f> Dataset<'f> { let dt = self.datatype()?; // A contiguous dataset is converted straight from the file bytes; going // through `read_raw` first copied the whole dataset an extra time. - if let Ok(Some(bytes)) = self.read_raw_ref() { - return Ok(data_read::read_as_u64(bytes, &dt)?); + if let Ok(Some(bytes)) = self.contiguous_raw() { + return Ok(data_read::read_as_u64(&bytes, &dt)?); } if let Some(values) = self.read_chunked_native::()? { return Ok(values); @@ -793,8 +990,8 @@ impl<'f> Dataset<'f> { // sparse) dataset — select from a fill-aware full read instead. (The // selection reader currently decodes the full dataset too, so this // costs nothing extra.) - let fill = clawhdf5_format::fill_value::dataset_fill_value_in( - self.file.data.as_bytes(), + let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage( + &self.file.data, &self.header.messages, self.file.offset_size(), self.file.length_size(), @@ -813,8 +1010,8 @@ impl<'f> Dataset<'f> { selection, )?); } - Ok(data_read::read_raw_data_selection( - self.file.data.as_bytes(), + Ok(data_read::read_raw_data_selection_in( + &self.file.data, &dl, &ds, &dt, @@ -871,12 +1068,26 @@ impl<'f> Dataset<'f> { return full(); } let dt = self.datatype()?; - if T::is_native(&dt) - && let Ok(Some(raw)) = self.read_raw_ref() + if T::is_native(&dt) && self.file.data.contiguous().is_some() { + if let Ok(Some(raw)) = self.read_raw_ref() { + let dims = self.dataspace()?.dimensions; + if let Some(values) = + data_read::read_selection_native::(raw, &dims, &dt, selection)? + { + return Ok(values); + } + } + } else if T::is_native(&dt) + && let (Ok(dl), Ok(ds)) = (self.data_layout(), self.dataspace()) { - let dims = self.dataspace()?.dimensions; - if let Some(values) = data_read::read_selection_native::(raw, &dims, &dt, selection)? - { + // Not in memory: only the selected runs are read. + if let Some(values) = data_read::read_selection_native_in::( + &self.file.data, + &dl, + &ds, + &dt, + selection, + )? { return Ok(values); } } @@ -893,10 +1104,46 @@ impl<'f> Dataset<'f> { let dl = self.data_layout()?; let ds = self.dataspace()?; let dt = self.datatype()?; - let slice = data_read::read_raw_data_zerocopy(self.file.data.as_bytes(), &dl, &ds, &dt)?; + let Some(bytes) = self.file.data.contiguous() else { + // The bytes are not in memory to borrow. + return match dl { + DataLayout::Contiguous { .. } => { + Err(Error::Format(FormatError::ContiguousStorageRequired( + "a zero-copy read (the file is not in memory)", + ))) + } + _ => Ok(None), + }; + }; + let slice = data_read::read_raw_data_zerocopy(bytes, &dl, &ds, &dt)?; Ok(slice) } + /// A contiguous dataset's stored bytes, for the typed readers' fast + /// path: borrowed from the file when it is in memory + /// ([`read_raw_ref`](Self::read_raw_ref)), read in one piece otherwise. + /// `Ok(None)` for other layouts; an error where `read_raw_ref` fails. + fn contiguous_raw(&self) -> Result>, Error> { + if self.file.data.contiguous().is_some() { + return Ok(self.read_raw_ref()?.map(Cow::Borrowed)); + } + let dl = self.data_layout()?; + if !matches!(dl, DataLayout::Contiguous { .. }) { + return Ok(None); + } + let ds = self.dataspace()?; + let dt = self.datatype()?; + Ok(Some(Cow::Owned(data_read::read_raw_data_full_in( + &self.file.data, + &dl, + &ds, + &dt, + None, + self.file.offset_size(), + self.file.length_size(), + )?))) + } + /// Zero-copy typed read of contiguous data as `&[T]`. /// /// Returns a borrowed slice of `T` directly from the file buffer with @@ -1083,7 +1330,7 @@ impl<'f> Dataset<'f> { pub fn attrs_with_errors( &self, ) -> Result<(HashMap, Vec), Error> { - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attrs( data, &self.header, @@ -1097,7 +1344,7 @@ impl<'f> Dataset<'f> { /// that name, found without reading the other attributes when they are /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { - let data = self.file.data.as_bytes(); + let data = &self.file.data; read_attr( data, &self.header, @@ -1124,8 +1371,8 @@ impl<'f> Dataset<'f> { /// result is not a tamper-evidence or authenticity guarantee. #[cfg(feature = "provenance")] pub fn verify_provenance(&self) -> Result { - Ok(clawhdf5_format::provenance::verify_dataset( - self.file.as_bytes(), + Ok(clawhdf5_format::provenance::verify_dataset_in( + &self.file.data, &self.header, self.file.offset_size(), self.file.length_size(), @@ -1144,8 +1391,8 @@ impl<'f> Dataset<'f> { .iter() .find(|m| m.msg_type == msg_type) .map(|msg| { - clawhdf5_format::shared_message::message_data( - self.file.as_bytes(), + clawhdf5_format::shared_message::message_data_in( + &self.file.data, msg, self.file.offset_size(), self.file.length_size(), @@ -1179,8 +1426,8 @@ impl<'f> Dataset<'f> { // one (`H5Dget_space`). if let Ok(dl @ DataLayout::Virtual { .. }) = self.data_layout() { let resolver = self.vds_resolver(); - ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent( - self.file.data.as_bytes(), + ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent_in( + &self.file.data, &dl, &ds, self.file.offset_size(), @@ -1225,9 +1472,9 @@ impl<'f> Dataset<'f> { } let ds = self.dataspace()?; let pipeline = self.filter_pipeline()?; - Ok(data_read::read_chunked_native::( + Ok(data_read::read_chunked_native_in::( &self.header.messages, - self.file.data.as_bytes(), + &self.file.data, &dl, &ds, &dt, @@ -1251,17 +1498,17 @@ impl<'f> Dataset<'f> { } // Unallocated storage reads as the dataset's fill value. - clawhdf5_format::fill_value::read_full_with_fill( + clawhdf5_format::fill_value::read_full_with_fill_in( &self.header.messages, - self.file.data.as_bytes(), + &self.file.data, &dl, &ds, dt.type_size() as usize, self.file.offset_size(), self.file.length_size(), || { - Ok(data_read::read_raw_data_cached( - self.file.data.as_bytes(), + Ok(data_read::read_raw_data_cached_in( + &self.file.data, &dl, &ds, &dt, @@ -1281,7 +1528,11 @@ impl<'f> Dataset<'f> { /// refused with an error rather than read as fill. fn vds_resolver(&self) -> impl Fn(&str) -> Result>, FormatError> + use<> { let base_dir = self.file.base_dir.clone(); + let custom = self.file.vds_resolver.clone(); move |name: &str| { + if let Some(resolve) = &custom { + return resolve(name); + } let Some(dir) = base_dir.as_ref() else { return Err(FormatError::ChunkedReadError(format!( "virtual dataset source file {name:?} cannot be resolved for an in-memory file" @@ -1310,15 +1561,15 @@ impl<'f> Dataset<'f> { ds: &Dataspace, dt: &Datatype, ) -> Result, Error> { - let fill = clawhdf5_format::fill_value::dataset_fill_value_in( - self.file.data.as_bytes(), + let fill = clawhdf5_format::fill_value::dataset_fill_value_from_storage( + &self.file.data, &self.header.messages, self.file.offset_size(), self.file.length_size(), )?; let resolver = self.vds_resolver(); - let v = clawhdf5_format::vds::read_virtual_dataset( - self.file.data.as_bytes(), + let v = clawhdf5_format::vds::read_virtual_dataset_in( + &self.file.data, dl, ds, dt, @@ -1439,7 +1690,7 @@ mod zero_copy_tests { let Backing::Mmap(r) = &f.data.backing else { return None; }; - let mapped = r.as_bytes()[f.data.base..].as_ptr(); + let mapped = r.as_bytes()[f.data.base as usize..].as_ptr(); match &f.data.patched { None => Some(std::ptr::eq(f.as_bytes().as_ptr(), mapped)), Some(p) => { diff --git a/crates/clawhdf5/src/types.rs b/crates/clawhdf5/src/types.rs index acae280..b3f7823 100644 --- a/crates/clawhdf5/src/types.rs +++ b/crates/clawhdf5/src/types.rs @@ -158,8 +158,8 @@ pub(crate) fn classify_datatype(dt: &clawhdf5_format::datatype::Datatype) -> DTy /// The attributes of the object with header `header` that could be read, /// and one error for each that could not (see /// [`extract_attributes_tolerant`](clawhdf5_format::attribute::extract_attributes_tolerant)). -pub(crate) fn read_attrs( - file_data: &[u8], +pub(crate) fn read_attrs( + file_data: &S, header: &clawhdf5_format::object_header::ObjectHeader, offset_size: u8, length_size: u8, @@ -170,7 +170,7 @@ pub(crate) fn read_attrs( ), crate::Error, > { - let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant( + let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant_in( file_data, header, offset_size, @@ -185,14 +185,14 @@ pub(crate) fn read_attrs( /// The attribute called `name` on the object with header `header`, decoded /// as [`read_attrs`] decodes it, or `None` (see /// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)). -pub(crate) fn read_attr( - file_data: &[u8], +pub(crate) fn read_attr( + file_data: &S, header: &clawhdf5_format::object_header::ObjectHeader, name: &str, offset_size: u8, length_size: u8, ) -> Result, crate::Error> { - let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file( + let Some(msg) = clawhdf5_format::attribute::find_attribute_in( file_data, header, name, @@ -211,9 +211,9 @@ pub(crate) fn read_attr( .remove(name)) } -pub(crate) fn attrs_to_map( +pub(crate) fn attrs_to_map( attrs: &[clawhdf5_format::attribute::AttributeMessage], - file_data: &[u8], + file_data: &S, offset_size: u8, length_size: u8, ) -> HashMap { @@ -266,9 +266,9 @@ fn decode_bool_enum(attr: &clawhdf5_format::attribute::AttributeMessage) -> Opti values.iter().all(|v| *v == 0 || *v == 1).then_some(values) } -fn decode_attr_value( +fn decode_attr_value( attr: &clawhdf5_format::attribute::AttributeMessage, - file_data: &[u8], + file_data: &S, offset_size: u8, length_size: u8, ) -> Option { @@ -311,7 +311,7 @@ fn decode_attr_value( is_string: true, .. } => { let strings = attr - .read_vl_strings(file_data, offset_size, length_size) + .read_vl_strings_in(file_data, offset_size, length_size) .ok()?; if strings.len() == 1 { Some(AttrValue::String(strings[0].clone())) diff --git a/crates/clawhdf5/src/vlen.rs b/crates/clawhdf5/src/vlen.rs index 01ba02d..f471b24 100644 --- a/crates/clawhdf5/src/vlen.rs +++ b/crates/clawhdf5/src/vlen.rs @@ -14,6 +14,7 @@ use clawhdf5_format::data_read; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; use clawhdf5_format::vl_data::{VlResolver, check_element_size}; use crate::error::Error; @@ -68,8 +69,8 @@ fn class_name(dt: &Datatype) -> &'static str { /// The strings in `raw`, elements of `dt`: fixed-length strings decoded as /// `read_string` always has, variable-length strings resolved in the heap. -pub(crate) fn decode_strings( - file_data: &[u8], +pub(crate) fn decode_strings( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -82,15 +83,15 @@ pub(crate) fn decode_strings( .. } => { check_element_size(*size, offset_size)?; - Ok(VlResolver::new(file_data, offset_size, length_size).strings(raw)?) + Ok(VlResolver::new_in(file_data, offset_size, length_size).strings(raw)?) } _ => Ok(data_read::read_as_strings(raw, dt)?), } } /// The exact bytes of the variable-length strings in `raw`. -pub(crate) fn decode_string_bytes( - file_data: &[u8], +pub(crate) fn decode_string_bytes( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -103,7 +104,7 @@ pub(crate) fn decode_string_bytes( .. } => { check_element_size(*size, offset_size)?; - Ok(VlResolver::new(file_data, offset_size, length_size).string_bytes(raw)?) + Ok(VlResolver::new_in(file_data, offset_size, length_size).string_bytes(raw)?) } other => Err(Error::Format(FormatError::TypeMismatch { expected: "variable-length string", @@ -114,8 +115,8 @@ pub(crate) fn decode_string_bytes( /// The sequences in `raw`, elements of the variable-length sequence type /// `dt`, converted to `T`. -pub(crate) fn decode_vlen( - file_data: &[u8], +pub(crate) fn decode_vlen( + file_data: &S, dt: &Datatype, raw: &[u8], offset_size: u8, @@ -135,7 +136,7 @@ pub(crate) fn decode_vlen( }; check_element_size(*size, offset_size)?; let base_size = base_type.type_size() as usize; - VlResolver::new(file_data, offset_size, length_size) + VlResolver::new_in(file_data, offset_size, length_size) .sequences(raw, base_size)? .iter() .map(|bytes| Ok(T::decode(bytes, base_type)?)) diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs new file mode 100644 index 0000000..7a2749b --- /dev/null +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -0,0 +1,434 @@ +//! `File::open_storage` over a read_at-only storage reads every file as +//! `File::open` does (range reads, milestone M2 in +//! `docs/design/range-reads.md`). +//! +//! Each file is read end to end twice — through `File::open` (the mmap +//! fast path) and through `File::open_storage` over a +//! [`CountingStorage`], which serves the file through `read_at` only +//! (`as_contiguous()` is `None`, so no reader can fall back to a slice of +//! the whole file) — and the two transcripts must be identical: the tree +//! (every group's entries, followed by address), every object's attributes +//! (the whole map and each one by name), and every dataset's shape, types +//! and values (all bytes, as `f64`, a hyperslab of them, strings and +//! variable-length sequences). +//! +//! The storage also counts its `read_at` calls and bytes: what a remote +//! backend without a cache would be asked for. The totals and the files +//! that cost most are printed. +//! +//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every HDF5 file under those +//! directories (the conformance corpus is `conformance/.cache/corpus`); +//! `CLAWHDF5_STORAGE_REPORT=1` prints every file's counts. + +use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clawhdf5::{DType, File, Selection}; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::CountingStorage; + +/// Objects visited per file. +const MAX_OBJECTS: usize = 2000; +/// Datasets with more bytes than this are not read (their metadata is). +const MAX_DATA_BYTES: u64 = 64 << 20; + +/// A short, stable digest of a value's `Debug` form. +fn digest(v: &T) -> String { + let s = format!("{v:?}"); + if s.len() <= 200 { + return s; + } + let mut h = 0xcbf2_9ce4_8422_2325u64; + for b in s.bytes() { + h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); + } + format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len()) +} + +/// A data read's result: its value, or just `Err` — a full read goes +/// through the file's chunk cache, which lists a damaged dataset's chunks +/// in hash-map order, so which failing chunk it reports varies from one +/// `File` to the next (the format crate's harness compares these errors on +/// the uncached path). +fn value(r: &Result) -> String { + match r { + Ok(v) => digest(v), + Err(_) => "Err".into(), + } +} + +fn transcript(file: &File) -> String { + let mut out = String::new(); + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]); + while let Some((path, addr)) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let entries = group.entries(); + writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap(); + match file.dataset_at(addr) { + Ok(ds) => dataset(&mut out, &path, &ds), + Err(e) => writeln!(out, "{path} dataset_at {e:?}").unwrap(), + } + let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap(); + if let Ok((attrs, _)) = &attrs { + for name in attrs.keys().take(50) { + writeln!(out, "{path} attr {name:?} {}", digest(&group.attr(name))).unwrap(); + } + } + if let Ok(entries) = entries { + for (name, child) in entries { + queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child)); + // Name lookups (through the name index of a dense group). + if queue.len() < 64 { + writeln!( + out, + "{path} group({name:?}) {}", + digest(&group.group(&name).map(|_| ())) + ) + .unwrap(); + } + } + } + } + out +} + +fn sorted(m: std::collections::HashMap) -> BTreeMap { + m.into_iter().collect() +} + +fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { + let shape = ds.shape(); + let dtype = ds.dtype(); + writeln!( + out, + "{path} shape {} max {} dtype {} raw {}", + digest(&shape), + digest(&ds.max_dimensions()), + digest(&dtype), + digest(&ds.raw_datatype()) + ) + .unwrap(); + let attrs = ds.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} dataset attrs {}", digest(&attrs)).unwrap(); + let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else { + return; + }; + let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d)); + let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size()))); + if bytes.is_none_or(|b| b > MAX_DATA_BYTES) { + writeln!(out, "{path} too large to read").unwrap(); + return; + } + writeln!( + out, + "{path} all {}", + value(&ds.read_selection(&Selection::All)) + ) + .unwrap(); + let numeric = matches!( + dtype, + DType::F32 + | DType::F64 + | DType::I8 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::U8 + | DType::U16 + | DType::U32 + | DType::U64 + ); + if numeric { + writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap(); + writeln!(out, "{path} f32 {}", value(&ds.read_f32())).unwrap(); + writeln!(out, "{path} i64 {}", value(&ds.read_i64())).unwrap(); + if let Some(&d0) = shape.first() { + let rank = shape.len(); + let sel = Selection::Hyperslab { + start: std::iter::once(d0 / 3) + .chain(std::iter::repeat_n(0, rank - 1)) + .collect(), + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(3)) + .chain(shape[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + writeln!( + out, + "{path} f64 third {}", + value(&ds.read_f64_selection(&sel)) + ) + .unwrap(); + writeln!( + out, + "{path} bytes third {}", + value(&ds.read_selection(&sel)) + ) + .unwrap(); + } + } + match &raw_dt { + clawhdf5_format::datatype::Datatype::String { .. } + | clawhdf5_format::datatype::Datatype::VariableLength { + is_string: true, .. + } => { + writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap(); + writeln!(out, "{path} string {}", value(&ds.read_string())).unwrap(); + } + clawhdf5_format::datatype::Datatype::VariableLength { .. } => { + writeln!(out, "{path} vlen {}", value(&ds.read_vlen::())).unwrap(); + } + _ => {} + } +} + +/// External virtual-dataset sources as `File::open` finds them: files in +/// the same directory. +fn sibling_resolver(dir: Option) -> clawhdf5::VdsResolver { + Arc::new(move |name: &str| { + let Some(dir) = dir.as_ref() else { + return Err(FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} cannot be resolved for an in-memory file" + ))); + }; + let p = Path::new(name); + if name.is_empty() + || !p.components().all(|c| { + matches!( + c, + std::path::Component::Normal(_) | std::path::Component::CurDir + ) + }) + { + return Err(FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} is outside the virtual file's \ + directory and is not followed" + ))); + } + match std::fs::read(dir.join(p)) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(FormatError::ChunkedReadError(format!( + "cannot read virtual dataset source file {name:?}: {e}" + ))), + } + }) +} + +#[derive(Default)] +struct Totals { + files: usize, + opened: usize, + /// The comparison's reads (each dataset read several ways). + reads: u64, + bytes: u64, + /// One pass: open, list every group, read every attribute and every + /// dataset once (`read_selection(All)`). + pass_reads: u64, + pass_bytes: u64, + file_bytes: u64, + /// (one-pass reads, one-pass bytes, file size, name) per file. + per_file: Vec<(u64, u64, u64, String)>, +} + +/// Open the file and read everything once, as a tree viewer that then +/// shows every value would. +fn one_pass(file: &File) { + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([file.superblock().root_group_address]); + while let Some(addr) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let _ = group.attrs(); + if let Ok(ds) = file.dataset_at(addr) { + let small = ds.shape().ok().and_then(|s| { + let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; + let size = u64::from(ds.raw_datatype().ok()?.type_size()); + n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) + }); + if small.is_some() { + let _ = ds.read_selection(&Selection::All); + } + } + if let Ok(entries) = group.entries() { + queue.extend(entries.into_iter().map(|(_, a)| a)); + } + } +} + +fn check(path: &Path, totals: &mut Totals) { + let Ok(bytes) = std::fs::read(path) else { + return; + }; + let name = path.display().to_string(); + let local = File::open(path); + let storage = Arc::new(CountingStorage::new(bytes.clone())); + let resolver = sibling_resolver(path.parent().map(Path::to_path_buf)); + let remote = File::open_storage(storage.clone()).map(|mut f| { + f.set_vds_resolver(resolver.clone()); + f + }); + totals.files += 1; + let (local, remote) = match (local, remote) { + (Ok(l), Ok(r)) => (l, r), + (l, r) => { + // Both refuse the file, with the same error. + assert_eq!( + format!("{:?}", l.map(|_| ())), + format!("{:?}", r.map(|_| ())), + "{name}: open" + ); + return; + } + }; + totals.opened += 1; + assert!(remote.contiguous_bytes().is_none(), "{name}"); + assert_eq!(local.user_block_size(), remote.user_block_size(), "{name}"); + let want = transcript(&local); + let got = transcript(&remote); + if want != got { + let first = want + .lines() + .zip(got.lines()) + .find(|(w, g)| w != g) + .map(|(w, g)| format!("\n local: {w}\n storage: {g}")) + .unwrap_or_else(|| { + format!( + "\n {} vs {} lines", + want.lines().count(), + got.lines().count() + ) + }); + panic!("{name}: File::open_storage differs from File::open{first}"); + } + totals.reads += storage.reads(); + totals.bytes += storage.bytes_read(); + totals.file_bytes += bytes.len() as u64; + + let pass = Arc::new(CountingStorage::new(bytes.clone())); + if let Ok(mut f) = File::open_storage(pass.clone()) { + f.set_vds_resolver(resolver); + one_pass(&f); + } + let (reads, read_bytes) = (pass.reads(), pass.bytes_read()); + totals.pass_reads += reads; + totals.pass_bytes += read_bytes; + if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") { + eprintln!( + "{reads:>9} reads {read_bytes:>12} bytes {:>12} file {name}", + bytes.len() + ); + } + totals + .per_file + .push((reads, read_bytes, bytes.len() as u64, name)); +} + +fn report(what: &str, totals: &mut Totals) { + eprintln!( + "{what}: {} files ({} open, {} bytes); comparison: {} read_at calls, {} bytes; \ + one pass (list, attributes, every dataset once): {} read_at calls, {} bytes", + totals.files, + totals.opened, + totals.file_bytes, + totals.reads, + totals.bytes, + totals.pass_reads, + totals.pass_bytes + ); + totals.per_file.sort_by_key(|a| std::cmp::Reverse(a.0)); + for (reads, bytes, size, name) in totals.per_file.iter().take(10) { + eprintln!(" {reads:>9} reads {bytes:>12} bytes (file {size:>11}) {name}"); + } +} +fn hdf5_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + hdf5_files(&p, out); + } else if p + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf")) + { + out.push(p); + } + } +} + +#[test] +fn fixtures_read_identically_through_open_storage() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + files.sort(); + assert!(files.len() >= 45, "{} fixtures", files.len()); + let mut totals = Totals::default(); + for f in &files { + check(f, &mut totals); + } + report("fixtures", &mut totals); + assert!(totals.opened >= 40, "{}", totals.opened); + assert!(totals.reads > 0); +} + +#[test] +fn corpus_reads_identically_through_open_storage() { + let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else { + eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus"); + return; + }; + let mut files = Vec::new(); + for d in std::env::split_paths(&dirs) { + hdf5_files(&d, &mut files); + } + files.sort(); + let mut totals = Totals::default(); + for f in &files { + check(f, &mut totals); + } + report("corpus", &mut totals); + assert!(totals.files > 0); +} + +/// A user block, a metadata cache image and a Storage that is itself in +/// memory: the in-memory view of a storage that has one is used as is. +#[test] +fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let path = root.join("tests/fixtures/h5clear_mdc_image.h5"); + let bytes = std::fs::read(&path).unwrap(); + let local = File::open(&path).unwrap(); + // A Vec is a Storage with a contiguous view; the image still has + // to be laid over it, so the view is not used. + let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap(); + assert!(in_memory.contiguous_bytes().is_none()); + assert_eq!(transcript(&local), transcript(&in_memory)); + + let plain = root.join("../clawhdf5-format/tests/fixtures/chunked_2d.h5"); + let bytes = std::fs::read(&plain).unwrap(); + let in_memory = File::open_storage(Arc::new(bytes.clone())).unwrap(); + assert_eq!(in_memory.contiguous_bytes(), Some(&bytes[..])); + let counting = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap(); + assert!(counting.contiguous_bytes().is_none()); + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| counting.as_bytes().len())); + assert!( + result.is_err(), + "as_bytes over a range storage must not answer" + ); +} -- 2.54.0 From f191dc09d5537e90eb141a0f04077cdbf0cef64f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:49:42 -0500 Subject: [PATCH 06/49] =?UTF-8?q?docs:=20range-read=20milestone=20M2=20?= =?UTF-8?q?=E2=80=94=20changelog,=20limits,=20design=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHANGELOG (Unreleased): File::open_storage, raw data and v2 B-trees over Storage, the tests and their corpus results (2026-09-26, tank; conformance 600 of 697, results.json identical to 8f59b2e). known-issues: what open_storage does not do yet (no remote backend or block cache, read_at counts of a one-pass read, v1 group lookups, whole-file VDS sources, zero-copy methods, SWMR growth, hash-order error choice on damaged chunked datasets). Design: M2 status and the choices made. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 94 +++++++++++++++++++++++++++++++++++++- docs/design/range-reads.md | 35 ++++++++++++-- docs/known-issues.md | 38 +++++++++++++++ 3 files changed, 163 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa0a30f..661a2eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,97 @@ ## Unreleased +### Range reads, milestone M2: raw data and `File::open_storage` (2026-09-26) +- **`clawhdf5::File::open_storage(Arc)`** opens + a file served by any `clawhdf5_format::storage::Storage` and gives the + whole read API over it: groups and paths, datasets, attributes, the + `read_*` methods, selections, variable-length strings and sequences, and + virtual datasets. Every byte comes through `Storage::read_at` / + `read_ranges`; a user block is found and skipped, nothing past the + superblock's end of file is read, and a metadata cache image is laid over + the reads it covers (new `CacheImage::entries`). External virtual-dataset + sources are read through the new `File::set_vds_resolver` (any `File`; + without one a storage-backed file cannot follow them). `File::open` and + `File::from_bytes` keep their mmap and in-memory paths: the file's view + is now a `Storage` whose `as_contiguous()` is that buffer, and every hot + loop takes it. New exports: `clawhdf5::{Storage, SharedStorage, + VdsResolver}`, `File::contiguous_bytes()`. + - Over a storage without the whole file in memory the zero-copy methods + (`read_raw_ref`, `read_as_slice`, `read_*_zerocopy`) answer + `FormatError::ContiguousStorageRequired`, and `File::as_bytes` panics + (documented; use `contiguous_bytes`). The typed readers keep their fast + paths: a contiguous dataset is read in one piece and converted, and a + contiguous selection of a native type reads only its runs + (`data_read::read_selection_native_in`). +- **Nothing in the format crate needs the whole file any more.** The + structures M1 left to `ContiguousStorageRequired` read through `Storage`: + v2 B-trees (`BTreeV2Header::parse_in`, `collect_btree_v2_records_in`, + `find_btree_v2_records_in`; one bounded read per node, whose size is known + before it is read), hence dense attributes, a SOHM B-tree index, huge + fractal-heap objects, and dense groups; v1 and v2 group listings, lookups + and paths (`group_v2::resolve_group_children_in`, `resolve_child_in`, + `resolve_path_any_in`, `group_v1::*_in`). +- **Raw data reads through `Storage`**, each with a generic `*_in` core and + its `&[u8]` function as a thin wrapper (callers do not change): + `data_read` (`read_raw_data*_in`, `read_raw_data_selection_in`, + `read_chunked_native_in`), `chunked_read` (the v1 B-tree chunk index — + one read of each node's header, one of its entries — `list_chunks_in`, + and the full, cached, sweep and indexed reads), `parallel_read`, + `partial_read`, `fill_value` (`read_full_with_fill_in`, + `apply_to_unallocated_chunks_in`; `dataset_fill_value_from_storage` is + now generic, so a `&dyn Storage` still works), `vds` + (`read_virtual_dataset_in`, `virtual_dataset_extent_in`: the virtual + file through `Storage`, external source files still loaded whole through + the resolver), `vl_data` (`VlResolver<'a, S = [u8]>` with `new_in`; + `read_vl_strings_in`, `read_vl_bytes_in`), + `AttributeMessage::read_vl_strings_in`, `provenance::verify_dataset_in`. + - A chunked read first lists the chunks it needs, then fetches all their + stored bytes with **one `read_ranges` call** (per 64 MiB of stored + data), so a remote backend can coalesce and parallelise them, then + decodes as before (in parallel with the `parallel` feature). Chunks the + file's chunk cache already holds are not fetched. A selection fetches + only the chunks its bounding box overlaps; a contiguous selection only + its runs (adjacent ones merged). A global-heap collection is read once + per resolver and kept (within the resolver's 32 MiB budget). + - Each extent's bounds error is the one the slice readers gave, reported + when the read reaches that extent, so a damaged file fails with the + same error, in the same order, through either path. +- **No behaviour change for in-memory and mapped files:** with + `as_contiguous()` every path slices the file as before (checked below). +- Tests (2026-09-26, tank): + - `clawhdf5-format/tests/storage_equivalence.rs` now also reads every + dataset — whole, fill-aware, through a chunk cache (twice) and the + indexed path, three selections, virtual datasets with their sibling + sources, VL strings, sequences and bytes — through the read_at-only + `CountingStorage` and requires the slice results, and fails on any + `ContiguousStorageRequired`. With + `CLAWHDF5_STORAGE_CORPUS=conformance/.cache/corpus`, all 653 HDF5 + files of the corpus agree (82 396 checks). The cached and indexed + paths are compared on values only when a read fails: they order + chunks by hash map, so which failing chunk a damaged dataset reports + varies between two caches even for the same slice (seen on + `cve-2025-2310.h5`; see `docs/known-issues.md`). + - A misbehaving storage (fails its N-th read; serves short reads) over + every fixture: each listing and dataset read is an error or exactly the + in-memory result, never other data (1 137 runs). + - A chunked read issues one `read_ranges` call with one range per chunk, + and a one-chunk selection one call with one range. + - `clawhdf5/tests/storage_equivalence.rs` reads every fixture (61 files) + and, with `CLAWHDF5_STORAGE_CORPUS`, every corpus file (701 files, 621 + that open) through `File::open` and through `File::open_storage` over + `CountingStorage`: the tree, every attribute (all, and each by name), + every dataset's shape, types and values (all bytes, `f64`, `f32`, + `i64`, a hyperslab, strings, VL sequences) must be identical, and are. + It also counts what one pass — open, list, read every attribute and + every dataset once — asks of a storage with no cache: 176 092 `read_at` + calls and 208 MB for the 621 corpus files (254 MB of files); the most + are `h5stat_newgrat.h5` (35 001 groups: 92 489 calls) and + `ref_hdf5_compat1.nc` (16 062). A remote backend needs the block cache + of milestone M3. Command: `CLAWHDF5_STORAGE_CORPUS=… cargo test + --release -p clawhdf5 --test storage_equivalence -- --nocapture`. + - Conformance sweep (`conformance/run.sh --no-fetch`): 600 of 697 files + ok, `results.json` byte-identical to `8f59b2e`. + ### Name lookups through the name index (2026-09-26) - **Finding one link or attribute by name reads the name index, not every entry.** In a dense group (links in a fractal heap) the v2 B-tree name @@ -108,7 +199,8 @@ v2 B-tree and dense groups come with milestone M3); over a backend without the whole file in memory they are the clean `ContiguousStorageRequired` error, never a partial result. Raw data, chunk B-tree (v1) indexes and VL - data are milestone M2. + data are milestone M2. (All of them read through `Storage` since M2, + above.) - **No behaviour change**, checked three ways (2026-09-26, tank): every existing test passes unchanged; the conformance sweep (`conformance/run.sh --no-fetch`) gives a byte-identical `results.json` diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1c7fa55..1098922 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -1,9 +1,12 @@ # Design: range reads (reading HDF5 without holding the whole file) Status: proposal, 2026-09-26; the plan for Phase 3's largest architectural -change. Progress: M1, first part (the `Storage` trait and the metadata -parsers listed in `CHANGELOG.md` under "Range reads, milestone M1") is done; -group B-tree v2 lookups, dense groups and the facade are not converted yet. Every count below was +change. Progress: M0 and M1 are done, and so is M2 (branch +`feat/p3-m2-raw-data`): every read path of the format crate works through +`Storage`, v2 B-trees, dense groups and raw data included, and +`File::open_storage` gives the facade's read API over any `Storage` (see +`CHANGELOG.md`, "Range reads, milestone M2"). M3 (a remote backend with +its block cache) is next. Every count in §1–§2 was taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given next to it. No timing numbers appear here on purpose: the machine was shared with other build jobs when this was written. @@ -421,6 +424,32 @@ fast path within benchmark noise. on other backends (they already return `Option`/`Result`). - Facade: `File::open_storage(Box)`; `File::open` keeps mmap and `from_bytes` keeps `Vec`, both through `impl Storage for [u8]`. +- *Status 2026-09-26:* done on branch `feat/p3-m2-raw-data`. As planned, + with these choices: + - `File::open_storage` takes an `Arc` (the + file handle is shared by its datasets and may be sent across threads). + The file's view (user block skipped, bounded by the recorded end of + file, cache image laid over its reads) is itself a `Storage`, and the + facade calls the generic cores with it; for a `Vec` or an mmap its + `as_contiguous()` is the buffer, so the local paths are the slice code + (checked: identical conformance results; the bench gate below still + has to be run on an idle machine). + - Chunked reads fetch in batches of at most 64 MiB of stored bytes, one + `read_ranges` call each, so a remote read never holds more than that + undecoded; chunks already in the chunk cache are not fetched. + - The typed readers' zero-copy fast path became "read the contiguous + bytes once": over a range storage a contiguous `read_f64` is one read, + and a native contiguous selection reads only its runs. + - External VDS source files stay whole-file, through a resolver that + returns bytes (`File::set_vds_resolver`). + - The v2 B-tree and dense groups were done here rather than in M3, so no + format-crate path answers `ContiguousStorageRequired` any more; only the + facade's zero-copy methods do. + - Measured with the M2 harness (`crates/clawhdf5/tests/storage_equivalence.rs`, + tank, 2026-09-26): one pass over the 621 corpus files that open — list, + every attribute, every dataset once — is 176 092 `read_at` calls and + 208 MB through a storage with no cache (254 MB of files). The block + cache of M3 is what turns that into requests (§2). **M3 — HTTP/S3 backend (1–2 weeks).** - `clawhdf5-io`, feature `remote` (off by default, so the default tree stays diff --git a/docs/known-issues.md b/docs/known-issues.md index dcb365f..313bc8e 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -763,6 +763,44 @@ the same agent-store interop test. **Fix:** an empty contiguous dataset gets the undefined address (all `0xff`), which is what libhdf5 itself writes. +## Range reads (`File::open_storage`) limits + +**Status:** open (added 2026-09-26, milestone M2 of +`docs/design/range-reads.md`). `File::open_storage` reads any +`clawhdf5_format::storage::Storage` through the whole read API, and every +format-crate read path works through `Storage::read_at`/`read_ranges`, but: + +- **No remote backend and no block cache yet** (milestone M3). A `Storage` + is asked for each structure as the parsers need it, several times over + for some (an object header is re-read by each lookup through it): one + pass over the conformance corpus — open, list, every attribute, every + dataset once — is 176 092 `read_at` calls for 621 files, 92 489 of them + for the 35 001-group `h5stat_newgrat.h5` (2026-09-26, tank, + `crates/clawhdf5/tests/storage_equivalence.rs` with + `CLAWHDF5_STORAGE_CORPUS`). A backend over a network needs a cache in + front of it. `Storage::read_ranges` defaults to one `read_at` per range; + coalescing is the backend's job. +- A group lookup by name in a version-1 (symbol-table) group lists the whole + group (dense groups use their name index). Over a range backend that is + one read per symbol-table node and name, per lookup. +- External virtual-dataset source files are loaded whole through the + resolver (`File::set_vds_resolver`), as bytes; they are not read through + a `Storage`. +- The zero-copy methods (`Dataset::read_raw_ref`, `read_as_slice`, + `read_*_zerocopy`) need the file in memory and answer + `FormatError::ContiguousStorageRequired` otherwise; `File::as_bytes()` + panics for such a file (`File::contiguous_bytes()` is the fallible form). + `LazyFile`, `MmapFile`, the Python and wasm bindings and `h5rs` still read + a whole file. +- The file's length is read once, at open: a growing file (SWMR) is not + followed (milestone M5). +- Not new, but visible through the equivalence tests: a full read through + the file's chunk cache (`read_raw_data_cached`, `read_raw_data_indexed`, + and so `Dataset::read_*`) lists a damaged dataset's chunks in hash-map + order, so which failing chunk it reports can differ from one `File` to + the next (`cve-2025-2310.h5`); the values of a dataset that reads are + not affected. + ## `clawhdf5-wasm` (browser) limits **Status:** open (by design for now; added 2026-09-26). -- 2.54.0 From 7e5e920c727009ee2b8ec1e77c530d384dc2240e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:56:09 -0500 Subject: [PATCH 07/49] format: read object headers with long continuation chains libhdf5 follows any number of continuation chunks, and a header that is full gains one per message added (each new chunk holding the next continuation message), so a version-1 header with a few dozen attributes added one at a time is a chain dozens of chunks long. The reader recursed once per chunk and refused a chain deeper than 32 (NestingDepthExceeded): h5py read such files, we did not. Version-2 headers stopped at 256 continuation chunks. Version-1 chunks are now followed with an explicit stack (the same depth-first message order as before), version-2 ones as before; both refuse a chunk address seen twice (a cycle, what the limits guarded against) and more than 65 536 chunks. Regression: long_v1_continuation_chains_are_read (a 200-chunk chain), v1_continuation_cycles_are_refused; the dense-attribute interop test's 'earliest' case produces such a chain. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/object_header.rs | 164 +++++++++++++++----- 1 file changed, 122 insertions(+), 42 deletions(-) diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 0d031ed..383d553 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -1,7 +1,9 @@ //! HDF5 Object Header parsing (v1 and v2). #[cfg(not(feature = "std"))] -use alloc::vec::Vec; +use alloc::{collections::BTreeSet, vec, vec::Vec}; +#[cfg(feature = "std")] +use std::collections::BTreeSet; use byteorder::{ByteOrder, LittleEndian}; @@ -196,7 +198,6 @@ impl ObjectHeader { header_data_size, offset_size, length_size, - MAX_V1_CONTINUATION_DEPTH, &mut messages, )?; // libhdf5 reads every message in the first chunk and refuses a header @@ -230,49 +231,55 @@ impl ObjectHeader { /// 8; libhdf5 refuses a message that is not aligned, that runs past the /// end of the chunk, or leftover bytes too few for a message header (a /// "gap", which only version 2 allows). - #[allow(clippy::too_many_arguments)] + /// + /// Continuation chunks are followed depth-first, as met, with an + /// explicit stack: a chain of continuation chunks as long as libhdf5 + /// writes (each new chunk holding the next continuation message, one + /// per attribute added to a full header) is read without recursion. + /// A chunk address seen twice is a cycle and refused, as is a header of + /// more than [`MAX_V1_CHUNKS`] chunks. fn parse_v1_chunk( file: &S, offset: u64, length: usize, offset_size: u8, length_size: u8, - depth_remaining: u16, messages: &mut Vec, ) -> Result { - if depth_remaining == 0 { - return Err(FormatError::NestingDepthExceeded); - } - let chunk = read_exact_at(file, offset, length)?; - let data: &[u8] = &chunk; - let end = length; - let mut pos = 0usize; + let mut stack = vec![(read_exact_at(file, offset, length)?, 0usize)]; + let mut seen = BTreeSet::new(); + seen.insert(offset); let mut count = 0usize; - while pos < end { - if end - pos < V1_MSG_HEADER_SIZE { + while let Some((chunk, pos)) = stack.last_mut() { + let data: &[u8] = chunk; + let end = data.len(); + if *pos >= end { + stack.pop(); + continue; + } + if end - *pos < V1_MSG_HEADER_SIZE { return Err(FormatError::InvalidObjectHeader( "gap found in early version of file format", )); } - let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); - let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; - let msg_flags = data[pos + 4]; - // reserved(3) at pos+5..pos+8 - pos += V1_MSG_HEADER_SIZE; + let p = *pos; + let msg_type_raw = LittleEndian::read_u16(&data[p..p + 2]); + let msg_data_size = LittleEndian::read_u16(&data[p + 2..p + 4]) as usize; + let msg_flags = data[p + 4]; + // reserved(3) at p+5..p+8 + let p = p + V1_MSG_HEADER_SIZE; if !msg_data_size.is_multiple_of(8) { return Err(FormatError::InvalidObjectHeader("message not aligned")); } - if msg_data_size > end - pos { + if msg_data_size > end - p { return Err(FormatError::InvalidObjectHeader( "message size exceeds buffer end", )); } - let body = &data[pos..pos + msg_data_size]; + let body = &data[p..p + msg_data_size]; check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; - count += 1; - let msg_type = MessageType::from_u16(msg_type_raw); if msg_type != MessageType::Nil { messages.push(HeaderMessage { @@ -283,22 +290,26 @@ impl ObjectHeader { data: body.to_vec(), }); } - pos += msg_data_size; - // Follow continuations (v1 continuation chunks are just raw // messages, no signature); check_message has checked the body. - if msg_type == MessageType::ObjectHeaderContinuation { - let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?; - let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?; - Self::parse_v1_chunk( - file, - cont_offset as u64, - cont_length, - offset_size, - length_size, - depth_remaining - 1, - messages, - )?; + let cont = if msg_type == MessageType::ObjectHeaderContinuation { + Some(( + read_offset(body, 0, offset_size)?, + to_usize(read_offset(body, offset_size as usize, length_size)?)?, + )) + } else { + None + }; + *pos = p + msg_data_size; + // Only the first chunk's messages are held to the prefix count. + if stack.len() == 1 { + count += 1; + } + if let Some((cont_offset, cont_length)) = cont { + if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { + return Err(FormatError::NestingDepthExceeded); + } + stack.push((read_exact_at(file, cont_offset, cont_length)?, 0)); } } @@ -425,13 +436,15 @@ impl ObjectHeader { &mut continuations, )?; - // Follow continuations (limit to prevent cycles in malformed data) - let mut cont_remaining = 256u16; + // Follow continuations. A chunk address seen twice is a cycle in + // malformed data; a valid header can have many chunks (libhdf5 adds + // one whenever a message no longer fits), up to the same bound as a + // version-1 header. + let mut seen = BTreeSet::new(); while let Some((cont_offset, cont_length)) = continuations.pop() { - if cont_remaining == 0 { + if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { return Err(FormatError::NestingDepthExceeded); } - cont_remaining -= 1; Self::parse_v2_continuation( file, cont_offset as u64, @@ -594,8 +607,10 @@ const V2_PREFIX_MAX: usize = 34; /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3). const V1_MSG_HEADER_SIZE: usize = 8; -/// How deep version-1 continuation chunks may chain (malformed-data guard). -const MAX_V1_CONTINUATION_DEPTH: u16 = 32; +/// Most chunks a version-1 object header may have (malformed-data guard; +/// libhdf5 has no limit, and a header that gains one continuation chunk per +/// attribute added can have many). +const MAX_V1_CHUNKS: usize = 1 << 16; /// Every defined version-2 object header status flag (libhdf5 /// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation @@ -901,6 +916,71 @@ mod tests { assert_eq!(hdr.messages[1].data[..2], [5, 6]); } + /// A version-1 header whose continuation chunks form a chain: chunk k + /// holds a Dataspace message `[k]` and the continuation to chunk k + 1. + /// With `cycle`, the last chunk points back at the first continuation + /// chunk. + fn v1_chain(n: usize, cycle: bool) -> Vec { + // Each continuation chunk: dataspace (8 + 8) + continuation (8 + 16). + let chunk_len = 40u64; + let first = 64u64; + let cont = |addr: u64| { + let mut b = addr.to_le_bytes().to_vec(); + b.extend_from_slice(&chunk_len.to_le_bytes()); + b + }; + let mut data = build_v1_header(&[(0x0010, &cont(first)[..], 0)], 8, 8); + data.resize(first as usize, 0); + for k in 0..n { + let mut c = Vec::new(); + c.extend_from_slice(&1u16.to_le_bytes()); + c.extend_from_slice(&8u16.to_le_bytes()); + c.extend_from_slice(&[0; 4]); + c.extend_from_slice(&(k as u64).to_le_bytes()); + let next = if k + 1 < n { + first + (k as u64 + 1) * chunk_len + } else if cycle { + first + } else { + // The last chunk ends in a NIL message instead. + c.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]); + c.extend_from_slice(&[0; 16]); + data.extend_from_slice(&c); + continue; + }; + c.extend_from_slice(&0x10u16.to_le_bytes()); + c.extend_from_slice(&16u16.to_le_bytes()); + c.extend_from_slice(&[0; 4]); + c.extend_from_slice(&cont(next)); + data.extend_from_slice(&c); + } + data + } + + /// libhdf5 reads any chain of continuation chunks (a header grows one + /// per attribute added when full); the reader used to stop at 32. + #[test] + fn long_v1_continuation_chains_are_read() { + let data = v1_chain(200, false); + let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); + let spaces: Vec = hdr + .messages + .iter() + .filter(|m| m.msg_type == MessageType::Dataspace) + .map(|m| m.data[0]) + .collect(); + assert_eq!(spaces, (0..200).map(|k| k as u8).collect::>()); + } + + #[test] + fn v1_continuation_cycles_are_refused() { + let data = v1_chain(5, true); + assert!(matches!( + ObjectHeader::parse(&data, 0, 8, 8), + Err(FormatError::NestingDepthExceeded) + )); + } + #[test] fn parse_v1_unknown_message_ok() { let messages = [(0x00FFu16, &[0xAA, 0xBB][..], 0u8)]; -- 2.54.0 From 773f427f1685cd882e5f82fcd0e9d9d62e1a298b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:58:08 -0500 Subject: [PATCH 08/49] edit: dense attributes, compact-to-dense transition, creation order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FileEditor::set_attr now handles every attribute storage libhdf5 uses for version-2 object headers: - objects that track (and index) attribute creation order: compact attributes carry their creation index in the message header, the Attribute Info message its maximum; - the move to dense storage when an object reaches its compact limit (or an attribute is too large for a header message), as H5O__attr_create does it: a new fractal heap, name index (v2 B-tree type 8) and, when creation order is indexed, creation-order index (type 9); the compact attributes moved over in header message order, their messages freed; - objects already in dense storage (h5py- or clawhdf5-written): new attributes inserted (H5A__dense_insert), an attribute replaced by one of the same encoded size rewritten in its heap object (H5A__dense_write), otherwise removed from both indexes and the heap and inserted anew. edit/fheap.rs follows H5HF: managed objects go to the best-fitting free section of the heap's free-space manager (FSHD/FSSE, kept as libhdf5 keeps it — sorted sections, counts, section info reallocated when its size changes, the manager deleted when empty); otherwise to a new direct block: the root direct block of an empty heap, else the block at the allocation iterator in the root indirect block (created from the root direct block, doubled as needed), with libhdf5's managed-space, allocated space, free space and iterator bookkeeping. Objects above the managed limit are huge objects in their own space, indexed by the huge-object B-tree (type 1). Removed objects return their space merged with adjacent free space. Refused before anything is written: heaps with I/O filters, child indirect blocks, an object larger than the next heap block (libhdf5 skips blocks and records them as free space), free sections other than those inside direct blocks, removing a direct block's last object (libhdf5 frees the block), directly addressed huge objects. Attributes are encoded as libhdf5 does when h5py opens a file r+ (low bound "earliest"): message version 1 (3 for non-ASCII names), simple dataspaces with their maximum dimensions. Header chunks are now visited in libhdf5's order (FIFO), which is also the order attributes move to dense storage in. Tests (edit_coverage_interop): 40 attributes on each of a plain group, a group tracking and indexing creation order, and a dataset (earliest, v110, latest), some above the 4 KiB managed limit, then same-size rewrites: the heap statistics, free-space sections and both index B-trees node for node equal libhdf5's doing the same through h5py; then replacements of other sizes, h5py adds/deletes/rewrites; h5py, h5dump, h5rs check and our reader agree throughout, h5py's attribute count included. clawhdf5-written dense storage (tracked and untracked) is extended the same way; refusals leave the file byte for byte as it was. edit_interop's attribute test now expects dense storage and tracked creation order to work. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 495 +++++++- crates/clawhdf5-tools/tests/edit_interop.rs | 19 +- crates/clawhdf5/src/edit/attrs.rs | 513 ++++++++ crates/clawhdf5/src/edit/btree2.rs | 35 +- crates/clawhdf5/src/edit/fheap.rs | 1121 +++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 190 +-- crates/clawhdf5/src/edit/ohdr.rs | 8 +- 7 files changed, 2165 insertions(+), 216 deletions(-) create mode 100644 crates/clawhdf5/src/edit/attrs.rs create mode 100644 crates/clawhdf5/src/edit/fheap.rs diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index f2c27fd..036839f 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -213,7 +213,6 @@ fn tmpdir() -> tempfile::TempDir { tempfile::TempDir::new_in(base).unwrap() } -#[allow(dead_code)] fn unsupported(r: Result) { match r { Err(Error::Unsupported(_)) => {} @@ -830,3 +829,497 @@ fn btree2_random_chunk_order() { verify(&path, "x", &m); check_tools(&path, false); } + +// ---- dense attributes ---- + +/// An attribute value both sides can write with the same encoded size: a +/// 1-D int64 array or a fixed-length string. +#[derive(Clone, Debug, PartialEq)] +enum AV { + Ints(Vec), + Str(String), +} + +impl AV { + fn value(&self) -> clawhdf5::AttrValue { + match self { + AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()), + AV::Str(s) => clawhdf5::AttrValue::String(s.clone()), + } + } + + fn py(&self) -> String { + match self { + AV::Ints(v) => format!("np.array({v:?}, dtype=' format!("np.bytes_({s:?})"), + } + } + + fn py_expect(&self) -> String { + match self { + AV::Ints(v) => format!("{v:?}"), + AV::Str(s) => format!("{s:?}"), + } + } + + fn make(i: u64, salt: u64) -> Self { + match i % 11 { + 5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)), + 1 | 4 | 8 => AV::Str( + (0..10 + (i * 13 + salt) % 300) + .map(|k| (b'a' + ((k + salt) % 26) as u8) as char) + .collect(), + ), + _ => AV::Ints( + (0..1 + (i + salt) % 9) + .map(|k| (k * 31 + salt) as i64) + .collect(), + ), + } + } +} + +/// An object's dense attribute storage as libhdf5 would compare it: the +/// heap's statistics and root shape, its free sections (heap offsets and +/// sizes), and the shapes of the name and creation-order index B-trees. +fn dense_info(path: &Path, obj: &str) -> String { + use clawhdf5_format::message_type::MessageType; + use clawhdf5_format::object_header::ObjectHeader; + let f = File::open(path).unwrap(); + let sb = f.superblock(); + let (os, ls) = (sb.offset_size, sb.length_size); + assert_eq!((os, ls), (8, 8)); + let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap(); + let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap(); + let Some(m) = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::AttributeInfo) + else { + return "no attribute info".into(); + }; + let d = &m.data; + let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 }; + let heap = le(&d[p..p + 8]); + p += 8; + let name = le(&d[p..p + 8]); + let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16])); + if heap == u64::MAX { + return "compact".into(); + } + let b = std::fs::read(path).unwrap(); + let h = &b[heap as usize..]; + assert_eq!(&h[0..4], b"FRHP"); + // next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge + // size, huge objs (8-byte fields from offset 14). + let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]); + let fs = stat(3); + // Root rows: after 12 8-byte fields, width, start, max direct, max + // index, start rows, root address. + let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8; + let mut out = format!( + "heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}", + stat(0), + stat(2), + stat(4), + stat(5), + stat(6), + stat(7), + stat(8), + stat(9), + le(&h[rows_at..rows_at + 2]) + ); + if fs != u64::MAX { + let s = &b[fs as usize..]; + assert_eq!(&s[0..4], b"FSHD"); + let tot = le(&s[6..14]); + let n = le(&s[14..22]); + // counts (4 x 8), 4 x u16, max section size, section info address. + let at = 6 + 32 + 8 + 8; + let sect_addr = le(&s[at..at + 8]); + let sect_size = le(&s[at + 8..at + 16]); + let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize]; + // Sections after the prefix (signature, version, header address); + // trailing zero padding and the checksum are left out. + let body = &ss[13..ss.len() - 4]; + let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count(); + out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used])); + } else { + out += " no-fs"; + } + out += &format!(" names={:?}", bt2_shape_at(&b, name as usize)); + if let Some(o) = order { + out += &format!(" order={:?}", bt2_shape_at(&b, o as usize)); + } + out +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +/// h5py and our reader both see `want` on each object (other attributes +/// may exist; these must have these values), and h5py's attribute count +/// agrees with libhdf5's object info. +fn check_attr_values(path: &Path, want: &[(String, String, AV)]) { + let f = File::open(path).unwrap(); + for (o, n, v) in want { + let attrs = if o == "d" { + f.dataset(o).unwrap().attrs().unwrap() + } else { + f.group(o).unwrap().attrs().unwrap() + }; + let got = attrs + .get(n.as_str()) + .unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len())); + match (got, v) { + (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"), + (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"), + (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"), + other => panic!("{o}/{n}: {other:?}"), + } + } + let exp: Vec = want + .iter() + .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect())) + .collect(); + let script = format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + want = [{w}]\n\ + for o, n, v in want:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 a = [a] if isinstance(a, int) else a\n\ + \x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\ + for o in set(x[0] for x in want):\n\ + \x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\ + \x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n", + p = path.to_str().unwrap(), + w = exp.join(", ") + ); + let sp = path.with_extension("check.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); +} + +/// Run python statements (inside `with h5py.File(path, 'r+') as f:`). +fn py_r_plus(path: &Path, lines: &[String]) { + let script = format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n{l}", + p = path.to_str().unwrap(), + l = lines.concat() + ); + let sp = path.with_extension("ops.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "h5py workload failed:\n{}", text(&o)); +} + +type AttrOp = (String, String, AV); + +fn set_want(want: &mut Vec, o: &str, n: &str, v: AV) { + want.retain(|(wo, wn, _)| !(wo == o && wn == n)); + want.push((o.into(), n.into(), v)); +} + +/// The same attribute workload by libhdf5 and by the editor on objects +/// with compact attributes that move to dense storage (a plain group, a +/// group tracking and indexing creation order, a dataset): 40 new +/// attributes each (some larger than the heap's 4 KiB managed limit), then +/// same-size rewrites — the heaps, their free space and both index B-trees +/// must come out as libhdf5 makes them; then replacements of another size, +/// then h5py adds, deletes and rewrites attributes. +fn dense_workload(libver: &str, h5dump: bool, tag: &str) { + let dir = tmpdir(); + let a = dir.path().join(format!("dense_{tag}_h5py.h5")); + let b = dir.path().join(format!("dense_{tag}_edit.h5")); + for p in [&a, &b] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \ + f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + for o in objs { + for i in 0..3i64 { + want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i]))); + } + } + // Phase 1: new attributes. + let mut ops: Vec = Vec::new(); + for i in 0..40u64 { + for (k, o) in objs.iter().enumerate() { + ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64))); + } + } + let lines: Vec = ops + .iter() + .map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py())) + .collect(); + py_r_plus(&a, &lines); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, n, v) in &ops { + ed.set_attr(o, n, &v.value()) + .unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}")); + set_want(&mut want, o, n, v.clone()); + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + check_attr_values(&a, &want); + for o in objs { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{tag}: dense storage of {o} differs from libhdf5's after insertions" + ); + } + // Phase 2: same-size rewrites (H5Awrite in place). + let ops2: Vec = ops + .iter() + .step_by(4) + .map(|(o, n, v)| { + let nv = match v { + AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()), + AV::Str(s) => AV::Str(s.chars().rev().collect()), + }; + (o.clone(), n.clone(), nv) + }) + .collect(); + let lines: Vec = ops2 + .iter() + .map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py())) + .collect(); + py_r_plus(&a, &lines); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, n, v) in &ops2 { + ed.set_attr(o, n, &v.value()).unwrap(); + set_want(&mut want, o, n, v.clone()); + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + for o in objs { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{tag}: dense storage of {o} differs from libhdf5's after rewrites" + ); + } + // Phase 3: replacements of another size (values only: h5py replaces + // through a temporary attribute and a rename). + let mut ed = FileEditor::open(&b).unwrap(); + for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) { + let nv = match v { + AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()), + AV::Str(s) => AV::Str(format!("{s}-{k}")), + }; + let before = std::fs::read(&b).unwrap(); + match ed.set_attr(o, n, &nv.value()) { + Ok(()) => set_want(&mut want, o, n, nv), + Err(Error::Unsupported(msg)) => { + assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}"); + assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote"); + } + Err(e) => panic!("{tag}: replace {o}/{n}: {e}"), + } + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + // libhdf5 goes on: adds, deletes and rewrites. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['g', 't', 'd']:\n\ + \x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\ + \x20 del f[o].attrs['n3']\n\ + \x20 del f[o].attrs['c1']\n\ + \x20 f[o].attrs['n7'] = 'rewritten by h5py'\n", + p = b.to_str().unwrap() + )); + want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7"); + check_tools(&b, h5dump); + check_attr_values(&b, &want); +} + +#[test] +fn dense_attributes_match_libhdf5() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] + .iter() + .enumerate() + { + dense_workload(lv, *dump, &format!("{i}")); + } +} + +/// Dense attributes in files clawhdf5 writes (its own heap and index +/// layout, no free-space manager), with and without tracked creation +/// order: attributes added (moving a dataset's attributes to dense storage +/// too), rewritten and replaced, then h5py goes on. +#[test] +fn dense_attributes_on_clawhdf5_files() { + if !tools_ok() { + return; + } + for track in [false, true] { + let dir = tmpdir(); + let path = dir.path().join(format!("ours_dense_{track}.h5")); + let mut b = clawhdf5::FileBuilder::new(); + b.track_order(track); + for i in 0..12i64 { + b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3])); + } + b.create_dataset("d") + .with_i32_data(&[1, 2, 3]) + .set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5])); + b.write(&path).unwrap(); + let mut want: Vec = (0..12i64) + .map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3]))) + .collect(); + want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5]))); + let mut ed = FileEditor::open(&path).unwrap(); + for i in 0..30u64 { + for (k, o) in ["/", "d"].iter().enumerate() { + // Small attributes: a larger one needs a heap block bigger than + // the next (see dense_attribute_refusals_change_nothing). + let v = match AV::make(i, k as u64 + 7) { + AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()), + v => v, + }; + ed.set_attr(o, &format!("n{i}"), &v.value()) + .unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}")); + set_want(&mut want, o, &format!("n{i}"), v); + } + } + // Rewrites in place, then replacements of another size. + for i in (0..12i64).step_by(3) { + let v = AV::Ints(vec![-i; 3]); + ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap(); + set_want(&mut want, "/", &format!("r{i}"), v); + } + for i in (1..30u64).step_by(7) { + let v = AV::Str(format!("replaced {i}")); + match ed.set_attr("d", &format!("n{i}"), &v.value()) { + Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v), + Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"), + Err(e) => panic!("{e}"), + } + } + drop(ed); + check_tools(&path, true); + let want_py: Vec = want + .iter() + .map(|(o, n, v)| { + let o = if o == "/" { "/".to_string() } else { o.clone() }; + (o, n.clone(), v.clone()) + }) + .collect(); + check_root_and_attrs(&path, &want_py); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'd']:\n\ + \x20 f[o].attrs['from_h5py'] = np.arange(3)\n\ + \x20 del f[o].attrs['n2']\n", + p = path.to_str().unwrap() + )); + want.retain(|(_, n, _)| n != "n2"); + check_tools(&path, true); + check_root_and_attrs(&path, &want); + } +} + +/// `check_attr_values`, with the root group as "/". +fn check_root_and_attrs(path: &Path, want: &[AttrOp]) { + let f = File::open(path).unwrap(); + for (o, n, v) in want { + let attrs = match o.as_str() { + "/" => f.root().attrs().unwrap(), + "d" => f.dataset(o).unwrap().attrs().unwrap(), + _ => f.group(o).unwrap().attrs().unwrap(), + }; + let got = attrs + .get(n.as_str()) + .unwrap_or_else(|| panic!("{o}/{n} missing")); + match (got, v) { + (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"), + (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"), + (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"), + other => panic!("{o}/{n}: {other:?}"), + } + } + let exp: Vec = want + .iter() + .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect())) + .collect(); + let script = format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + want = [{w}]\n\ + for o, n, v in want:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 a = [a] if isinstance(a, int) else a\n\ + \x20 assert a == v, (o, n)\n\ + for o in set(x[0] for x in want):\n\ + \x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n", + p = path.to_str().unwrap(), + w = exp.join(", ") + ); + let sp = path.with_extension("check.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); +} + +/// What the editor refuses in dense storage — an object larger than the +/// next heap block (libhdf5 would skip blocks and record their space as +/// free, which this editor does not do) — is `Error::Unsupported`, and the +/// file is left byte for byte as it was. +#[test] +fn dense_attribute_refusals_change_nothing() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("dense_refuse.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n", + p = path.to_str().unwrap() + )); + let before = std::fs::read(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000)))); + drop(ed); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit wrote" + ); + // What libhdf5 does with it instead works on the untouched file. + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['g'].attrs['big'] = 'x' * 2000\n\ + \x20 assert len(f['g'].attrs) == 13\n", + p = path.to_str().unwrap() + )); + check_tools(&path, true); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 6a2dbf6..bc4c9a6 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -836,19 +836,16 @@ fn attributes_in_place() { .unwrap(); want.push(("d", "units".into(), AttrValue::String("km".into()))); if *lv != "'earliest'" { - // Up to the compact limit (8) and no further; attributes in - // dense storage and tracked creation order are refused, and a - // refused edit writes nothing. + // Up to the compact limit (8), then into dense storage; objects + // already in dense storage and ones tracking creation order. ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap(); want.push(("g", "eighth".into(), AttrValue::I64(8))); - let before = std::fs::read(&path).unwrap(); - unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9))); - unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1))); - unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1))); - assert!( - std::fs::read(&path).unwrap() == before, - "a refused edit changed the file" - ); + ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap(); + want.push(("g", "ninth".into(), AttrValue::I64(9))); + ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap(); + want.push(("dense", "k0".into(), AttrValue::I64(1))); + ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap(); + want.push(("tracked", "b".into(), AttrValue::I64(1))); } drop(ed); check_tools(&path, *dump); diff --git a/crates/clawhdf5/src/edit/attrs.rs b/crates/clawhdf5/src/edit/attrs.rs new file mode 100644 index 0000000..5772db3 --- /dev/null +++ b/crates/clawhdf5/src/edit/attrs.rs @@ -0,0 +1,513 @@ +//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do: +//! compact attributes are object header messages (with their creation +//! index in the message header when the object tracks creation order); +//! when an object reaches its compact limit (or an attribute is too large +//! for a header message) its attributes move to dense storage — a fractal +//! heap for the encoded messages, a version-2 B-tree indexing them by name +//! hash (record type 8) and, when creation order is indexed, a second one +//! by creation index (type 9) — and the Attribute Info message points at +//! them. + +use std::cmp::Ordering; + +use clawhdf5_format::attribute::AttributeMessage; +use clawhdf5_format::dataspace::DataspaceType; + +use crate::edit::btree2::Bt2; +use crate::edit::fheap::Heap; +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::edit::ohdr::{Header, MSG_ATTRIBUTE}; +use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain}; +use crate::error::Error; +use crate::reader::File; +use crate::types::AttrValue; + +/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage. +const MESG_MAX_SIZE: usize = 65536; +/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object +/// that does not track creation order. +const NO_CRT_IDX: u16 = u16::MAX; +/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`, +/// `H5A_CORDER_BT2_*`). +const NAME_BT2_TYPE: u8 = 8; +const CORDER_BT2_TYPE: u8 = 9; +const ATTR_BT2_NODE: u32 = 512; +/// Heap IDs in attribute records. +const ID_LEN: usize = 8; + +/// An object's Attribute Info message. +#[derive(Debug, Clone)] +struct AInfo { + /// Its message index in the header. + idx: usize, + track: bool, + index: bool, + max_crt: u16, + fheap: u64, + name_bt2: u64, + corder_bt2: u64, +} + +impl AInfo { + fn load(img: &Image<'_>, hdr: &Header) -> Result, Error> { + let Some(idx) = hdr.find(MSG_ATTR_INFO) else { + return Ok(None); + }; + if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute info message".into())); + } + let d = hdr.data(img, idx)?; + let os = img.os as usize; + let short = || Error::Unsupported("short attribute info message".into()); + if d.first() != Some(&0) { + return Err(Error::Unsupported("attribute info message version".into())); + } + let flags = *d.get(1).ok_or_else(short)?; + let track = flags & 0x01 != 0; + let index = flags & 0x02 != 0; + let mut p = 2; + let mut max_crt = 0; + if track { + let b = d.get(p..p + 2).ok_or_else(short)?; + max_crt = u16::from_le_bytes([b[0], b[1]]); + p += 2; + } + let n = if index { 3 } else { 2 }; + if d.len() < p + n * os { + return Err(short()); + } + Ok(Some(Self { + idx, + track, + index, + max_crt, + fheap: get_uint(&d[p..], img.os), + name_bt2: get_uint(&d[p + os..], img.os), + corder_bt2: if index { + get_uint(&d[p + 2 * os..], img.os) + } else { + undef(img.os) + }, + })) + } + + fn dense(&self, os: u8) -> bool { + self.fheap != undef(os) + } + + /// Store the changeable fields back into the message. + fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> { + let os = img.os as usize; + let mut p = 2; + if self.track { + hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?; + p += 2; + } + let mut a = vec![0u8; os]; + for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2] + .into_iter() + .enumerate() + .take(if self.index { 3 } else { 2 }) + { + put_uint(&mut a, v, img.os); + hdr.patch(img, self.idx, p + k * os, &a)?; + } + Ok(()) + } + + /// The next creation index (`H5O__attr_create`), or libhdf5's "none". + fn next_crt(&mut self) -> Result { + if !self.track { + return Ok(NO_CRT_IDX); + } + if self.max_crt == NO_CRT_IDX { + return Err(Error::Unsupported( + "object's attribute creation index is exhausted".into(), + )); + } + self.max_crt += 1; + Ok(self.max_crt - 1) + } +} + +/// The name bytes of an attribute message body (without the NUL). +pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> { + let bad = || Error::Unsupported("malformed attribute message".into()); + let (len, at) = match d.first() { + Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), + Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), + _ => return Err(bad()), + }; + let name = d.get(at..at + len).ok_or_else(bad)?; + Ok(name.split(|&b| b == 0).next().unwrap_or(name)) +} + +/// A new Attribute Info message for a version-2 header with flags +/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order +/// tracked / indexed as the header's flags say, the maximum creation index, +/// and no dense storage (undefined fractal heap and B-tree addresses). +fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec { + let track = hdr_flags & 0x04 != 0; + let index = hdr_flags & 0x08 != 0; + let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; + if track { + b.extend_from_slice(&max_crt.to_le_bytes()); + } + let undef_addr = vec![0xffu8; os as usize]; + b.extend_from_slice(&undef_addr); + b.extend_from_slice(&undef_addr); + if index { + b.extend_from_slice(&undef_addr); + } + b +} + +/// A version-2 header's limit on compact attributes: stored when its flags +/// say so, else libhdf5's default of 8. +fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { + if hdr.flags & 0x10 == 0 { + return Ok(8); + } + let mut p = hdr.addr + 6; + if hdr.flags & 0x20 != 0 { + p += 16; + } + let b = img.read(p, 2)?; + Ok(u16::from_le_bytes([b[0], b[1]])) +} + +/// A version-1 attribute message (what libhdf5 writes in a version-1 object +/// header): name, datatype and dataspace each padded to 8 bytes, the +/// dataspace as a version-1 dataspace message. +fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { + let mut name = a.name.as_bytes().to_vec(); + name.push(0); + let dt = a.datatype.serialize(); + let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; + if a.dataspace.space_type == DataspaceType::Simple { + let mut b = vec![0u8; ls as usize]; + for &d in &a.dataspace.dimensions { + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + if let Some(max) = &a.dataspace.max_dimensions { + ds[2] = 0x01; + for &d in max { + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + } + } else { + ds[1] = 0; + } + let mut out = vec![1u8, 0]; + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); + out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); + for part in [&name, &dt, &ds] { + out.extend_from_slice(part); + out.resize(out.len().next_multiple_of(8), 0); + } + out.extend_from_slice(&a.raw_data); + out +} + +/// Dense storage opened for changes. +struct Dense { + heap: Heap, + names: Bt2, + order: Option, +} + +/// `H5_checksum_lookup3` of a name, as the name index keys it. +fn name_hash(name: &[u8]) -> u32 { + clawhdf5_format::checksum::jenkins_lookup3(name) +} + +/// Compare attribute `name` (hash `hash`) with a name-index record +/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name). +fn cmp_name( + heap: &Heap, + img: &Image<'_>, + hash: u32, + name: &[u8], + rec: &[u8], +) -> Result { + let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]); + match hash.cmp(&theirs) { + Ordering::Equal => { + if rec[ID_LEN] & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported( + "shared attribute in dense storage".into(), + )); + } + let obj = heap.read(img, &rec[..ID_LEN])?; + Ok(name.cmp(attr_name(&obj)?)) + } + o => Ok(o), + } +} + +fn corder_of(rec: &[u8]) -> u32 { + u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]]) +} + +impl Dense { + fn open(img: &Image<'_>, ai: &AInfo) -> Result { + let heap = Heap::open(img, ai.fheap)?; + let names = Bt2::open(img, ai.name_bt2)?; + if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 { + return Err(Error::Unsupported("attribute name index layout".into())); + } + let order = if ai.index { + let t = Bt2::open(img, ai.corder_bt2)?; + if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 { + return Err(Error::Unsupported( + "attribute creation-order index layout".into(), + )); + } + Some(t) + } else { + None + }; + Ok(Self { heap, names, order }) + } + + /// `H5A__dense_create`: heap, name index, [creation-order index]. + fn create(img: &mut Image<'_>, index: bool) -> Result { + let heap = Heap::create_attribute_heap(img)?; + let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?; + let order = if index { + Some(Bt2::create( + img, + CORDER_BT2_TYPE, + ATTR_BT2_NODE, + ID_LEN + 5, + 100, + 40, + )?) + } else { + None + }; + Ok(Self { heap, names, order }) + } + + /// `H5A__dense_insert` of an encoded attribute message. + fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> { + let name = attr_name(body)?.to_vec(); + let id = self.heap.insert(img, body)?; + if id.len() != ID_LEN { + return Err(Error::Unsupported("attribute heap ID length".into())); + } + let hash = name_hash(&name); + let mut rec = id.clone(); + rec.push(0); + rec.extend_from_slice(&u32::from(crt).to_le_bytes()); + rec.extend_from_slice(&hash.to_le_bytes()); + let heap = &self.heap; + self.names + .insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?; + if let Some(t) = &mut self.order { + let key = u32::from(crt); + t.insert( + img, + &mut |_, r| Ok(key.cmp(&corder_of(r))), + &rec[..ID_LEN + 5], + )?; + } + Ok(()) + } + + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + self.heap.finish(img)?; + self.names.finish(img)?; + if let Some(t) = &mut self.order { + t.finish(img)?; + } + Ok(()) + } +} + +/// Set attribute `name` of the object at `path` to `value`. +pub(super) fn set_attr( + f: &File, + img: &mut Image<'_>, + path: &str, + name: &str, + value: &AttrValue, +) -> Result<(), Error> { + let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let mut hdr = Header::load(img, addr)?; + let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value); + check_plain(&msg.datatype)?; + // libhdf5 encodes a simple dataspace with its maximum dimensions (the + // current ones when none were given), so an attribute takes the same + // space in a header or heap as when libhdf5 writes it. + if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() { + msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone()); + } + // H5A__set_version: version 1 unless the name is not ASCII (then 3), + // raised to the file's low bound — which is the earliest for a file + // libhdf5 opens without a libver setting (h5py's `r+`). + let body = if hdr.version == 1 || name.is_ascii() { + encode_attr_v1(&msg, img.ls) + } else { + let mut b = msg.serialize_v3(img.ls); + if !name.is_ascii() { + b[8] = 1; // UTF-8 name + } + b + }; + let mut ainfo = if hdr.version == 2 { + AInfo::load(img, &hdr)? + } else { + None + }; + if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) { + let mut ai = ai.clone(); + set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?; + return hdr.finish(img); + } + + let mut existing = None; + let mut count = 0usize; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + count += 1; + if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { + existing = Some(i); + } + } + if let Some(i) = existing { + hdr.delete(img, i)?; + count -= 1; + } + if hdr.version == 1 { + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + return hdr.finish(img); + } + let tracked = hdr.flags & 0x04 != 0; + // H5O__attr_create: a missing Attribute Info message starts from + // nothing (and is added below, holding the new maximum creation index). + let new_ainfo = ainfo.is_none(); + let mut ai = ainfo.take().unwrap_or(AInfo { + idx: usize::MAX, + track: tracked, + index: hdr.flags & 0x08 != 0, + max_crt: 0, + fheap: undef(img.os), + name_bt2: undef(img.os), + corder_bt2: undef(img.os), + }); + let max_compact = usize::from(max_compact_attrs(img, &hdr)?); + if count == max_compact || body.len() >= MESG_MAX_SIZE { + if new_ainfo { + return Err(Error::Unsupported( + "dense attribute storage for an object without an Attribute Info message".into(), + )); + } + to_dense(img, &mut hdr, &mut ai)?; + set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?; + return hdr.finish(img); + } + let crt = ai.next_crt()?; + let corder = tracked.then_some(crt); + if new_ainfo { + // libhdf5 appends the Attribute Info message before the attribute + // when free space holds both, else after it, so that a new + // continuation chunk made for the attribute has room for it too. + let a = attr_info_message(hdr.flags, ai.max_crt, img.os); + let first = hdr.has_free(a.len() + hdr.hsize() + body.len()); + if first { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?; + } + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?; + if !first { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?; + } + } else { + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?; + ai.store(img, &mut hdr)?; + } + hdr.finish(img) +} + +/// Move every compact attribute of the object into new dense storage, in +/// header message order (`H5O__attr_to_dense_cb`), leaving free space where +/// the messages were. +fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> { + let mut dense = Dense::create(img, ai.index)?; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + let body = hdr.data(img, i)?; + let crt = if ai.track { + hdr.msgs[i].corder.unwrap_or(0) + } else { + NO_CRT_IDX + }; + dense.insert(img, &body, crt)?; + hdr.delete(img, i)?; + } + dense.finish(img)?; + ai.fheap = dense.heap.address(); + ai.name_bt2 = dense.names.address(); + if let Some(t) = &dense.order { + ai.corder_bt2 = t.address(); + } + ai.store(img, hdr) +} + +/// Set an attribute of an object whose attributes are in dense storage: an +/// attribute of that name whose new encoding has the old one's size is +/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one +/// is removed (`H5A__dense_remove`: name index, creation-order index, heap +/// object) and the new one inserted with the next creation index. +fn set_dense( + img: &mut Image<'_>, + hdr: &mut Header, + ai: &mut AInfo, + name: &[u8], + body: &[u8], +) -> Result<(), Error> { + let mut dense = Dense::open(img, ai)?; + let hash = name_hash(name); + let found = { + let heap = &dense.heap; + dense + .names + .find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))? + }; + if let Some(rec) = found { + if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? { + return dense.finish(img); + } + { + let heap = &dense.heap; + dense + .names + .remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?; + } + if let Some(t) = &mut dense.order { + let key = corder_of(&rec); + t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))? + .ok_or_else(|| { + Error::Unsupported("attribute missing from its creation-order index".into()) + })?; + } + dense.heap.remove(img, &rec[..ID_LEN])?; + } + let crt = ai.next_crt()?; + dense.insert(img, body, crt)?; + dense.finish(img)?; + ai.store(img, hdr) +} diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs index 8ff7205..80be9f0 100644 --- a/crates/clawhdf5/src/edit/btree2.rs +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -180,11 +180,6 @@ impl Bt2 { self.tree_type } - /// Records in the whole tree. - pub(crate) fn len(&self) -> u64 { - self.root.all - } - /// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`, /// `H5B2__split_root`). fn compute_levels(&mut self, os: u8) -> Result<(), Error> { @@ -389,35 +384,6 @@ impl Bt2 { } } - /// Every record, in key order. - pub(crate) fn records(&mut self, img: &Image<'_>) -> Result>, Error> { - let mut out = Vec::new(); - if self.root.addr != undef(img.os) && self.root.nrec > 0 { - self.walk(img, self.root, self.depth, &mut out)?; - } - Ok(out) - } - - fn walk( - &mut self, - img: &Image<'_>, - p: Ptr, - depth: u16, - out: &mut Vec>, - ) -> Result<(), Error> { - self.load(img, p, depth)?; - let n = self.peek(p.addr).clone(); - for i in 0..=n.recs.len() { - if depth > 0 { - self.walk(img, n.ptrs[i], depth - 1, out)?; - } - if i < n.recs.len() { - out.push(n.recs[i].clone()); - } - } - Ok(()) - } - /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). pub(crate) fn update( &mut self, @@ -665,6 +631,7 @@ impl Bt2 { /// parent records between them) out over the children `lo..` of /// `parent`, with `counts[k]` records in the k-th; the records between /// them go back into the parent. + #[allow(clippy::too_many_arguments)] fn relayout( &mut self, parent: u64, diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs new file mode 100644 index 0000000..7c7b72a --- /dev/null +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -0,0 +1,1121 @@ +//! Changing a fractal heap (`FRHP`) in place, as libhdf5's `H5HF` code does +//! for the heaps that hold dense attributes: managed objects go into the +//! best-fitting free section the heap's free-space manager records (a new +//! direct block when none fits: the root direct block of an empty heap, the +//! next block of the root indirect block — created from the root direct +//! block, and doubled, as needed), huge objects (larger than the heap's +//! managed maximum) into their own file space tracked by the huge-object +//! version-2 B-tree; removed objects return their space to the free-space +//! manager, merged with adjacent free space. +//! +//! The heap's free-space manager (`FSHD` header, `FSSE` section info) is +//! kept exactly as libhdf5 keeps it: sections sorted by size then offset, +//! the header's counts and sizes, section info moved when its size changes, +//! the manager deleted when it tracks nothing. Header statistics (managed +//! space, allocated space, free space, allocation iterator, object counts) +//! follow libhdf5's arithmetic. +//! +//! What libhdf5 would do differently is refused ([`Error::Unsupported`], +//! before anything is written): I/O filters on the heap, child indirect +//! blocks, skipped blocks (a first object too large for the next block), +//! free sections other than "single" ones, freeing a whole direct block, +//! directly addressed huge objects, tiny objects. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::edit::btree2::Bt2; +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +fn unsupported(why: &str) -> Error { + Error::Unsupported(format!("fractal heap: {why}")) +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("fractal heap: {why}"), + )) +} + +/// `H5VM_log2_gen`: floor(log2(v)). +fn log2(v: u64) -> u32 { + 63 - v.max(1).leading_zeros() +} + +/// `H5VM_limit_enc_size`. +fn enc_size(v: u64) -> usize { + (log2(v) / 8 + 1) as usize +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} + +/// Free-space client ID for fractal heaps. +const FS_CLIENT_FHEAP: u8 = 0; +/// `H5HF_FSPACE_SHRINK` / `H5HF_FSPACE_EXPAND`. +const FS_SHRINK: u16 = 80; +const FS_EXPAND: u16 = 120; +/// Section classes the heap registers (single, first row, normal row, +/// indirect). +const FS_NCLASSES: u16 = 4; +/// Huge-object B-tree (`H5HF_HUGE_BT2_*`): record type 1 (indirectly +/// accessed, unfiltered), node size and split/merge percentages. +const HUGE_BT2_TYPE: u8 = 1; +const HUGE_BT2_NODE: u32 = 512; + +/// The heap's free-space manager: only "single" sections (free space inside +/// a direct block), keyed by heap offset. +struct FreeSpace { + addr: u64, + max_sect_addr: u16, + max_sect_size: u64, + shrink: u16, + expand: u16, + nclasses: u16, + sect_addr: u64, + sect_size: u64, + alloc_sect_size: u64, + sects: BTreeMap, +} + +impl FreeSpace { + fn hdr_len(os: u8, ls: u8) -> usize { + 6 + 4 * ls as usize + 8 + ls as usize + os as usize + 2 * ls as usize + 4 + } + + fn open(img: &Image<'_>, addr: u64) -> Result { + let (os, ls) = (img.os, img.ls); + let len = Self::hdr_len(os, ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"FSHD" || d[4] != 0 { + return Err(bad("bad free-space header")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("free-space header checksum mismatch")); + } + if d[5] != FS_CLIENT_FHEAP { + return Err(bad("free-space manager of another client")); + } + let l = ls as usize; + let mut p = 6; + let mut next = |w: usize| { + let v = le(&d[p..p + w]); + p += w; + v + }; + let tot_space = next(l); + let tot_count = next(l); + let serial = next(l); + let ghost = next(l); + let nclasses = next(2) as u16; + let shrink = next(2) as u16; + let expand = next(2) as u16; + let max_sect_addr = next(2) as u16; + let max_sect_size = next(l); + let sect_addr = next(os as usize); + let sect_size = next(l); + let alloc_sect_size = next(l); + if ghost != 0 || tot_count != serial { + return Err(unsupported("free space in row or indirect sections")); + } + let mut fs = Self { + addr, + max_sect_addr, + max_sect_size, + shrink, + expand, + nclasses, + sect_addr, + sect_size, + alloc_sect_size, + sects: BTreeMap::new(), + }; + if serial == 0 { + return Ok(fs); + } + if sect_addr == undef(os) || sect_size < 9 + os as u64 || sect_size > alloc_sect_size { + return Err(bad("bad free-space section info")); + } + let n = usize::try_from(sect_size).map_err(|_| bad("section info too large"))?; + let s = img.read(sect_addr, n)?; + let stored = u32::from_le_bytes(s[n - 4..].try_into().unwrap_or([0; 4])); + if &s[0..4] != b"FSSE" + || s[4] != 0 + || get_uint(&s[5..], os) != addr + || clawhdf5_format::checksum::jenkins_lookup3(&s[..n - 4]) != stored + { + return Err(bad("bad free-space section info")); + } + let cnt_w = enc_size(serial); + let len_w = enc_size(max_sect_size); + let off_w = (usize::from(max_sect_addr)).div_ceil(8); + let mut q = 5 + os as usize; + let mut seen = 0u64; + let mut total = 0u64; + while seen < serial { + if q + cnt_w + len_w > n - 4 { + return Err(bad("section info ends early")); + } + let count = le(&s[q..q + cnt_w]); + q += cnt_w; + let size = le(&s[q..q + len_w]); + q += len_w; + if count == 0 || size == 0 { + return Err(bad("empty section size node")); + } + for _ in 0..count { + if q + off_w + 1 > n - 4 { + return Err(bad("section info ends early")); + } + let off = le(&s[q..q + off_w]); + let class = s[q + off_w]; + q += off_w + 1; + if class != 0 { + return Err(unsupported("free space in row or indirect sections")); + } + if fs.sects.insert(off, size).is_some() { + return Err(bad("duplicate free section")); + } + seen += 1; + total += size; + } + } + if total != tot_space { + return Err(bad("free-space total disagrees with its sections")); + } + Ok(fs) + } + + /// Best fit (`H5FS__sect_find_node`): the smallest section of at least + /// `size` bytes, the lowest offset among equals. + fn find(&self, size: u64) -> Option<(u64, u64)> { + self.sects + .iter() + .filter(|&(_, &s)| s >= size) + .min_by_key(|&(&o, &s)| (s, o)) + .map(|(&o, &s)| (o, s)) + } + + /// The serialized section info's size (`H5FS__sect_serialize_size`). + fn needed(&self, os: u8) -> u64 { + let n = self.sects.len() as u64; + let prefix = 4 + 1 + u64::from(os) + 4; + if n == 0 { + return prefix; + } + let sizes: BTreeSet = self.sects.values().copied().collect(); + prefix + + sizes.len() as u64 * (enc_size(n) + enc_size(self.max_sect_size)) as u64 + + n * (u64::from(self.max_sect_addr).div_ceil(8) + 1) + } + + fn serialize(&self, os: u8) -> Vec { + let n = self.sects.len() as u64; + let mut by_size: BTreeMap> = BTreeMap::new(); + for (&o, &s) in &self.sects { + by_size.entry(s).or_default().push(o); + } + let cnt_w = enc_size(n); + let len_w = enc_size(self.max_sect_size); + let off_w = usize::from(self.max_sect_addr).div_ceil(8); + let mut d = Vec::new(); + d.extend_from_slice(b"FSSE"); + d.push(0); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.addr, os); + d.extend_from_slice(&a); + for (size, offs) in by_size { + d.extend_from_slice(&(offs.len() as u64).to_le_bytes()[..cnt_w]); + d.extend_from_slice(&size.to_le_bytes()[..len_w]); + for o in offs { + d.extend_from_slice(&o.to_le_bytes()[..off_w]); + d.push(0); + } + } + d + } + + fn write_header(&self, img: &mut Image<'_>) -> Result<(), Error> { + let (os, ls) = (img.os, img.ls); + let len = Self::hdr_len(os, ls); + let l = ls as usize; + let n = self.sects.len() as u64; + let tot: u64 = self.sects.values().sum(); + let mut d = Vec::with_capacity(len); + d.extend_from_slice(b"FSHD"); + d.push(0); + d.push(FS_CLIENT_FHEAP); + for v in [tot, n, n, 0] { + d.extend_from_slice(&v.to_le_bytes()[..l]); + } + for v in [self.nclasses, self.shrink, self.expand, self.max_sect_addr] { + d.extend_from_slice(&v.to_le_bytes()); + } + d.extend_from_slice(&self.max_sect_size.to_le_bytes()[..l]); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.sect_addr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&self.sect_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.alloc_sect_size.to_le_bytes()[..l]); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d) + } +} + +/// An open fractal heap. +pub(crate) struct Heap { + addr: u64, + id_len: u16, + flags: u8, + max_man_size: u32, + next_huge_id: u64, + huge_bt2: u64, + total_man_free: u64, + fs_addr: u64, + man_size: u64, + man_alloc_size: u64, + man_iter_off: u64, + man_nobjs: u64, + huge_size: u64, + huge_nobjs: u64, + tiny_size: u64, + tiny_nobjs: u64, + width: u16, + start_block_size: u64, + max_direct_size: u64, + max_index: u16, + start_root_rows: u16, + root: u64, + root_rows: u16, + heap_off_size: usize, + heap_len_size: usize, + max_direct_rows: usize, + max_root_rows: usize, + /// Root indirect block children (all rows), when the root is one. + ents: Vec, + iblock_dirty: bool, + fs: Option, + fs_dirty: bool, + /// Direct blocks changed (address -> size), rechecksummed by `finish`. + dirty_dblocks: BTreeMap, +} + +impl Heap { + /// Header size (unfiltered heaps): the fixed fields, ten lengths and + /// two addresses of statistics, the doubling table, the checksum. + fn header_len(os: u8, ls: u8) -> usize { + 26 + 12 * ls as usize + 3 * os as usize + } + + /// Open the heap whose header is at `addr`. + pub(crate) fn open(img: &Image<'_>, addr: u64) -> Result { + let (os, ls) = (img.os, img.ls); + let o = os as usize; + let l = ls as usize; + let len = Self::header_len(os, ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"FRHP" || d[4] != 0 { + return Err(bad("bad header")); + } + let filter_len = u16::from_le_bytes([d[7], d[8]]); + if filter_len != 0 { + return Err(unsupported("heaps with I/O filters")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let mut p = 14usize; + let mut next = |w: usize| { + let v = le(&d[p..p + w]); + p += w; + v + }; + let next_huge_id = next(l); + let huge_bt2 = next(o); + let total_man_free = next(l); + let fs_addr = next(o); + let man_size = next(l); + let man_alloc_size = next(l); + let man_iter_off = next(l); + let man_nobjs = next(l); + let huge_size = next(l); + let huge_nobjs = next(l); + let tiny_size = next(l); + let tiny_nobjs = next(l); + let width = next(2) as u16; + let start_block_size = next(l); + let max_direct_size = next(l); + let max_index = next(2) as u16; + let start_root_rows = next(2) as u16; + let root = next(o); + let root_rows = next(2) as u16; + let pow2 = |v: u64| v != 0 && v.is_power_of_two(); + if !pow2(u64::from(width)) + || !pow2(start_block_size) + || !pow2(max_direct_size) + || max_direct_size < start_block_size + || !(1..=64).contains(&max_index) + { + return Err(bad("bad doubling table")); + } + let start_bits = log2(start_block_size) as usize; + let first_row_bits = start_bits + log2(u64::from(width)) as usize; + if usize::from(max_index) < first_row_bits { + return Err(bad("bad doubling table")); + } + let max_direct_rows = log2(max_direct_size) as usize - start_bits + 2; + let max_root_rows = usize::from(max_index) - first_row_bits + 1; + let heap_off_size = usize::from(max_index).div_ceil(8); + // H5HF__hdr_finish_init_phase1: object lengths are encoded in the + // bytes a direct block offset, or a managed object's size, needs. + let max_man_size = u32::from_le_bytes([d[10], d[11], d[12], d[13]]); + let len_bits = |v: u64| (log2(v) as usize).div_ceil(8); + let heap_len_size = len_bits(max_direct_size).min(len_bits(u64::from(max_man_size))); + let id_len = u16::from_le_bytes([d[5], d[6]]); + let mut h = Self { + addr, + id_len, + flags: d[9], + max_man_size, + next_huge_id, + huge_bt2, + total_man_free, + fs_addr, + man_size, + man_alloc_size, + man_iter_off, + man_nobjs, + huge_size, + huge_nobjs, + tiny_size, + tiny_nobjs, + width, + start_block_size, + max_direct_size, + max_index, + start_root_rows, + root, + root_rows, + heap_off_size, + heap_len_size, + max_direct_rows, + max_root_rows, + ents: Vec::new(), + iblock_dirty: false, + fs: None, + fs_dirty: false, + dirty_dblocks: BTreeMap::new(), + }; + if usize::from(id_len) != 1 + heap_off_size + heap_len_size && id_len < 8 { + return Err(unsupported("unusual heap ID length")); + } + if root_rows > 0 { + if root == undef(os) || usize::from(root_rows) > max_root_rows { + return Err(bad("bad root indirect block")); + } + let n = usize::from(root_rows) * usize::from(width); + let blen = h.iblock_len(root_rows, os); + let b = img.read(root, blen)?; + let stored = u32::from_le_bytes(b[blen - 4..].try_into().unwrap_or([0; 4])); + if &b[0..4] != b"FHIB" + || b[4] != 0 + || get_uint(&b[5..], os) != addr + || le(&b[5 + o..5 + o + heap_off_size]) != 0 + || clawhdf5_format::checksum::jenkins_lookup3(&b[..blen - 4]) != stored + { + return Err(bad("bad root indirect block")); + } + let at = 5 + o + heap_off_size; + h.ents = (0..n).map(|i| get_uint(&b[at + i * o..], os)).collect(); + let direct = h.max_direct_rows * usize::from(width); + if h.ents.iter().skip(direct).any(|&a| a != undef(os)) { + return Err(unsupported("child indirect blocks")); + } + } + if fs_addr != undef(os) { + let fs = FreeSpace::open(img, fs_addr)?; + if fs.max_sect_addr != max_index { + return Err(bad("free-space manager does not match the heap")); + } + h.fs = Some(fs); + } + Ok(h) + } + + /// Create an empty heap with libhdf5's attribute-heap parameters + /// (`H5A__dense_create`: width 4, 1 KiB starting blocks, 64 KiB direct + /// blocks, 40-bit offsets, 1 starting root row, checksummed direct + /// blocks, 4 KiB managed objects, 8-byte IDs). + pub(crate) fn create_attribute_heap(img: &mut Image<'_>) -> Result { + let (os, ls) = (img.os, img.ls); + let addr = img.alloc_reusing(Self::header_len(os, ls) as u64)?; + let mut h = Self { + addr, + id_len: 8, + flags: 0x02, + max_man_size: 4096, + next_huge_id: 0, + huge_bt2: undef(os), + total_man_free: 0, + fs_addr: undef(os), + man_size: 0, + man_alloc_size: 0, + man_iter_off: 0, + man_nobjs: 0, + huge_size: 0, + huge_nobjs: 0, + tiny_size: 0, + tiny_nobjs: 0, + width: 4, + start_block_size: 1024, + max_direct_size: 65536, + max_index: 40, + start_root_rows: 1, + root: undef(os), + root_rows: 0, + heap_off_size: 5, + heap_len_size: 2, + max_direct_rows: 8, + max_root_rows: 29, + ents: Vec::new(), + iblock_dirty: false, + fs: None, + fs_dirty: false, + dirty_dblocks: BTreeMap::new(), + }; + h.write_header(img)?; + Ok(h) + } + + pub(crate) fn address(&self) -> u64 { + self.addr + } + + fn overhead(&self, os: u8) -> u64 { + 5 + u64::from(os) + self.heap_off_size as u64 + if self.flags & 0x02 != 0 { 4 } else { 0 } + } + + fn row_size(&self, row: usize) -> u64 { + if row == 0 { + self.start_block_size + } else { + self.start_block_size << (row - 1) + } + } + + fn row_off(&self, row: usize) -> u64 { + if row == 0 { + 0 + } else { + (self.start_block_size * u64::from(self.width)) << (row - 1) + } + } + + fn iblock_len(&self, rows: u16, os: u8) -> usize { + 5 + os as usize + + self.heap_off_size + + usize::from(rows) * usize::from(self.width) * os as usize + + 4 + } + + fn first_row_bits(&self) -> u32 { + log2(self.start_block_size) + log2(u64::from(self.width)) + } + + /// Row and column of heap offset `off` in the root indirect block + /// (`H5HF__dtable_lookup`). + fn lookup(&self, off: u64) -> (usize, usize) { + let w = u64::from(self.width); + if off < self.start_block_size * w { + (0, (off / self.start_block_size) as usize) + } else { + let hb = log2(off); + let row = (hb - self.first_row_bits() + 1) as usize; + (row, ((off - (1u64 << hb)) / self.row_size(row)) as usize) + } + } + + /// The direct block holding heap offset `off`: (address, size, heap + /// offset of the block). + fn dblock_of(&self, os: u8, off: u64) -> Result<(u64, u64, u64), Error> { + if self.root == undef(os) { + return Err(bad("empty heap")); + } + if self.root_rows == 0 { + if off >= self.start_block_size { + return Err(bad("offset outside the heap")); + } + return Ok((self.root, self.start_block_size, 0)); + } + let (row, col) = self.lookup(off); + if row >= self.max_direct_rows || row >= usize::from(self.root_rows) { + return Err(unsupported("objects below child indirect blocks")); + } + let a = self.ents[row * usize::from(self.width) + col]; + if a == undef(os) { + return Err(bad("object in an unallocated block")); + } + Ok(( + a, + self.row_size(row), + self.row_off(row) + self.row_size(row) * col as u64, + )) + } + + /// Decode a managed heap ID: (offset, length). + fn man_id(&self, id: &[u8]) -> (u64, u64) { + let o = &id[1..]; + ( + le(&o[..self.heap_off_size]), + le(&o[self.heap_off_size..self.heap_off_size + self.heap_len_size]), + ) + } + + fn huge_id(&self, id: &[u8], os: u8, ls: u8) -> Result { + let w = usize::from(self.id_len - 1).min(8); + if self.huge_ids_direct(os, ls) { + return Err(unsupported("directly addressed huge objects")); + } + Ok(le(&id[1..1 + w])) + } + + fn huge_ids_direct(&self, os: u8, ls: u8) -> bool { + usize::from(os) + usize::from(ls) < usize::from(self.id_len) + } + + /// Where an object lives: (file address, length). + fn locate(&self, img: &Image<'_>, id: &[u8]) -> Result<(u64, u64), Error> { + if id.len() != usize::from(self.id_len) || id[0] >> 6 != 0 { + return Err(bad("bad heap ID")); + } + match (id[0] >> 4) & 0x03 { + 0 => { + let (off, len) = self.man_id(id); + let (a, size, boff) = self.dblock_of(img.os, off)?; + let within = off - boff; + if within < self.overhead(img.os) || within + len > size { + return Err(bad("object outside its block")); + } + Ok((a + within, len)) + } + 1 => { + if self.huge_ids_direct(img.os, img.ls) { + return Err(unsupported("directly addressed huge objects")); + } + let key = self.huge_id(id, img.os, img.ls)?; + let mut t = Bt2::open(img, self.huge_bt2)?; + let rec = t + .find( + img, + &mut |_, r| Ok(key.cmp(&huge_rec_id(r, img.os, img.ls))), + )? + .ok_or_else(|| bad("huge object missing from its B-tree"))?; + Ok(( + get_uint(&rec, img.os), + get_uint(&rec[img.os as usize..], img.ls), + )) + } + _ => Err(unsupported("tiny objects")), + } + } + + /// The object `id` names. + pub(crate) fn read(&self, img: &Image<'_>, id: &[u8]) -> Result, Error> { + let (a, len) = self.locate(img, id)?; + let n = usize::try_from(len).map_err(|_| bad("object too large"))?; + img.read(a, n) + } + + /// Overwrite managed object `id` with `obj` of the same length + /// (`H5HF_write`). + pub(crate) fn write_in_place( + &mut self, + img: &mut Image<'_>, + id: &[u8], + obj: &[u8], + ) -> Result { + if (id[0] >> 4) & 0x03 == 1 { + // H5HF__huge_write: an unfiltered huge object in place. + let (a, len) = self.locate(img, id)?; + if len != obj.len() as u64 { + return Ok(false); + } + img.write(a, obj)?; + return Ok(true); + } + if (id[0] >> 4) & 0x03 != 0 { + return Ok(false); + } + let (off, len) = self.man_id(id); + if len != obj.len() as u64 { + return Ok(false); + } + let (a, size, boff) = self.dblock_of(img.os, off)?; + img.write(a + (off - boff), obj)?; + self.dirty_dblocks.insert(a, size); + Ok(true) + } + + /// Insert `obj`; returns its heap ID (`H5HF_insert`). + pub(crate) fn insert(&mut self, img: &mut Image<'_>, obj: &[u8]) -> Result, Error> { + let os = img.os; + let n = obj.len() as u64; + if n == 0 { + return Err(bad("empty object")); + } + if n > u64::from(self.max_man_size) { + return self.huge_insert(img, obj); + } + let tiny_max = u64::from(self.id_len) - 1; + if n <= tiny_max { + return Err(unsupported("tiny objects")); + } + let (off, size) = match self.fs.as_ref().and_then(|fs| fs.find(n)) { + Some((o, s)) => { + self.fs.as_mut().expect("found above").sects.remove(&o); + (o, s) + } + None => self.dblock_new(img, n)?, + }; + // H5HF__sect_single_reduce: the object goes at the section's start. + if size > n { + self.fs_add(img, off + n, size - n)?; + } + self.fs_dirty = true; + let (a, bsize, boff) = self.dblock_of(os, off)?; + img.write(a + (off - boff), obj)?; + self.dirty_dblocks.insert(a, bsize); + self.man_nobjs += 1; + self.total_man_free = self.total_man_free.saturating_sub(n); + let mut id = vec![0u8; usize::from(self.id_len)]; + id[1..1 + self.heap_off_size].copy_from_slice(&off.to_le_bytes()[..self.heap_off_size]); + id[1 + self.heap_off_size..1 + self.heap_off_size + self.heap_len_size] + .copy_from_slice(&n.to_le_bytes()[..self.heap_len_size]); + Ok(id) + } + + /// Add a single section (no merging: `H5FS_sect_add` without + /// `H5FS_ADD_RETURNED_SPACE`), creating the free-space manager if the + /// heap has none. + fn fs_add(&mut self, img: &mut Image<'_>, off: u64, size: u64) -> Result<(), Error> { + if self.fs.is_none() { + let addr = img.alloc_reusing(FreeSpace::hdr_len(img.os, img.ls) as u64)?; + self.fs = Some(FreeSpace { + addr, + max_sect_addr: self.max_index, + max_sect_size: self.max_direct_size, + shrink: FS_SHRINK, + expand: FS_EXPAND, + nclasses: FS_NCLASSES, + sect_addr: undef(img.os), + sect_size: 0, + alloc_sect_size: 0, + sects: BTreeMap::new(), + }); + self.fs_addr = addr; + } + self.fs + .as_mut() + .expect("created above") + .sects + .insert(off, size); + self.fs_dirty = true; + Ok(()) + } + + /// `H5HF__man_dblock_new`: a direct block for an object of `request` + /// bytes; returns its free section (not in the free-space manager). + fn dblock_new(&mut self, img: &mut Image<'_>, request: u64) -> Result<(u64, u64), Error> { + let os = img.os; + let mut min = if request < self.start_block_size { + self.start_block_size + } else { + 1u64 << (1 + log2(request)) + }; + if min < self.overhead(os) + request { + min *= 2; + } + if min > self.max_direct_size { + return Err(bad("object larger than a direct block")); + } + if self.root == undef(os) && min == self.start_block_size { + let a = self.dblock_create(img, 0, self.start_block_size)?; + self.root = a; + self.root_rows = 0; + self.man_size = self.start_block_size; + self.total_man_free += self.start_block_size - self.overhead(os); + return Ok((self.overhead(os), self.start_block_size - self.overhead(os))); + } + if self.root == undef(os) { + return Err(unsupported( + "a first object too large for the starting block", + )); + } + // H5HF__hdr_update_iter. + if self.root_rows == 0 { + self.root_create(img, min)?; + } + let (mut row, mut col) = self.iter_pos(); + let min_row = self.size_to_row(min); + if min_row > row && row < usize::from(self.root_rows) { + return Err(unsupported("skipping blocks too small for an object")); + } + while row >= usize::from(self.root_rows) { + self.root_double(img, min)?; + (row, col) = self.iter_pos(); + } + if row >= self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + let size = self.row_size(row); + if min > size { + return Err(unsupported("skipping blocks too small for an object")); + } + self.man_iter_off += size; + let entry = row * usize::from(self.width) + col; + let block_off = self.row_off(row) + size * col as u64; + let a = self.dblock_create(img, block_off, size)?; + self.ents[entry] = a; + self.iblock_dirty = true; + Ok((block_off + self.overhead(os), size - self.overhead(os))) + } + + /// The allocation iterator's (row, column) in the root indirect block. + fn iter_pos(&self) -> (usize, usize) { + if self.man_iter_off >= self.man_size { + (usize::from(self.root_rows), 0) + } else { + self.lookup(self.man_iter_off) + } + } + + /// `H5HF__dtable_size_to_row`. + fn size_to_row(&self, size: u64) -> usize { + if size == self.start_block_size { + 0 + } else { + (log2(size) - log2(self.start_block_size) + 1) as usize + } + } + + /// Allocate and write an empty direct block at heap offset `block_off`. + fn dblock_create( + &mut self, + img: &mut Image<'_>, + block_off: u64, + size: u64, + ) -> Result { + let os = img.os; + let a = img.alloc_reusing(size)?; + let n = usize::try_from(size).map_err(|_| bad("block too large"))?; + let mut d = vec![0u8; n]; + d[0..4].copy_from_slice(b"FHDB"); + put_uint(&mut d[5..], self.addr, os); + let at = 5 + os as usize; + d[at..at + self.heap_off_size] + .copy_from_slice(&block_off.to_le_bytes()[..self.heap_off_size]); + img.write(a, &d)?; + self.dirty_dblocks.insert(a, size); + self.man_alloc_size += size; + Ok(a) + } + + /// `H5HF__man_iblock_root_create`: the root direct block becomes entry + /// 0 of a new root indirect block. + fn root_create(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + let os = img.os; + let mut nrows = if self.start_root_rows == 0 { + self.max_root_rows + } else { + usize::from(self.start_root_rows) + }; + if self.start_root_rows != 0 { + let mut block_row_off = (log2(min) - log2(self.start_block_size)) as usize; + if block_row_off > 0 { + block_row_off += 1; + } + nrows = nrows.max(1 + block_row_off); + } + if nrows > self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + if min > self.start_block_size { + return Err(unsupported("skipping blocks too small for an object")); + } + let have_direct = self.root != undef(os); + let rows = u16::try_from(nrows).map_err(|_| bad("too many rows"))?; + let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + self.ents = vec![undef(os); nrows * usize::from(self.width)]; + if have_direct { + self.ents[0] = self.root; + self.man_iter_off = self.start_block_size; + } else { + self.man_iter_off = 0; + } + self.iblock_dirty = true; + self.root_rows = rows; + self.root = a; + let w = u64::from(self.width); + let ov = self.overhead(os); + let mut acc: u64 = (0..nrows).map(|u| (self.row_size(u) - ov) * w).sum(); + if have_direct { + acc -= self.row_size(0) - ov; + } + self.man_size = self.row_off(nrows); + self.total_man_free += acc; + Ok(()) + } + + /// `H5HF__man_iblock_root_double`: the root indirect block gets twice + /// its rows, at a new address. + fn root_double(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + let os = img.os; + let old = usize::from(self.root_rows); + let (row, _) = self.iter_pos(); + let next_size = self.row_size(row.min(self.max_root_rows - 1)); + if old < self.max_direct_rows && min > next_size { + return Err(unsupported("skipping blocks too small for an object")); + } + let new = (2 * old).min(self.max_root_rows); + if new > self.max_direct_rows || new == old { + return Err(unsupported("child indirect blocks")); + } + img.free(self.root, self.iblock_len(self.root_rows, os) as u64); + let rows = u16::try_from(new).map_err(|_| bad("too many rows"))?; + let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + let w = usize::from(self.width); + self.ents.resize(new * w, undef(os)); + let ov = self.overhead(os); + let acc: u64 = (old * w..new * w).map(|u| self.row_size(u / w) - ov).sum(); + self.root_rows = rows; + self.root = a; + self.iblock_dirty = true; + self.man_size = 2 * self.row_off(new - 1); + self.total_man_free += acc; + Ok(()) + } + + /// Remove object `id` (`H5HF_remove`): a managed object's space goes + /// back to the free-space manager, merged with free space next to it; a + /// huge object's file space is freed. + pub(crate) fn remove(&mut self, img: &mut Image<'_>, id: &[u8]) -> Result<(), Error> { + match (id[0] >> 4) & 0x03 { + 0 => { + let os = img.os; + let (off, len) = self.man_id(id); + let (_, size, boff) = self.dblock_of(os, off)?; + let (mut lo, mut hi) = (off, off + len); + if let Some(fs) = &self.fs { + if let Some((&o, &s)) = fs.sects.range(..off).next_back() + && o + s == off + { + lo = o; + } + if let Some(&s) = fs.sects.get(&hi) { + hi += s; + } + } + if hi - lo == size - self.overhead(os) && boff <= lo { + return Err(unsupported( + "removing the last object of a direct block (libhdf5 frees the block)", + )); + } + if let Some(fs) = &mut self.fs { + fs.sects.remove(&lo); + fs.sects.remove(&(off + len)); + } + self.fs_add(img, lo, hi - lo)?; + self.total_man_free += len; + self.man_nobjs -= 1; + Ok(()) + } + 1 => { + let key = self.huge_id(id, img.os, img.ls)?; + let (os, ls) = (img.os, img.ls); + let mut t = Bt2::open(img, self.huge_bt2)?; + let rec = t + .remove(img, &mut |_, r| Ok(key.cmp(&huge_rec_id(r, os, ls))))? + .ok_or_else(|| bad("huge object missing from its B-tree"))?; + t.finish(img)?; + let len = get_uint(&rec[os as usize..], ls); + img.free(get_uint(&rec, os), len); + self.huge_size = self.huge_size.saturating_sub(len); + self.huge_nobjs = self.huge_nobjs.saturating_sub(1); + Ok(()) + } + _ => Err(unsupported("tiny objects")), + } + } + + /// `H5HF__huge_insert` (unfiltered heap, IDs that index the huge-object + /// B-tree). + fn huge_insert(&mut self, img: &mut Image<'_>, obj: &[u8]) -> Result, Error> { + let (os, ls) = (img.os, img.ls); + if self.huge_ids_direct(os, ls) { + return Err(unsupported("directly addressed huge objects")); + } + let rs = os as usize + 2 * ls as usize; + let mut t = if self.huge_bt2 == undef(os) { + let t = Bt2::create(img, HUGE_BT2_TYPE, HUGE_BT2_NODE, rs, 100, 40)?; + self.huge_bt2 = t.address(); + t + } else { + Bt2::open(img, self.huge_bt2)? + }; + if t.record_size() != rs || t.tree_type() != HUGE_BT2_TYPE { + return Err(unsupported("huge-object B-tree layout")); + } + let n = obj.len() as u64; + let a = img.alloc_reusing(n)?; + img.write(a, obj)?; + let w = usize::from(self.id_len - 1).min(8); + let max_id = if w >= 8 { + u64::MAX + } else { + (1u64 << (8 * w)) - 1 + }; + if self.flags & 0x01 != 0 || self.next_huge_id >= max_id { + return Err(unsupported("huge object IDs wrapped")); + } + self.next_huge_id += 1; + let key = self.next_huge_id; + if key == max_id { + self.flags |= 0x01; + } + let mut rec = vec![0u8; rs]; + put_uint(&mut rec, a, os); + put_uint(&mut rec[os as usize..], n, ls); + put_uint(&mut rec[os as usize + ls as usize..], key, ls); + t.insert(img, &mut |_, r| Ok(key.cmp(&huge_rec_id(r, os, ls))), &rec)?; + t.finish(img)?; + self.huge_size += n; + self.huge_nobjs += 1; + let mut id = vec![0u8; usize::from(self.id_len)]; + id[0] = 0x10; + id[1..1 + w].copy_from_slice(&key.to_le_bytes()[..w]); + Ok(id) + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let (os, ls) = (img.os, img.ls); + let l = ls as usize; + let mut d = Vec::with_capacity(Self::header_len(os, ls)); + d.extend_from_slice(b"FRHP"); + d.push(0); + d.extend_from_slice(&self.id_len.to_le_bytes()); + d.extend_from_slice(&0u16.to_le_bytes()); + d.push(self.flags); + d.extend_from_slice(&self.max_man_size.to_le_bytes()); + let addr = |d: &mut Vec, v: u64| { + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, v, os); + d.extend_from_slice(&a); + }; + d.extend_from_slice(&self.next_huge_id.to_le_bytes()[..l]); + addr(&mut d, self.huge_bt2); + d.extend_from_slice(&self.total_man_free.to_le_bytes()[..l]); + addr(&mut d, self.fs_addr); + for v in [ + self.man_size, + self.man_alloc_size, + self.man_iter_off, + self.man_nobjs, + self.huge_size, + self.huge_nobjs, + self.tiny_size, + self.tiny_nobjs, + ] { + d.extend_from_slice(&v.to_le_bytes()[..l]); + } + d.extend_from_slice(&self.width.to_le_bytes()); + d.extend_from_slice(&self.start_block_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.max_direct_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.max_index.to_le_bytes()); + d.extend_from_slice(&self.start_root_rows.to_le_bytes()); + addr(&mut d, self.root); + d.extend_from_slice(&self.root_rows.to_le_bytes()); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d) + } + + /// Write back everything that changed: direct block checksums, the + /// root indirect block, the free-space manager, the header. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + if self.flags & 0x02 != 0 { + let at = 5 + os as usize + self.heap_off_size; + for (a, size) in std::mem::take(&mut self.dirty_dblocks) { + let n = usize::try_from(size).map_err(|_| bad("block too large"))?; + let mut b = img.read(a, n)?; + b[at..at + 4].fill(0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&b); + img.write(a + at as u64, &sum.to_le_bytes())?; + } + } + if self.iblock_dirty && self.root_rows > 0 { + let len = self.iblock_len(self.root_rows, os); + let mut b = Vec::with_capacity(len); + b.extend_from_slice(b"FHIB"); + b.push(0); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.addr, os); + b.extend_from_slice(&a); + b.extend_from_slice(&vec![0u8; self.heap_off_size]); + for &e in &self.ents { + put_uint(&mut a, e, os); + b.extend_from_slice(&a); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&b); + b.extend_from_slice(&sum.to_le_bytes()); + img.write(self.root, &b)?; + self.iblock_dirty = false; + } + if self.fs_dirty { + self.fs_dirty = false; + if let Some(mut fs) = self.fs.take() { + let hdr_len = FreeSpace::hdr_len(os, img.ls) as u64; + if fs.sects.is_empty() { + // H5HF__space_close: a manager that tracks nothing is + // deleted. + img.free(fs.addr, hdr_len); + if fs.sect_addr != undef(os) { + img.free(fs.sect_addr, fs.alloc_sect_size); + } + self.fs_addr = undef(os); + } else { + let need = fs.needed(os); + if fs.sect_addr == undef(os) || need != fs.alloc_sect_size { + if fs.sect_addr != undef(os) { + img.free(fs.sect_addr, fs.alloc_sect_size); + } + fs.sect_addr = img.alloc_reusing(need)?; + fs.alloc_sect_size = need; + } + fs.sect_size = fs.alloc_sect_size; + let mut s = fs.serialize(os); + let n = usize::try_from(fs.sect_size).map_err(|_| bad("section info"))?; + s.resize(n - 4, 0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&s); + s.extend_from_slice(&sum.to_le_bytes()); + img.write(fs.sect_addr, &s)?; + fs.write_header(img)?; + self.fs = Some(fs); + } + } + } + self.write_header(img) + } +} + +/// The ID in a huge-object B-tree record (type 1: address, length, ID). +fn huge_rec_id(r: &[u8], os: u8, ls: u8) -> u64 { + get_uint(&r[os as usize + ls as usize..], ls) +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index b14244a..c32431a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -15,10 +15,12 @@ //! file) is dropped before the first write, so no `&[u8]` over the mapping //! is alive while the file changes (see `image`). +mod attrs; mod btree1; mod btree2; mod earray; mod farray; +mod fheap; mod image; mod ohdr; mod select; @@ -27,7 +29,6 @@ use std::collections::{BTreeMap, HashMap}; use std::fs::{OpenOptions, TryLockError}; use std::path::{Path, PathBuf}; -use clawhdf5_format::attribute::AttributeMessage; use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::data_read::NativeElement; @@ -43,8 +44,8 @@ use btree1::{BTree1, Key}; use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; -use image::{Image, get_uint, put_uint, undef}; -use ohdr::{Header, MSG_ATTRIBUTE}; +use image::{Image, put_uint, undef}; +use ohdr::Header; const MSG_DATASPACE: u16 = 0x01; const MSG_LAYOUT: u16 = 0x08; @@ -918,101 +919,29 @@ impl FileEditor { /// Set attribute `name` of the object at `path` (a group or dataset; /// `"/"` is the root group) to `value`, replacing an attribute of that - /// name. The attribute goes into free space in the object header, or a - /// new header continuation chunk at the end of the file. + /// name, as libhdf5 creates attributes (`H5O__attr_create`). /// - /// [`Error::Unsupported`] for an object whose attributes are in dense - /// storage (or would have to move there: more than the object's - /// compact-attribute limit), one that tracks attribute creation order, - /// or one with shared attribute messages. + /// A compact attribute goes into free space in the object header, or a + /// new header continuation chunk; an object that tracks creation order + /// gives it the next creation index. When the object reaches its + /// compact-attribute limit (or the attribute is too large for a header + /// message) its attributes move to dense storage, and objects already + /// using dense storage get the attribute there: a fractal heap object + /// (in free heap space, a new heap block, or its own file space when + /// larger than the heap's managed limit) indexed by name and, when the + /// object indexes creation order, by creation index. An attribute + /// replaced by one of the same encoded size is rewritten where it is. + /// + /// [`Error::Unsupported`] for shared attribute messages, heaps this + /// editor cannot extend the way libhdf5 would (child indirect blocks, + /// skipped blocks, free space other than within direct blocks, freeing + /// a whole block), and version-1 object headers asked for an attribute + /// larger than a header message holds. pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { if name.is_empty() { return Err(Error::InvalidArgument("empty attribute name".into())); } - self.edit(|f, img| { - let addr = - clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; - let mut hdr = Header::load(img, addr)?; - if hdr.version == 2 && hdr.flags & 0x04 != 0 { - return Err(Error::Unsupported( - "object tracks attribute creation order".into(), - )); - } - if let Some(i) = hdr.find(MSG_ATTR_INFO) { - let d = hdr.data(img, i)?; - // version(1) flags(1) [max creation index(2)] fractal heap - // address, name index address, [order index address]. - let mut p = 2; - if d.get(1).is_some_and(|f| f & 0x01 != 0) { - p += 2; - } - let os = img.os as usize; - if d.len() < p + os { - return Err(Error::Unsupported("short attribute info message".into())); - } - if get_uint(&d[p..], img.os) != undef(img.os) { - return Err(Error::Unsupported( - "object with attributes in dense storage".into(), - )); - } - } - let mut existing = None; - let mut count = 0usize; - for i in 0..hdr.msgs.len() { - if hdr.msgs[i].mtype != MSG_ATTRIBUTE { - continue; - } - if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { - return Err(Error::Unsupported("shared attribute message".into())); - } - count += 1; - if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { - existing = Some(i); - } - } - if existing.is_none() && hdr.version == 2 { - let max_compact = max_compact_attrs(img, &hdr)?; - if count + 1 > usize::from(max_compact) { - return Err(Error::Unsupported(format!( - "object already has {count} compact attributes (its limit is \ - {max_compact}); more need dense storage" - ))); - } - } - let msg = clawhdf5_format::type_builders::build_attr_message(name, value); - check_plain(&msg.datatype)?; - let body = if hdr.version == 1 { - encode_attr_v1(&msg, img.ls) - } else { - let mut b = msg.serialize_v3(img.ls); - if !name.is_ascii() { - b[8] = 1; // UTF-8 name - } - b - }; - if let Some(i) = existing { - hdr.delete(img, i)?; - } - // libhdf5 counts a version-2 header's attributes through its - // Attribute Info message and reports none without one; like - // H5O__attr_create, add it when missing: before the attribute - // when free space holds both (libhdf5's order), else after it, - // so that a new continuation chunk made for the attribute has - // room for it too. - let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none()) - .then(|| attr_info_message(hdr.flags, img.os)); - let ainfo_first = ainfo - .as_ref() - .is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len())); - if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) { - hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; - } - hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; - if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) { - hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; - } - hdr.finish(img) - }) + self.edit(|f, img| attrs::set_attr(f, img, path, name, value)) } } @@ -1097,81 +1026,6 @@ fn set_superblock_eof( Ok(()) } -/// The name bytes of an attribute message body (without the NUL). -fn attr_name(d: &[u8]) -> Result<&[u8], Error> { - let bad = || Error::Unsupported("malformed attribute message".into()); - let (len, at) = match d.first() { - Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), - Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), - _ => return Err(bad()), - }; - let name = d.get(at..at + len).ok_or_else(bad)?; - Ok(name.split(|&b| b == 0).next().unwrap_or(name)) -} - -/// A new Attribute Info message for a version-2 header with flags -/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order -/// tracked / indexed as the header's flags say, maximum creation index 0, -/// and no dense storage (undefined fractal heap and B-tree addresses). -fn attr_info_message(hdr_flags: u8, os: u8) -> Vec { - let track = hdr_flags & 0x04 != 0; - let index = hdr_flags & 0x08 != 0; - let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; - if track { - b.extend_from_slice(&0u16.to_le_bytes()); - } - let undef_addr = vec![0xffu8; os as usize]; - b.extend_from_slice(&undef_addr); - b.extend_from_slice(&undef_addr); - if index { - b.extend_from_slice(&undef_addr); - } - b -} - -/// A version-2 header's limit on compact attributes: stored when its flags -/// say so, else libhdf5's default of 8. -fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { - if hdr.flags & 0x10 == 0 { - return Ok(8); - } - let mut p = hdr.addr + 6; - if hdr.flags & 0x20 != 0 { - p += 16; - } - let b = img.read(p, 2)?; - Ok(u16::from_le_bytes([b[0], b[1]])) -} - -/// A version-1 attribute message (what libhdf5 writes in a version-1 object -/// header): name, datatype and dataspace each padded to 8 bytes, the -/// dataspace as a version-1 dataspace message. -fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { - let mut name = a.name.as_bytes().to_vec(); - name.push(0); - let dt = a.datatype.serialize(); - let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; - if a.dataspace.space_type == DataspaceType::Simple { - for &d in &a.dataspace.dimensions { - let mut b = vec![0u8; ls as usize]; - put_uint(&mut b, d, ls); - ds.extend_from_slice(&b); - } - } else { - ds[1] = 0; - } - let mut out = vec![1u8, 0]; - out.extend_from_slice(&(name.len() as u16).to_le_bytes()); - out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); - out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); - for part in [&name, &dt, &ds] { - out.extend_from_slice(part); - out.resize(out.len().next_multiple_of(8), 0); - } - out.extend_from_slice(&a.raw_data); - out -} - fn write_selection( f: &File, img: &mut Image<'_>, diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs index f3abfe0..8aa2a15 100644 --- a/crates/clawhdf5/src/edit/ohdr.rs +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -59,7 +59,7 @@ pub(crate) struct Header { added: usize, } -const MAX_CHUNKS: usize = 1024; +const MAX_CHUNKS: usize = 1 << 16; fn corrupt(why: &'static str) -> Error { Error::Format(FormatError::InvalidObjectHeader(why)) @@ -118,7 +118,11 @@ impl Header { }); h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?; } - while let Some((caddr, clen)) = pending.pop() { + // Continuation chunks in the order their messages are found, as + // H5O_protect loads them (so messages keep libhdf5's order). + let mut next = 0; + while let Some(&(caddr, clen)) = pending.get(next) { + next += 1; if h.chunks.len() >= MAX_CHUNKS { return Err(corrupt("too many object header chunks")); } -- 2.54.0 From dc9cfba6bb39b1b4481ca86d577d7e6c26aacf7e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:04:15 -0500 Subject: [PATCH 09/49] edit: reuse space freed earlier in the editing session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A FileEditor now keeps the space its edits free — a filtered chunk that moved, chunks a shrink removed, B-tree nodes merged away, a heap's replaced root indirect block or free-space section info, huge objects replaced — and later edits allocate from it (best fit, lowest address among equals, zeroed) before growing the file. An edit never reuses what it frees itself: until it is committed the file still refers to that space. Reused blocks are written in the commit's first phase with the space past the old end of file (nothing on disk refers to them yet), before any existing byte changes, so the crash-safety ordering holds. Space still free when the editor is dropped is leaked, as libhdf5 leaks it without a persistent free-space manager (files that have one, or use paged aggregation, are still refused at open). FileEditor::reusable_bytes reports what is left to reuse. Tests: FreeList merging and best fit, the edit-local rule and the commit split (image unit tests); a shrink followed by regrowth writing the same data reuses every removed chunk and leaves the file size unchanged, while one editor per edit grows the file, h5py/h5dump/h5rs check read both and h5py continues (freed_space_is_reused_within_a_session). measure_append_waste (edit_interop, ignored), same workload, one editor, before -> after (bytes; libhdf5 in brackets), on tank 2026-09-26: gzip chunks 1024, 1000 appends of 100: 307210 -> 306780 (306058); gzip chunks 4096, 2000 appends of 10: 119684 -> 79829 (50292); unfiltered unchanged (no space is freed). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 73 ++++++ crates/clawhdf5-tools/tests/edit_interop.rs | 7 +- crates/clawhdf5/src/edit/btree2.rs | 2 +- crates/clawhdf5/src/edit/fheap.rs | 14 +- crates/clawhdf5/src/edit/image.rs | 226 +++++++++++++++--- crates/clawhdf5/src/edit/mod.rs | 58 +++-- 6 files changed, 317 insertions(+), 63 deletions(-) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 036839f..332dffa 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1323,3 +1323,76 @@ fn dense_attribute_refusals_change_nothing() { )); check_tools(&path, true); } + +/// Space one edit frees is reused by later edits of the same editor: the +/// chunks a shrink removes are where the chunks of the following growth +/// go, so the file does not grow; with a new editor per edit (nothing to +/// reuse) it does. h5py, h5dump and `h5rs check` read the result, and +/// h5py goes on. +#[test] +fn freed_space_is_reused_within_a_session() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let mut sizes = Vec::new(); + for session in [true, false] { + let path = dir.path().join(format!("reuse_{session}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 f.create_dataset('x', data=np.zeros(2000, dtype=' = (0..2000).map(|_| rng.next() as i32).collect(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.write_values("x", &Selection::All, &vals).unwrap(); + drop(ed); + let len0 = std::fs::metadata(&path).unwrap().len(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("x", &[1000]).unwrap(); + if session { + assert!(ed.reusable_bytes() > 0); + } else { + ed = { + drop(ed); + FileEditor::open(&path).unwrap() + }; + } + ed.resize("x", &[2000]).unwrap(); + ed.write_values("x", &block(&[1000], &[1000]), &vals[1000..]) + .unwrap(); + if session { + assert_eq!(ed.reusable_bytes(), 0, "every freed chunk is reused"); + } + drop(ed); + let len1 = std::fs::metadata(&path).unwrap().len(); + sizes.push((len0, len1)); + let m = Model { + shape: vec![2000], + data: vals.clone(), + }; + verify(&path, "x", &m); + check_tools(&path, true); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['x'].resize((2100,))\n\ + \x20 f['x'][2000:] = 9\n", + p = path.to_str().unwrap() + )); + let mut m = m; + m.resize(&[2100], 0); + m.write_block(&[2000], &[100], &[9; 100]); + verify(&path, "x", &m); + check_tools(&path, true); + } + let (reuse, fresh) = (sizes[0], sizes[1]); + assert_eq!( + reuse.1, reuse.0, + "a session reusing freed chunks does not grow the file" + ); + assert!(fresh.1 > fresh.0, "without reuse the file grows"); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index bc4c9a6..cb345f8 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1122,9 +1122,10 @@ fn out_of_order_chunk_creation_matches_libhdf5() { } } -/// Not a check: prints how much space an append workload leaks (the editor -/// never reuses space), against libhdf5 doing the same appends and against -/// `h5repack` of each. Run with `--ignored --nocapture`. +/// Not a check: prints how much space an append workload leaks (one +/// editor for the whole workload, which reuses the space it frees but not +/// space it cannot fit a grown chunk into), against libhdf5 doing the same +/// appends and against `h5repack` of each. Run with `--ignored --nocapture`. #[test] #[ignore] fn measure_append_waste() { diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs index 80be9f0..5761e1e 100644 --- a/crates/clawhdf5/src/edit/btree2.rs +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -316,7 +316,7 @@ impl Bt2 { } fn new_node(&mut self, img: &mut Image<'_>, depth: u16) -> Result { - let addr = img.alloc_reusing(u64::from(self.node_size))?; + let addr = img.alloc(u64::from(self.node_size))?; self.nodes.insert( addr, Node { diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs index 7c7b72a..2fab657 100644 --- a/crates/clawhdf5/src/edit/fheap.rs +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -455,7 +455,7 @@ impl Heap { /// blocks, 4 KiB managed objects, 8-byte IDs). pub(crate) fn create_attribute_heap(img: &mut Image<'_>) -> Result { let (os, ls) = (img.os, img.ls); - let addr = img.alloc_reusing(Self::header_len(os, ls) as u64)?; + let addr = img.alloc(Self::header_len(os, ls) as u64)?; let mut h = Self { addr, id_len: 8, @@ -706,7 +706,7 @@ impl Heap { /// heap has none. fn fs_add(&mut self, img: &mut Image<'_>, off: u64, size: u64) -> Result<(), Error> { if self.fs.is_none() { - let addr = img.alloc_reusing(FreeSpace::hdr_len(img.os, img.ls) as u64)?; + let addr = img.alloc(FreeSpace::hdr_len(img.os, img.ls) as u64)?; self.fs = Some(FreeSpace { addr, max_sect_addr: self.max_index, @@ -813,7 +813,7 @@ impl Heap { size: u64, ) -> Result { let os = img.os; - let a = img.alloc_reusing(size)?; + let a = img.alloc(size)?; let n = usize::try_from(size).map_err(|_| bad("block too large"))?; let mut d = vec![0u8; n]; d[0..4].copy_from_slice(b"FHDB"); @@ -851,7 +851,7 @@ impl Heap { } let have_direct = self.root != undef(os); let rows = u16::try_from(nrows).map_err(|_| bad("too many rows"))?; - let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + let a = img.alloc(self.iblock_len(rows, os) as u64)?; self.ents = vec![undef(os); nrows * usize::from(self.width)]; if have_direct { self.ents[0] = self.root; @@ -889,7 +889,7 @@ impl Heap { } img.free(self.root, self.iblock_len(self.root_rows, os) as u64); let rows = u16::try_from(new).map_err(|_| bad("too many rows"))?; - let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + let a = img.alloc(self.iblock_len(rows, os) as u64)?; let w = usize::from(self.width); self.ents.resize(new * w, undef(os)); let ov = self.overhead(os); @@ -973,7 +973,7 @@ impl Heap { return Err(unsupported("huge-object B-tree layout")); } let n = obj.len() as u64; - let a = img.alloc_reusing(n)?; + let a = img.alloc(n)?; img.write(a, obj)?; let w = usize::from(self.id_len - 1).min(8); let max_id = if w >= 8 { @@ -1096,7 +1096,7 @@ impl Heap { if fs.sect_addr != undef(os) { img.free(fs.sect_addr, fs.alloc_sect_size); } - fs.sect_addr = img.alloc_reusing(need)?; + fs.sect_addr = img.alloc(need)?; fs.alloc_sect_size = need; } fs.sect_size = fs.alloc_sect_size; diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs index 722534d..08d389c 100644 --- a/crates/clawhdf5/src/edit/image.rs +++ b/crates/clawhdf5/src/edit/image.rs @@ -35,8 +35,61 @@ pub(crate) struct Image<'a> { /// Width of addresses and lengths in the file. pub(crate) os: u8, pub(crate) ls: u8, - /// Space the edit stopped using. + /// Space the edit stopped using. Not reused by this edit: until the + /// edit is committed, the file's metadata still points at it. freed: Vec<(u64, u64)>, + /// Space earlier edits of the session freed, available to this one. + reusable: FreeList, + /// Blocks this edit took from `reusable`: nothing on disk refers to + /// them, so they are written with the new space, before the changes + /// that link them in (see [`Plan::commit`]). + fresh: Vec<(u64, u64)>, +} + +/// Free space, address -> length, adjacent blocks merged. +#[derive(Debug, Clone, Default)] +pub(crate) struct FreeList(BTreeMap); + +impl FreeList { + /// Add `[addr, addr + len)`, merged with neighbours it touches. + pub(crate) fn add(&mut self, addr: u64, len: u64) { + if len == 0 { + return; + } + let (mut lo, mut hi) = (addr, addr.saturating_add(len)); + if let Some((&a, &l)) = self.0.range(..=lo).next_back() + && a + l >= lo + { + lo = a; + hi = hi.max(a + l); + self.0.remove(&a); + } + while let Some((&a, &l)) = self.0.range(lo..=hi).next() { + hi = hi.max(a + l); + self.0.remove(&a); + } + self.0.insert(lo, hi - lo); + } + + /// Take `size` bytes from the smallest block that holds them (the + /// lowest address among equals), from its start. + fn take(&mut self, size: u64) -> Option { + let (&a, &l) = self + .0 + .iter() + .filter(|&(_, &l)| l >= size) + .min_by_key(|&(&a, &l)| (l, a))?; + self.0.remove(&a); + if l > size { + self.0.insert(a + size, l - size); + } + Some(a) + } + + /// Total bytes. + pub(crate) fn total(&self) -> u64 { + self.0.values().sum() + } } impl<'a> Image<'a> { @@ -50,9 +103,23 @@ impl<'a> Image<'a> { os, ls, freed: Vec::new(), + reusable: FreeList::default(), + fresh: Vec::new(), } } + /// Let the edit allocate from `free` (space earlier edits freed). + pub(crate) fn with_reusable(mut self, free: FreeList) -> Self { + // Only space inside the file as it is now. + self.reusable = FreeList( + free.0 + .into_iter() + .filter(|&(a, l)| a.saturating_add(l) <= self.old_eoa) + .collect(), + ); + self + } + pub(crate) fn eoa(&self) -> u64 { self.eoa } @@ -66,11 +133,24 @@ impl<'a> Image<'a> { !self.patches.is_empty() || self.eoa != self.old_eoa } - /// Allocate `size` bytes at the end of the file. The space reads as - /// zeros until written. Nothing is ever freed: space an edit stops - /// using (a relocated chunk, say) is leaked, as there is no free-space - /// manager. + /// Allocate `size` bytes: from space an earlier edit of this session + /// freed when a block holds them (best fit), else at the end of the + /// file. The space reads as zeros until written. pub(crate) fn alloc(&mut self, size: u64) -> Result { + if size > 0 + && let Some(a) = self.reusable.take(size) + { + self.fresh.push((a, size)); + let n = usize::try_from(size) + .map_err(|_| Error::Unsupported("allocation too large".into()))?; + self.write(a, &vec![0u8; n])?; + return Ok(a); + } + self.alloc_end(size) + } + + /// Allocate `size` bytes at the end of the file. + fn alloc_end(&mut self, size: u64) -> Result { let addr = self.eoa; let end = addr .checked_add(size) @@ -80,14 +160,8 @@ impl<'a> Image<'a> { Ok(addr) } - /// Allocate `size` bytes for metadata or data, from space an earlier - /// edit of this session freed when there is a block that fits, - /// otherwise at the end of the file. - pub(crate) fn alloc_reusing(&mut self, size: u64) -> Result { - self.alloc(size) - } - - /// Note that the edit no longer uses `[addr, addr + len)`. + /// Note that the edit no longer uses `[addr, addr + len)`; later edits + /// of the session may reuse it. pub(crate) fn free(&mut self, addr: u64, len: u64) { if len > 0 { self.freed.push((addr, len)); @@ -108,7 +182,7 @@ impl<'a> Image<'a> { } let old_end = self.eoa; self.eoa = addr; - if let Err(e) = self.alloc(new_len) { + if let Err(e) = self.alloc_end(new_len) { self.eoa = old_end; return Err(e); } @@ -206,12 +280,24 @@ impl<'a> Image<'a> { /// The edit's writes, detached from the base bytes (see the module's /// invariant: the reader that owns them can then be dropped before /// anything is written). - pub(crate) fn into_plan(self) -> Plan { - Plan { - patches: self.patches, - eoa: self.eoa, - old_eoa: self.old_eoa, + pub(crate) fn into_plan(self) -> (Plan, FreeList) { + // What the session may reuse once this edit is committed: what it + // did not take, and what it freed. + let mut free = self.reusable; + for (a, l) in self.freed { + free.add(a, l); } + let mut fresh = self.fresh; + fresh.sort_unstable(); + ( + Plan { + patches: self.patches, + eoa: self.eoa, + old_eoa: self.old_eoa, + fresh, + }, + free, + ) } } @@ -220,33 +306,57 @@ pub(crate) struct Plan { patches: BTreeMap>, eoa: u64, old_eoa: u64, + /// Reused blocks (sorted): written with the new space. + fresh: Vec<(u64, u64)>, } impl Plan { + /// Whether `addr` is in space nothing on disk refers to yet (past the + /// old end of file, or in a reused block), and up to where (before + /// `end`) that stays so. + fn new_space(&self, addr: u64, end: u64) -> (bool, u64) { + if addr >= self.old_eoa { + return (true, end); + } + let limit = end.min(self.old_eoa); + // The reused block holding `addr`, or the next one after it. + let i = self.fresh.partition_point(|&(a, l)| a + l <= addr); + match self.fresh.get(i) { + Some(&(a, l)) if a <= addr => (true, limit.min(a + l)), + Some(&(a, _)) => (false, limit.min(a)), + None => (false, limit), + } + } + /// Write the edit to `file`, whose superblock is at `user_block`. /// /// Order: first everything in newly allocated space (new chunks, new - /// index blocks, relocated structures), which nothing on disk refers to - /// yet, then a sync; then the changes to existing bytes — raw data - /// overwritten in place and the metadata that links the new space in - /// (superblock end of file, chunk index entries, object header + /// index blocks, relocated structures — past the old end of file, or in + /// space an earlier edit of the session freed), which nothing on disk + /// refers to yet, then a sync; then the changes to existing bytes — raw + /// data overwritten in place and the metadata that links the new space + /// in (superblock end of file, chunk index entries, object header /// messages) — then a sync. A crash during the first phase leaves the - /// file as it was (plus unreferenced bytes past its end of file); a - /// crash during the second can leave it inconsistent, as with libhdf5 - /// without SWMR: there is no journal. + /// file as it was (plus unreferenced bytes); a crash during the second + /// can leave it inconsistent, as with libhdf5 without SWMR: there is no + /// journal. pub(crate) fn commit(self, file: &mut std::fs::File, user_block: u64) -> Result<(), Error> { let old_eoa = self.old_eoa; let mut in_place: Vec<(u64, &[u8])> = Vec::new(); for (&addr, bytes) in &self.patches { - // A patch may run from existing bytes into new space (writes - // merge); its new part goes with the new space. - let split = old_eoa.saturating_sub(addr).min(bytes.len() as u64) as usize; - let (old, new) = bytes.split_at(split); - if !new.is_empty() { - write_at(file, user_block + addr + split as u64, new)?; - } - if !old.is_empty() { - in_place.push((addr, old)); + // A patch may run across new and existing space (writes + // merge): split it where that changes. + let end = addr + bytes.len() as u64; + let mut at = addr; + while at < end { + let (new, upto) = self.new_space(at, end); + let part = &bytes[(at - addr) as usize..(upto - addr) as usize]; + if new { + write_at(file, user_block + at, part)?; + } else { + in_place.push((at, part)); + } + at = upto; } } if self.eoa > old_eoa { @@ -322,6 +432,52 @@ mod tests { assert!(img.write(42, &[1]).is_err()); } + #[test] + fn free_list_merges_and_takes_best_fit() { + let mut f = FreeList::default(); + f.add(100, 10); + f.add(120, 5); + f.add(110, 10); // joins both neighbours + assert_eq!( + f.0.iter().map(|(&a, &l)| (a, l)).collect::>(), + [(100, 25)] + ); + f.add(300, 8); + f.add(200, 40); + // Best fit: the 8-byte block for 6 bytes, from its start. + assert_eq!(f.take(6), Some(300)); + assert_eq!(f.take(30), Some(200)); + assert_eq!(f.take(26), None); + assert_eq!(f.total(), 25 + 2 + 10); + } + + /// An edit allocates from space earlier edits freed (zeroed), never + /// from what it frees itself; the plan writes reused blocks with the + /// new space. + #[test] + fn reuse_across_edits_only() { + let base = vec![7u8; 64]; + let mut free = FreeList::default(); + free.add(8, 16); + let mut img = Image::new(&base, 8, 8).with_reusable(free); + img.free(32, 16); // freed by this edit: not reusable yet + let a = img.alloc(16).unwrap(); + assert_eq!(a, 8); + assert_eq!(img.read(8, 16).unwrap(), vec![0u8; 16]); + let b = img.alloc(8).unwrap(); + assert_eq!(b, 64, "the edit's own freed space is not reused"); + img.write(4, &[1; 8]).unwrap(); // existing bytes 4..8, reused 8..12 + let (plan, next) = img.into_plan(); + assert_eq!(plan.new_space(4, 12), (false, 8)); + assert_eq!(plan.new_space(8, 12), (true, 12)); + assert_eq!(plan.new_space(30, 40), (false, 40)); + assert_eq!(plan.new_space(64, 72), (true, 72)); + assert_eq!( + next.0.iter().map(|(&a, &l)| (a, l)).collect::>(), + [(32, 16)] + ); + } + /// Random reads and writes against a flat copy of the bytes. #[test] fn matches_a_flat_model() { diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index c32431a..336fbb8 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -44,7 +44,7 @@ use btree1::{BTree1, Key}; use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; -use image::{Image, put_uint, undef}; +use image::{FreeList, Image, put_uint, undef}; use ohdr::Header; const MSG_DATASPACE: u16 = 0x01; @@ -74,16 +74,18 @@ const MSG_FLAG_DONTSHARE: u8 = 0x04; /// contiguous or chunked dataset with values of the dataset's own /// datatype, under any selection. Chunks are decoded, updated and /// re-encoded; an unfiltered chunk is rewritten in place, a filtered one -/// in place when it still fits and otherwise at the end of the file. New -/// chunks are added to the chunk index: version-1 B-tree (layout v1-v3, -/// what h5py's default `libver` writes), Extensible Array, Fixed Array and -/// single-chunk indexes. A version-2 B-tree index (two or more unlimited -/// dimensions) can only have existing chunks overwritten in place, and an -/// implicit index only in place. -/// - [`resize`](Self::resize): grow a chunked dataset up to its maximum -/// dimensions (h5py's `Dataset.resize`); new chunks come with the writes. +/// in place when it still fits and otherwise in new space. New chunks are +/// added to the chunk index: version-1 B-tree (layout v1-v3, what h5py's +/// default `libver` writes), Extensible Array, Fixed Array, version-2 +/// B-tree (two or more unlimited dimensions) and single-chunk indexes, as +/// libhdf5 adds them (the same splits and blocks). An implicit index can +/// only be written in place. +/// - [`resize`](Self::resize): grow or shrink a chunked dataset within its +/// maximum dimensions (h5py's `Dataset.resize`), pruning the chunks a +/// shrink leaves outside the extent as libhdf5 does. /// - [`set_attr`](Self::set_attr): add or replace an attribute of any -/// object whose attributes are stored in its object header. +/// object, in its object header or in dense storage (moving attributes +/// there when the object reaches its compact limit). /// /// Anything else is an [`Error::Unsupported`] and leaves the file untouched. /// @@ -95,13 +97,20 @@ const MSG_FLAG_DONTSHARE: u8 = 0x04; /// before that point leaves the file as it was; a crash while the existing /// structures are being patched can leave the file inconsistent. /// -/// Space is never reused: a filtered chunk that grows moves to the end of -/// the file and its old bytes are leaked, as are index blocks that are -/// replaced. `h5repack` reclaims such space. +/// Space an edit stops using — a filtered chunk that moved, chunks a +/// shrink removed, index nodes a B-tree merged away, a heap's replaced +/// blocks — is reused by later edits of the same editor (best fit, at the +/// lowest address), never by the edit that freed it: until that edit is +/// committed the file still refers to it. Space reused this way is written +/// with the new space, before any existing byte changes. Freed space still +/// unused when the editor is dropped is lost, as it is when libhdf5 closes +/// a file without a persistent free-space manager; `h5repack` reclaims it. #[derive(Debug)] pub struct FileEditor { path: PathBuf, file: std::fs::File, + /// Space edits of this session freed, which later ones reuse. + free: FreeList, } /// Where a layout message keeps the fields an edit may change (offsets in @@ -754,7 +763,11 @@ impl FileEditor { } Err(TryLockError::Error(e)) => return Err(Error::Io(e)), } - let ed = Self { path, file }; + let ed = Self { + path, + file, + free: FreeList::default(), + }; let f = File::open(&ed.path)?; check_editable(&f)?; Ok(ed) @@ -765,6 +778,12 @@ impl FileEditor { &self.path } + /// Bytes earlier edits of this editor freed that later ones can still + /// reuse. + pub fn reusable_bytes(&self) -> u64 { + self.free.total() + } + /// Plan an edit over the file's current bytes, then commit it. /// /// The reader (a memory map of the file, with the `mmap` feature) is @@ -779,7 +798,8 @@ impl FileEditor { check_editable(&f)?; let sb = f.superblock().clone(); let user_block = f.user_block_size(); - let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); + let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size) + .with_reusable(self.free.clone()); let r = op(&f, &mut img).map_err(unsupported_filter)?; let plan = if img.is_dirty() { if img.eoa() != img.old_eoa() { @@ -790,10 +810,14 @@ impl FileEditor { None }; drop(f); - if let Some(plan) = plan { + if let Some((plan, free)) = plan { #[cfg(test)] tests::note_commit(&self.path); + // A commit that fails part-way leaves the file in an unknown + // state: reuse nothing after it. + self.free = FreeList::default(); plan.commit(&mut self.file, user_block)?; + self.free = free; } Ok(r) } @@ -1524,7 +1548,7 @@ fn store_chunk( }) } _ => { - let a = img.alloc_reusing(len)?; + let a = img.alloc(len)?; img.write(a, &bytes)?; if let Some(info) = existing { img.free(info.address, u64::from(info.chunk_size)); -- 2.54.0 From 1ffd013de910012444d89b648fb082154551ee84 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:11:36 -0500 Subject: [PATCH 10/49] edit: delete a heap's huge-object B-tree with its last huge object libhdf5 deletes a fractal heap's huge-object B-tree when the heap is closed with no huge object left (H5HF__huge_term) and starts huge IDs over. The editor left the empty tree, and a read-only libhdf5 then failed to list the object's attributes: closing the heap tried to delete the tree ("no write intent on file"), and h5dump failed the same way. Replacing an object's last huge attribute (one above the heap's 4 KiB managed limit) now deletes the tree (header and nodes freed), resets the next huge ID and the wrapped flag, as libhdf5 does; a later huge attribute creates a new tree. Bt2::delete frees a whole tree. Found by the extended random-operation test; regression: last_huge_attribute_replaced (fails before: h5dump, h5py listing), which also compares the heap with libhdf5's after the same replacement. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 54 +++++++++++++++++++ crates/clawhdf5/src/edit/btree2.rs | 26 +++++++++ crates/clawhdf5/src/edit/fheap.rs | 9 ++++ 3 files changed, 89 insertions(+) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 332dffa..839e32c 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1396,3 +1396,57 @@ fn freed_space_is_reused_within_a_session() { ); assert!(fresh.1 > fresh.0, "without reuse the file grows"); } + +/// Replacing the last huge attribute (larger than the heap's managed +/// limit) of an object with a small one deletes the heap's huge-object +/// B-tree, as libhdf5 does when it closes the heap (`H5HF__huge_term`). A +/// heap left with an empty huge-object B-tree made read-only libhdf5 fail +/// to list the attributes ("no write intent on file"). +#[test] +fn last_huge_attribute_replaced() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let a = dir.path().join("huge_h5py.h5"); + let b = dir.path().join("huge_edit.h5"); + for p in [&a, &b] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i in range(10): g.attrs.create(f'k{{i}}', np.array([i], dtype=' = (0..10i64) + .map(|i| ("g".to_string(), format!("k{i}"), AV::Ints(vec![i]))) + .collect(); + want.push(("g".into(), "big".into(), AV::Ints(vec![1, 2]))); + check_attr_values(&b, &want); + let info = |p: &Path| { + let s = dense_info(p, "g"); + s[..s.find(" fs ").or(s.find(" no-fs")).unwrap()].to_string() + }; + assert_eq!(info(&b), info(&a), "heap after the replacement"); + // A huge attribute again starts the huge-object B-tree over. + let mut ed = FileEditor::open(&b).unwrap(); + ed.set_attr("g", "big2", &clawhdf5::AttrValue::String("x".repeat(7000))) + .unwrap(); + drop(ed); + want.push(("g".into(), "big2".into(), AV::Str("x".repeat(7000)))); + check_tools(&b, true); + check_attr_values(&b, &want); +} diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs index 5761e1e..71bd9b2 100644 --- a/crates/clawhdf5/src/edit/btree2.rs +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -384,6 +384,32 @@ impl Bt2 { } } + /// Delete the whole tree (`H5B2_delete`): every node and the header go + /// to the image's free list. + pub(crate) fn delete(mut self, img: &mut Image<'_>) -> Result<(), Error> { + if self.root.addr != undef(img.os) && self.root.nrec > 0 { + let mut level = vec![self.root]; + let mut depth = self.depth; + loop { + let mut next = Vec::new(); + for p in level { + self.load(img, p, depth)?; + if depth > 0 { + next.extend(self.peek(p.addr).ptrs.iter().copied()); + } + img.free(p.addr, u64::from(self.node_size)); + } + if depth == 0 { + break; + } + depth -= 1; + level = next; + } + } + img.free(self.addr, Self::header_len(img.os, img.ls) as u64); + Ok(()) + } + /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). pub(crate) fn update( &mut self, diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs index 2fab657..03e6b1e 100644 --- a/crates/clawhdf5/src/edit/fheap.rs +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -948,6 +948,15 @@ impl Heap { img.free(get_uint(&rec, os), len); self.huge_size = self.huge_size.saturating_sub(len); self.huge_nobjs = self.huge_nobjs.saturating_sub(1); + // H5HF__huge_term: with no huge object left, the huge-object + // B-tree is deleted and IDs start over (libhdf5 does it when + // it closes the heap, and a read-only libhdf5 fails to). + if self.huge_nobjs == 0 { + t.delete(img)?; + self.huge_bt2 = undef(os); + self.next_huge_id = 0; + self.flags &= !0x01; + } Ok(()) } _ => Err(unsupported("tiny objects")), -- 2.54.0 From 304aed581309fccce336088c7238389baae2a342 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:11:44 -0500 Subject: [PATCH 11/49] tests: random editor operations include shrinking, 2-D growth, dense attributes random_operations_match_a_model (edit_interop) now drives, on every libver (earliest, v114, latest) and filter set (none, gzip + shuffle + fletcher32, LZF): - a dataset with one unlimited dimension and one with two (a version-2 B-tree chunk index under v114/latest), resized to random shapes that shrink and grow any resizable dimension, with block and point writes; - attributes on the first dataset under 20 names with values of random types and sizes (scalars, int64 arrays, short strings, strings above the heap's managed limit), so they move to dense storage on version-2 headers and are replaced by values of other sizes; against a model where shrunk-away elements that come back read as the fill value, compared with our reader and with h5py/numpy every 40 steps, with h5dump and h5rs check; h5py then grows both datasets and adds an attribute. The attribute check also compares h5py's attribute count with libhdf5's object info. CLAWHDF5_EDIT_SEED reruns the workloads with other random choices (seeds 1000-4000 pass). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-tools/tests/edit_interop.rs | 183 ++++++++++++++------ 1 file changed, 129 insertions(+), 54 deletions(-) diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index cb345f8..3a37e83 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -113,7 +113,8 @@ impl Model { .fold(0u64, |a, (&x, &d)| a * d + x) as usize } - /// Grow to `shape`, new elements `fill`. + /// Change the extent to `shape`: elements inside both keep their + /// values, new ones are `fill`. fn resize(&mut self, shape: &[u64], fill: i32) { let old = self.clone(); *self = Self::new(shape, |_| fill); @@ -125,8 +126,10 @@ impl Model { c[d] = r % old.shape[d]; r /= old.shape[d]; } - let i = self.index(&c); - self.data[i] = old.data[flat as usize]; + if c.iter().zip(shape).all(|(x, s)| x < s) { + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } } } @@ -293,9 +296,24 @@ fn append_many_gzip() { } } -/// Random operations — grow, hyperslab writes, point writes, attributes — -/// on a 2-D dataset with one unlimited dimension, checked against a model -/// after every few operations. +/// A random attribute value: a scalar, an int64 array, a short string or +/// one larger than a heap's managed-object limit (a huge heap object once +/// the attributes are in dense storage). +fn random_attr(rng: &mut Rng) -> AttrValue { + match rng.below(8) { + 0..=2 => AttrValue::I64(rng.next() as i64 >> 3), + 3..=4 => AttrValue::I64Array((0..1 + rng.below(40)).map(|k| k as i64 * 7).collect()), + 5..=6 => AttrValue::String("s".repeat(1 + rng.below(200) as usize)), + _ => AttrValue::String("h".repeat(5000 + rng.below(100) as usize)), + } +} + +/// Random operations — growth and shrinking along any dimension, hyperslab +/// and point writes, attributes (enough names to move them to dense storage +/// on version-2 object headers, replaced with values of any size) — on a +/// 2-D dataset with one unlimited dimension and one with two (a version-2 +/// B-tree chunk index under `v114`/`latest`), checked against a model (and +/// through h5py, numpy) after every few operations. fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { let dir = tmpdir(); let path = dir.path().join(format!("rand_{tag}.h5")); @@ -304,106 +322,159 @@ fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ \x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \ dtype=' = Vec::new(); + let mut models = [Model::new(&[4, 7], |_| -9), Model::new(&[5, 6], |_| 3)]; + models[0].write_block(&[1, 2], &[2, 4], &[5; 8]); + models[1].write_block(&[0, 1], &[4, 4], &[8; 16]); + let fills = [-9, 3]; + let names = ["m", "b"]; + let mut attrs: Vec<(String, AttrValue)> = Vec::new(); let mut rng = Rng(seed); let mut ed = FileEditor::open(&path).unwrap(); - for step in 0..120 { - match rng.below(10) { - 0..=1 => { - let rows = m.shape[0] + 1 + rng.below(5); - ed.resize("m", &[rows, 7]).unwrap(); - m.resize(&[rows, 7], -9); + for step in 0..160 { + let d = rng.below(2) as usize; + let name = names[d]; + let m = &mut models[d]; + match rng.below(12) { + 0..=2 => { + // Grow or shrink: dimension 1 of "m" is fixed at 7. + let rows = rng.below(m.shape[0] + 6); + let cols = if d == 0 { 7 } else { rng.below(m.shape[1] + 5) }; + ed.resize(name, &[rows, cols]).unwrap(); + m.resize(&[rows, cols], fills[d]); } - 2..=6 => { + 3..=7 if m.shape.iter().all(|&s| s > 0) => { let r0 = rng.below(m.shape[0]); - let c0 = rng.below(7); + let c0 = rng.below(m.shape[1]); let cnt = [ 1 + rng.below((m.shape[0] - r0).min(6)), - 1 + rng.below(7 - c0), + 1 + rng.below((m.shape[1] - c0).min(6)), ]; let n = cnt[0] * cnt[1]; let vals: Vec = (0..n).map(|_| (rng.next() % 100_000) as i32).collect(); - ed.write_values("m", &block(&[r0, c0], &cnt), &vals) + ed.write_values(name, &block(&[r0, c0], &cnt), &vals) .unwrap(); m.write_block(&[r0, c0], &cnt, &vals); } - 7 => { + 8 if m.shape.iter().all(|&s| s > 0) => { let pts: Vec> = (0..1 + rng.below(4)) - .map(|_| vec![rng.below(m.shape[0]), rng.below(7)]) + .map(|_| vec![rng.below(m.shape[0]), rng.below(m.shape[1])]) .collect(); let vals: Vec = pts.iter().map(|_| rng.next() as i32).collect(); - ed.write_values("m", &Selection::Points(pts.clone()), &vals) + ed.write_values(name, &Selection::Points(pts.clone()), &vals) .unwrap(); for (p, v) in pts.iter().zip(&vals) { let i = m.index(p); m.data[i] = *v; } } - _ => { - let k = rng.below(6); - let name = format!("a{k}"); - let v = rng.next() as i64; - match ed.set_attr("m", &name, &AttrValue::I64(v)) { + 9..=11 => { + let k = rng.below(20); + let aname = format!("a{k}"); + let v = random_attr(&mut rng); + match ed.set_attr("m", &aname, &v) { Ok(()) => { - attrs.retain(|(n, _)| *n != name); - attrs.push((name, v)); + attrs.retain(|(n, _)| *n != aname); + attrs.push((aname, v)); } - Err(e) => panic!("set_attr {name}: {e}"), + // Replacing the only attribute in a heap block with one + // of another size would have libhdf5 free the block. + Err(Error::Unsupported(msg)) if msg.contains("last object") => {} + Err(e) => panic!("set_attr {aname}: {e}"), } } + _ => {} } - if step % 30 == 29 { + if step % 40 == 39 { drop(ed); - verify(&path, "m", &m); + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_tools(&path, h5dump); check_attrs(&path, "m", &attrs); ed = FileEditor::open(&path).unwrap(); } } drop(ed); - verify(&path, "m", &m); + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_attrs(&path, "m", &attrs); py(&format!( "import h5py, numpy as np\n\ with h5py.File({p:?}, 'r+') as f:\n\ - \x20 d = f['m']\n\ - \x20 n = d.shape[0]\n\ - \x20 d.resize((n + 3, 7))\n\ - \x20 d[n:, :] = 42\n\ - \x20 d.attrs['from_h5py'] = 1.5\n", + \x20 for name, cols in (('m', 7), ('b', None)):\n\ + \x20 d = f[name]\n\ + \x20 n, c = d.shape\n\ + \x20 d.resize((n + 3, cols or c + 2))\n\ + \x20 d[n:, :] = 42\n\ + \x20 f['m'].attrs['from_h5py'] = 1.5\n", p = path.to_str().unwrap() )); - let n = m.shape[0]; - m.resize(&[n + 3, 7], -9); - m.write_block(&[n, 0], &[3, 7], &[42; 21]); - verify(&path, "m", &m); + for (d, m) in models.iter_mut().enumerate() { + let (n, c) = (m.shape[0], m.shape[1]); + let c2 = if d == 0 { 7 } else { c + 2 }; + m.resize(&[n + 3, c2], fills[d]); + m.write_block(&[n, 0], &[3, c2], &vec![42; (3 * c2) as usize]); + } + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_tools(&path, h5dump); + check_attrs(&path, "m", &attrs); } -fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) { +/// Our reader and h5py see `attrs` on dataset `obj` (and h5py's count of +/// its attributes agrees with libhdf5's object info). +fn check_attrs(path: &Path, obj: &str, attrs: &[(String, AttrValue)]) { let f = File::open(path).unwrap(); let got = f.dataset(obj).unwrap().attrs().unwrap(); for (n, v) in attrs { - match got.get(n) { - Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"), - other => panic!("attribute {n}: {other:?}"), - } + let g = got + .get(n) + .unwrap_or_else(|| panic!("attribute {n} missing")); + // Our reader reports a one-element array as a scalar. + let v = match v { + AttrValue::I64Array(a) if a.len() == 1 => &AttrValue::I64(a[0]), + v => v, + }; + assert_eq!(format!("{g:?}"), format!("{v:?}"), "attribute {n}"); } - let want: Vec = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect(); - py(&format!( + let want: Vec = attrs + .iter() + .map(|(n, v)| { + let pv = match v { + AttrValue::I64(x) => format!("{x}"), + AttrValue::I64Array(a) => format!("{a:?}"), + AttrValue::String(s) => format!("{s:?}"), + other => panic!("{other:?}"), + }; + format!("{n:?}: {pv}") + }) + .collect(); + let script = format!( "import h5py\n\ f = h5py.File({p:?}, 'r')\n\ want = {{{w}}}\n\ - got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\ - assert got == want, (got, want)\n", + a = f[{obj:?}].attrs\n\ + def norm(v):\n\ + \x20 v = v.decode() if isinstance(v, bytes) else v\n\ + \x20 return v.tolist() if hasattr(v, 'tolist') else v\n\ + got = {{k: norm(v) for k, v in a.items() if k in want}}\n\ + assert got == want, sorted(set(want) ^ set(got))\n\ + assert len(a) == h5py.h5o.get_info(f[{obj:?}].id).num_attrs == len(list(a))\n", p = path.to_str().unwrap(), w = want.join(", ") - )); + ); + let sp = path.with_extension("attrs.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); } #[test] @@ -411,7 +482,11 @@ fn random_operations_match_a_model() { if !tools_ok() { return; } - let mut seed = 1; + // CLAWHDF5_EDIT_SEED runs the same workloads with other random choices. + let mut seed = std::env::var("CLAWHDF5_EDIT_SEED") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(1); for (i, (lv, dump)) in LIBVERS.iter().enumerate() { // h5dump has no LZF decoder (h5py's own filter). for (j, (extra, lzf)) in [ -- 2.54.0 From 955fdb660d5121211d06192101d443c12d06fa11 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:12:14 -0500 Subject: [PATCH 12/49] tests: resize and write clawhdf5-written datasets on every chunk index Version-2 B-tree (its writer's own node size and single-leaf layout), Extensible Array and Fixed Array datasets written by FileBuilder, with and without deflate, resized up and down along both dimensions and written at random against a model; h5py, h5dump and h5rs check read the result, and h5py resizes and rewrites every dataset afterwards. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 839e32c..ffe13d9 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1450,3 +1450,83 @@ fn last_huge_attribute_replaced() { check_tools(&b, true); check_attr_values(&b, &want); } + +/// Datasets clawhdf5 writes — a version-2 B-tree index (two unlimited +/// dimensions, its own node size and a single leaf sized to its records), +/// an Extensible Array, a Fixed Array (fixed maximum shape), deflated and +/// not — resized up and down and written at random, against a model; h5py, +/// h5dump and `h5rs check` read the result and h5py goes on. +#[test] +fn resize_clawhdf5_written_datasets() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("ours_resize.h5"); + let mut b = clawhdf5::FileBuilder::new(); + let grid: Vec = (0..48).collect(); + let specs: [(&str, [u64; 2], [u64; 2], bool); 4] = [ + ("bt2", [6, 8], [u64::MAX, u64::MAX], false), + ("bt2_gz", [6, 8], [u64::MAX, u64::MAX], true), + ("ea", [6, 8], [u64::MAX, 8], true), + ("fa", [6, 8], [20, 12], false), + ]; + for (name, shape, max, gz) in specs { + let d = b + .create_dataset(name) + .with_i32_data(&grid) + .with_shape(&shape) + .with_maxshape(&max) + .with_chunks(&[2, 3]); + if gz { + d.with_deflate(4); + } + } + b.write(&path).unwrap(); + assert_eq!(chunk_index(&path, "bt2").0, 5, "version-2 B-tree index"); + assert_eq!(chunk_index(&path, "ea").0, 4, "Extensible Array index"); + let mut models: Vec = specs + .iter() + .map(|_| Model::new(&[6, 8], |i| i as i32)) + .collect(); + let mut rng = Rng(77); + let mut ed = FileEditor::open(&path).unwrap(); + for step in 0..120 { + let k = rng.below(4) as usize; + let (name, _, max, _) = specs[k]; + let m = &mut models[k]; + if step % 4 == 0 { + let s: Vec = (0..2) + .map(|d| rng.below((m.shape[d] * 2 + 4).min(max[d]) + 1)) + .collect(); + ed.resize(name, &s).unwrap(); + m.resize(&s, 0); + } else if m.shape.iter().all(|&s| s > 0) { + let st: Vec = m.shape.iter().map(|&s| rng.below(s)).collect(); + let cnt: Vec = (0..2) + .map(|d| 1 + rng.below((m.shape[d] - st[d]).min(5))) + .collect(); + let vals: Vec = (0..cnt[0] * cnt[1]).map(|_| rng.next() as i32).collect(); + ed.write_values(name, &block(&st, &cnt), &vals).unwrap(); + m.write_block(&st, &cnt, &vals); + } + } + drop(ed); + for ((name, ..), m) in specs.iter().zip(&models) { + verify(&path, name, m); + } + check_tools(&path, true); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for n in ['bt2', 'bt2_gz', 'ea', 'fa']:\n\ + \x20 d = f[n]\n\ + \x20 d.resize((6, 8))\n\ + \x20 d[...] = np.arange(48, dtype=' Date: Sat, 26 Sep 2026 17:13:28 -0500 Subject: [PATCH 13/49] clawhdf5-remote: block cache and HTTP range reads (open_url) Range-read milestone M3, first half: a new crate with the block cache the design makes mandatory for remote files and an HTTP backend, so open_url("http://...") gives a clawhdf5::File over File::open_storage. BlockCache wraps any Storage: aligned blocks (1 MiB by default, the size docs/design/range-reads.md section 2 measured), LRU with a byte budget, the missing blocks of one read_at/read_ranges fetched with one backend read_ranges call as runs of consecutive blocks (a one-block gap filled to merge runs, each request at most 8 MiB), and reads that miss more than half the budget not kept. Thread-safe without holding the lock across a fetch: a block being fetched is in flight, a second reader waits for it instead of fetching it again, and a failed fetch fails its waiters and is not cached. A backend holding the file in memory passes through. HttpStorage (ureq, no TLS by default; `https` adds rustls with ring): opening is one ranged GET of the first block, whose Content-Range gives the length (the cache keeps the bytes). The file is pinned by a strong ETag (If-Match), else Last-Modified (If-Unmodified-Since), and its length, checked on every response: a change is RemoteError::FileChanged, never mixed data. A server that ignores Range is refused without reading the body unless a full download is allowed. Connection errors, timeouts, 408/429/5xx and short bodies are retried with exponential backoff; Accept-Encoding: identity, and an encoded body is refused. read_ranges fetches its ranges in parallel. Tests (a std-only HTTP/1.1 server in tests/common/server.rs, also the range_server example): every fixture read over HTTP gives File::open's transcript (CLAWHDF5_REMOTE_CORPUS adds the conformance corpus), with request counts per file with and without the cache; an h5py-written file against libhdf5's values; a multi-block file fetched in whole blocks, each once; a server ignoring Range; a file replaced mid-read (ETag, Last-Modified, length only); truncated bodies and 503s (retried, then an error, never cached); a slow server with 8 concurrent readers (no block fetched twice); bad URLs, 404, encoded bodies, non-HDF5 data. The cache has unit tests for coalescing, splitting, LRU order, large reads, failures and concurrent in-flight dedup. ci-test.sh: clawhdf5-remote joins the no-C default-build check, and its https feature is linted. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 15 +- Cargo.toml | 1 + crates/clawhdf5-remote/Cargo.toml | 27 + crates/clawhdf5-remote/README.md | 71 ++ .../clawhdf5-remote/examples/range_server.rs | 56 ++ crates/clawhdf5-remote/examples/read_url.rs | 45 + crates/clawhdf5-remote/src/cache.rs | 825 ++++++++++++++++++ crates/clawhdf5-remote/src/error.rs | 125 +++ crates/clawhdf5-remote/src/http.rs | 565 ++++++++++++ crates/clawhdf5-remote/src/lib.rs | 127 +++ crates/clawhdf5-remote/tests/common/mod.rs | 302 +++++++ crates/clawhdf5-remote/tests/common/server.rs | 331 +++++++ crates/clawhdf5-remote/tests/http.rs | 517 +++++++++++ scripts/ci-test.sh | 23 +- 14 files changed, 3024 insertions(+), 6 deletions(-) create mode 100644 crates/clawhdf5-remote/Cargo.toml create mode 100644 crates/clawhdf5-remote/README.md create mode 100644 crates/clawhdf5-remote/examples/range_server.rs create mode 100644 crates/clawhdf5-remote/examples/read_url.rs create mode 100644 crates/clawhdf5-remote/src/cache.rs create mode 100644 crates/clawhdf5-remote/src/error.rs create mode 100644 crates/clawhdf5-remote/src/http.rs create mode 100644 crates/clawhdf5-remote/src/lib.rs create mode 100644 crates/clawhdf5-remote/tests/common/mod.rs create mode 100644 crates/clawhdf5-remote/tests/common/server.rs create mode 100644 crates/clawhdf5-remote/tests/http.rs diff --git a/CLAUDE.md b/CLAUDE.md index f493066..57a5fa4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persist ## Architecture -Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): +Cargo workspace with 19 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature): | Crate | Role | |-------|------| @@ -26,6 +26,7 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-py` | PyO3 Python bindings | | `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` | +| `clawhdf5-remote` | Remote files: `open_url` over HTTP(S) range requests through a mandatory block cache (`BlockCache`) | | `clawhdf5-bench` | Benchmark suite | ## Key Features @@ -157,6 +158,18 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F `docs/known-issues.md`). Test changes with `cargo test -p clawhdf5-tools --test edit_interop` (h5py, h5dump, `h5rs check`). +- Remote files (`clawhdf5-remote`, range-read milestone M3 of + `docs/design/range-reads.md`): `open_url("http://…")` gives a + `clawhdf5::File` over `File::open_storage`, read through `BlockCache` + (1 MiB blocks, LRU byte budget, per-block in-flight dedup across threads, + runs coalesced into parallel requests). `HttpStorage` pins the file by + ETag/Last-Modified and length (a change is `RemoteError::FileChanged`), + refuses servers that ignore `Range` unless a full download is allowed, + and retries transient failures. Default build is plain HTTP with no C; + `https` (rustls + ring) is opt-in. Tests run a std-only HTTP server + (`tests/common/server.rs`, also the `range_server` example); + `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus` compares every corpus + file over HTTP with `File::open`. - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only - Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. diff --git a/Cargo.toml b/Cargo.toml index 0801b3d..dbb2f5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/clawhdf5-bench", "crates/clawhdf5-tools", "crates/clawhdf5-wasm", + "crates/clawhdf5-remote", "crates/libaec-sys", ] resolver = "2" diff --git a/crates/clawhdf5-remote/Cargo.toml b/crates/clawhdf5-remote/Cargo.toml new file mode 100644 index 0000000..6e68a73 --- /dev/null +++ b/crates/clawhdf5-remote/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "clawhdf5-remote" +version = "2.7.0" +edition = "2024" +rust-version.workspace = true +description = "Read HDF5 files over HTTP(S) range requests with clawhdf5, through a block cache" +license = "MIT" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" +readme = "README.md" +keywords = ["hdf5", "http", "s3", "range-requests", "science"] +categories = ["science", "network-programming"] + +[features] +# Plain-HTTP range reads and the block cache: pure Rust, no TLS stack, no C. +default = ["http"] +http = ["dep:ureq"] +# HTTPS through rustls (ring provider, Mozilla roots). ring compiles C and +# assembly, so this is not part of the default build. +https = ["http", "ureq/rustls"] + +[dependencies] +clawhdf5 = { path = "../clawhdf5", version = "2.7.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +ureq = { version = "3.4", optional = true, default-features = false } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md new file mode 100644 index 0000000..bb20f71 --- /dev/null +++ b/crates/clawhdf5-remote/README.md @@ -0,0 +1,71 @@ +# clawhdf5-remote + +Read HDF5 files where they live — on an HTTP(S) server — with +[clawhdf5](../../README.md), without downloading them first. + +```rust +let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/data.h5")?; +let temps = file.dataset("/grid/temperature")?.read_f64()?; +``` + +The result is an ordinary `clawhdf5::File`: groups, datasets, attributes, +selections, variable-length data. Only the bytes an operation needs are +fetched, by `Range` requests, through a block cache. + +## How it reads + +- **Block cache** (`BlockCache`, mandatory for remote files; see + `docs/design/range-reads.md` §2): aligned blocks of 1 MiB by default, LRU + with a byte budget (64 MiB by default), the missing blocks of one read + fetched as runs of consecutive blocks (a gap of one block is fetched to + merge two runs), each request at most 8 MiB, the requests of one read in + parallel. A read that misses more than half the budget is not kept, so a + big dataset read does not evict the metadata. Readers on several threads + share one cache; a block is never fetched twice at once — a second reader + waits for the first one's request. +- **Opening costs one request**: a `GET` of the first block, whose + `Content-Range` gives the file's length. HDF5 files keep the superblock + and usually the root group's metadata there, so listing a small file + often needs nothing more. +- **Pinned to one version of the file**: a strong `ETag` is sent back as + `If-Match` (else `Last-Modified` as `If-Unmodified-Since`) and checked on + every response, as is the length. A file replaced while open is an error + (`RemoteError::FileChanged`), never a mix of old and new bytes. A server + with neither validator can only be checked by length; + `HttpOptions::require_validator` refuses it. +- **Servers that ignore `Range`** (answer `200` with the whole file) are + refused (`RemoteError::RangeNotSupported`) without reading the body, + unless `HttpOptions::allow_full_download` is set; then the file is + downloaded once and read from memory. +- **Retries**: connection failures, timeouts, `408`/`429`/`5xx` and bodies + that end early are retried with exponential backoff (3 retries, from + 200 ms). Bodies are requested with `Accept-Encoding: identity`; an encoded + body is refused. + +The zero-copy methods of `clawhdf5` (`read_raw_ref`, `read_*_zerocopy`, +`File::as_bytes`) borrow the whole file from memory, so they are errors +(`as_bytes` a panic; use `File::contiguous_bytes`) on a remote file. + +## Features + +| Feature | What | C code | +|---|---|---| +| `http` (default) | `http://` through `ureq`, no TLS | none | +| `https` | `https://` through rustls, ring provider, Mozilla roots | ring (C and assembly) | + +The default build compiles no C (`scripts/ci-test.sh` checks it). + +## Counting requests + +```rust +use clawhdf5_remote::{storage_for_url, Options}; +let storage = storage_for_url(url, &Options::default())?; +let file = clawhdf5::File::open_storage(storage.clone())?; +// ... read ... +let s = storage.stats(); // requests, bytes_fetched, hits, misses, cached_bytes, ... +``` + +`cargo run -p clawhdf5-remote --example range_server -- DIR` serves a +directory with range support (the server the tests use), and +`cargo run -p clawhdf5-remote --example read_url -- URL [DATASET]` lists a +file and prints what it cost. diff --git a/crates/clawhdf5-remote/examples/range_server.rs b/crates/clawhdf5-remote/examples/range_server.rs new file mode 100644 index 0000000..609693f --- /dev/null +++ b/crates/clawhdf5-remote/examples/range_server.rs @@ -0,0 +1,56 @@ +//! Serve the HDF5 files of a directory over HTTP with range support — the +//! server the tests use — to try `clawhdf5_remote` and `h5rs` on URLs. +//! +//! ```text +//! cargo run -p clawhdf5-remote --example range_server -- DIR [127.0.0.1:8000] +//! ``` +//! +//! Every file under `DIR` is served at its path relative to `DIR`. The +//! files are read into memory at start. Requests are logged to stderr. + +#[path = "../tests/common/server.rs"] +mod server; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(dir) = args.next() else { + eprintln!("usage: range_server DIR [ADDR]"); + std::process::exit(2); + }; + let addr = args.next().unwrap_or_else(|| "127.0.0.1:8000".into()); + let root = std::path::PathBuf::from(&dir); + let mut files = Vec::new(); + let mut stack = vec![root.clone()]; + while let Some(d) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&d) else { + continue; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else if let (Ok(rel), Ok(bytes)) = (p.strip_prefix(&root), std::fs::read(&p)) { + let url = format!("/{}", rel.to_string_lossy().replace('\\', "/")); + files.push((url, bytes)); + } + } + } + files.sort(); + let server = server::Server::bind(&addr, files.clone()); + for (path, bytes) in &files { + eprintln!("{} ({} bytes)", server.url(path), bytes.len()); + } + eprintln!("serving {} files on http://{}", files.len(), server.addr); + let mut logged = 0; + loop { + std::thread::sleep(std::time::Duration::from_millis(200)); + let log = server.log(); + for (path, range) in &log[logged.min(log.len())..] { + match range { + Some((a, b)) => eprintln!("GET {path} bytes={a}-{b}"), + None => eprintln!("GET {path} (whole file)"), + } + } + logged = log.len(); + } +} diff --git a/crates/clawhdf5-remote/examples/read_url.rs b/crates/clawhdf5-remote/examples/read_url.rs new file mode 100644 index 0000000..145b4ce --- /dev/null +++ b/crates/clawhdf5-remote/examples/read_url.rs @@ -0,0 +1,45 @@ +//! List a remote HDF5 file and read one dataset, then print what it cost. +//! +//! ```text +//! cargo run -p clawhdf5-remote --example read_url -- URL [DATASET] +//! ``` + +use clawhdf5::File; +use clawhdf5_remote::{Options, storage_for_url}; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let url = args.next().ok_or("usage: read_url URL [DATASET]")?; + let dataset = args.next(); + + let storage = storage_for_url(&url, &Options::default())?; + let file = File::open_storage(storage.clone())?; + + // Walk the tree. + let mut stack = vec![("/".to_string(), file.root())]; + while let Some((path, group)) = stack.pop() { + for (name, addr) in group.entries()? { + let child = format!("{}/{name}", path.trim_end_matches('/')); + match file.dataset_at(addr) { + Ok(ds) => println!("{child} dataset {:?} {:?}", ds.shape()?, ds.dtype()?), + Err(_) => { + println!("{child} group"); + stack.push((child, file.group_at(addr))); + } + } + } + } + + if let Some(name) = dataset { + let values = file.dataset(&name)?.read_f64()?; + let shown: Vec<_> = values.iter().take(8).collect(); + println!("{name}: {} values, first {shown:?}", values.len()); + } + + let s = storage.stats(); + println!( + "{} range requests (the one at open included), {} bytes fetched, {} bytes cached", + s.requests, s.bytes_fetched, s.cached_bytes + ); + Ok(()) +} diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs new file mode 100644 index 0000000..233938c --- /dev/null +++ b/crates/clawhdf5-remote/src/cache.rs @@ -0,0 +1,825 @@ +//! The block cache every remote backend is read through. +//! +//! `docs/design/range-reads.md` §2 measured why it is required: the parsers +//! revisit the same structures many times (410 reads to list the 21 objects +//! of a 7.7 MB file), so mapping reads one-to-one onto requests is hopeless, +//! while a cache of 1 MiB blocks lists the same file in 2 requests. +//! +//! [`BlockCache`] wraps any [`Storage`] (the backend that does the fetching) +//! and serves reads from fixed-size, aligned blocks: +//! +//! - **LRU with a byte budget.** Blocks are evicted least-recently-used +//! first once the cached bytes exceed [`CacheConfig::capacity`]. Reads +//! hold their blocks by reference count, so eviction never invalidates +//! data a read is still using. +//! - **Coalescing.** All the blocks one `read_at`/`read_ranges` call misses +//! are fetched with one `read_ranges` call on the backend, as runs of +//! consecutive blocks (a gap of up to [`CacheConfig::coalesce_gap`] bytes +//! of uncached blocks is fetched too, to merge two runs), each run at +//! most [`CacheConfig::max_request`] bytes. A backend fetches the runs of +//! one call in parallel. +//! - **Concurrency.** The cache's lock is held only to look blocks up and +//! to insert them, never across a fetch. A block being fetched is marked +//! in flight: a second reader that needs it waits for that fetch instead +//! of issuing its own, so concurrent readers never fetch a block twice. +//! A failed fetch fails every reader waiting for it, and is not cached. +//! - **Large reads do not flush the cache.** A call whose missing blocks +//! add up to more than half the budget is served without keeping them +//! (a big chunked read would otherwise evict all the metadata). +//! - **Local storages pass through.** A backend that holds the whole file +//! in memory ([`Storage::as_contiguous`]) is read directly. + +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; + +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; + +/// Default block size: 1 MiB, the size `docs/design/range-reads.md` §2 +/// measured (metadata of the test files in 1–15 blocks; the chunk index of a +/// netCDF file spread over 14 blocks of a 48 MB file). +pub const DEFAULT_BLOCK_SIZE: u64 = 1 << 20; + +/// Settings of a [`BlockCache`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheConfig { + /// Size of a block, in bytes; every fetch is whole, aligned blocks + /// (the last block of the file is shorter). + pub block_size: u64, + /// Byte budget of cached blocks. + pub capacity: u64, + /// Two runs of missing blocks separated by at most this many bytes of + /// uncached blocks are fetched as one request (the gap included). + pub coalesce_gap: u64, + /// Largest single request, in bytes (rounded down to whole blocks, at + /// least one block). Longer runs are split, so a backend can fetch the + /// pieces in parallel. + pub max_request: u64, +} + +impl Default for CacheConfig { + fn default() -> Self { + CacheConfig { + block_size: DEFAULT_BLOCK_SIZE, + capacity: 64 << 20, + coalesce_gap: DEFAULT_BLOCK_SIZE, + max_request: 8 << 20, + } + } +} + +/// What a [`BlockCache`] has done so far. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CacheStats { + /// Reads served (`read_at` calls plus ranges of `read_ranges` calls). + pub reads: u64, + /// Block lookups that found the block cached. + pub hits: u64, + /// Block lookups that had to fetch the block (or wait for it). + pub misses: u64, + /// Block lookups that waited for another reader's fetch of the block. + pub waits: u64, + /// Ranges requested from the backend: for HTTP, one request each. + pub requests: u64, + /// `read_ranges` calls made on the backend. + pub fetch_calls: u64, + /// Bytes fetched from the backend. + pub bytes_fetched: u64, + /// Blocks evicted to stay within the budget. + pub evictions: u64, + /// Bytes currently cached. + pub cached_bytes: u64, +} + +type Block = Arc<[u8]>; + +/// A fetch in progress: the readers waiting for a block wait on this. +struct Flight { + result: Mutex>>, + done: Condvar, +} + +impl Flight { + fn new() -> Arc { + Arc::new(Flight { + result: Mutex::new(None), + done: Condvar::new(), + }) + } + + fn finish(&self, r: Result) { + let mut slot = lock(&self.result); + if slot.is_none() { + *slot = Some(r); + } + self.done.notify_all(); + } + + fn wait(&self) -> Result { + let mut slot = lock(&self.result); + loop { + if let Some(r) = slot.as_ref() { + return r.clone(); + } + slot = self + .done + .wait(slot) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } +} + +enum Slot { + Ready { data: Block, tick: u64 }, + Pending(Arc), +} + +#[derive(Default)] +struct State { + blocks: HashMap, + /// tick -> block index, oldest first. + lru: BTreeMap, + tick: u64, + bytes: u64, +} + +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[derive(Default)] +struct Counters { + reads: AtomicU64, + hits: AtomicU64, + misses: AtomicU64, + waits: AtomicU64, + requests: AtomicU64, + fetch_calls: AtomicU64, + bytes_fetched: AtomicU64, + evictions: AtomicU64, +} + +/// A [`Storage`] that serves reads of `inner` from an LRU cache of +/// fixed-size blocks. See the [module documentation](self). +pub struct BlockCache { + inner: S, + config: CacheConfig, + len: u64, + state: Mutex, + counters: Counters, +} + +/// Fails every flight a fetch claimed and did not complete (an error or a +/// panic in the backend), so no reader waits forever. +struct FlightGuard<'a, S> { + cache: &'a BlockCache, + flights: Vec<(u64, Arc)>, +} + +impl Drop for FlightGuard<'_, S> { + fn drop(&mut self) { + if self.flights.is_empty() { + return; + } + let mut st = lock(&self.cache.state); + for (i, f) in &self.flights { + if matches!(st.blocks.get(i), Some(Slot::Pending(p)) if Arc::ptr_eq(p, f)) { + st.blocks.remove(i); + } + } + drop(st); + for (_, f) in &self.flights { + f.finish(Err("the fetch of this block failed".into())); + } + } +} + +impl BlockCache { + /// Cache `inner` with the given settings. The length of the file is + /// taken from `inner` once, here: a remote file is pinned at open. + pub fn new(inner: S, mut config: CacheConfig) -> Self { + config.block_size = config.block_size.max(512); + config.capacity = config.capacity.max(config.block_size); + config.max_request = (config.max_request / config.block_size).max(1) * config.block_size; + let len = inner.len(); + BlockCache { + inner, + config, + len, + state: Mutex::new(State::default()), + counters: Counters::default(), + } + } + + /// The backend. + pub fn inner(&self) -> &S { + &self.inner + } + + /// The settings in use (after rounding). + pub fn config(&self) -> &CacheConfig { + &self.config + } + + /// Counters since the cache was made (or last [`reset_stats`](Self::reset_stats)). + pub fn stats(&self) -> CacheStats { + let c = &self.counters; + CacheStats { + reads: c.reads.load(Ordering::Relaxed), + hits: c.hits.load(Ordering::Relaxed), + misses: c.misses.load(Ordering::Relaxed), + waits: c.waits.load(Ordering::Relaxed), + requests: c.requests.load(Ordering::Relaxed), + fetch_calls: c.fetch_calls.load(Ordering::Relaxed), + bytes_fetched: c.bytes_fetched.load(Ordering::Relaxed), + evictions: c.evictions.load(Ordering::Relaxed), + cached_bytes: lock(&self.state).bytes, + } + } + + /// Zero the counters (the cached blocks stay). + pub fn reset_stats(&self) { + let c = &self.counters; + for a in [ + &c.reads, + &c.hits, + &c.misses, + &c.waits, + &c.requests, + &c.fetch_calls, + &c.bytes_fetched, + &c.evictions, + ] { + a.store(0, Ordering::Relaxed); + } + } + + /// Drop every cached block. + pub fn clear(&self) { + let mut st = lock(&self.state); + st.blocks.retain(|_, s| matches!(s, Slot::Pending(_))); + st.lru.clear(); + st.bytes = 0; + } + + /// Fetch the blocks covering `[offset, offset + len)` now (readahead), + /// keeping them cached. + pub fn prefetch(&self, offset: u64, len: u64) -> Result<(), FormatError> { + if self.inner.as_contiguous().is_some() { + return Ok(()); + } + let Some(blocks) = self.block_span(offset, len) else { + return Ok(()); + }; + self.blocks(&blocks.collect::>(), true).map(|_| ()) + } + + /// Put bytes already fetched (such as the first block, which a backend + /// may get while it probes the file's length) into the cache: `bytes` + /// are the file's bytes at `offset`. Only whole blocks (or the file's + /// last, short block) are kept; a block already cached is left alone. + /// The bytes count as one request in [`CacheStats`]. + pub fn insert(&self, offset: u64, bytes: &[u8]) { + self.counters.requests.fetch_add(1, Ordering::Relaxed); + self.counters + .bytes_fetched + .fetch_add(bytes.len() as u64, Ordering::Relaxed); + let bs = self.config.block_size; + let end = offset.saturating_add(bytes.len() as u64).min(self.len); + let mut i = offset.div_ceil(bs); + let mut st = lock(&self.state); + while i * bs < end { + let start = i * bs; + let block_end = (start + bs).min(self.len); + if block_end > end { + break; + } + if !st.blocks.contains_key(&i) { + let rel = (start - offset) as usize..(block_end - offset) as usize; + self.keep(&mut st, i, Arc::from(&bytes[rel])); + } + i += 1; + } + self.evict(&mut st); + } + + /// Block indices covering `[offset, offset + len)`, clamped to the file. + fn block_span(&self, offset: u64, len: u64) -> Option> { + let end = offset.saturating_add(len).min(self.len); + if offset >= end { + return None; + } + let bs = self.config.block_size; + Some(offset / bs..(end - 1) / bs + 1) + } + + fn block_len(&self, i: u64) -> u64 { + let start = i * self.config.block_size; + (start + self.config.block_size).min(self.len) - start + } + + fn keep(&self, st: &mut State, i: u64, data: Block) { + st.tick += 1; + let tick = st.tick; + st.bytes += <[u8]>::len(&data) as u64; + st.lru.insert(tick, i); + st.blocks.insert(i, Slot::Ready { data, tick }); + } + + fn evict(&self, st: &mut State) { + while st.bytes > self.config.capacity { + let Some((_, i)) = st.lru.pop_first() else { + break; + }; + if let Some(Slot::Ready { data, .. }) = st.blocks.remove(&i) { + st.bytes -= <[u8]>::len(&data) as u64; + self.counters.evictions.fetch_add(1, Ordering::Relaxed); + } + } + } + + /// The blocks `wanted` (sorted, distinct), fetching the missing ones. + fn blocks(&self, wanted: &[u64], may_keep: bool) -> Result, FormatError> { + let mut have = HashMap::with_capacity(wanted.len()); + let mut waits = Vec::new(); + let mut guard = FlightGuard { + cache: self, + flights: Vec::new(), + }; + { + let mut st = lock(&self.state); + for &i in wanted { + match st.blocks.get(&i) { + Some(Slot::Ready { data, tick }) => { + let (data, old) = (data.clone(), *tick); + st.tick += 1; + let tick = st.tick; + st.lru.remove(&old); + st.lru.insert(tick, i); + if let Some(Slot::Ready { tick: t, .. }) = st.blocks.get_mut(&i) { + *t = tick; + } + self.counters.hits.fetch_add(1, Ordering::Relaxed); + have.insert(i, data); + } + Some(Slot::Pending(f)) => { + self.counters.misses.fetch_add(1, Ordering::Relaxed); + self.counters.waits.fetch_add(1, Ordering::Relaxed); + waits.push((i, f.clone())); + } + None => { + self.counters.misses.fetch_add(1, Ordering::Relaxed); + let f = Flight::new(); + st.blocks.insert(i, Slot::Pending(f.clone())); + guard.flights.push((i, f)); + } + } + } + // Fill small gaps between runs with blocks nobody has, so the + // runs merge into one request. + if self.config.coalesce_gap > 0 && guard.flights.len() > 1 { + let gap_blocks = self.config.coalesce_gap / self.config.block_size; + let mut extra = Vec::new(); + for w in guard.flights.windows(2) { + let (a, b) = (w[0].0, w[1].0); + let gap = b - a - 1; + if gap == 0 || gap > gap_blocks { + continue; + } + if (a + 1..b).all(|j| !st.blocks.contains_key(&j)) { + extra.extend(a + 1..b); + } + } + for j in extra { + let f = Flight::new(); + st.blocks.insert(j, Slot::Pending(f.clone())); + guard.flights.push((j, f)); + } + guard.flights.sort_by_key(|(i, _)| *i); + } + } + + if !guard.flights.is_empty() { + let runs = self.runs(&guard.flights); + let missing_bytes: u64 = runs.iter().map(|r| r.end - r.start).sum(); + let keep = may_keep && missing_bytes <= self.config.capacity / 2; + self.counters.fetch_calls.fetch_add(1, Ordering::Relaxed); + self.counters + .requests + .fetch_add(runs.len() as u64, Ordering::Relaxed); + let fetched = self.inner.read_ranges(&runs)?; + if fetched.len() != runs.len() { + return Err(FormatError::Storage(format!( + "backend returned {} ranges for {} requested", + fetched.len(), + runs.len() + ))); + } + let mut got: Vec<(u64, Block)> = Vec::with_capacity(guard.flights.len()); + for (run, bytes) in runs.iter().zip(&fetched) { + self.counters + .bytes_fetched + .fetch_add(bytes.len() as u64, Ordering::Relaxed); + if bytes.len() as u64 != run.end - run.start { + return Err(FormatError::Storage(format!( + "short read from the backend: {} of {} bytes at offset {}", + bytes.len(), + run.end - run.start, + run.start + ))); + } + let bs = self.config.block_size; + let mut i = run.start / bs; + let mut pos = 0usize; + while pos < bytes.len() { + let n = self.block_len(i) as usize; + got.push((i, Arc::from(&bytes[pos..pos + n]))); + pos += n; + i += 1; + } + } + drop(fetched); + let flights = std::mem::take(&mut guard.flights); + let mut st = lock(&self.state); + let mut by_index: HashMap = got.into_iter().collect(); + for (i, f) in &flights { + let Some(data) = by_index.remove(i) else { + continue; + }; + let ours = matches!(st.blocks.get(i), Some(Slot::Pending(p)) if Arc::ptr_eq(p, f)); + if ours { + if keep { + self.keep(&mut st, *i, data.clone()); + } else { + st.blocks.remove(i); + } + } + f.finish(Ok(data.clone())); + have.insert(*i, data); + } + self.evict(&mut st); + drop(st); + // Any flight without data (cannot happen: every run is split + // into its blocks) is failed rather than left waiting. + guard.flights = flights + .into_iter() + .filter(|(i, _)| !have.contains_key(i)) + .collect(); + if !guard.flights.is_empty() { + return Err(FormatError::Storage( + "backend did not return every block".into(), + )); + } + } + + for (i, f) in waits { + match f.wait() { + Ok(data) => { + have.insert(i, data); + } + Err(e) => return Err(FormatError::Storage(e)), + } + } + Ok(have) + } + + /// Byte ranges of the runs of consecutive blocks in `flights` (sorted), + /// each at most `max_request` long. + fn runs(&self, flights: &[(u64, Arc)]) -> Vec> { + let bs = self.config.block_size; + let per_request = self.config.max_request / bs; + let mut runs: Vec<(u64, u64)> = Vec::new(); + for &(i, _) in flights { + match runs.last_mut() { + Some((first, last)) if *last + 1 == i && i - *first < per_request => *last = i, + _ => runs.push((i, i)), + } + } + runs.into_iter() + .map(|(a, b)| a * bs..(b * bs + self.block_len(b))) + .collect() + } + + /// Copy `[offset, end)` out of `blocks`. + fn assemble(&self, offset: u64, end: u64, blocks: &HashMap) -> Vec { + let bs = self.config.block_size; + let mut out = Vec::with_capacity((end - offset) as usize); + let mut pos = offset; + while pos < end { + let i = pos / bs; + let block = &blocks[&i]; + let from = (pos - i * bs) as usize; + let to = ((end - i * bs) as usize).min(<[u8]>::len(block)); + out.extend_from_slice(&block[from..to]); + pos = i * bs + to as u64; + } + out + } +} + +impl Storage for BlockCache { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if let Some(all) = self.inner.as_contiguous() { + return all.read_at(offset, len); + } + self.counters.reads.fetch_add(1, Ordering::Relaxed); + let Some(span) = self.block_span(offset, len as u64) else { + return Ok(Cow::Owned(Vec::new())); + }; + let end = offset.saturating_add(len as u64).min(self.len); + let blocks = self.blocks(&span.collect::>(), true)?; + Ok(Cow::Owned(self.assemble(offset, end, &blocks))) + } + + fn len(&self) -> u64 { + self.len + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if let Some(all) = self.inner.as_contiguous() { + return all.read_ranges(ranges); + } + self.counters + .reads + .fetch_add(ranges.len() as u64, Ordering::Relaxed); + let mut wanted = BTreeSet::new(); + for r in ranges { + if r.end < r.start { + return Err(FormatError::Storage( + "read range ends before it starts".into(), + )); + } + if let Some(span) = self.block_span(r.start, r.end - r.start) { + wanted.extend(span); + } + } + let wanted: Vec = wanted.into_iter().collect(); + let blocks = self.blocks(&wanted, true)?; + Ok(ranges + .iter() + .map(|r| { + let end = r.end.min(self.len); + if r.start >= end { + Cow::Owned(Vec::new()) + } else { + Cow::Owned(self.assemble(r.start, end, &blocks)) + } + }) + .collect()) + } + + fn as_contiguous(&self) -> Option<&[u8]> { + self.inner.as_contiguous() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clawhdf5_format::storage::CountingStorage; + + fn data(n: usize) -> Vec { + (0..n).map(|i| (i * 7 + i / 251) as u8).collect() + } + + fn cache(n: usize, config: CacheConfig) -> (Vec, BlockCache) { + let d = data(n); + (d.clone(), BlockCache::new(CountingStorage::new(d), config)) + } + + fn small() -> CacheConfig { + CacheConfig { + block_size: 1024, + capacity: 8 * 1024, + coalesce_gap: 0, + max_request: 4 * 1024, + } + } + + #[test] + fn reads_match_the_data_and_hit_the_cache() { + let mut cfg = small(); + cfg.capacity = 64 * 1024; + let (d, c) = cache(10_000, cfg); + for (off, len) in [ + (0, 10), + (1000, 100), + (1020, 10), + (9990, 100), + (20_000, 5), + (0, 10_000), + ] { + let got = c.read_at(off, len).unwrap(); + let s = (off as usize).min(d.len()); + let e = (off as usize + len).min(d.len()); + assert_eq!(&*got, &d[s..e], "{off} {len}"); + } + let before = c.inner().reads(); + assert_eq!(&*c.read_at(5000, 3000).unwrap(), &d[5000..8000]); + assert_eq!(c.inner().reads(), before, "all cached"); + assert!(c.stats().hits > 0); + } + + #[test] + fn read_ranges_coalesces_consecutive_missing_blocks() { + let (d, c) = cache(20_000, small()); + let ranges = [100..200, 1500..1600, 2100..3000, 9000..9100]; + let got = c.read_ranges(&ranges).unwrap(); + for (r, g) in ranges.iter().zip(&got) { + assert_eq!(&**g, &d[r.start as usize..r.end as usize]); + } + // Blocks 0-2 are one run, block 8 another: one backend call, two ranges. + assert_eq!(c.stats().fetch_calls, 1); + assert_eq!(c.stats().requests, 2); + assert_eq!(c.inner().reads(), 2); + } + + #[test] + fn a_small_gap_is_fetched_to_merge_runs() { + let mut cfg = small(); + cfg.coalesce_gap = 1024; + let (d, c) = cache(20_000, cfg); + let ranges = [0..10, 2048..2058, 5000..5010]; + let got = c.read_ranges(&ranges).unwrap(); + for (r, g) in ranges.iter().zip(&got) { + assert_eq!(&**g, &d[r.start as usize..r.end as usize]); + } + // Blocks 0 and 2 merge over block 1; block 4 is two blocks away. + assert_eq!(c.stats().requests, 2); + } + + #[test] + fn long_runs_are_split_at_max_request() { + let (d, c) = cache(20_000, small()); + assert_eq!(&*c.read_at(0, 10_000).unwrap(), &d[..10_000]); + // 10 blocks, 4 per request: 3 requests in one call. + assert_eq!(c.stats().requests, 3); + assert_eq!(c.stats().fetch_calls, 1); + } + + #[test] + fn lru_stays_within_budget_and_evicts_oldest() { + let (d, c) = cache(40_000, small()); + for i in 0..8u64 { + c.read_at(i * 1024, 1).unwrap(); + } + c.read_at(0, 1).unwrap(); // block 0 is now the newest + c.read_at(8 * 1024, 1).unwrap(); // evicts block 1 + let s = c.stats(); + assert!(s.cached_bytes <= 8 * 1024); + assert_eq!(s.evictions, 1); + let before = c.inner().reads(); + c.read_at(0, 1).unwrap(); + assert_eq!(c.inner().reads(), before, "block 0 kept"); + assert_eq!(&*c.read_at(1024, 5).unwrap(), &d[1024..1029]); + assert_eq!(c.inner().reads(), before + 1, "block 1 refetched"); + } + + #[test] + fn large_reads_do_not_flush_the_cache() { + let (d, c) = cache(40_000, small()); + c.read_at(0, 1).unwrap(); + assert_eq!(&*c.read_at(10_000, 20_000).unwrap(), &d[10_000..30_000]); + assert_eq!(c.stats().cached_bytes, 1024, "only block 0 kept"); + } + + #[test] + fn insert_keeps_whole_blocks_only() { + let (d, c) = cache(5000, small()); + c.insert(0, &d[..2500]); + assert_eq!(c.stats().cached_bytes, 2048); + c.insert(4096, &d[4096..]); + assert_eq!(c.stats().cached_bytes, 2048 + 904); + assert_eq!(&*c.read_at(0, 2048).unwrap(), &d[..2048]); + assert_eq!(&*c.read_at(4500, 500).unwrap(), &d[4500..]); + assert_eq!(c.inner().reads(), 0); + } + + /// A backend that fails its first `n` fetches. + struct Flaky { + data: Vec, + fail: AtomicU64, + } + + impl Storage for Flaky { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if self + .fail + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Err(FormatError::Storage("boom".into())); + } + self.data + .as_slice() + .read_at(offset, len) + .map(|c| Cow::Owned(c.into_owned())) + } + fn len(&self) -> u64 { + self.data.len() as u64 + } + } + + #[test] + fn a_failed_fetch_is_an_error_and_is_not_cached() { + let d = data(5000); + let c = BlockCache::new( + Flaky { + data: d.clone(), + fail: AtomicU64::new(1), + }, + small(), + ); + assert!(c.read_at(0, 10).is_err()); + assert_eq!(&*c.read_at(0, 10).unwrap(), &d[..10]); + } + + /// A backend that returns fewer bytes than asked inside the file. + struct Short(Vec); + + impl Storage for Short { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let got = self.0.as_slice().read_at(offset, len)?; + Ok(Cow::Owned(got[..got.len() / 2].to_vec())) + } + fn len(&self) -> u64 { + self.0.len() as u64 + } + } + + #[test] + fn a_short_backend_read_is_an_error() { + let c = BlockCache::new(Short(data(5000)), small()); + assert!(c.read_at(0, 10).is_err()); + assert!(c.read_at(0, 10).is_err(), "not cached"); + } + + #[test] + fn contiguous_backends_pass_through() { + let d = data(5000); + let c = BlockCache::new(d.clone(), small()); + assert_eq!(c.as_contiguous(), Some(&d[..])); + assert!(matches!(c.read_at(0, 10).unwrap(), Cow::Borrowed(_))); + assert_eq!(c.stats().requests, 0); + } + + /// A slow backend: concurrent readers of the same blocks share fetches. + struct Slow { + data: Vec, + fetched: Mutex>>, + } + + impl Storage for Slow { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + std::thread::sleep(std::time::Duration::from_millis(30)); + lock(&self.fetched).push(offset..offset + len as u64); + self.data + .as_slice() + .read_at(offset, len) + .map(|c| Cow::Owned(c.into_owned())) + } + fn len(&self) -> u64 { + self.data.len() as u64 + } + } + + #[test] + fn concurrent_readers_never_fetch_a_block_twice() { + let d = data(64 * 1024); + let cfg = CacheConfig { + block_size: 1024, + capacity: 1 << 20, + coalesce_gap: 0, + max_request: 1024, + }; + let c = BlockCache::new( + Slow { + data: d.clone(), + fetched: Mutex::new(Vec::new()), + }, + cfg, + ); + std::thread::scope(|s| { + for t in 0..8u64 { + let (c, d) = (&c, &d); + s.spawn(move || { + for k in 0..16u64 { + let off = ((k * 3 + t) % 32) * 1024 + 100; + let got = c.read_at(off, 2000).unwrap(); + assert_eq!(&*got, &d[off as usize..off as usize + 2000]); + } + }); + } + }); + let fetched = lock(&c.inner().fetched).clone(); + let mut blocks: Vec = fetched.iter().map(|r| r.start / 1024).collect(); + let n = blocks.len(); + blocks.sort_unstable(); + blocks.dedup(); + assert_eq!(n, blocks.len(), "a block was fetched twice: {fetched:?}"); + assert!(c.stats().waits > 0, "readers should have shared fetches"); + } +} diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs new file mode 100644 index 0000000..d5d1feb --- /dev/null +++ b/crates/clawhdf5-remote/src/error.rs @@ -0,0 +1,125 @@ +//! Errors of the remote backends. + +use clawhdf5_format::error::FormatError; + +/// Why a remote file could not be opened or read. +/// +/// Inside a [`clawhdf5::File`] read these arrive as +/// `clawhdf5::Error::Format(FormatError::Storage(message))`, the message +/// being this error's `Display`. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum RemoteError { + /// The URL is malformed. + InvalidUrl(String), + /// The URL's scheme is not supported by this build (for example + /// `s3://` without the `s3` feature, or `https://` without `https`). + UnsupportedScheme(String), + /// The server answered a range request with the whole file (status + /// 200), i.e. it does not support ranges, and a full download was not + /// allowed ([`HttpOptions::allow_full_download`](crate::HttpOptions)). + RangeNotSupported(String), + /// The file changed since it was opened (a different ETag, + /// Last-Modified or length, or a failed `If-Match` precondition). + /// Nothing read after the change is returned. + FileChanged(String), + /// The server answered with an unexpected status. + Status { + /// The HTTP status code. + code: u16, + /// What was being requested. + what: String, + }, + /// A response did not carry what was asked for (a wrong + /// `Content-Range`, a body shorter or longer than announced), after + /// every retry. + BadResponse(String), + /// A network failure (connection, timeout, reset) after every retry. + Transport(String), + /// An error from the object store. + ObjectStore(String), + /// Called in a way the backend cannot serve (for example a blocking + /// read from inside an async runtime). + Usage(String), +} + +impl RemoteError { + /// Whether retrying the same request may succeed. + #[cfg_attr(not(feature = "http"), allow(dead_code))] + pub(crate) fn is_transient(&self) -> bool { + match self { + RemoteError::Transport(_) | RemoteError::BadResponse(_) => true, + RemoteError::Status { code, .. } => { + matches!(code, 408 | 429 | 500 | 502 | 503 | 504) + } + _ => false, + } + } +} + +impl std::fmt::Display for RemoteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RemoteError::InvalidUrl(s) => write!(f, "invalid URL: {s}"), + RemoteError::UnsupportedScheme(s) => write!(f, "unsupported URL: {s}"), + RemoteError::RangeNotSupported(s) => { + write!(f, "the server does not support range requests: {s}") + } + RemoteError::FileChanged(s) => write!(f, "the remote file changed while open: {s}"), + RemoteError::Status { code, what } => write!(f, "HTTP status {code} for {what}"), + RemoteError::BadResponse(s) => write!(f, "bad response: {s}"), + RemoteError::Transport(s) => write!(f, "network error: {s}"), + RemoteError::ObjectStore(s) => write!(f, "object store: {s}"), + RemoteError::Usage(s) => write!(f, "{s}"), + } + } +} + +impl std::error::Error for RemoteError {} + +impl From for FormatError { + fn from(e: RemoteError) -> Self { + FormatError::Storage(e.to_string()) + } +} + +/// An error of [`open_url`](crate::open_url): the remote side, or the file +/// itself. +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// Reaching or reading the remote file failed. + Remote(RemoteError), + /// The bytes were read, but they are not an HDF5 file clawhdf5 can open. + Hdf5(clawhdf5::Error), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Remote(e) => e.fmt(f), + Error::Hdf5(e) => e.fmt(f), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::Remote(e) => Some(e), + Error::Hdf5(e) => Some(e), + } + } +} + +impl From for Error { + fn from(e: RemoteError) -> Self { + Error::Remote(e) + } +} + +impl From for Error { + fn from(e: clawhdf5::Error) -> Self { + Error::Hdf5(e) + } +} diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs new file mode 100644 index 0000000..ce3d68f --- /dev/null +++ b/crates/clawhdf5-remote/src/http.rs @@ -0,0 +1,565 @@ +//! HTTP(S) range requests: [`HttpStorage`]. +//! +//! Every read is a `GET` with a `Range: bytes=a-b` header, answered `206 +//! Partial Content`. The file is pinned when it is opened: +//! +//! - its length comes from the `Content-Range` of the first request (which +//! also fetches the first block, so opening costs one request); +//! - a strong `ETag` is sent back as `If-Match` on every later request, and +//! compared with the `ETag` of every response; without one, `Last-Modified` +//! is sent as `If-Unmodified-Since` and compared; the length in every +//! `Content-Range` must stay the same. A file that changes while it is +//! open is [`RemoteError::FileChanged`], never a mix of old and new bytes. +//! (A server that sends neither validator cannot be checked beyond the +//! length; [`HttpOptions::require_validator`] refuses such servers.) +//! - a server that ignores `Range` and answers `200` with the whole file is +//! refused with [`RemoteError::RangeNotSupported`], unless +//! [`HttpOptions::allow_full_download`] is set: then the file is +//! downloaded once, at open, and read from memory. +//! +//! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and +//! a body shorter or longer than its `Content-Range` — are retried with +//! exponential backoff. Responses are requested with +//! `Accept-Encoding: identity`, since a compressed body cannot be a byte +//! range of the file. +//! +//! `HttpStorage` itself does not cache: each `read_at` is one request. Read +//! it through [`BlockCache`](crate::BlockCache) (which [`open_url`](crate::open_url) +//! does); its `read_ranges` fetches the ranges of one call in parallel. + +use std::borrow::Cow; +use std::io::Read; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::time::Duration; + +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; + +use crate::error::RemoteError; + +/// Settings of an [`HttpStorage`]. +#[derive(Debug, Clone)] +pub struct HttpOptions { + /// Retries of a request that failed transiently (so up to `retries + 1` + /// attempts). + pub retries: u32, + /// Delay before the first retry; doubled for each further one. + pub backoff: Duration, + /// Timeout of one request, from connecting to the end of the body. + pub timeout: Duration, + /// Requests of one `read_ranges` call in flight at once. + pub max_parallel: usize, + /// Bytes fetched by the first request, from offset 0 (the superblock and + /// usually the root group's metadata); at least 1. + pub first_request: u64, + /// When the server ignores `Range` (answers `200`), download the whole + /// file once and read it from memory, instead of failing. + pub allow_full_download: bool, + /// Largest file [`allow_full_download`](Self::allow_full_download) will + /// download. + pub max_full_download: u64, + /// Refuse a server that sends neither a strong `ETag` nor + /// `Last-Modified`, since a change of the file could then go unnoticed + /// (only its length is checked). + pub require_validator: bool, + /// Extra headers sent with every request (for example + /// `Authorization`). + pub headers: Vec<(String, String)>, +} + +impl Default for HttpOptions { + fn default() -> Self { + HttpOptions { + retries: 3, + backoff: Duration::from_millis(200), + timeout: Duration::from_secs(60), + max_parallel: 8, + first_request: crate::cache::DEFAULT_BLOCK_SIZE, + allow_full_download: false, + max_full_download: 1 << 30, + require_validator: false, + headers: Vec::new(), + } + } +} + +/// Requests and bytes an [`HttpStorage`] has used. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct HttpStats { + /// HTTP requests sent (retries included). + pub requests: u64, + /// Requests that were retries. + pub retries: u64, + /// Response body bytes received. + pub bytes: u64, +} + +/// How the file is pinned. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Validator { + ETag(String), + LastModified(String), + None, +} + +/// An HTTP(S) file read by range requests. See the [module +/// documentation](self). +pub struct HttpStorage { + agent: ureq::Agent, + url: String, + len: u64, + validator: Validator, + options: HttpOptions, + /// The whole file, when the server ignores ranges and a full download + /// was allowed. + full: Option>, + requests: AtomicU64, + retries: AtomicU64, + bytes: AtomicU64, +} + +impl std::fmt::Debug for HttpStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HttpStorage") + .field("url", &self.url) + .field("len", &self.len) + .field("validator", &self.validator) + .field("full_download", &self.full.is_some()) + .finish() + } +} + +/// A parsed `Content-Range: bytes a-b/total`. +fn content_range(v: &str) -> Option<(u64, u64, Option)> { + let rest = v.trim().strip_prefix("bytes")?.trim_start(); + let (span, total) = rest.split_once('/')?; + let (a, b) = span.trim().split_once('-')?; + let a: u64 = a.trim().parse().ok()?; + let b: u64 = b.trim().parse().ok()?; + if b < a { + return None; + } + let total = match total.trim() { + "*" => None, + t => Some(t.parse().ok()?), + }; + Some((a, b, total)) +} + +fn header<'a>(resp: &'a ureq::http::Response, name: &str) -> Option<&'a str> { + resp.headers().get(name).and_then(|v| v.to_str().ok()) +} + +/// A body with a `Content-Encoding` is not a byte range of the file. +fn check_identity(url: &str, resp: &ureq::http::Response) -> Result<(), RemoteError> { + match header(resp, "content-encoding") { + Some(enc) if !enc.trim().eq_ignore_ascii_case("identity") => { + Err(RemoteError::Usage(format!( + "{url}: the server sent a {enc}-encoded body despite Accept-Encoding: identity" + ))) + } + _ => Ok(()), + } +} + +fn transport(e: ureq::Error) -> RemoteError { + match e { + ureq::Error::StatusCode(code) => RemoteError::Status { + code, + what: "request".into(), + }, + ureq::Error::BadUri(s) => RemoteError::InvalidUrl(s), + other => RemoteError::Transport(other.to_string()), + } +} + +impl HttpStorage { + /// Open `url` (`http://`, or `https://` with the `https` feature): + /// one ranged `GET` of the first [`HttpOptions::first_request`] bytes, + /// which gives the file's length and validators. Returns the storage + /// and the bytes that request fetched (the file's start), for a + /// [`BlockCache`](crate::BlockCache) to keep. + pub fn open(url: &str, options: HttpOptions) -> Result<(HttpStorage, Vec), RemoteError> { + let lower = url.to_ascii_lowercase(); + if lower.starts_with("https://") { + if !cfg!(feature = "https") { + return Err(RemoteError::UnsupportedScheme(format!( + "{url}: https:// needs the `https` feature of clawhdf5-remote" + ))); + } + } else if !lower.starts_with("http://") { + return Err(RemoteError::UnsupportedScheme(url.to_string())); + } + let config = ureq::Agent::config_builder() + .http_status_as_error(false) + .timeout_global(Some(options.timeout)) + .build(); + let mut storage = HttpStorage { + agent: ureq::Agent::new_with_config(config), + url: url.to_string(), + len: 0, + validator: Validator::None, + options, + full: None, + requests: AtomicU64::new(0), + retries: AtomicU64::new(0), + bytes: AtomicU64::new(0), + }; + let first = storage.with_retries(|| storage.probe())?; + let (len, validator, bytes, full) = first; + storage.len = len; + storage.validator = validator; + if full { + storage.full = Some(bytes); + return Ok((storage, Vec::new())); + } + if storage.options.require_validator && storage.validator == Validator::None { + return Err(RemoteError::Usage(format!( + "{url}: the server sends neither a strong ETag nor Last-Modified, so a change \ + of the file could not be detected (HttpOptions::require_validator)" + ))); + } + Ok((storage, bytes)) + } + + /// The URL. + pub fn url(&self) -> &str { + &self.url + } + + /// Whether the file was downloaded whole, because the server does not + /// support ranges and [`HttpOptions::allow_full_download`] was set. + pub fn is_full_download(&self) -> bool { + self.full.is_some() + } + + /// The `ETag` the file is pinned to, if the server sent a strong one. + pub fn etag(&self) -> Option<&str> { + match &self.validator { + Validator::ETag(e) => Some(e), + _ => None, + } + } + + /// Requests and bytes so far. + pub fn stats(&self) -> HttpStats { + HttpStats { + requests: self.requests.load(Ordering::Relaxed), + retries: self.retries.load(Ordering::Relaxed), + bytes: self.bytes.load(Ordering::Relaxed), + } + } + + fn with_retries( + &self, + mut attempt: impl FnMut() -> Result, + ) -> Result { + let mut delay = self.options.backoff; + let mut n = 0; + loop { + match attempt() { + Ok(v) => return Ok(v), + Err(e) if e.is_transient() && n < self.options.retries => { + n += 1; + self.retries.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(delay); + delay = delay.saturating_mul(2); + } + Err(e) => return Err(e), + } + } + } + + fn request( + &self, + range: Option<(u64, u64)>, + ) -> ureq::RequestBuilder { + let mut req = self + .agent + .get(&self.url) + .header("Accept-Encoding", "identity"); + if let Some((a, b)) = range { + req = req.header("Range", format!("bytes={a}-{b}")); + } + match &self.validator { + Validator::ETag(e) => req = req.header("If-Match", e), + Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t), + Validator::None => {} + } + for (k, v) in &self.options.headers { + req = req.header(k, v); + } + req + } + + /// Read a body of exactly `want` bytes (or up to `limit` when `want` is + /// unknown). + fn body( + &self, + resp: ureq::http::Response, + want: Option, + limit: u64, + ) -> Result, RemoteError> { + let cap = want.unwrap_or(limit); + let mut buf = Vec::with_capacity(usize::try_from(cap.min(64 << 20)).unwrap_or(0)); + let reader = resp.into_body().into_reader(); + let got = reader + .take(cap.saturating_add(1)) + .read_to_end(&mut buf) + .map_err(|e| RemoteError::Transport(format!("{}: reading the body: {e}", self.url))); + self.bytes.fetch_add(buf.len() as u64, Ordering::Relaxed); + got?; + match want { + Some(n) if buf.len() as u64 != n => Err(RemoteError::BadResponse(format!( + "{}: body of {} bytes, expected {n}", + self.url, + buf.len() + ))), + None if buf.len() as u64 > limit => Err(RemoteError::Usage(format!( + "{}: the file is larger than HttpOptions::max_full_download ({limit} bytes)", + self.url + ))), + _ => Ok(buf), + } + } + + /// The first request: length, validators and the file's first bytes. + /// The last field is true when the server ignored the range and sent + /// the whole file (only kept when a full download is allowed). + fn probe(&self) -> Result<(u64, Validator, Vec, bool), RemoteError> { + let n = self.options.first_request.max(1); + self.requests.fetch_add(1, Ordering::Relaxed); + let resp = self.request(Some((0, n - 1))).call().map_err(transport)?; + let status = resp.status().as_u16(); + check_identity(&self.url, &resp)?; + let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) { + (Some(e), _) if !e.starts_with("W/") => Validator::ETag(e.to_string()), + (_, Some(t)) => Validator::LastModified(t.to_string()), + _ => Validator::None, + }; + match status { + 206 => { + let cr = header(&resp, "content-range").ok_or_else(|| { + RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + })?; + let (a, b, total) = content_range(cr).ok_or_else(|| { + RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + })?; + let total = total.ok_or_else(|| { + RemoteError::BadResponse(format!( + "{}: the server does not report the file's length (Content-Range {cr:?})", + self.url + )) + })?; + if a != 0 || b >= total || b > n - 1 { + return Err(RemoteError::BadResponse(format!( + "{}: asked for bytes 0-{}, got Content-Range {cr:?}", + self.url, + n - 1 + ))); + } + let bytes = self.body(resp, Some(b - a + 1), 0)?; + Ok((total, validator, bytes, false)) + } + 200 => { + if !self.options.allow_full_download { + return Err(RemoteError::RangeNotSupported(format!( + "{} answered a range request with the whole file (status 200); set \ + HttpOptions::allow_full_download to download it", + self.url + ))); + } + let want = header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); + if want.is_some_and(|w: u64| w > self.options.max_full_download) { + return Err(RemoteError::Usage(format!( + "{}: the file is larger than HttpOptions::max_full_download ({} bytes)", + self.url, self.options.max_full_download + ))); + } + let bytes = self.body(resp, want, self.options.max_full_download)?; + Ok((bytes.len() as u64, validator, bytes, true)) + } + 416 => Err(RemoteError::Usage(format!( + "{}: status 416 for the first bytes (an empty file?)", + self.url + ))), + code => Err(RemoteError::Status { + code, + what: self.url.clone(), + }), + } + } + + /// One request for `[start, end)` (inside the file, non-empty). + fn fetch_once(&self, start: u64, end: u64) -> Result, RemoteError> { + self.requests.fetch_add(1, Ordering::Relaxed); + let resp = self + .request(Some((start, end - 1))) + .call() + .map_err(transport)?; + let changed = |why: String| RemoteError::FileChanged(format!("{}: {why}", self.url)); + check_identity(&self.url, &resp)?; + match resp.status().as_u16() { + 206 => {} + 200 => { + return Err(RemoteError::RangeNotSupported(format!( + "{} answered a range request with the whole file (status 200)", + self.url + ))); + } + 412 => { + return Err(changed( + "If-Match/If-Unmodified-Since failed (status 412)".into(), + )); + } + 416 => return Err(changed("range no longer satisfiable (status 416)".into())), + code => { + return Err(RemoteError::Status { + code, + what: format!("{} bytes {start}-{}", self.url, end - 1), + }); + } + } + match &self.validator { + Validator::ETag(e) => { + if let Some(got) = header(&resp, "etag") + && got != e + { + return Err(changed(format!("ETag {got} instead of {e}"))); + } + } + Validator::LastModified(t) => { + if let Some(got) = header(&resp, "last-modified") + && got != t + { + return Err(changed(format!("Last-Modified {got} instead of {t}"))); + } + } + Validator::None => {} + } + let cr = header(&resp, "content-range").ok_or_else(|| { + RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + })?; + let (a, b, total) = content_range(cr).ok_or_else(|| { + RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + })?; + if let Some(total) = total + && total != self.len + { + return Err(changed(format!("length {total} instead of {}", self.len))); + } + if a != start || b != end - 1 { + return Err(RemoteError::BadResponse(format!( + "{}: asked for bytes {start}-{}, got Content-Range {cr:?}", + self.url, + end - 1 + ))); + } + self.body(resp, Some(end - start), 0) + } + + fn fetch(&self, start: u64, end: u64) -> Result, RemoteError> { + self.with_retries(|| self.fetch_once(start, end)) + } + + /// `[offset, offset + len)` clamped to the file, or `None` if empty. + fn clamp(&self, offset: u64, len: u64) -> Option<(u64, u64)> { + let end = offset.saturating_add(len).min(self.len); + (offset < end).then_some((offset, end)) + } +} + +impl Storage for HttpStorage { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + if let Some(all) = &self.full { + return all.as_slice().read_at(offset, len); + } + match self.clamp(offset, len as u64) { + None => Ok(Cow::Owned(Vec::new())), + Some((a, b)) => Ok(Cow::Owned(self.fetch(a, b)?)), + } + } + + fn len(&self) -> u64 { + self.len + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if let Some(all) = &self.full { + return all.as_slice().read_ranges(ranges); + } + let parallel = self.options.max_parallel.clamp(1, ranges.len().max(1)); + if parallel <= 1 { + return ranges + .iter() + .map(|r| self.read_at(r.start, (r.end.saturating_sub(r.start)) as usize)) + .collect(); + } + let next = AtomicUsize::new(0); + let failed = std::sync::atomic::AtomicBool::new(false); + type Slot = Option, RemoteError>>; + let results: std::sync::Mutex> = + std::sync::Mutex::new((0..ranges.len()).map(|_| None).collect()); + std::thread::scope(|s| { + for _ in 0..parallel { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= ranges.len() || failed.load(Ordering::Relaxed) { + break; + } + let r = &ranges[i]; + let got = match self.clamp(r.start, r.end.saturating_sub(r.start)) { + None => Ok(Vec::new()), + Some((a, b)) => self.fetch(a, b), + }; + if got.is_err() { + failed.store(true, Ordering::Relaxed); + } + results + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner)[i] = Some(got); + } + }); + } + }); + let results = results + .into_inner() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut out = Vec::with_capacity(ranges.len()); + for r in results { + match r { + Some(Ok(v)) => out.push(Cow::Owned(v)), + Some(Err(e)) => return Err(e.into()), + // Not fetched because another range failed first. + None => continue, + } + } + if out.len() != ranges.len() { + return Err(FormatError::Storage(format!( + "{}: a parallel range read failed", + self.url + ))); + } + Ok(out) + } + + fn as_contiguous(&self) -> Option<&[u8]> { + self.full.as_deref() + } +} + +#[cfg(test)] +mod tests { + use super::content_range; + + #[test] + fn content_range_parses() { + assert_eq!(content_range("bytes 0-99/1000"), Some((0, 99, Some(1000)))); + assert_eq!(content_range("bytes 5-5/*"), Some((5, 5, None))); + assert_eq!(content_range("bytes 9-5/10"), None); + assert_eq!(content_range("items 0-1/2"), None); + assert_eq!(content_range("bytes */1000"), None); + } +} diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs new file mode 100644 index 0000000..597fb63 --- /dev/null +++ b/crates/clawhdf5-remote/src/lib.rs @@ -0,0 +1,127 @@ +//! Read HDF5 files where they are — on an HTTP(S) server or in an object +//! store — without downloading them first. +//! +//! This is milestone M3 of `docs/design/range-reads.md`: remote backends for +//! [`clawhdf5::File::open_storage`], each read through a mandatory +//! [`BlockCache`]. +//! +//! ```no_run +//! let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/data.h5")?; +//! let temperature = file.dataset("/grid/temperature")?.read_f64()?; +//! # Ok::<(), Box>(()) +//! ``` +//! +//! - [`open_url`] / [`open_url_with`]: `http://` (default feature `http`), +//! `https://` (feature `https`) → a [`clawhdf5::File`] with the whole +//! read API. +//! - [`storage_for_url`] gives the cached storage itself, to open with +//! [`clawhdf5::File::open_storage`] and to read its [`CacheStats`]. +//! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried +//! with backoff), and [`BlockCache`] over any +//! [`Storage`](clawhdf5_format::storage::Storage). +//! +//! What costs what: opening costs one request (it also fetches the first +//! block, 1 MiB by default); listing a file whose metadata sits in its +//! first blocks costs nothing more; reading a chunked dataset costs one +//! parallel batch of requests for the blocks holding its chunk index, then +//! one for its chunks. The zero-copy methods of `clawhdf5` +//! (`read_raw_ref`, `read_*_zerocopy`, `File::as_bytes`) need the file in +//! memory and are errors (`as_bytes` a panic) on a remote file. + +#![warn(missing_docs)] + +pub mod cache; +pub mod error; +#[cfg(feature = "http")] +pub mod http; + +use std::sync::Arc; + +use clawhdf5::File; +use clawhdf5_format::storage::Storage; + +pub use cache::{BlockCache, CacheConfig, CacheStats}; +pub use error::{Error, RemoteError}; +#[cfg(feature = "http")] +pub use http::{HttpOptions, HttpStats, HttpStorage}; + +/// A backend a [`BlockCache`] can read through. +pub type Backend = Box; + +/// The storage [`storage_for_url`] returns: a block cache over the URL's +/// backend. +pub type RemoteStorage = BlockCache; + +/// Settings of [`open_url_with`] and [`storage_for_url`]. +#[derive(Debug, Clone, Default)] +pub struct Options { + /// The block cache. + pub cache: CacheConfig, + /// HTTP(S) requests. + #[cfg(feature = "http")] + pub http: HttpOptions, +} + +/// Open the HDF5 file at `url` with default [`Options`]. +/// +/// `http://…` needs the (default) `http` feature, `https://…` the `https` +/// feature. +pub fn open_url(url: &str) -> Result { + open_url_with(url, &Options::default()) +} + +/// [`open_url`] with explicit [`Options`]. +pub fn open_url_with(url: &str, options: &Options) -> Result { + let storage = storage_for_url(url, options)?; + Ok(File::open_storage(storage)?) +} + +/// The cached storage for `url`, with the first block already fetched: +/// open it with [`clawhdf5::File::open_storage`] (a clone of the `Arc`), +/// and read its [`BlockCache::stats`] as you go. +pub fn storage_for_url(url: &str, options: &Options) -> Result, Error> { + let scheme = url + .split_once("://") + .map(|(s, _)| s.to_ascii_lowercase()) + .ok_or_else(|| RemoteError::InvalidUrl(format!("{url}: no scheme")))?; + match scheme.as_str() { + "http" | "https" => http_storage(url, options), + "s3" | "s3a" | "gs" | "az" | "azure" | "abfs" | "abfss" | "adl" => { + cloud_storage(url, &scheme, options) + } + _ => Err(RemoteError::UnsupportedScheme(url.to_string()).into()), + } +} + +#[cfg(feature = "http")] +fn http_storage(url: &str, options: &Options) -> Result, Error> { + let mut http = options.http.clone(); + http.first_request = http.first_request.max(options.cache.block_size.max(1)); + let (storage, first) = HttpStorage::open(url, http)?; + let cache = BlockCache::new(Box::new(storage) as Backend, options.cache.clone()); + cache.insert(0, &first); + Ok(Arc::new(cache)) +} + +#[cfg(not(feature = "http"))] +fn http_storage(url: &str, _options: &Options) -> Result, Error> { + Err(RemoteError::UnsupportedScheme(format!( + "{url}: http(s):// needs the `http` feature of clawhdf5-remote" + )) + .into()) +} + +fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result, Error> { + Err(RemoteError::UnsupportedScheme(format!("{url}: {scheme}:// is not supported yet")).into()) +} + +/// A [`BlockCache`] over `backend` with its first block fetched (readahead +/// of the superblock and the metadata usually written next to it). +pub fn cached(backend: Backend, options: &Options) -> Result { + let cache = BlockCache::new(backend, options.cache.clone()); + let first = cache.config().block_size; + cache + .prefetch(0, first) + .map_err(|e| Error::Hdf5(clawhdf5::Error::Format(e)))?; + Ok(cache) +} diff --git a/crates/clawhdf5-remote/tests/common/mod.rs b/crates/clawhdf5-remote/tests/common/mod.rs new file mode 100644 index 0000000..498b603 --- /dev/null +++ b/crates/clawhdf5-remote/tests/common/mod.rs @@ -0,0 +1,302 @@ +//! Shared by the integration tests: the test server, a transcript of a +//! file (tree, attributes, values) to compare two ways of reading it, and +//! the test files. + +#![allow(dead_code)] + +pub mod server; + +use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; + +use clawhdf5::{DType, File, Selection}; +use clawhdf5_format::error::FormatError; + +/// Objects visited per file. +const MAX_OBJECTS: usize = 2000; +/// Datasets with more bytes than this are not read (their metadata is). +pub const MAX_DATA_BYTES: u64 = 64 << 20; + +/// A short, stable digest of a value's `Debug` form. +fn digest(v: &T) -> String { + let s = format!("{v:?}"); + if s.len() <= 200 { + return s; + } + let mut h = 0xcbf2_9ce4_8422_2325u64; + for b in s.bytes() { + h = (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); + } + format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len()) +} + +/// A data read's value, or `Err` (which chunk a damaged dataset reports +/// can vary between two `File`s: the chunk cache lists in hash order). +fn value(r: &Result) -> String { + match r { + Ok(v) => digest(v), + Err(_) => "Err".into(), + } +} + +fn sorted(m: std::collections::HashMap) -> BTreeMap { + m.into_iter().collect() +} + +/// Everything a reader sees in `file`: every group's entries, every +/// object's attributes, and every dataset's shape, types and values. +pub fn transcript(file: &File) -> String { + let mut out = String::new(); + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([(String::from("/"), file.superblock().root_group_address)]); + while let Some((path, addr)) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + let entries = group.entries(); + writeln!(out, "{path} @{addr} entries {}", digest(&entries)).unwrap(); + if let Ok(ds) = file.dataset_at(addr) { + dataset(&mut out, &path, &ds); + } + let attrs = group.attrs_with_errors().map(|(a, e)| (sorted(a), e)); + writeln!(out, "{path} attrs {}", digest(&attrs)).unwrap(); + if let Ok(entries) = entries { + for (name, child) in entries { + queue.push_back((format!("{}/{name}", path.trim_end_matches('/')), child)); + } + } + } + out +} + +fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { + let shape = ds.shape(); + let dtype = ds.dtype(); + writeln!( + out, + "{path} shape {} dtype {} raw {}", + digest(&shape), + digest(&dtype), + digest(&ds.raw_datatype()) + ) + .unwrap(); + let (Ok(shape), Ok(dtype), Ok(raw_dt)) = (shape, dtype, ds.raw_datatype()) else { + return; + }; + let elements = shape.iter().try_fold(1u64, |a, &d| a.checked_mul(d)); + let bytes = elements.and_then(|n| n.checked_mul(u64::from(raw_dt.type_size()))); + if bytes.is_none_or(|b| b > MAX_DATA_BYTES) { + writeln!(out, "{path} too large to read").unwrap(); + return; + } + writeln!( + out, + "{path} all {}", + value(&ds.read_selection(&Selection::All)) + ) + .unwrap(); + if matches!( + dtype, + DType::F32 + | DType::F64 + | DType::I8 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::U8 + | DType::U16 + | DType::U32 + | DType::U64 + ) { + writeln!(out, "{path} f64 {}", value(&ds.read_f64())).unwrap(); + if let Some(&d0) = shape.first() { + let rank = shape.len(); + let sel = Selection::Hyperslab { + start: std::iter::once(d0 / 3) + .chain(std::iter::repeat_n(0, rank - 1)) + .collect(), + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(3)) + .chain(shape[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + writeln!( + out, + "{path} f64 third {}", + value(&ds.read_f64_selection(&sel)) + ) + .unwrap(); + } + } + match &raw_dt { + clawhdf5_format::datatype::Datatype::String { .. } + | clawhdf5_format::datatype::Datatype::VariableLength { + is_string: true, .. + } => { + writeln!(out, "{path} strings {}", value(&ds.read_string_bytes())).unwrap(); + } + clawhdf5_format::datatype::Datatype::VariableLength { .. } => { + writeln!(out, "{path} vlen {}", value(&ds.read_vlen::())).unwrap(); + } + _ => {} + } +} + +/// Open, list every group, and read the first dataset found whose data is +/// at most `MAX_DATA_BYTES` (a tree view plus one plot). +pub fn list_and_read_one(file: &File) { + let mut seen = HashSet::new(); + let mut read_one = false; + let mut queue = VecDeque::from([file.superblock().root_group_address]); + while let Some(addr) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + let group = file.group_at(addr); + if let Ok(ds) = file.dataset_at(addr) { + let _ = (ds.shape(), ds.dtype()); + if !read_one { + let small = ds.shape().ok().and_then(|s| { + let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; + let size = u64::from(ds.raw_datatype().ok()?.type_size()); + n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) + }); + if small.is_some() { + let _ = ds.read_selection(&Selection::All); + read_one = true; + } + } + } + if let Ok(entries) = group.entries() { + queue.extend(entries.into_iter().map(|(_, a)| a)); + } + } +} + +/// External virtual-dataset sources read from `dir`, as `File::open` finds +/// them next to the file. +pub fn sibling_resolver(dir: PathBuf) -> clawhdf5::VdsResolver { + Arc::new(move |name: &str| { + let p = Path::new(name); + if name.is_empty() + || !p + .components() + .all(|c| matches!(c, std::path::Component::Normal(_))) + { + return Err(FormatError::Storage(format!("{name:?} not followed"))); + } + match std::fs::read(dir.join(p)) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(FormatError::Storage(e.to_string())), + } + }) +} + +/// HDF5 files under `dir`, recursively. +pub fn hdf5_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + hdf5_files(&p, out); + } else if p + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf")) + { + out.push(p); + } + } +} + +/// The repository's HDF5 test fixtures. +pub fn fixtures() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("../clawhdf5/tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + files.sort(); + files +} + +/// Files of `CLAWHDF5_REMOTE_CORPUS` (directories separated like `PATH`). +pub fn corpus() -> Option> { + let dirs = std::env::var("CLAWHDF5_REMOTE_CORPUS").ok()?; + let mut files = Vec::new(); + for d in std::env::split_paths(&dirs) { + hdf5_files(&d, &mut files); + } + files.sort(); + Some(files) +} + +pub fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +pub fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +/// Whether python3 with h5py and numpy runs; panics when interop is +/// required and it does not. +pub fn have_h5py() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + assert!( + ok || !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + if !ok { + eprintln!("SKIP: python3 with h5py not available"); + } + ok +} + +/// Run a Python script; its stdout. +pub fn run_python(script: &str, args: &[&str]) -> String { + let out = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() +} + +/// A clawhdf5-written file with a multi-block chunked dataset (`/big`, +/// 1 000 000 f64 in chunks of 10 000, deflated), a contiguous one and a +/// group — several blocks of 1 MiB, with no Python needed. +pub fn multi_block_file() -> Vec { + let mut b = clawhdf5::FileBuilder::new(); + b.set_attr("title", clawhdf5::AttrValue::String("remote test".into())); + let big: Vec = (0..1_000_000u64) + .map(|i| ((i * 2_654_435_761) % 1_000_003) as f64 * 0.5) + .collect(); + b.create_dataset("big") + .with_f64_data(&big) + .with_chunks(&[10_000]) + .with_deflate(1); + let flat: Vec = (0..300_000u64).map(|i| i as f64).collect(); + b.create_dataset("flat").with_f64_data(&flat); + let mut g = b.create_group("grp"); + g.create_dataset("small").with_f64_data(&[1.0, 2.0, 3.0]); + b.add_group(g.finish()); + b.finish().unwrap() +} diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs new file mode 100644 index 0000000..e556d7f --- /dev/null +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -0,0 +1,331 @@ +//! A small HTTP/1.1 file server on 127.0.0.1 for tests and the example: +//! `Range: bytes=a-b` requests (206, 416), `HEAD`, keep-alive, strong ETags +//! and Last-Modified with `If-Match` / `If-Unmodified-Since` (412), and +//! switches to misbehave — ignore ranges (200 with the whole file), cut +//! bodies short, answer 503, respond slowly, send no validators. It counts +//! requests and body bytes, and logs every range asked for. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; + +/// A logged GET: the path and the range asked for (`None`: whole file). +pub type LogEntry = (String, Option<(u64, u64)>); + +struct Resource { + data: Arc>, + etag: String, + last_modified: String, +} + +/// Switches and counters shared with the connection threads. +#[derive(Default)] +pub struct Shared { + files: RwLock>, + version: AtomicU64, + /// Answer every request with 200 and the whole file. + pub ignore_range: AtomicBool, + /// Send no ETag and no Last-Modified. + pub no_validators: AtomicBool, + /// Send a weak ETag only (no Last-Modified). + pub weak_etag: AtomicBool, + /// Send Last-Modified but no ETag. + pub no_etag: AtomicBool, + /// Label bodies `Content-Encoding: gzip` (they are not). + pub gzip_label: AtomicBool, + /// Cut the body of the next N ranged responses in half (then close the + /// connection). + pub truncate_next: AtomicU32, + /// Answer the next N requests with 503. + pub fail_next: AtomicU32, + /// Sleep this long before answering each request. + pub delay_ms: AtomicU64, + /// Requests served (every status). + pub requests: AtomicU64, + /// Body bytes sent. + pub bytes: AtomicU64, + /// (path, range) of every GET. + pub log: Mutex>, + stop: AtomicBool, +} + +/// A running server; stops accepting when dropped. +pub struct Server { + pub addr: std::net::SocketAddr, + pub shared: Arc, +} + +impl Server { + /// Serve `files` (URL path such as `/a.h5` → bytes) on `127.0.0.1`, on + /// a free port. + pub fn start(files: Vec<(String, Vec)>) -> Server { + Server::bind("127.0.0.1:0", files) + } + + /// Serve `files` on `addr`. + pub fn bind(addr: &str, files: Vec<(String, Vec)>) -> Server { + let listener = TcpListener::bind(addr).expect("bind"); + let addr = listener.local_addr().unwrap(); + let shared = Arc::new(Shared::default()); + for (path, data) in files { + shared.put(&path, data); + } + let s = shared.clone(); + std::thread::spawn(move || { + for conn in listener.incoming() { + if s.stop.load(Ordering::SeqCst) { + break; + } + let Ok(conn) = conn else { continue }; + let s = s.clone(); + std::thread::spawn(move || { + let _ = serve(conn, &s); + }); + } + }); + Server { addr, shared } + } + + /// The URL of `path` (which starts with `/`). + pub fn url(&self, path: &str) -> String { + format!("http://{}{path}", self.addr) + } + + /// Requests served so far. + pub fn requests(&self) -> u64 { + self.shared.requests.load(Ordering::SeqCst) + } + + /// Body bytes sent so far. + pub fn bytes(&self) -> u64 { + self.shared.bytes.load(Ordering::SeqCst) + } + + /// Zero the counters and the log. + pub fn reset(&self) { + self.shared.requests.store(0, Ordering::SeqCst); + self.shared.bytes.store(0, Ordering::SeqCst); + self.shared.log.lock().unwrap().clear(); + } + + /// The ranges asked for so far. + pub fn log(&self) -> Vec { + self.shared.log.lock().unwrap().clone() + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.shared.stop.store(true, Ordering::SeqCst); + let _ = TcpStream::connect(self.addr); + } +} + +impl Shared { + /// Add or replace a file (a replacement gets a new ETag and + /// Last-Modified). + pub fn put(&self, path: &str, data: Vec) { + let v = self.version.fetch_add(1, Ordering::SeqCst) + 1; + let secs = 1_700_000_000 + v; + self.files.write().unwrap().insert( + path.to_string(), + Resource { + data: Arc::new(data), + etag: format!("\"v{v}-{path}\""), + last_modified: http_date(secs), + }, + ); + } +} + +/// An IMF-fixdate for `secs` since the epoch (enough for distinct values). +fn http_date(secs: u64) -> String { + const DAYS: [&str; 7] = ["Thu", "Fri", "Sat", "Sun", "Mon", "Tue", "Wed"]; + const MONTHS: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + let days = secs / 86_400; + let rem = secs % 86_400; + // Civil-from-days (Howard Hinnant). + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!( + "{}, {:02} {} {} {:02}:{:02}:{:02} GMT", + DAYS[(days % 7) as usize], + d, + MONTHS[(m - 1) as usize], + y, + rem / 3600, + rem % 3600 / 60, + rem % 60 + ) +} + +fn parse_range(v: &str, len: u64) -> Option> { + let spec = v.trim().strip_prefix("bytes=")?; + if spec.contains(',') { + return None; // multiple ranges: serve the whole file + } + let (a, b) = spec.split_once('-')?; + let a: u64 = a.trim().parse().ok()?; + let b: u64 = match b.trim() { + "" => u64::MAX, + b => b.parse().ok()?, + }; + if b < a { + return None; + } + if a >= len { + return Some(Err(())); + } + Some(Ok((a, b.min(len - 1)))) +} + +fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { + conn.set_read_timeout(Some(Duration::from_secs(30)))?; + // One write per response and no Nagle delay: otherwise every request + // waits for a delayed ACK (~40 ms). + conn.set_nodelay(true)?; + let mut reader = BufReader::new(conn.try_clone()?); + let mut out = conn; + loop { + let mut line = String::new(); + if reader.read_line(&mut line)? == 0 { + return Ok(()); + } + let mut parts = line.split_whitespace(); + let method = parts.next().unwrap_or("").to_string(); + let path = parts.next().unwrap_or("").to_string(); + let mut headers = HashMap::new(); + loop { + let mut h = String::new(); + if reader.read_line(&mut h)? == 0 { + return Ok(()); + } + let h = h.trim_end(); + if h.is_empty() { + break; + } + if let Some((k, v)) = h.split_once(':') { + headers.insert(k.trim().to_ascii_lowercase(), v.trim().to_string()); + } + } + s.requests.fetch_add(1, Ordering::SeqCst); + let delay = s.delay_ms.load(Ordering::SeqCst); + if delay > 0 { + std::thread::sleep(Duration::from_millis(delay)); + } + let close = headers + .get("connection") + .is_some_and(|v| v.eq_ignore_ascii_case("close")); + if s.fail_next + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + write!( + out, + "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n" + )?; + continue; + } + let res = { + let files = s.files.read().unwrap(); + files + .get(&path) + .map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone())) + }; + let Some((data, etag, lm)) = res else { + write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?; + continue; + }; + let len = data.len() as u64; + let mut validators = String::new(); + if s.weak_etag.load(Ordering::SeqCst) { + validators = format!("ETag: W/{etag}\r\n"); + } else if s.no_etag.load(Ordering::SeqCst) { + validators = format!("Last-Modified: {lm}\r\n"); + } else if !s.no_validators.load(Ordering::SeqCst) { + validators = format!("ETag: {etag}\r\nLast-Modified: {lm}\r\n"); + } + let precondition_failed = headers.get("if-match").is_some_and(|v| v != &etag) + || headers.get("if-unmodified-since").is_some_and(|v| v != &lm); + if precondition_failed { + write!( + out, + "HTTP/1.1 412 Precondition Failed\r\n{validators}Content-Length: 0\r\n\r\n" + )?; + continue; + } + let range = if s.ignore_range.load(Ordering::SeqCst) { + None + } else { + headers.get("range").and_then(|v| parse_range(v, len)) + }; + if method == "GET" { + s.log + .lock() + .unwrap() + .push((path.clone(), range.and_then(Result::ok))); + } + let (status, body, extra) = match range { + Some(Err(())) => { + write!( + out, + "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{len}\r\n\ + {validators}Content-Length: 0\r\n\r\n" + )?; + continue; + } + Some(Ok((a, b))) => ( + "206 Partial Content", + &data[a as usize..=b as usize], + format!("Content-Range: bytes {a}-{b}/{len}\r\n"), + ), + None => ("200 OK", &data[..], String::new()), + }; + let extra = if s.gzip_label.load(Ordering::SeqCst) { + format!("{extra}Content-Encoding: gzip\r\n") + } else { + extra + }; + let head = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nAccept-Ranges: bytes\r\n{extra}{validators}\r\n", + body.len() + ); + if method == "HEAD" { + out.write_all(head.as_bytes())?; + continue; + } + let truncate = status.starts_with("206") + && s.truncate_next + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok(); + let body = if truncate { + &body[..body.len() / 2] + } else { + body + }; + let mut response = head.into_bytes(); + response.extend_from_slice(body); + out.write_all(&response)?; + out.flush()?; + s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); + if truncate || close { + let _ = out.shutdown(std::net::Shutdown::Both); + return Ok(()); + } + } +} diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs new file mode 100644 index 0000000..e7faed8 --- /dev/null +++ b/crates/clawhdf5-remote/tests/http.rs @@ -0,0 +1,517 @@ +//! HTTP range reads against a local server (no internet): every value read +//! over HTTP equals `File::open`'s, and the server's misbehaviour — no +//! range support, a file replaced mid-read, cut-off bodies, errors, slow +//! answers under concurrent readers — is an error or the right data, never +//! wrong data. +//! +//! `CLAWHDF5_REMOTE_CORPUS=dir[:dir...]` adds every HDF5 file under those +//! directories (the conformance corpus is `conformance/.cache/corpus`), and +//! `CLAWHDF5_REMOTE_REPORT=1` prints the per-file request counts. + +#![cfg(feature = "http")] + +mod common; + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use clawhdf5::File; +use clawhdf5_remote::{ + CacheConfig, Error, HttpOptions, HttpStorage, Options, RemoteError, open_url, open_url_with, + storage_for_url, +}; +use common::server::Server; +use common::{list_and_read_one, multi_block_file, transcript}; + +/// Options for tests: fast retries. +fn quick() -> Options { + Options { + http: HttpOptions { + backoff: Duration::from_millis(5), + ..HttpOptions::default() + }, + ..Options::default() + } +} + +fn changed(e: &str) -> bool { + e.contains("changed while open") +} + +/// Serve `files`, open each by URL and through `File::open`, and require +/// identical transcripts. Returns (files compared, files both refused). +fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) { + let served: Vec<(String, Vec)> = files + .iter() + .enumerate() + .filter_map(|(i, p)| { + let bytes = std::fs::read(p).ok()?; + let name = p.file_name()?.to_str()?.replace(' ', "_"); + Some((format!("/f{i}/{name}"), bytes)) + }) + .collect(); + let server = Server::start(served.clone()); + let (mut same, mut refused) = (0, 0); + let mut totals = [0u64; 5]; + for (i, p) in files.iter().enumerate() { + let Some((url_path, bytes)) = served + .iter() + .find(|(u, _)| u.starts_with(&format!("/f{i}/"))) + else { + continue; + }; + let url = server.url(url_path); + let local = File::open(p); + let remote = storage_for_url(&url, &quick()) + .map_err(|e| e.to_string()) + .and_then(|s| { + File::open_storage(s.clone()) + .map(|mut f| { + f.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into())); + (f, s) + }) + .map_err(|e| e.to_string()) + }); + let (local, (remote, storage)) = match (local, remote) { + (Ok(l), Ok(r)) => (l, r), + (Err(l), Err(r)) => { + assert!( + !r.contains("HTTP") && !r.contains("network"), + "{url}: {r} ({l})" + ); + refused += 1; + continue; + } + (l, r) => panic!( + "{}: File::open {:?}, open_url {:?}", + p.display(), + l.map(|_| ()), + r.map(|_| ()) + ), + }; + let want = transcript(&local); + let got = transcript(&remote); + if want != got { + let first = want + .lines() + .zip(got.lines()) + .find(|(w, g)| w != g) + .map(|(w, g)| format!("\n local: {w}\n remote: {g}")) + .unwrap_or_default(); + panic!("{}: open_url differs from File::open{first}", p.display()); + } + assert!(storage.stats().reads > 0, "read through the cache"); + same += 1; + + // Cost of "list + read one dataset": with the block cache, and + // with none (every read a request). + let with = storage_for_url(&url, &quick()).unwrap(); + server.reset(); + if let Ok(f) = File::open_storage(with.clone()) { + list_and_read_one(&f); + } + let (cached_requests, cached_bytes) = (server.requests(), server.bytes()); + let (bare, _) = HttpStorage::open(&url, quick().http).unwrap(); + let bare = Arc::new(bare); + server.reset(); + if let Ok(f) = File::open_storage(bare.clone()) { + list_and_read_one(&f); + } + let uncached_requests = server.requests(); + totals[0] += 1 + cached_requests; // + the open probe + totals[1] += cached_bytes + with.config().block_size.min(bytes.len() as u64); + totals[2] += 1 + uncached_requests; + totals[3] += bytes.len() as u64; + totals[4] += 1; + if report { + eprintln!( + "requests cached {:>6} uncached {:>8} bytes {:>12} of {:>12} {}", + 1 + cached_requests, + 1 + uncached_requests, + cached_bytes + with.config().block_size.min(bytes.len() as u64), + bytes.len(), + p.display() + ); + } + // Files within one block: opening fetched everything. + if bytes.len() as u64 <= with.config().block_size { + assert_eq!(cached_requests, 0, "{}", p.display()); + } + } + eprintln!( + "list + read one dataset over {} files: {} requests with the 1 MiB block cache \ + ({} bytes transferred, files {} bytes), {} requests without a cache", + totals[4], totals[0], totals[1], totals[3], totals[2] + ); + (same, refused) +} + +#[test] +fn fixtures_read_identically_over_http() { + let files = common::fixtures(); + assert!(files.len() >= 45, "{} fixtures", files.len()); + let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1"); + let (same, _) = compare(&files, report); + assert!(same >= 40, "{same}"); +} + +#[test] +fn corpus_reads_identically_over_http() { + let Some(files) = common::corpus() else { + eprintln!("CLAWHDF5_REMOTE_CORPUS not set; skipping the corpus"); + return; + }; + let report = std::env::var("CLAWHDF5_REMOTE_REPORT").is_ok_and(|v| v == "1"); + let (same, refused) = compare(&files, report); + eprintln!("corpus: {same} files identical, {refused} refused by both"); + assert!(same > 0); +} + +#[test] +fn multi_block_file_values_and_request_budget() { + let bytes = multi_block_file(); + assert!(bytes.len() > 3 << 20, "{}", bytes.len()); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let local = File::from_bytes(bytes.clone()).unwrap(); + let storage = storage_for_url(&url, &quick()).unwrap(); + assert_eq!(server.requests(), 1, "opening is one request"); + let remote = File::open_storage(storage.clone()).unwrap(); + assert!(remote.contiguous_bytes().is_none()); + let mut names = remote.root().datasets().unwrap(); + names.sort(); + assert_eq!(names, ["big", "flat"]); + assert_eq!(remote.root().groups().unwrap(), ["grp"]); + assert_eq!( + remote.dataset("grp/small").unwrap().read_f64().unwrap(), + [1.0, 2.0, 3.0] + ); + let big = remote.dataset("big").unwrap().read_f64().unwrap(); + assert_eq!(big, local.dataset("big").unwrap().read_f64().unwrap()); + let flat = remote.dataset("flat").unwrap().read_f64().unwrap(); + assert_eq!(flat.len(), 300_000); + assert_eq!(flat[299_999], 299_999.0); + // Every byte was fetched at most once, in whole 1 MiB blocks. + let log = server.log(); + let mut blocks: Vec = Vec::new(); + for (_, r) in &log { + let (a, b) = r.expect("every request is ranged"); + assert_eq!(a % (1 << 20), 0, "block-aligned"); + blocks.extend(a >> 20..=b >> 20); + } + let n = blocks.len(); + blocks.sort_unstable(); + blocks.dedup(); + assert_eq!(n, blocks.len(), "a block was fetched twice: {log:?}"); + assert!( + server.requests() <= (bytes.len() as u64 >> 20) + 2, + "{} requests for a {} byte file", + server.requests(), + bytes.len() + ); + // Reading again costs nothing. + let before = server.requests(); + assert_eq!(transcript(&remote), transcript(&local)); + assert_eq!(server.requests(), before); +} + +#[test] +fn h5py_written_file_over_http() { + if !common::have_h5py() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("py.h5"); + common::run_python( + r#" +import sys, h5py, numpy as np +with h5py.File(sys.argv[1], "w", libver="latest") as f: + f.attrs["title"] = "h5py remote test" + g = f.create_group("sensors") + for i in range(40): + d = g.create_dataset(f"s{i:02d}", data=np.arange(1000, dtype="().unwrap()); + assert_eq!(vl[1], it.next().unwrap()); + assert_eq!( + remote.dataset("link").unwrap().read_i32().unwrap(), + local.dataset("sensors/s03").unwrap().read_i32().unwrap() + ); +} + +#[test] +fn a_server_that_ignores_range_is_refused_or_downloaded_when_allowed() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + server.shared.ignore_range.store(true, Ordering::SeqCst); + let url = server.url("/m.h5"); + let err = open_url_with(&url, &quick()).unwrap_err(); + assert!( + matches!(err, Error::Remote(RemoteError::RangeNotSupported(_))), + "{err}" + ); + assert!( + server.bytes() < bytes.len() as u64, + "refusing must not download the file ({} bytes read)", + server.bytes() + ); + let mut opts = quick(); + opts.http.allow_full_download = true; + let storage = storage_for_url(&url, &opts).unwrap(); + let f = File::open_storage(storage.clone()).unwrap(); + assert_eq!(f.contiguous_bytes(), Some(&bytes[..]), "read from memory"); + assert_eq!(server.requests(), 2); + let local = File::from_bytes(bytes).unwrap(); + assert_eq!(transcript(&f), transcript(&local)); + assert_eq!(server.requests(), 2, "no further requests"); + // Too large for the download limit. + opts.http.max_full_download = 1000; + assert!(open_url_with(&url, &opts).is_err()); +} + +#[test] +fn a_file_replaced_mid_read_is_an_error_not_mixed_data() { + for mode in ["etag", "last-modified", "none"] { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + match mode { + "last-modified" => server.shared.no_etag.store(true, Ordering::SeqCst), + "none" => server.shared.no_validators.store(true, Ordering::SeqCst), + _ => {} + } + let url = server.url("/m.h5"); + let f = open_url_with(&url, &quick()).unwrap(); + assert_eq!(f.root().groups().unwrap(), ["grp"], "{mode}"); + // Replace the file: same layout, other values (and, for "none", + // one byte longer, which is all that can be checked). + let mut other = bytes.clone(); + let n = other.len(); + for b in &mut other[n / 2..n / 2 + 1000] { + *b ^= 0xff; + } + if mode == "none" { + other.push(0); + } + server.shared.put("/m.h5", other); + let err = f + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!(changed(&err), "{mode}: {err}"); + } +} + +#[test] +fn a_weak_etag_or_no_validator_is_refused_when_required() { + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + server.shared.weak_etag.store(true, Ordering::SeqCst); + let mut opts = quick(); + opts.http.require_validator = true; + assert!(open_url_with(&server.url("/m.h5"), &opts).is_err()); + opts.http.require_validator = false; + assert!(open_url_with(&server.url("/m.h5"), &opts).is_ok()); +} + +#[test] +fn truncated_bodies_are_retried_then_an_error() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let local = File::from_bytes(bytes).unwrap(); + let want = local.dataset("big").unwrap().read_f64().unwrap(); + + // One cut-off body: retried, right values. + let (http, first) = HttpStorage::open(&url, quick().http).unwrap(); + assert_eq!(first.len(), 1 << 20); + let storage = Arc::new(clawhdf5_remote::BlockCache::new( + http, + CacheConfig::default(), + )); + let f = File::open_storage(storage.clone()).unwrap(); + server.shared.truncate_next.store(1, Ordering::SeqCst); + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); + assert_eq!(storage.inner().stats().retries, 1); + + // Every body cut off: an error, never partial data. + let f = open_url_with(&url, &quick()).unwrap(); + server.shared.truncate_next.store(1000, Ordering::SeqCst); + let err = f + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!( + err.contains("network error") || err.contains("bad response"), + "{err}" + ); + server.shared.truncate_next.store(0, Ordering::SeqCst); + // The failure was not cached: the next read succeeds. + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); +} + +#[test] +fn server_errors_are_retried_with_backoff() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + server.shared.fail_next.store(2, Ordering::SeqCst); + let url = server.url("/m.h5"); + let f = open_url_with(&url, &quick()).unwrap(); + let local = File::from_bytes(bytes).unwrap(); + server.shared.fail_next.store(3, Ordering::SeqCst); + assert_eq!( + f.dataset("flat").unwrap().read_f64().unwrap(), + local.dataset("flat").unwrap().read_f64().unwrap() + ); + // More failures than retries: an error. + let f = open_url_with(&url, &quick()).unwrap(); + server.shared.fail_next.store(100, Ordering::SeqCst); + let err = f + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!(err.contains("503"), "{err}"); +} + +#[test] +fn slow_server_concurrent_readers_fetch_each_block_once() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let opts = Options { + cache: CacheConfig { + block_size: 256 << 10, + max_request: 256 << 10, + ..CacheConfig::default() + }, + ..quick() + }; + let storage = storage_for_url(&url, &opts).unwrap(); + let file = File::open_storage(storage.clone()).unwrap(); + let local = File::from_bytes(bytes).unwrap(); + let want_big = local.dataset("big").unwrap().read_f64().unwrap(); + let want_flat = local.dataset("flat").unwrap().read_f64().unwrap(); + server.shared.delay_ms.store(40, Ordering::SeqCst); + server.reset(); + std::thread::scope(|s| { + for t in 0..8 { + let (file, want_big, want_flat) = (&file, &want_big, &want_flat); + s.spawn(move || { + if t % 2 == 0 { + assert_eq!(&file.dataset("big").unwrap().read_f64().unwrap(), want_big); + } else { + assert_eq!( + &file.dataset("flat").unwrap().read_f64().unwrap(), + want_flat + ); + } + }); + } + }); + let log = server.log(); + let mut starts: Vec = log.iter().map(|(_, r)| r.unwrap().0).collect(); + let n = starts.len(); + starts.sort_unstable(); + starts.dedup(); + assert_eq!(n, starts.len(), "a block was fetched twice"); + assert!(storage.stats().waits > 0, "readers shared fetches"); +} + +#[test] +fn bad_urls_and_statuses_are_clean_errors() { + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + let err = open_url_with(&server.url("/missing.h5"), &quick()).unwrap_err(); + assert!( + matches!(err, Error::Remote(RemoteError::Status { code: 404, .. })), + "{err}" + ); + assert!(matches!( + open_url("ftp://example.com/a.h5"), + Err(Error::Remote(RemoteError::UnsupportedScheme(_))) + )); + assert!(matches!( + open_url("no-scheme"), + Err(Error::Remote(RemoteError::InvalidUrl(_))) + )); + assert!(matches!( + open_url("s3://bucket/key.h5"), + Err(Error::Remote(RemoteError::UnsupportedScheme(_))) + )); + #[cfg(not(feature = "https"))] + { + let e = open_url("https://example.com/a.h5") + .unwrap_err() + .to_string(); + assert!(e.contains("`https` feature"), "{e}"); + } + // A content-encoded body is not a byte range. + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + server.shared.gzip_label.store(true, Ordering::SeqCst); + let e = open_url(&server.url("/m.h5")).unwrap_err().to_string(); + assert!(e.contains("gzip-encoded"), "{e}"); + // Not HDF5. + let server = Server::start(vec![("/x.h5".into(), vec![7u8; 5000])]); + assert!(matches!( + open_url(&server.url("/x.h5")), + Err(Error::Hdf5(_)) + )); + // A server that closes every connection unanswered: a network error + // after the retries. (Not a closed port: another test's server could + // take it meanwhile.) + let port = { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + std::thread::spawn(move || { + for c in l.incoming() { + drop(c); + } + }); + port + }; + let err = open_url_with(&format!("http://127.0.0.1:{port}/a.h5"), &quick()).unwrap_err(); + assert!( + matches!(err, Error::Remote(RemoteError::Transport(_))), + "{err}" + ); +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index ad9af8b..2fce9c1 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -109,23 +109,36 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \ -- -D warnings +# clawhdf5-remote's HTTPS backend (rustls). +run_step "cargo clippy (remote, https)" cargo clippy \ + -p clawhdf5-remote \ + --all-targets \ + --features https \ + -- -D warnings + # The README promises that the core crates build no C by default. Hold it to # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. +# clawhdf5-remote is checked by default (plain HTTP); its https feature +# (ring) builds C and is opt-in. no_c_in_default_build() { - local crate found=0 - for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ + local entry crate features found=0 + for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ clawhdf5-tools \ - clawhdf5-wasm; do + clawhdf5-wasm \ + clawhdf5-remote; do + crate=${entry%%:*} + features=() + [ "$entry" != "$crate" ] && features=(--features "${entry#*:}") local c_deps - c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ + c_deps=$(cargo tree -q -p "$crate" "${features[@]}" -e normal,build --prefix none \ | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' \ | grep -v '^js-sys v' | sort -u) if [ -n "$c_deps" ]; then - echo "$crate pulls in C by default:" + echo "$entry pulls in C by default:" echo "$c_deps" | sed 's/^/ /' found=1 fi -- 2.54.0 From 0aca0eb724c2bb5ba2595501fadcefc106d88547 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:13:31 -0500 Subject: [PATCH 14/49] =?UTF-8?q?docs:=20editor=20coverage=20=E2=80=94=20v?= =?UTF-8?q?ersion-2=20B-trees,=20shrinking,=20dense=20attributes,=20reuse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHANGELOG (Unreleased): the new FileEditor operations, space reuse, and the two reader fixes (implicit index grid, object-header continuation chains). known-issues: the editor's remaining refusals (skipped heap blocks, heaps with filters or child indirect blocks, freeing a heap block, implicit-index insertions, ...) and the append-waste sizes before and after reuse (measure_append_waste, tank 2026-09-26; file sizes are deterministic). range-reads design: status note on the reader changes. README and CLAUDE.md: what the editor covers and how to test it. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 69 +++++++++++++++++++++++++++++++++++++- CLAUDE.md | 16 ++++++--- README.md | 6 +++- docs/design/range-reads.md | 8 ++++- docs/known-issues.md | 57 ++++++++++++++++++------------- 5 files changed, 124 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa0a30f..551b079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,72 @@ ## Unreleased +### In-place editing: version-2 B-tree indexes, shrinking, dense attributes (2026-09-26) +- **`FileEditor` adds, moves and resizes chunks of datasets with two or + more unlimited dimensions** (version-2 B-tree chunk index, record types + 10/11), as libhdf5's `H5B2` code does: `H5B2_update`'s insert-or-modify, + the preemptive split/redistribute loop, `split1`/`split_root` (depth + growth), `redistribute2/3`, and removal with `merge2/3`, root collapse + and the internal-record swap; node pointer widths and cumulative record + counts per depth; a missing index is created from the layout message's + parameters. After the same growth libhdf5's and the editor's trees are + node for node the same (tested through a depth increase). +- **`FileEditor::resize` shrinks** along any dimension (h5py's + `Dataset.resize` to a smaller shape), as `H5D__chunk_prune_by_extent` + does, visiting the same chunks in the same order: chunks wholly outside + the new extent leave the index (version-1 B-tree removal with libhdf5's + sibling key and link fix-ups and empty-root case, version-2 B-tree + removal, Fixed/Extensible Array elements reset; an implicit index keeps + its chunks, as in libhdf5) and their space is freed; the part of a + partial edge chunk outside the extent is overwritten with the fill value, + so it reads as fill after a later growth. Growth under early allocation + now allocates and fills the new chunks (`H5D__chunk_allocate`), which an + implicit index needs. Shrinking was `Error::Unsupported`. +- **`FileEditor::set_attr` handles dense attribute storage and creation + order**: objects that track (and index) attribute creation order; the + move to dense storage when an object reaches its compact limit (or an + attribute is too large for a header message), as `H5O__attr_create` + does it (new fractal heap, name index, creation-order index when + indexed, compact attributes moved over in header order); objects + already in dense storage (h5py- or clawhdf5-written): insertion, + same-size rewrites in place, other replacements by removal and + insertion. The heap is changed as `H5HF` changes it — best-fit free + sections from its free-space manager (kept as libhdf5 keeps `FSHD`/ + `FSSE`), new direct blocks through the root indirect block (created, + doubled), huge objects through the huge-object B-tree (deleted with the + last huge object), removed objects' space merged back — with libhdf5's + statistics: after the same attribute workload the heap, its free space + and both index B-trees equal libhdf5's. Attributes are encoded as libhdf5 + encodes them for a file h5py opens `r+` (message version 1, 3 for + non-ASCII names; simple dataspaces with their maximum dimensions). + Still refused: see `docs/known-issues.md`. +- **Freed space is reused within an editing session.** A `FileEditor` + reuses (best fit, zeroed) what its earlier edits freed — moved filtered + chunks, pruned chunks, merged B-tree nodes, replaced heap blocks — never + what the current edit frees, and writes reused blocks with the new space + before any existing byte changes. `FileEditor::reusable_bytes`. The + append workload of `measure_append_waste` leaks less (sizes in + `docs/known-issues.md`). +- **Reader: implicit chunk indexes below their maximum shape.** libhdf5 + places an implicit index's chunks by their position in the *maximum* + chunk grid; the reader used the current grid and returned other chunks' + values from the second chunk row on (h5py early allocation with a fixed + `maxshape` larger than the shape). + `chunked_read::generate_implicit_chunks_in_grid` takes the maximum. +- **Reader: object headers with long continuation chains.** A version-1 + header whose continuation chunks chain more than 32 deep (a header that + gains a chunk per attribute added when full, as libhdf5 and the editor + grow it) was refused with `NestingDepthExceeded`; version-2 headers + stopped at 256 chunks. Chunks are now followed without recursion, in the + same order; a chunk address seen twice (a cycle) or more than 65 536 + chunks are refused. +- Tests: `crates/clawhdf5-tools/tests/edit_coverage_interop.rs` (h5py + `earliest`/`v110`/`latest` and clawhdf5-written files; structure + comparisons with libhdf5 for version-2 B-trees, shrink on every index, + and dense attribute heaps); the random-operation property test in + `edit_interop.rs` now shrinks, grows two unlimited dimensions and moves + attributes to dense storage (`CLAWHDF5_EDIT_SEED` for other seeds). + ### Name lookups through the name index (2026-09-26) - **Finding one link or attribute by name reads the name index, not every entry.** In a dense group (links in a fractal heap) the v2 B-tree name @@ -237,7 +303,8 @@ before any existing byte changes, then the metadata that links it in, then a second sync. There is no journal: a crash during the second phase can leave the file inconsistent (as with libhdf5 without SWMR). - Freed space is not reused (see `docs/known-issues.md`). + Freed space is not reused (see `docs/known-issues.md`; since reused + within an editing session, above). - Tests: `crates/clawhdf5-tools/tests/edit_interop.rs` (h5py `earliest`, `v114` and `latest` files and clawhdf5 files; after every round h5py reads the expected values, h5dump and `h5rs check --data` accept the diff --git a/CLAUDE.md b/CLAUDE.md index f493066..14463ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,12 +151,18 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F `MemorySource` for this bookkeeping is inferred from the caller-supplied `source_channel` string (a heuristic, not an authenticated trust boundary). - In-place modification: `clawhdf5::FileEditor` (`crates/clawhdf5/src/edit/`) - overwrites values, grows chunked datasets and sets attributes in existing - files (h5py- or clawhdf5-written) without rewriting them; anything it - cannot do safely is `Error::Unsupported` before any write (limits in + overwrites values, grows and shrinks chunked datasets (every chunk index, + version-2 B-trees included) and sets attributes (compact and dense + storage) in existing files (h5py- or clawhdf5-written) without rewriting + them, changing indexes and heaps as libhdf5 does (index shapes and heap + bookkeeping are compared with libhdf5's in the tests); space an edit + frees is reused by later edits of the same editor. Anything it cannot do + safely is `Error::Unsupported` before any write (limits in `docs/known-issues.md`). Test changes with - `cargo test -p clawhdf5-tools --test edit_interop` (h5py, h5dump, - `h5rs check`). + `cargo test -p clawhdf5-tools --test edit_interop --test + edit_coverage_interop` (h5py, h5dump, `h5rs check`, structure comparisons + with libhdf5; libhdf5 sources for the algorithms are at + github.com/HDFGroup/hdf5, tag `hdf5_1_14_6`). - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only - Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. diff --git a/README.md b/README.md index 88ad61b..a87a43b 100644 --- a/README.md +++ b/README.md @@ -444,9 +444,13 @@ ed.resize("x", &[1100])?; // h5py: ds.resize((1100,)) let sel = Selection::Hyperslab { start: vec![1000], stride: vec![1], count: vec![100], block: vec![1] }; ed.write_values("x", &sel, &[0.5f64; 100])?; // ds[1000:1100] = 0.5 ed.set_attr("x", "units", &AttrValue::String("m/s".into()))?; +ed.resize("x", &[900])?; // shrinking prunes chunks, like h5py ``` -Each call changes the file in place (no rewrite) and syncs it. What it +Each call changes the file in place (no rewrite) and syncs it. Any chunk +index (version-2 B-trees for several unlimited dimensions included) and +attributes in compact or dense storage are handled as libhdf5 handles +them; space an edit frees is reused by later edits of the same editor. What it cannot change safely is refused before anything is written; see [known issues](docs/known-issues.md) for the limits. diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1c7fa55..7490c7a 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -3,7 +3,13 @@ Status: proposal, 2026-09-26; the plan for Phase 3's largest architectural change. Progress: M1, first part (the `Storage` trait and the metadata parsers listed in `CHANGELOG.md` under "Range reads, milestone M1") is done; -group B-tree v2 lookups, dense groups and the facade are not converted yet. Every count below was +group B-tree v2 lookups, dense groups and the facade are not converted yet. +Later the same day (branch `feat/p3-editor-coverage`) two reader fixes touched +converted code without changing the plan: object-header continuation chunks +are followed without recursion (still one bounded `read_at` per chunk), and +implicit chunk indexes are addressed over the maximum chunk grid (in +`chunked_read`, an M2 module). The in-place editor (`FileEditor`) keeps +working on the whole file in memory; it is not part of this design. Every count below was taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given next to it. No timing numbers appear here on purpose: the machine was shared with other build jobs when this was written. diff --git a/docs/known-issues.md b/docs/known-issues.md index dcb365f..1b05553 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -26,40 +26,49 @@ libhdf5 modify them. ## In-place modification (`FileEditor`) limits -**Status:** open (documented 2026-09-26). `clawhdf5::FileEditor` refuses, -with `Error::Unsupported` and without writing anything: -- new, moved or resized chunks in a **version-2 B-tree** chunk index (what - libhdf5 uses for two or more unlimited dimensions) — existing unfiltered - chunks, and filtered ones that re-encode to the same size and filter - mask, are - overwritten in place; `resize` works — and new chunks in an **implicit** - index (it has all of its chunks from the start); -- **shrinking** a dataset; +**Status:** open (documented 2026-09-26, updated the same day when +version-2 B-tree chunk indexes, shrinking, dense attributes and space +reuse were added). `clawhdf5::FileEditor` refuses, with +`Error::Unsupported` and without writing anything: +- new chunks in an **implicit** index (it has all of its chunks from the + start; they are written in place, and allocated/filled on growth under + early allocation as libhdf5 does); - variable-length and reference data; - chunks through a filter this build cannot encode (scale-offset, N-Bit, SZIP, or a plugin filter it lacks), even an optional one: libhdf5 skips an optional filter only when its own build lacks it, which none does for these; -- attributes of an object in **dense storage**, past its compact limit (8 - by default) or with tracked **creation order**; +- attributes in dense storage when the heap cannot take them the way + libhdf5 would: an attribute that needs a heap block larger than the next + one (libhdf5 skips blocks and records them as free space — in practice an + attribute of roughly 1 to 4 KiB going into a young heap), a heap with I/O + filters or child indirect blocks (more than about 512 KiB of attributes), + free space the heap tracks outside direct blocks, replacing the last + attribute left in a heap block by one of another size (libhdf5 frees the + block), directly addressed huge objects; and shared attribute messages; +- version-1 object headers asked for an attribute larger than a header + message (they have no dense storage); - partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external raw data files, virtual datasets; - files with a metadata cache image, paged or persistent free-space management, a driver info block, or version-3 consistency flags set. -**Space is never reused.** There is no free-space manager: the old bytes of -a filtered chunk that grows and has to move, and of an attribute that is -replaced by a larger one, are leaked (`h5repack` reclaims them). A chunk -that is the last thing in the file grows in place instead, which covers the -usual append. Measured 2026-09-26 on tank with -`cargo test --release -p clawhdf5-tools --test edit_interop -- --ignored ---nocapture measure_append_waste` (file sizes are deterministic): 1000 -appends of 100 `f8` values to a 1-D dataset with 1024-element chunks give -810 504 bytes unfiltered, as libhdf5's file, and 307 210 bytes with gzip -(libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values with -4096-element gzip chunks give 119 684 bytes against libhdf5's 50 292 -(`h5repack`: 49 930), because the chunk being appended to is followed by -new index blocks and moves each time it grows. +**Space is reused only within one editor.** Space an edit frees (a filtered +chunk that moves, chunks a shrink removes, B-tree nodes merged away, a +heap's replaced blocks) is reused by later edits of the same `FileEditor`; +what is left when it is dropped is leaked, as libhdf5 leaks it without a +persistent free-space manager (`h5repack` reclaims it). A chunk that is the +last thing in the file grows in place, which covers the usual append. +Measured 2026-09-26 on tank with `cargo test -p clawhdf5-tools --test +edit_interop -- --ignored --nocapture measure_append_waste` (one editor for +the whole workload; file sizes are deterministic): 1000 appends of 100 `f8` +values to a 1-D dataset with 1024-element chunks give 810 504 bytes +unfiltered, as libhdf5's file, and 306 780 bytes with gzip (307 210 before +reuse; libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values +with 4096-element gzip chunks give 79 829 bytes (119 684 before reuse) +against libhdf5's 50 292 (`h5repack`: 49 930): the chunk being appended to +is followed by new index blocks and moves each time it grows, and the +space it leaves is too small for its next, larger version. **No journal.** A crash while an edit patches existing structures can leave the file inconsistent; see the `FileEditor` documentation. -- 2.54.0 From 4ff3e40fea420eee174c8b4201445fcc27180969 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:15:06 -0500 Subject: [PATCH 15/49] clawhdf5-remote: object stores through object_store (S3, GCS, Azure) ObjectStoreStorage (feature `object-store`, pure Rust) reads one object of any object_store store by ranged get_opts, pinned at open by a head request: If-Match with its ETag (and the ETag and size of every response compared), else its version or modification time. A change is RemoteError::FileChanged. object_store is async and Storage is not, so the storage owns a small multi-threaded tokio runtime (two workers) and blocks the calling thread on it; the ranges of one read_ranges call are fetched concurrently (up to 8). From inside another tokio runtime it refuses with RemoteError::Usage instead of blocking a worker, and it shuts its runtime down in the background on drop so dropping it in async code does not panic. open_object(store, path, options) opens a file through a block cache (first block prefetched); open_url accepts s3://, gs:// and az:// with the `s3`, `gcs` and `azure` features, configured from the environment by object_store's from_env builders. Those pull object_store's cloud clients and aws-lc-rs (C), so they are opt-in; without them the URL is a clean UnsupportedScheme error naming the feature. Tests against object_store's in-memory and local-file stores (no cloud): every fixture's transcript equals File::open's, a multi-block object is fetched in coalesced block runs, an object replaced while open is an error, and a missing object or a read from inside a runtime is a clean error. ci-test.sh lints all backends, runs these tests (with s3 for its URL parsing test) and checks object-store for C in the no-C step. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 9 +- crates/clawhdf5-remote/Cargo.toml | 14 +- crates/clawhdf5-remote/README.md | 32 ++- crates/clawhdf5-remote/src/lib.rs | 53 +++- crates/clawhdf5-remote/src/object.rs | 285 +++++++++++++++++++ crates/clawhdf5-remote/tests/http.rs | 9 +- crates/clawhdf5-remote/tests/object_store.rs | 118 ++++++++ scripts/ci-test.sh | 18 +- 8 files changed, 517 insertions(+), 21 deletions(-) create mode 100644 crates/clawhdf5-remote/src/object.rs create mode 100644 crates/clawhdf5-remote/tests/object_store.rs diff --git a/CLAUDE.md b/CLAUDE.md index 57a5fa4..9a4f5b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,7 +26,7 @@ Cargo workspace with 19 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-napi` | Node.js native addon bindings | | `clawhdf5-py` | PyO3 Python bindings | | `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` | -| `clawhdf5-remote` | Remote files: `open_url` over HTTP(S) range requests through a mandatory block cache (`BlockCache`) | +| `clawhdf5-remote` | Remote files: `open_url` over HTTP(S) range requests and object stores (`object_store`: S3, GCS, Azure) through a mandatory block cache (`BlockCache`) | | `clawhdf5-bench` | Benchmark suite | ## Key Features @@ -165,8 +165,11 @@ Cargo workspace with 19 crates under `crates/` (plus `libaec-sys`, an internal F runs coalesced into parallel requests). `HttpStorage` pins the file by ETag/Last-Modified and length (a change is `RemoteError::FileChanged`), refuses servers that ignore `Range` unless a full download is allowed, - and retries transient failures. Default build is plain HTTP with no C; - `https` (rustls + ring) is opt-in. Tests run a std-only HTTP server + and retries transient failures. `ObjectStoreStorage` (feature + `object-store`, pure Rust) blocks on a small owned tokio runtime and + refuses to run inside another runtime. Default build is plain HTTP with + no C; `https` (rustls + ring) and `s3`/`gcs`/`azure` (aws-lc-rs) are + opt-in. Tests run a std-only HTTP server (`tests/common/server.rs`, also the `range_server` example); `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus` compares every corpus file over HTTP with `File::open`. diff --git a/crates/clawhdf5-remote/Cargo.toml b/crates/clawhdf5-remote/Cargo.toml index 6e68a73..ab64c2c 100644 --- a/crates/clawhdf5-remote/Cargo.toml +++ b/crates/clawhdf5-remote/Cargo.toml @@ -3,7 +3,7 @@ name = "clawhdf5-remote" version = "2.7.0" edition = "2024" rust-version.workspace = true -description = "Read HDF5 files over HTTP(S) range requests with clawhdf5, through a block cache" +description = "Read HDF5 files over HTTP(S) range requests and object stores (S3, GCS, Azure) with clawhdf5, through a block cache" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" @@ -17,11 +17,23 @@ http = ["dep:ureq"] # HTTPS through rustls (ring provider, Mozilla roots). ring compiles C and # assembly, so this is not part of the default build. https = ["http", "ureq/rustls"] +# Any object_store backend (in-memory, local files, or one you configure), +# driven by a small tokio runtime the storage owns. Pure Rust. +object-store = ["dep:object_store", "dep:tokio", "dep:futures-util"] +# s3:// gs:// az:// URLs in open_url, credentials from the environment. +# object_store's cloud clients use aws-lc-rs (C), hence opt-in. +s3 = ["object-store", "object_store/aws"] +gcs = ["object-store", "object_store/gcp"] +azure = ["object-store", "object_store/azure"] [dependencies] clawhdf5 = { path = "../clawhdf5", version = "2.7.0" } clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } ureq = { version = "3.4", optional = true, default-features = false } +object_store = { version = "0.14", optional = true, default-features = false, features = ["fs"] } +tokio = { version = "1", optional = true, default-features = false, features = ["rt-multi-thread"] } +futures-util = { version = "0.3", optional = true, default-features = false, features = ["std"] } [dev-dependencies] tempfile = { workspace = true } +tokio = { version = "1", default-features = false, features = ["rt"] } diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md index bb20f71..b4a487d 100644 --- a/crates/clawhdf5-remote/README.md +++ b/crates/clawhdf5-remote/README.md @@ -1,6 +1,7 @@ # clawhdf5-remote -Read HDF5 files where they live — on an HTTP(S) server — with +Read HDF5 files where they live — on an HTTP(S) server or in an object +store (S3, GCS, Azure) — with [clawhdf5](../../README.md), without downloading them first. ```rust @@ -52,8 +53,35 @@ The zero-copy methods of `clawhdf5` (`read_raw_ref`, `read_*_zerocopy`, |---|---|---| | `http` (default) | `http://` through `ureq`, no TLS | none | | `https` | `https://` through rustls, ring provider, Mozilla roots | ring (C and assembly) | +| `object-store` | `ObjectStoreStorage` and `open_object` over any [`object_store`](https://docs.rs/object_store) store (in-memory, local files, or one you configure) | none | +| `s3`, `gcs`, `azure` | `s3://bucket/key`, `gs://bucket/key`, `az://container/key` in `open_url`, configured from the environment (`AWS_*`, `GOOGLE_*`, `AZURE_*`) as object_store's `from_env` builders read it | aws-lc-rs (object_store's cloud clients) | -The default build compiles no C (`scripts/ci-test.sh` checks it). +The default build and `object-store` compile no C (`scripts/ci-test.sh` +checks both). + +## Object stores + +`object_store` is async; `Storage` is synchronous (parsing is CPU work). +`ObjectStoreStorage` owns a small tokio runtime (two worker threads) and +blocks the calling thread on it for each read, so it is read from ordinary +threads, several at once. From inside an async runtime it refuses +(`RemoteError::Usage`) rather than block a worker: read in +`tokio::task::spawn_blocking`. The object is pinned by its ETag +(`If-Match`, and compared on every response), else its version or +modification time, and its size. The ranges of one read are fetched +concurrently (up to 8). + +```rust +use std::sync::Arc; +use clawhdf5_remote::object_store::{memory::InMemory, ObjectStore}; +let store: Arc = Arc::new(InMemory::new()); +// ... put a file at "data.h5" ... +let (file, cache) = clawhdf5_remote::open_object(store, "data.h5", &Default::default())?; +``` + +The tests use object_store's in-memory and local-file stores; no cloud +account is needed. The cloud schemes are only built (and unit-tested for +URL parsing) in CI, not run against a real bucket. ## Counting requests diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index 597fb63..f233954 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -12,12 +12,13 @@ //! ``` //! //! - [`open_url`] / [`open_url_with`]: `http://` (default feature `http`), -//! `https://` (feature `https`) → a [`clawhdf5::File`] with the whole -//! read API. +//! `https://` (feature `https`), `s3://`, `gs://`, `az://` (features `s3`, +//! `gcs`, `azure`) → a [`clawhdf5::File`] with the whole read API. //! - [`storage_for_url`] gives the cached storage itself, to open with //! [`clawhdf5::File::open_storage`] and to read its [`CacheStats`]. //! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried -//! with backoff), and [`BlockCache`] over any +//! with backoff), [`ObjectStoreStorage`] (any `object_store` store, +//! feature `object-store`), and [`BlockCache`] over any //! [`Storage`](clawhdf5_format::storage::Storage). //! //! What costs what: opening costs one request (it also fetches the first @@ -34,6 +35,8 @@ pub mod cache; pub mod error; #[cfg(feature = "http")] pub mod http; +#[cfg(feature = "object-store")] +pub mod object; use std::sync::Arc; @@ -44,6 +47,10 @@ pub use cache::{BlockCache, CacheConfig, CacheStats}; pub use error::{Error, RemoteError}; #[cfg(feature = "http")] pub use http::{HttpOptions, HttpStats, HttpStorage}; +#[cfg(feature = "object-store")] +pub use object::ObjectStoreStorage; +#[cfg(feature = "object-store")] +pub use object_store; /// A backend a [`BlockCache`] can read through. pub type Backend = Box; @@ -65,7 +72,9 @@ pub struct Options { /// Open the HDF5 file at `url` with default [`Options`]. /// /// `http://…` needs the (default) `http` feature, `https://…` the `https` -/// feature. +/// feature, `s3://bucket/key`, `gs://bucket/key` and `az://container/key` +/// the `s3`, `gcs` and `azure` features (credentials and region from the +/// environment, as `object_store`'s `from_env` builders read them). pub fn open_url(url: &str) -> Result { open_url_with(url, &Options::default()) } @@ -111,8 +120,24 @@ fn http_storage(url: &str, _options: &Options) -> Result, Err .into()) } +#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))] +fn cloud_storage(url: &str, _scheme: &str, options: &Options) -> Result, Error> { + let (store, path) = object::store_for_url(url)?; + let storage = ObjectStoreStorage::new(store, path)?; + Ok(Arc::new(cached(Box::new(storage), options)?)) +} + +#[cfg(not(any(feature = "s3", feature = "gcs", feature = "azure")))] fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result, Error> { - Err(RemoteError::UnsupportedScheme(format!("{url}: {scheme}:// is not supported yet")).into()) + let feature = match scheme { + "s3" | "s3a" => "s3", + "gs" => "gcs", + _ => "azure", + }; + Err(RemoteError::UnsupportedScheme(format!( + "{url}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote" + )) + .into()) } /// A [`BlockCache`] over `backend` with its first block fetched (readahead @@ -125,3 +150,21 @@ pub fn cached(backend: Backend, options: &Options) -> Result, + path: &str, + options: &Options, +) -> Result<(File, Arc), Error> { + let path = object_store::path::Path::parse(path) + .map_err(|e| RemoteError::InvalidUrl(format!("{path}: {e}")))?; + let storage = Arc::new(cached( + Box::new(ObjectStoreStorage::new(store, path)?), + options, + )?); + let file = File::open_storage(storage.clone())?; + Ok((file, storage)) +} diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs new file mode 100644 index 0000000..6bafe8d --- /dev/null +++ b/crates/clawhdf5-remote/src/object.rs @@ -0,0 +1,285 @@ +//! Object stores (S3, GCS, Azure, local files, memory) through the +//! [`object_store`] crate: [`ObjectStoreStorage`]. +//! +//! `object_store` is async and [`Storage`] is synchronous (parsing is CPU +//! work; `docs/design/range-reads.md` §3 (a)). The storage owns a small +//! multi-threaded tokio runtime (two worker threads) and blocks the calling +//! thread on it for each read, so it can be used from ordinary threads — +//! several at once. Calling it from inside another tokio runtime would +//! block that runtime's worker, so it refuses with +//! [`RemoteError::Usage`]: from async code, read in +//! `tokio::task::spawn_blocking`. +//! +//! The object is pinned when the storage is made: its size, and its ETag +//! (sent as `If-Match` with every read, and compared with every response) +//! or, without one, its version or modification time. A change while it is +//! open is [`RemoteError::FileChanged`]. The ranges of one `read_ranges` +//! call are fetched concurrently (at most 8 at a time). + +use std::borrow::Cow; +use std::ops::Range; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use clawhdf5_format::error::FormatError; +use clawhdf5_format::storage::Storage; +use futures_util::{StreamExt, TryStreamExt}; +use object_store::path::Path; +use object_store::{GetOptions, GetRange, ObjectMeta, ObjectStore, ObjectStoreExt}; + +use crate::error::RemoteError; + +/// Concurrent requests of one `read_ranges` call. +const MAX_CONCURRENT: usize = 8; + +/// One object of an [`ObjectStore`], read by ranged `get`s. See the +/// [module documentation](self). +pub struct ObjectStoreStorage { + store: Arc, + path: Path, + meta: ObjectMeta, + runtime: Option, + requests: AtomicU64, + bytes: AtomicU64, +} + +impl std::fmt::Debug for ObjectStoreStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ObjectStoreStorage") + .field("store", &self.store.to_string()) + .field("path", &self.path) + .field("size", &self.meta.size) + .field("e_tag", &self.meta.e_tag) + .finish() + } +} + +fn os_error(e: object_store::Error) -> RemoteError { + match e { + object_store::Error::Precondition { .. } | object_store::Error::NotModified { .. } => { + RemoteError::FileChanged(e.to_string()) + } + other => RemoteError::ObjectStore(other.to_string()), + } +} + +impl ObjectStoreStorage { + /// Open the object at `path` of `store`: one `head` request for its + /// size and validators. + pub fn new(store: Arc, path: Path) -> Result { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_name("clawhdf5-remote") + .enable_all() + .build() + .map_err(|e| RemoteError::Usage(format!("cannot start a tokio runtime: {e}")))?; + let mut s = ObjectStoreStorage { + store, + path, + meta: ObjectMeta { + location: Path::default(), + last_modified: Default::default(), + size: 0, + e_tag: None, + version: None, + }, + runtime: Some(runtime), + requests: AtomicU64::new(1), + bytes: AtomicU64::new(0), + }; + let (store, path) = (s.store.clone(), s.path.clone()); + s.meta = s.block_on(async move { store.head(&path).await.map_err(os_error) })?; + Ok(s) + } + + /// The object's metadata as pinned at open. + pub fn meta(&self) -> &ObjectMeta { + &self.meta + } + + /// Requests made (the `head` included) and bytes received. + pub fn stats(&self) -> (u64, u64) { + ( + self.requests.load(Ordering::Relaxed), + self.bytes.load(Ordering::Relaxed), + ) + } + + fn block_on( + &self, + fut: impl std::future::Future>, + ) -> Result { + if tokio::runtime::Handle::try_current().is_ok() { + return Err(RemoteError::Usage( + "ObjectStoreStorage blocks on its own runtime and cannot be read from inside \ + an async runtime; read in tokio::task::spawn_blocking" + .into(), + )); + } + let rt = self.runtime.as_ref().expect("runtime lives until drop"); + rt.block_on(fut) + } + + fn options(&self, range: Range) -> GetOptions { + let mut o = GetOptions { + range: Some(GetRange::Bounded(range)), + ..GetOptions::default() + }; + if let Some(e) = &self.meta.e_tag { + o.if_match = Some(e.clone()); + } else if let Some(v) = &self.meta.version { + o.version = Some(v.clone()); + } else { + o.if_unmodified_since = Some(self.meta.last_modified); + } + o + } + + fn fetch_all(&self, ranges: &[Range]) -> Result>, RemoteError> { + let len = self.meta.size; + let jobs: Vec<(usize, Range)> = ranges + .iter() + .enumerate() + .filter_map(|(i, r)| { + let end = r.end.min(len); + (r.start < end).then_some((i, r.start..end)) + }) + .collect(); + let store = self.store.clone(); + let path = self.path.clone(); + let pinned = self.meta.e_tag.clone(); + let reqs: Vec<(usize, Range, GetOptions)> = jobs + .into_iter() + .map(|(i, r)| { + let o = self.options(r.clone()); + (i, r, o) + }) + .collect(); + self.requests + .fetch_add(reqs.len() as u64, Ordering::Relaxed); + let fetched: Vec<(usize, Vec)> = self.block_on(async move { + futures_util::stream::iter(reqs) + .map(|(i, r, o)| { + let (store, path, pinned) = (store.clone(), path.clone(), pinned.clone()); + async move { + let got = store.get_opts(&path, o).await.map_err(os_error)?; + if let (Some(want), Some(have)) = (&pinned, &got.meta.e_tag) + && want != have + { + return Err(RemoteError::FileChanged(format!( + "{path}: ETag {have} instead of {want}" + ))); + } + if got.meta.size != len { + return Err(RemoteError::FileChanged(format!( + "{path}: size {} instead of {len}", + got.meta.size + ))); + } + let bytes = got.bytes().await.map_err(os_error)?; + if bytes.len() as u64 != r.end - r.start { + return Err(RemoteError::BadResponse(format!( + "{path}: {} bytes for range {r:?}", + bytes.len() + ))); + } + Ok::<_, RemoteError>((i, bytes.to_vec())) + } + }) + .buffer_unordered(MAX_CONCURRENT) + .try_collect() + .await + })?; + let mut out = vec![Vec::new(); ranges.len()]; + for (i, b) in fetched { + self.bytes.fetch_add(b.len() as u64, Ordering::Relaxed); + out[i] = b; + } + Ok(out) + } +} + +impl Drop for ObjectStoreStorage { + fn drop(&mut self) { + // Dropping a runtime blocks, which panics inside an async context. + if let Some(rt) = self.runtime.take() { + rt.shutdown_background(); + } + } +} + +impl Storage for ObjectStoreStorage { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let range = offset..offset.saturating_add(len as u64); + let mut v = self.fetch_all(std::slice::from_ref(&range))?; + Ok(Cow::Owned(v.pop().unwrap_or_default())) + } + + fn len(&self) -> u64 { + self.meta.size + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + if ranges.iter().any(|r| r.end < r.start) { + return Err(FormatError::Storage( + "read range ends before it starts".into(), + )); + } + Ok(self + .fetch_all(ranges)? + .into_iter() + .map(Cow::Owned) + .collect()) + } +} + +/// The store and object path for a cloud URL (`s3://bucket/key`, +/// `gs://bucket/key`, `az://container/key`, ...), configured from the +/// environment as `object_store`'s `from_env` builders do (`AWS_*`, +/// `GOOGLE_*`, `AZURE_*`). +#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))] +pub(crate) fn store_for_url(url: &str) -> Result<(Arc, Path), RemoteError> { + let parsed = object_store::path::Path::parse( + url.split_once("://") + .and_then(|(_, rest)| rest.split_once('/')) + .map(|(_, key)| key) + .unwrap_or(""), + ) + .map_err(|e| RemoteError::InvalidUrl(format!("{url}: {e}")))?; + let scheme = url.split_once("://").map(|(s, _)| s.to_ascii_lowercase()); + let store: Arc = match scheme.as_deref() { + #[cfg(feature = "s3")] + Some("s3" | "s3a") => Arc::new( + object_store::aws::AmazonS3Builder::from_env() + .with_url(url) + .build() + .map_err(os_error)?, + ), + #[cfg(feature = "gcs")] + Some("gs") => Arc::new( + object_store::gcp::GoogleCloudStorageBuilder::from_env() + .with_url(url) + .build() + .map_err(os_error)?, + ), + #[cfg(feature = "azure")] + Some("az" | "azure" | "abfs" | "abfss" | "adl") => Arc::new( + object_store::azure::MicrosoftAzureBuilder::from_env() + .with_url(url) + .build() + .map_err(os_error)?, + ), + _ => return Err(RemoteError::UnsupportedScheme(url.to_string())), + }; + Ok((store, parsed)) +} + +#[cfg(all(test, feature = "s3"))] +mod tests { + #[test] + fn s3_urls_name_the_bucket_and_key() { + let (store, path) = super::store_for_url("s3://my-bucket/dir/file.h5").unwrap(); + assert_eq!(path.as_ref(), "dir/file.h5"); + assert!(store.to_string().contains("my-bucket"), "{store}"); + } +} diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index e7faed8..197e268 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -474,10 +474,11 @@ fn bad_urls_and_statuses_are_clean_errors() { open_url("no-scheme"), Err(Error::Remote(RemoteError::InvalidUrl(_))) )); - assert!(matches!( - open_url("s3://bucket/key.h5"), - Err(Error::Remote(RemoteError::UnsupportedScheme(_))) - )); + #[cfg(not(feature = "s3"))] + { + let e = open_url("s3://bucket/key.h5").unwrap_err().to_string(); + assert!(e.contains("`s3` feature"), "{e}"); + } #[cfg(not(feature = "https"))] { let e = open_url("https://example.com/a.h5") diff --git a/crates/clawhdf5-remote/tests/object_store.rs b/crates/clawhdf5-remote/tests/object_store.rs new file mode 100644 index 0000000..0d2f5fa --- /dev/null +++ b/crates/clawhdf5-remote/tests/object_store.rs @@ -0,0 +1,118 @@ +//! `ObjectStoreStorage` against object_store's in-memory and local-file +//! backends (no cloud needed): values equal `File::open`'s, the block cache +//! coalesces, and an object replaced while open is an error. + +#![cfg(feature = "object-store")] + +mod common; + +use std::sync::Arc; + +use clawhdf5::File; +use clawhdf5_format::storage::Storage; +use clawhdf5_remote::object_store::memory::InMemory; +use clawhdf5_remote::object_store::path::Path as ObjectPath; +use clawhdf5_remote::object_store::{ObjectStore, ObjectStoreExt, PutPayload}; +use clawhdf5_remote::{ObjectStoreStorage, Options, open_object}; +use common::{multi_block_file, transcript}; + +fn put(store: &dyn ObjectStore, path: &str, bytes: Vec) { + let rt = tokio_rt(); + rt.block_on(store.put(&ObjectPath::from(path), PutPayload::from(bytes))) + .unwrap(); +} + +fn tokio_rt() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +#[test] +fn in_memory_store_reads_like_file_open() { + let store: Arc = Arc::new(InMemory::new()); + let mut compared = 0; + for (i, p) in common::fixtures().iter().enumerate() { + let bytes = std::fs::read(p).unwrap(); + let key = format!("f{i}.h5"); + put(store.as_ref(), &key, bytes); + let (Ok(local), Ok((mut remote, cache))) = ( + File::open(p), + open_object(store.clone(), &key, &Options::default()), + ) else { + continue; + }; + remote.set_vds_resolver(common::sibling_resolver(p.parent().unwrap().into())); + assert!(remote.contiguous_bytes().is_none()); + assert_eq!(transcript(&remote), transcript(&local), "{}", p.display()); + assert!(cache.stats().reads > 0); + compared += 1; + } + assert!(compared >= 40, "{compared}"); +} + +#[test] +fn multi_block_object_is_fetched_in_coalesced_blocks() { + let store: Arc = Arc::new(InMemory::new()); + let bytes = multi_block_file(); + put(store.as_ref(), "m.h5", bytes.clone()); + let (remote, cache) = open_object(store, "m.h5", &Options::default()).unwrap(); + let local = File::from_bytes(bytes.clone()).unwrap(); + assert_eq!( + remote.dataset("big").unwrap().read_f64().unwrap(), + local.dataset("big").unwrap().read_f64().unwrap() + ); + let s = cache.stats(); + let blocks = (bytes.len() as u64).div_ceil(1 << 20); + assert!(s.bytes_fetched <= bytes.len() as u64); + assert!(s.requests <= blocks, "{s:?}"); + assert_eq!(cache.inner().len(), bytes.len() as u64); +} + +#[test] +fn local_file_store_reads_like_file_open() { + let dir = tempfile::tempdir().unwrap(); + let bytes = multi_block_file(); + std::fs::write(dir.path().join("m.h5"), &bytes).unwrap(); + let store: Arc = Arc::new( + clawhdf5_remote::object_store::local::LocalFileSystem::new_with_prefix(dir.path()).unwrap(), + ); + let (remote, _) = open_object(store, "m.h5", &Options::default()).unwrap(); + let local = File::from_bytes(bytes).unwrap(); + assert_eq!(transcript(&remote), transcript(&local)); +} + +#[test] +fn an_object_replaced_while_open_is_an_error() { + let store: Arc = Arc::new(InMemory::new()); + let bytes = multi_block_file(); + put(store.as_ref(), "m.h5", bytes.clone()); + let (remote, _) = open_object(store.clone(), "m.h5", &Options::default()).unwrap(); + assert_eq!(remote.root().groups().unwrap(), ["grp"]); + let mut other = bytes; + let n = other.len(); + other[n / 2] ^= 0xff; + put(store.as_ref(), "m.h5", other); + let err = remote + .dataset("big") + .unwrap() + .read_f64() + .unwrap_err() + .to_string(); + assert!(err.contains("changed while open"), "{err}"); +} + +#[test] +fn missing_objects_and_async_callers_are_clean_errors() { + let store: Arc = Arc::new(InMemory::new()); + assert!(ObjectStoreStorage::new(store.clone(), ObjectPath::from("nope.h5")).is_err()); + put(store.as_ref(), "m.h5", multi_block_file()); + let storage = ObjectStoreStorage::new(store, ObjectPath::from("m.h5")).unwrap(); + // From inside a runtime: refused, not a panic or a deadlock. + let rt = tokio_rt(); + let r = rt.block_on(async { storage.read_at(0, 10).map(|b| b.len()) }); + assert!(r.unwrap_err().to_string().contains("spawn_blocking")); + // Dropping the storage inside a runtime does not panic. + rt.block_on(async move { drop(storage) }); +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 2fce9c1..fe7242c 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -109,11 +109,12 @@ run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \ -- -D warnings -# clawhdf5-remote's HTTPS backend (rustls). -run_step "cargo clippy (remote, https)" cargo clippy \ +# clawhdf5-remote's optional backends: object_store (in-memory and local +# stores in the tests), HTTPS through rustls, and the cloud stores. +run_step "cargo clippy (remote, all backends)" cargo clippy \ -p clawhdf5-remote \ --all-targets \ - --features https \ + --features object-store,https,s3,gcs,azure \ -- -D warnings # The README promises that the core crates build no C by default. Hold it to @@ -121,15 +122,16 @@ run_step "cargo clippy (remote, https)" cargo clippy \ # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. -# clawhdf5-remote is checked by default (plain HTTP); its https feature -# (ring) builds C and is opt-in. +# clawhdf5-remote is checked by default (plain HTTP) and with its +# object-store feature; its https (ring) and s3/gcs/azure (aws-lc-rs) +# features build C and are opt-in. no_c_in_default_build() { local entry crate features found=0 for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ clawhdf5-tools \ clawhdf5-wasm \ - clawhdf5-remote; do + clawhdf5-remote clawhdf5-remote:object-store; do crate=${entry%%:*} features=() [ "$entry" != "$crate" ] && features=(--features "${entry#*:}") @@ -204,6 +206,10 @@ run_step "cargo test (facade parallel)" cargo test \ -p clawhdf5 \ --features parallel +run_step "cargo test (remote, object_store backend, s3 URLs)" cargo test \ + -p clawhdf5-remote \ + --features object-store,s3 + run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ --features parallel -- 2.54.0 From c54c64cc9bf3638f347df247055f6a22c6a6f87d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:16:35 -0500 Subject: [PATCH 16/49] clawhdf5: File::storage gives the file's view as a Storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bytes File::as_bytes returns (from the superblock on, bounded by the recorded end of file, a metadata cache image laid over), as a &(dyn Storage + Send + Sync) for every backend. Code that parses the file itself with the clawhdf5_format *_in functions — h5rs does — can then read a file opened with File::open_storage (a remote file) as well as a local one; in memory its as_contiguous() is as_bytes(), so local reads stay slices. Test: for every fixture, File::open's storage() is as_bytes() as its contiguous view, and File::open_storage over a read_at-only storage gives the same bytes through storage().read_at. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/src/reader.rs | 11 +++++++ crates/clawhdf5/tests/storage_equivalence.rs | 30 ++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 6bd7b35..2a9140c 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -523,6 +523,17 @@ impl File { self.data.contiguous() } + /// The bytes [`as_bytes`](Self::as_bytes) returns, as a [`Storage`], + /// for every backend: from the superblock on, bounded by the recorded + /// end of file, with a metadata cache image laid over them. Code that + /// parses the file itself with the `clawhdf5_format` `*_in` functions + /// reads through this, so it works on remote files too; for a file in + /// memory its [`Storage::as_contiguous`] is [`as_bytes`](Self::as_bytes) + /// (and every read a slice of it). + pub fn storage(&self) -> &(dyn Storage + Send + Sync) { + &self.data + } + /// The error of a metadata cache image libhdf5 cannot load, when the /// file has one. Such a file opens, as in libhdf5, and every object /// lookup fails with this error; code that parses [`Self::as_bytes`] diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs index 7a2749b..6793d47 100644 --- a/crates/clawhdf5/tests/storage_equivalence.rs +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -432,3 +432,33 @@ fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() { "as_bytes over a range storage must not answer" ); } + +/// `File::storage` is the view `as_bytes` gives, for every backend: the +/// user block skipped, bounded by the end of file, a cache image laid over. +#[test] +fn file_storage_is_the_as_bytes_view_for_every_backend() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + let mut compared = 0; + for p in files { + let Ok(local) = File::open(&p) else { continue }; + let bytes = std::fs::read(&p).unwrap(); + let remote = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap(); + let want = local.as_bytes(); + let view = local.storage(); + assert_eq!(view.as_contiguous(), Some(want), "{}", p.display()); + let got = remote.storage(); + assert!(got.as_contiguous().is_none()); + assert_eq!(got.len(), want.len() as u64, "{}", p.display()); + assert_eq!( + &*got.read_at(0, want.len()).unwrap(), + want, + "{}", + p.display() + ); + compared += 1; + } + assert!(compared >= 40, "{compared}"); +} -- 2.54.0 From a4f586e6574791b49180a2e35e6cca12368b85ea Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:20:50 -0500 Subject: [PATCH 17/49] format: VlResolver::element_in and string_element_in over any Storage VlResolver::element and string_element return slices of the whole file, so they exist only for a resolver over &[u8]. Their *_in forms work for any Storage (a remote file): the element's bytes borrowed from the resolver's cache of heap collections, with the same null-element, NUL and size checks. h5rs decodes variable-length values with them. Test: over a read_at-only storage they give what element/string_element give over the slice, for a string with an embedded NUL, a null element and an element whose heap object has the wrong size. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/vl_data.rs | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index f37638b..378ada4 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -305,6 +305,24 @@ impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> { Ok(Some(data)) } + /// [`VlResolver::element`] over any storage: the element's bytes + /// (borrowed from the resolver's cache of heap collections, so they + /// live until the next call), or `None` for a null element. + pub fn element_in( + &mut self, + elem: &[u8], + base_size: usize, + ) -> Result, FormatError> { + let vl = parse_vl_references(elem, 1, self.offset_size)?; + self.resolve(&vl[0], base_size) + } + + /// [`VlResolver::string_element`] over any storage (see + /// [`element_in`](Self::element_in)). + pub fn string_element_in(&mut self, elem: &[u8]) -> Result, FormatError> { + Ok(self.element_in(elem, 1)?.map(cut_at_nul)) + } + /// The strings of the variable-length string elements in `raw`, as /// bytes. A string ends at its first NUL, as libhdf5 returns it (it /// converts each to a C string); a null element is empty. @@ -645,6 +663,32 @@ mod tests { } } + #[test] + fn element_in_over_a_storage_matches_element_over_a_slice() { + let mut file_data = vec![0u8; 512]; + build_gcol_at(&mut file_data, 256, &[(1, b"Alice\0x"), (2, b"Bob")]); + let mut raw = build_vl_refs(&["Alice\0x", "Bob"], 256, 1, 8); + raw.extend(element(0, 0, 0, 8)); // null + raw.extend(element(9, 256, 1, 8)); // wrong length: an error + let storage = crate::storage::CountingStorage::new(file_data.clone()); + let dynamic: &dyn crate::storage::Storage = &storage; + let mut slice = VlResolver::new(&file_data, 8, 8); + let mut any = VlResolver::new_in(dynamic, 8, 8); + for e in raw.chunks(16) { + let want = slice.element(e, 1).map(|o| o.map(<[u8]>::to_vec)); + let got = any.element_in(e, 1).map(|o| o.map(<[u8]>::to_vec)); + assert_eq!(format!("{want:?}"), format!("{got:?}")); + let want = slice.string_element(e).map(|o| o.map(<[u8]>::to_vec)); + let got = any.string_element_in(e).map(|o| o.map(<[u8]>::to_vec)); + assert_eq!(format!("{want:?}"), format!("{got:?}")); + } + assert_eq!( + any.string_element_in(&raw[..16]).unwrap(), + Some(&b"Alice"[..]) + ); + assert!(storage.reads() > 0); + } + #[test] fn null_vl_element_zero_address() { let mut raw = Vec::new(); -- 2.54.0 From c513f7e6d787db81ce3329e57b59b3daaddde291 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:23:47 -0500 Subject: [PATCH 18/49] h5rs: URLs as FILE arguments (feature `remote`) With the `remote` feature (`remote-https` for https://), ls, dump, stat and diff take an http(s):// (or s3://, gs://, az:// with those clawhdf5-remote features) URL wherever they take a file, and read it by range requests through clawhdf5-remote's block cache. check validates every byte, so it downloads a remote file whole and checks it as before. Without the feature a URL is a clean error naming it. The tools read the file through File::storage instead of as_bytes: object headers, shared messages, attributes, v1 and v2 group links, dense storage (fractal heaps and v2 B-trees), path resolution, chunk listings and variable-length values go through the format crate's *_in functions, and the fractal-heap block verifier reads each block through the storage (a read failure of a remote file is reported as a problem, not as "past the end of the file"). A local file's storage is its mapped bytes, so its reads are still slices. stat's file size comes from the opened file, so it is right for a URL. Tests: tests/remote.rs serves fixtures (old and new formats, a paged file, a metadata cache image, a multi-block fractal heap, compounds, v1 groups) with the clawhdf5-remote test server and requires every subcommand's output and exit status for the URL to equal the local file's, and diff of the two to be clean; 404s, non-HDF5 bodies and https without its feature are clean errors. Local output is unchanged: the old and new h5rs print the same for ls -r -v, dump, stat and check --data on the 747 conformance and CVE corpus files (tank, 2026-09-26; the dumps of h5diff_hyper1/2.h5 were too large for the comparison script, their ls, stat and check agree), except cve-2025-2310.h5, whose dump error messages differ between runs of the old binary too (which failing chunk is reported first). ci-test.sh lints h5rs with remote-https, runs the URL tests and checks h5rs with remote for C. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-tools/Cargo.toml | 7 ++ crates/clawhdf5-tools/README.md | 20 ++++ crates/clawhdf5-tools/src/check.rs | 25 ++-- crates/clawhdf5-tools/src/diff.rs | 2 +- crates/clawhdf5-tools/src/dump.rs | 2 +- crates/clawhdf5-tools/src/h5.rs | 139 +++++++++++++++++++---- crates/clawhdf5-tools/src/heap_blocks.rs | 82 ++++++++++--- crates/clawhdf5-tools/src/info.rs | 6 +- crates/clawhdf5-tools/src/ls.rs | 2 +- crates/clawhdf5-tools/src/stat.rs | 4 +- crates/clawhdf5-tools/src/value.rs | 11 +- crates/clawhdf5-tools/tests/remote.rs | 100 ++++++++++++++++ scripts/ci-test.sh | 18 ++- 13 files changed, 357 insertions(+), 61 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/remote.rs diff --git a/crates/clawhdf5-tools/Cargo.toml b/crates/clawhdf5-tools/Cargo.toml index d54f39d..d6cf7d5 100644 --- a/crates/clawhdf5-tools/Cargo.toml +++ b/crates/clawhdf5-tools/Cargo.toml @@ -14,9 +14,16 @@ readme = "README.md" name = "h5rs" path = "src/main.rs" +[features] +# FILE arguments may be URLs: http:// with `remote` (no C), https:// with +# `remote-https` (rustls + ring, which compiles C). +remote = ["dep:clawhdf5-remote"] +remote-https = ["remote", "clawhdf5-remote/https"] + [dependencies] clawhdf5 = { path = "../clawhdf5", version = "2.7.0" } clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +clawhdf5-remote = { path = "../clawhdf5-remote", version = "2.7.0", optional = true } serde_json = "1" [dev-dependencies] diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 8f089c3..42d829d 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -24,6 +24,26 @@ Every command takes `--max-bytes N` where it reads values (default 1 GiB): a dataset whose dataspace claims more than that is reported instead of read, so a corrupt size cannot exhaust memory. +### Remote files + +Built with the `remote` feature, every FILE argument may be an `http://` +URL (`remote-https` adds `https://`, through rustls and ring, which compiles +C; the default build has neither). The file is read by range requests +through [clawhdf5-remote](../clawhdf5-remote/README.md)'s block cache, so +`ls` of a large file fetches its metadata blocks, not the file: + +```bash +cargo install --path crates/clawhdf5-tools --features remote +h5rs ls -r -v http://127.0.0.1:8000/file.h5 +h5rs dump http://127.0.0.1:8000/file.h5 +h5rs diff local.h5 http://127.0.0.1:8000/file.h5 +``` + +A URL names the whole file (`FILE/OBJECT` suffixes are for local paths). +`check` validates every byte, so it downloads a remote file whole first. +The output is the local file's (`tests/remote.rs` compares every +subcommand). + ## `h5rs ls` ```console diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index da90247..e7d97e5 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -148,13 +148,24 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { return args.usage_error(out, "missing FILE", USAGE); }; let path = std::path::Path::new(&file); - if !path.is_file() { - writeln!(out.e, "h5rs check: {file}: no such file")?; - return Ok(2); - } - let mut h5 = match H5::open(path) { - Ok(h) => h, - Err(_) => return unopenable(path, out), + let mut h5 = if crate::h5::is_url(&file) { + // check validates every byte, so a remote file is downloaded whole. + match H5::open_arg_whole(&file) { + Ok(h) => h, + Err(e) => { + writeln!(out.e, "h5rs check: {e}")?; + return Ok(2); + } + } + } else { + if !path.is_file() { + writeln!(out.e, "h5rs check: {file}: no such file")?; + return Ok(2); + } + match H5::open(path) { + Ok(h) => h, + Err(_) => return unopenable(path, out), + } }; if let Some(m) = max_bytes { h5.max_bytes = m; diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index 98d4ba5..af684c5 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -174,7 +174,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } let mut files = Vec::new(); for f in &pos[..2] { - match H5::open(std::path::Path::new(f)) { + match H5::open_arg(f) { Ok(mut h) => { if let Some(m) = max_bytes { h.max_bytes = m; diff --git a/crates/clawhdf5-tools/src/dump.rs b/crates/clawhdf5-tools/src/dump.rs index 7b1d8e7..4b59f00 100644 --- a/crates/clawhdf5-tools/src/dump.rs +++ b/crates/clawhdf5-tools/src/dump.rs @@ -82,7 +82,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let Some(file) = file else { return args.usage_error(out, "missing FILE", USAGE); }; - let mut h5 = match H5::open(std::path::Path::new(&file)) { + let mut h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs dump: {e}")?; diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index 5f71a14..c63ccb3 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -10,9 +10,9 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use clawhdf5::File; -use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant}; +use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant_in}; use clawhdf5_format::attribute_info::AttributeInfoMessage; -use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records_in}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; @@ -24,6 +24,7 @@ use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::link_message::{LinkMessage, LinkTarget}; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::storage::Storage; use clawhdf5_format::superblock::Superblock; use clawhdf5_format::symbol_table::SymbolTableMessage; @@ -186,8 +187,11 @@ pub struct Link { /// An open file. pub struct H5 { + /// The file's path, or its URL for a remote file. pub path: PathBuf, pub file: File, + /// Size of the whole file in bytes (user block included). + pub size: u64, pub max_bytes: u64, /// Fractal heaps whose blocks were verified: `None` = sound. verified_heaps: RefCell>>, @@ -204,19 +208,90 @@ impl H5 { path.display() )) })?; - Ok(H5 { - path: path.to_path_buf(), + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + Ok(H5::new(path.to_path_buf(), file, size)) + } + + fn new(path: PathBuf, file: File, size: u64) -> H5 { + H5 { + path, file, + size, max_bytes: DEFAULT_MAX_BYTES, verified_heaps: RefCell::new(HashMap::new()), - }) + } + } + + /// Open a command-line FILE argument: a path, or with the `remote` + /// feature an `http(s)://` (or `s3://`, `gs://`, `az://`) URL, read by + /// range requests through a block cache. + pub fn open_arg(arg: &str) -> Result
{ + if !is_url(arg) { + return H5::open(Path::new(arg)); + } + #[cfg(feature = "remote")] + { + let storage = + clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; + let size = storage.len(); + let file = File::open_storage(storage).map_err(|e| { + Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + })?; + Ok(H5::new(PathBuf::from(arg), file, size)) + } + #[cfg(not(feature = "remote"))] + Err(Error::new(format!( + "{arg}: URLs need h5rs built with the `remote` feature" + ))) + } + + /// [`H5::open_arg`], with a remote file downloaded whole into memory + /// first — for `check`, which validates every byte of the file anyway + /// and parses it as one slice ([`H5::data`]). + pub fn open_arg_whole(arg: &str) -> Result
{ + let h5 = H5::open_arg(arg)?; + if h5.file.contiguous_bytes().is_some() { + return Ok(h5); + } + #[cfg(feature = "remote")] + { + let storage = + clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; + let len = usize::try_from(storage.len()) + .map_err(|_| Error::new(format!("{arg}: too large to download")))?; + let bytes = storage + .read_at(0, len) + .map_err(|e| Error::new(format!("{arg}: {e}")))? + .into_owned(); + let size = bytes.len() as u64; + let file = File::from_bytes(bytes).map_err(|e| { + Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + })?; + Ok(H5::new(PathBuf::from(arg), file, size)) + } + #[cfg(not(feature = "remote"))] + unreachable!("open_arg refuses URLs without the remote feature") } /// The file's bytes from the superblock on: what every address indexes. + /// + /// # Panics + /// + /// For a remote file opened by [`H5::open_arg`]; read it through + /// [`H5::store`], or open it with [`H5::open_arg_whole`]. pub fn data(&self) -> &[u8] { self.file.as_bytes() } + /// The same bytes as a [`Storage`], for local and remote files alike: + /// in memory a read is a slice of [`H5::data`], remote it is served by + /// the block cache. + pub fn store(&self) -> &dyn Storage { + self.file.storage() + } + pub fn sb(&self) -> &Superblock { self.file.superblock() } @@ -239,8 +314,8 @@ impl H5 { if let Some(e) = self.file.cache_image_error() { return Err(Error::at(addr, format!("metadata cache image: {e}"))); } - let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?; - ObjectHeader::parse(self.data(), off, self.os(), self.ls()) + to_usize(addr)?; + ObjectHeader::parse_in(self.store(), addr, self.os(), self.ls()) .map_err(|e| Error::at(addr, format!("object header: {e}"))) } @@ -249,11 +324,14 @@ impl H5 { pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result>> { match h.messages.iter().find(|m| m.msg_type == t) { None => Ok(None), - Some(m) => { - clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls()) - .map(|c| Some(c.into_owned())) - .map_err(|e| Error::new(format!("{t:?} message: {e}"))) - } + Some(m) => clawhdf5_format::shared_message::message_data_in( + self.store(), + m, + self.os(), + self.ls(), + ) + .map(|c| Some(c.into_owned())) + .map_err(|e| Error::new(format!("{t:?} message: {e}"))), } } @@ -305,8 +383,9 @@ impl H5 { self.verified_heap(fh) .map_err(|e| e.context("dense attribute storage"))?; } - let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls()) - .map_err(|e| Error::new(format!("attributes: {e}")))?; + let (mut attrs, errs) = + extract_attributes_tolerant_in(self.store(), h, self.os(), self.ls()) + .map_err(|e| Error::new(format!("attributes: {e}")))?; attrs.sort_by(|a, b| a.name.cmp(&b.name)); Ok((attrs, errs.iter().map(|e| e.to_string()).collect())) } @@ -316,7 +395,7 @@ impl H5 { pub fn links(&self, h: &ObjectHeader) -> Result> { let os = self.os(); let ls = self.ls(); - let data = self.data(); + let data = self.store(); let mut out = Vec::new(); if let Some(m) = h .messages @@ -325,7 +404,7 @@ impl H5 { { let stm = SymbolTableMessage::parse(&m.data, os) .map_err(|e| Error::new(format!("symbol table message: {e}")))?; - let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls) + let entries = group_v1::resolve_v1_group_entries_in(data, &stm, os, ls) .map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?; let has_soft = entries.iter().any(group_v1::is_v1_soft_link); for e in entries { @@ -337,7 +416,7 @@ impl H5 { } } if has_soft { - let soft = group_v1::v1_soft_links(data, &stm, os, ls) + let soft = group_v1::v1_soft_links_in(data, &stm, os, ls) .map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?; for (name, target) in soft { out.push(Link { @@ -387,15 +466,15 @@ impl H5 { name_type: u8, ) -> Result>> { self.verified_heap(heap)?; - let data = self.data(); + let data = self.store(); let os = self.os(); let ls = self.ls(); - let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls) + let fh = FractalHeapHeader::parse_in(data, to_usize(heap).map(|_| heap)?, os, ls) .map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?; let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?; - let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls) + let hdr = BTreeV2Header::parse_in(data, to_usize(bt).map(|_| bt)?, os, ls) .map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?; - let recs = collect_btree_v2_records(data, &hdr, os, ls) + let recs = collect_btree_v2_records_in(data, &hdr, os, ls) .map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?; // Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID. let skip = if hdr.tree_type == name_type { 4 } else { 8 }; @@ -407,7 +486,7 @@ impl H5 { .get(skip..skip + idlen) .ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?; let obj = fh - .read_managed_object(data, id, os) + .read_managed_object_in(data, id, os) .map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?; out.push(obj); } @@ -561,7 +640,7 @@ impl H5 { if p.is_empty() { return Ok(self.root()); } - clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p) + clawhdf5_format::group_v2::resolve_path_any_in(self.store(), self.sb(), p) .map_err(|e| Error::new(format!("{path}: {e}"))) } } @@ -681,7 +760,9 @@ pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result { /// Split a `FILE[/object/path]` argument the way h5ls does: the longest /// prefix that is an existing file is the file. pub fn split_file_arg(arg: &str) -> (String, Option) { - if Path::new(arg).is_file() { + // A URL names the file only (its path cannot be split against the + // local file system). + if is_url(arg) || Path::new(arg).is_file() { return (arg.to_string(), None); } let mut idx: Vec = arg.match_indices('/').map(|(i, _)| i).collect(); @@ -694,3 +775,13 @@ pub fn split_file_arg(arg: &str) -> (String, Option) { } (arg.to_string(), None) } + +/// Whether a FILE argument is a URL (`scheme://...`) rather than a path. +pub fn is_url(arg: &str) -> bool { + arg.split_once("://").is_some_and(|(scheme, _)| { + !scheme.is_empty() + && scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + }) +} diff --git a/crates/clawhdf5-tools/src/heap_blocks.rs b/crates/clawhdf5-tools/src/heap_blocks.rs index a1c40cd..59cc6d3 100644 --- a/crates/clawhdf5-tools/src/heap_blocks.rs +++ b/crates/clawhdf5-tools/src/heap_blocks.rs @@ -4,10 +4,12 @@ //! heap header's flag says so). The library reads only the blocks an object //! lives in and does not verify block checksums, so `check` does it here. +use std::borrow::Cow; use std::collections::HashSet; use clawhdf5_format::checksum::jenkins_lookup3; use clawhdf5_format::fractal_heap::FractalHeapHeader; +use clawhdf5_format::storage::{Storage, read_exact_at}; use crate::h5::{Error, H5}; @@ -26,7 +28,7 @@ pub struct HeapReport { } struct Walk<'a> { - data: &'a [u8], + data: &'a dyn Storage, heap: u64, fh: FractalHeapHeader, checksum_dblocks: bool, @@ -60,14 +62,14 @@ fn log2(v: u64) -> u32 { /// parsed (and its checksum verified) by the library; an error there is /// returned as the only problem. pub fn verify(h5: &H5, heap: u64) -> HeapReport { - let data = h5.data(); + let data = h5.store(); let Ok(off) = usize::try_from(heap) else { return HeapReport { problems: vec![Error::at(heap, "fractal heap address out of range")], ..Default::default() }; }; - let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) { + let fh = match FractalHeapHeader::parse_in(data, heap, h5.os(), h5.ls()) { Ok(f) => f, Err(e) => { return HeapReport { @@ -77,7 +79,15 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport { } }; // Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1). - let flags = data.get(off + 9).copied().unwrap_or(0); + let flags = match data.read_at(off as u64 + 9, 1) { + Ok(b) => b.first().copied().unwrap_or(0), + Err(e) => { + return HeapReport { + problems: vec![Error::at(heap, format!("fractal heap header: {e}"))], + ..Default::default() + }; + } + }; let mut w = Walk { data, heap, @@ -107,11 +117,42 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport { w.r } -impl Walk<'_> { +impl<'a> Walk<'a> { fn problem(&mut self, addr: u64, msg: impl Into) { self.r.problems.push(Error::at(addr, msg)); } + /// Bytes `[start, end)` of the file: `Ok(None)` when they run past its + /// end (what a slice `get` of the whole file answered), `Err` when the + /// storage fails to read them (a remote file). + fn get(&self, start: usize, end: usize) -> Result>, String> { + let Some(len) = end.checked_sub(start) else { + return Ok(None); + }; + if end as u64 > self.data.len() { + return Ok(None); + } + read_exact_at(self.data, start as u64, len) + .map(Some) + .map_err(|e| e.to_string()) + } + + /// [`Walk::get`], recording a read failure as a problem at `addr`. + fn get_or_note( + &mut self, + addr: u64, + start: usize, + end: usize, + ) -> Option>> { + match self.get(start, end) { + Ok(b) => Some(b), + Err(e) => { + self.problem(addr, format!("fractal heap block: {e}")); + None + } + } + } + fn row_size(&self, row: usize) -> Option { let s = self.fh.starting_block_size; if row <= 1 { @@ -155,10 +196,11 @@ impl Walk<'_> { return None; }; let hdr_len = 5 + self.os + self.boff_bytes; - let Some(b) = start - .checked_add(hdr_len) - .and_then(|e| self.data.get(start..e)) - else { + let b = match start.checked_add(hdr_len) { + Some(e) => self.get_or_note(addr, start, e)?, + None => None, + }; + let Some(b) = b else { self.problem( addr, format!("fractal heap {what} block lies past the end of the file"), @@ -209,7 +251,10 @@ impl Walk<'_> { self.problem(addr, "fractal heap direct block size out of range"); return; }; - let Some(block) = self.data.get(start..end) else { + let Some(block) = self.get_or_note(addr, start, end) else { + return; + }; + let Some(block) = block else { self.problem( addr, "fractal heap direct block extends past the end of the file", @@ -259,14 +304,17 @@ impl Walk<'_> { }; let direct = row < direct_rows; for _ in 0..width { - let Some(b) = self.data.get(pos..pos + self.os) else { + let Some(b) = self.get_or_note(addr, pos, pos + self.os) else { + return; + }; + let Some(b) = b else { self.problem( addr, "fractal heap indirect block extends past the end of the file", ); return; }; - let child = le(b); + let child = le(&b); pos += self.os; if direct && filtered { pos += self.ls + 4; @@ -277,7 +325,10 @@ impl Walk<'_> { off = off.saturating_add(rs); } } - let Some(stored) = self.data.get(pos..pos + 4) else { + let Some(stored) = self.get_or_note(addr, pos, pos + 4) else { + return; + }; + let Some(stored) = stored else { self.problem( addr, "fractal heap indirect block extends past the end of the file", @@ -285,7 +336,10 @@ impl Walk<'_> { return; }; let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]); - let computed = jenkins_lookup3(&self.data[start..pos]); + let Some(Some(body)) = self.get_or_note(addr, start, pos) else { + return; + }; + let computed = jenkins_lookup3(&body); self.r.checksums += 1; if computed != stored { self.problem( diff --git a/crates/clawhdf5-tools/src/info.rs b/crates/clawhdf5-tools/src/info.rs index 606457b..d169f73 100644 --- a/crates/clawhdf5-tools/src/info.rs +++ b/crates/clawhdf5-tools/src/info.rs @@ -1,7 +1,7 @@ //! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text, //! layout, filters and storage. -use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; +use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks_in}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; @@ -189,8 +189,8 @@ pub fn chunks( let Some(addr) = *btree_address else { return Ok(Vec::new()); }; - list_chunks( - h5.data(), + list_chunks_in( + h5.store(), layout, ds, dt.type_size() as usize, diff --git a/crates/clawhdf5-tools/src/ls.rs b/crates/clawhdf5-tools/src/ls.rs index ef966f6..203cce3 100644 --- a/crates/clawhdf5-tools/src/ls.rs +++ b/crates/clawhdf5-tools/src/ls.rs @@ -63,7 +63,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { return args.usage_error(out, "missing FILE", USAGE); }; let (file, obj) = split_file_arg(&target); - let mut h5 = match H5::open(std::path::Path::new(&file)) { + let mut h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs ls: {e}")?; diff --git a/crates/clawhdf5-tools/src/stat.rs b/crates/clawhdf5-tools/src/stat.rs index c7e44da..32e91fe 100644 --- a/crates/clawhdf5-tools/src/stat.rs +++ b/crates/clawhdf5-tools/src/stat.rs @@ -86,7 +86,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let Some(file) = file else { return args.usage_error(out, "missing FILE", USAGE); }; - let h5 = match H5::open(std::path::Path::new(&file)) { + let h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs stat: {e}")?; @@ -291,7 +291,7 @@ fn report(h5: &H5, file: &str, s: &Stats, out: &mut Out) -> std::io::Result<()> s.attr_objects )?; writeln!(o, "\tMax. # of attributes to objects: {}", s.max_attrs)?; - let total = std::fs::metadata(&h5.path).map(|m| m.len()).unwrap_or(0); + let total = h5.size; let ub = h5.file.user_block_size(); writeln!(o, "Summary of file space information:")?; writeln!(o, " User block: {ub} bytes")?; diff --git a/crates/clawhdf5-tools/src/value.rs b/crates/clawhdf5-tools/src/value.rs index d9255f1..c4f4af6 100644 --- a/crates/clawhdf5-tools/src/value.rs +++ b/crates/clawhdf5-tools/src/value.rs @@ -151,14 +151,14 @@ pub struct Decoder<'a> { pub h5: &'a H5, /// Variable-length elements are resolved as the library resolves them /// (so as libhdf5 does), not by a decoder of our own. - vl: RefCell>, + vl: RefCell>, } impl<'a> Decoder<'a> { pub fn new(h5: &'a H5) -> Self { Self { h5, - vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())), + vl: RefCell::new(VlResolver::new_in(h5.store(), h5.os(), h5.ls())), } } @@ -268,7 +268,7 @@ impl<'a> Decoder<'a> { /// has it. fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value { if is_string { - return match self.vl.borrow_mut().string_element(b) { + return match self.vl.borrow_mut().string_element_in(b) { Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()), Ok(None) => Value::NullStr, Err(e) => Value::Error(e.to_string()), @@ -278,8 +278,9 @@ impl<'a> Decoder<'a> { if bs == 0 { return Value::Error("VL base type of size 0".into()); } - let obj = match self.vl.borrow_mut().element(b, bs) { - Ok(o) => o.unwrap_or(&[]), + // Copied out: decoding an element may resolve nested ones. + let obj = match self.vl.borrow_mut().element_in(b, bs) { + Ok(o) => o.unwrap_or(&[]).to_vec(), Err(e) => return Value::Error(e.to_string()), }; Value::Seq( diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs new file mode 100644 index 0000000..d14bd61 --- /dev/null +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -0,0 +1,100 @@ +//! `h5rs` on URLs (feature `remote`): every subcommand prints for +//! `http://…/file.h5` what it prints for the local file (the name aside). +//! The files are served by the range-request test server of +//! clawhdf5-remote on 127.0.0.1. + +#![cfg(feature = "remote")] + +#[path = "../../clawhdf5-remote/tests/common/server.rs"] +mod server; + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn h5rs(args: &[&str]) -> (String, i32) { + let out = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(args) + .output() + .expect("run h5rs"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (text, out.status.code().unwrap_or(-1)) +} + +fn fixtures() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + [ + "../clawhdf5/tests/fixtures/tall.h5", + "../clawhdf5/tests/fixtures/written_by_v2_7_0.h5", + "../clawhdf5/tests/fixtures/written_by_v2_7_0_paged.h5", + "../clawhdf5/tests/fixtures/h5clear_mdc_image.h5", + "../clawhdf5-format/tests/fixtures/fractal_heap_multiblock.h5", + "../clawhdf5-format/tests/fixtures/legacy/tcompound.h5", + "../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5", + ] + .iter() + .map(|p| root.join(p)) + .collect() +} + +#[test] +fn every_subcommand_reads_a_url_like_the_local_file() { + let files = fixtures(); + let served: Vec<(String, Vec)> = files + .iter() + .enumerate() + .map(|(i, p)| { + let name = p.file_name().unwrap().to_str().unwrap(); + (format!("/{i}/{name}"), std::fs::read(p).unwrap()) + }) + .collect(); + let server = server::Server::start(served.clone()); + for (p, (url_path, _)) in files.iter().zip(&served) { + let url = server.url(url_path); + let local = p.to_str().unwrap(); + for cmd in [ + &["ls", "-r", "-v"][..], + &["dump"], + &["dump", "--json"], + &["stat"], + &["check", "--data"], + ] { + fn args<'a>(cmd: &[&'a str], f: &'a str) -> Vec<&'a str> { + cmd.iter().copied().chain([f]).collect() + } + let (want, want_rc) = h5rs(&args(cmd, local)); + let (got, got_rc) = h5rs(&args(cmd, &url)); + assert_eq!( + got.replace(&url, local), + want, + "h5rs {} {url}", + cmd.join(" ") + ); + assert_eq!(got_rc, want_rc, "h5rs {} {url}", cmd.join(" ")); + } + let (a, rc) = h5rs(&["diff", local, &url]); + assert_eq!(rc, 0, "h5rs diff {local} {url}: {a}"); + } +} + +#[test] +fn url_errors_are_clean() { + let server = server::Server::start(vec![("/x.h5".into(), vec![1u8; 100])]); + let (out, rc) = h5rs(&["ls", &server.url("/missing.h5")]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("404"), "{out}"); + let (out, rc) = h5rs(&["ls", &server.url("/x.h5")]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("not an HDF5 file"), "{out}"); + let (out, rc) = h5rs(&["check", &server.url("/missing.h5")]); + assert_eq!(rc, 2, "{out}"); + #[cfg(not(feature = "remote-https"))] + { + let (out, rc) = h5rs(&["ls", "https://example.com/a.h5"]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("`https` feature"), "{out}"); + } +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index fe7242c..4f678ad 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -117,21 +117,28 @@ run_step "cargo clippy (remote, all backends)" cargo clippy \ --features object-store,https,s3,gcs,azure \ -- -D warnings +# h5rs with URL arguments. +run_step "cargo clippy (h5rs remote)" cargo clippy \ + -p clawhdf5-tools \ + --all-targets \ + --features remote-https \ + -- -D warnings + # The README promises that the core crates build no C by default. Hold it to # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. # clawhdf5-remote is checked by default (plain HTTP) and with its -# object-store feature; its https (ring) and s3/gcs/azure (aws-lc-rs) -# features build C and are opt-in. +# object-store feature, and h5rs with URL support (remote); the https +# (ring) and s3/gcs/azure (aws-lc-rs) features build C and are opt-in. no_c_in_default_build() { local entry crate features found=0 for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ clawhdf5-tools \ clawhdf5-wasm \ - clawhdf5-remote clawhdf5-remote:object-store; do + clawhdf5-remote clawhdf5-remote:object-store clawhdf5-tools:remote; do crate=${entry%%:*} features=() [ "$entry" != "$crate" ] && features=(--features "${entry#*:}") @@ -210,6 +217,11 @@ run_step "cargo test (remote, object_store backend, s3 URLs)" cargo test \ -p clawhdf5-remote \ --features object-store,s3 +run_step "cargo test (h5rs on URLs)" cargo test \ + -p clawhdf5-tools \ + --features remote \ + --test remote + run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ --features parallel -- 2.54.0 From 4e8109770d41ac22f6f8cc19c7d9bc1d5571d350 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:25:28 -0500 Subject: [PATCH 19/49] tests: shrink and regrow datasets allocated early Datasets with early allocation and unlimited dimensions (Extensible Array, version-2 B-tree, version-1 B-tree under earliest), unfiltered and deflated: the random resize workload gives the values h5py gets and the same chunk index shape, with every chunk a growth brings in allocated and filled as H5D__chunk_allocate does. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index ffe13d9..7fc309a 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -717,6 +717,42 @@ fn shrink_matches_libhdf5() { false, 99, ); + // Early allocation with unlimited dimensions (Extensible Array, + // version-2 B-tree; version-1 B-tree under `earliest`), filtered and + // not: growth allocates and fills every new chunk, as libhdf5 does. + for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] + .iter() + .enumerate() + { + for (j, (max, z)) in [ + ("(None, 9)", ""), + ("(None, None)", ""), + ("(None, 9)", ", compression='gzip'"), + ("(None, None)", ", compression='gzip'"), + ] + .iter() + .enumerate() + { + shrink_workload( + &format!("early_{i}_{j}"), + lv, + &format!( + "\x20 f.create_dataset('x', data=np.arange(72, dtype=' Date: Sat, 26 Sep 2026 17:26:57 -0500 Subject: [PATCH 20/49] clawhdf5-remote tests: open + list and a dataset read counted apart The per-file report now separates a tree view (open, every group's entries, every dataset's shape and type) from reading the largest dataset under 64 MiB, and checks the budget the design's testing section asks for: listing the IMERG file (file A of docs/design/range-reads.md section 2) takes at most 3 requests when CLAWHDF5_REMOTE_CORPUS includes it. The test server now counts a response's bytes before sending it: a client could read a body and reset the counters before the server thread had added it, so the counts of the next file were occasionally too high. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/tests/common/mod.rs | 44 +++++++++----- crates/clawhdf5-remote/tests/common/server.rs | 4 +- crates/clawhdf5-remote/tests/http.rs | 58 ++++++++++++------- 3 files changed, 69 insertions(+), 37 deletions(-) diff --git a/crates/clawhdf5-remote/tests/common/mod.rs b/crates/clawhdf5-remote/tests/common/mod.rs index 498b603..9aeb1bc 100644 --- a/crates/clawhdf5-remote/tests/common/mod.rs +++ b/crates/clawhdf5-remote/tests/common/mod.rs @@ -147,11 +147,13 @@ fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { } } -/// Open, list every group, and read the first dataset found whose data is -/// at most `MAX_DATA_BYTES` (a tree view plus one plot). -pub fn list_and_read_one(file: &File) { +/// List the file as a tree view does — every group's entries, every +/// dataset's shape and type — and return the largest dataset whose data +/// is at most `MAX_DATA_BYTES` (address, bytes), the one a viewer would +/// plot. +pub fn list(file: &File) -> Option<(u64, u64)> { let mut seen = HashSet::new(); - let mut read_one = false; + let mut largest: Option<(u64, u64)> = None; let mut queue = VecDeque::from([file.superblock().root_group_address]); while let Some(addr) = queue.pop_front() { if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { @@ -160,22 +162,36 @@ pub fn list_and_read_one(file: &File) { let group = file.group_at(addr); if let Ok(ds) = file.dataset_at(addr) { let _ = (ds.shape(), ds.dtype()); - if !read_one { - let small = ds.shape().ok().and_then(|s| { - let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; - let size = u64::from(ds.raw_datatype().ok()?.type_size()); - n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) - }); - if small.is_some() { - let _ = ds.read_selection(&Selection::All); - read_one = true; - } + let bytes = ds.shape().ok().and_then(|s| { + let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?; + let size = u64::from(ds.raw_datatype().ok()?.type_size()); + n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES) + }); + if let Some(b) = bytes + && largest.is_none_or(|(_, l)| b > l) + { + largest = Some((addr, b)); } } if let Ok(entries) = group.entries() { queue.extend(entries.into_iter().map(|(_, a)| a)); } } + largest +} + +/// Read the dataset at `addr` whole. +pub fn read_one(file: &File, addr: u64) { + if let Ok(ds) = file.dataset_at(addr) { + let _ = ds.read_selection(&Selection::All); + } +} + +/// [`list`], then [`read_one`] of the dataset it picks. +pub fn list_and_read_one(file: &File) { + if let Some((addr, _)) = list(file) { + read_one(file, addr); + } } /// External virtual-dataset sources read from `dir`, as `File::open` finds diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index e556d7f..4749f50 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -320,9 +320,11 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { }; let mut response = head.into_bytes(); response.extend_from_slice(body); + // Counted before the client can have the bytes, so a test that + // resets the counters after a read never sees them arrive late. + s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); out.write_all(&response)?; out.flush()?; - s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); if truncate || close { let _ = out.shutdown(std::net::Shutdown::Both); return Ok(()); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 197e268..b0a0e34 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -53,7 +53,7 @@ fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) { .collect(); let server = Server::start(served.clone()); let (mut same, mut refused) = (0, 0); - let mut totals = [0u64; 5]; + let mut totals = [0u64; 7]; for (i, p) in files.iter().enumerate() { let Some((url_path, bytes)) = served .iter() @@ -104,45 +104,59 @@ fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) { assert!(storage.stats().reads > 0, "read through the cache"); same += 1; - // Cost of "list + read one dataset": with the block cache, and - // with none (every read a request). - let with = storage_for_url(&url, &quick()).unwrap(); + // Cost of "open + list" (a tree view) and of then reading the + // largest dataset (a plot): with the block cache, and with none + // (every read a request). Requests and bytes as the server saw + // them, the open included. server.reset(); - if let Ok(f) = File::open_storage(with.clone()) { - list_and_read_one(&f); + let with = storage_for_url(&url, &quick()).unwrap(); + let f = File::open_storage(with.clone()).unwrap(); + let pick = common::list(&f); + let (list_requests, list_bytes) = (server.requests(), server.bytes()); + if let Some((addr, _)) = pick { + common::read_one(&f, addr); } let (cached_requests, cached_bytes) = (server.requests(), server.bytes()); - let (bare, _) = HttpStorage::open(&url, quick().http).unwrap(); - let bare = Arc::new(bare); server.reset(); - if let Ok(f) = File::open_storage(bare.clone()) { + let (bare, _) = HttpStorage::open(&url, quick().http).unwrap(); + if let Ok(f) = File::open_storage(Arc::new(bare)) { list_and_read_one(&f); } let uncached_requests = server.requests(); - totals[0] += 1 + cached_requests; // + the open probe - totals[1] += cached_bytes + with.config().block_size.min(bytes.len() as u64); - totals[2] += 1 + uncached_requests; - totals[3] += bytes.len() as u64; - totals[4] += 1; + for (t, v) in totals.iter_mut().zip([ + list_requests, + list_bytes, + cached_requests, + cached_bytes, + uncached_requests, + bytes.len() as u64, + 1, + ]) { + *t += v; + } if report { eprintln!( - "requests cached {:>6} uncached {:>8} bytes {:>12} of {:>12} {}", - 1 + cached_requests, - 1 + uncached_requests, - cached_bytes + with.config().block_size.min(bytes.len() as u64), + "open+list {list_requests:>4} req {list_bytes:>10} B | +read {:>5} req \ + {cached_bytes:>10} B | uncached {uncached_requests:>7} req | file {:>10} B {}", + cached_requests, bytes.len(), p.display() ); } // Files within one block: opening fetched everything. if bytes.len() as u64 <= with.config().block_size { - assert_eq!(cached_requests, 0, "{}", p.display()); + assert_eq!(cached_requests, 1, "{}", p.display()); + } + // The budget of docs/design/range-reads.md (Testing): listing the + // IMERG file (file A of section 2) takes at most 3 requests. + if p.ends_with("xarray-data/imerghh_730.hdf5") { + assert!(list_requests <= 3, "{}: {list_requests}", p.display()); } } eprintln!( - "list + read one dataset over {} files: {} requests with the 1 MiB block cache \ - ({} bytes transferred, files {} bytes), {} requests without a cache", - totals[4], totals[0], totals[1], totals[3], totals[2] + "{} files ({} bytes): open + list {} requests, {} bytes; + read the largest \ + dataset {} requests, {} bytes (1 MiB block cache); without a cache {} requests", + totals[6], totals[5], totals[0], totals[1], totals[2], totals[3], totals[4] ); (same, refused) } -- 2.54.0 From 955dd1c691fa127237358bb8fba50119f9fcad66 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:29:09 -0500 Subject: [PATCH 21/49] =?UTF-8?q?docs:=20range-read=20milestone=20M3=20?= =?UTF-8?q?=E2=80=94=20remote=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README: "Reading remote files" (open_url, the range_server and read_url examples with their real output against the fixtures, h5rs on URLs), the crate in the crate map and the unreleased highlights. CHANGELOG: the clawhdf5-remote crate, h5rs URLs, File::storage and VlResolver::element_in, with the request counts over the conformance corpus (tank, 2026-09-26, the command given). known-issues: the M2 range-read entry updated (the cache now exists; h5rs reads through storage) and a new entry for the remote backends' limits (no Python or browser URLs yet, fixed block size, cloud stores not run against a real bucket, validators, credentials). Design doc: M3 status with the choices that differ from the plan (a crate rather than a clawhdf5-io feature, ureq for HTTP so the default build has no C) and the corpus counts. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 56 +++++++++++++++++++++++++++++++ README.md | 56 +++++++++++++++++++++++++++++-- docs/design/range-reads.md | 43 ++++++++++++++++++++++-- docs/known-issues.md | 68 +++++++++++++++++++++++++++++--------- 4 files changed, 203 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 661a2eb..69941c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,62 @@ ## Unreleased +### Range reads, milestone M3: remote files (2026-09-26) +- **New crate `clawhdf5-remote`.** `open_url("http://host/file.h5")` gives + a `clawhdf5::File` (through `File::open_storage`) that reads the file by + HTTP `Range` requests; `storage_for_url` returns the cached storage to + read its statistics. Features: `http` (default; `ureq` without TLS, no + C), `https` (rustls with ring, which compiles C), `object-store` (any + `object_store` store, pure Rust), `s3`/`gcs`/`azure` (`s3://`, `gs://`, + `az://` URLs, configured from the environment; object_store's cloud + clients use aws-lc-rs, C). + - **`BlockCache`** wraps any `Storage`: aligned 1 MiB blocks (the size + `docs/design/range-reads.md` §2 measured), LRU with a byte budget + (64 MiB), the blocks one read misses fetched with one backend + `read_ranges` call as runs of consecutive blocks (a one-block gap is + fetched to merge two runs; at most 8 MiB per request). Readers on + several threads share it without holding its lock across a request, and + a block being fetched is waited for, not fetched again. A read that + misses more than half the budget is not kept (a large dataset does not + evict the metadata). A failed fetch is an error for every reader waiting + on it and is not cached. + - **`HttpStorage`**: opening is one ranged `GET` of the first block, + whose `Content-Range` gives the length. The file is pinned by its strong + `ETag` (`If-Match`) or else `Last-Modified` (`If-Unmodified-Since`), and + its length, checked on every response: a file changed while open is + `RemoteError::FileChanged`, never mixed data. A server that ignores + `Range` is refused without reading the body unless + `HttpOptions::allow_full_download`. Connection failures, timeouts, + `408`/`429`/`5xx` and cut-off bodies are retried with exponential + backoff; bodies are asked for with `Accept-Encoding: identity` and an + encoded one is refused. The ranges of one call are fetched in parallel. + - **`ObjectStoreStorage`** reads one object of any `object_store` store, + pinned by ETag (else version or modification time) and size. It blocks + on a small tokio runtime it owns; called from inside another runtime it + refuses (`RemoteError::Usage`) instead of blocking a worker. + `open_object(store, path, options)` opens a file through a block cache. + - Counted on the conformance corpus (tank, 2026-09-26, + `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus CLAWHDF5_REMOTE_REPORT=1 + cargo test --release -p clawhdf5-remote --test http -- --nocapture corpus`): + the 621 files that open (254 MB) read over HTTP exactly as through + `File::open`; opening and listing them all (every group's entries, every + dataset's shape and type) took 640 requests and 55.5 MB, and then + reading each file's largest dataset under 64 MiB 96 more requests + (171 MB in all). Without the cache the same work is 141 936 requests. + Listing the 7.7 MB IMERG file (file A of the design's §2) takes 2 + requests; the tests hold it to at most 3. +- **`h5rs` takes URLs** with the new `remote` feature (`remote-https` for + `https://`): `ls`, `dump`, `stat` and `diff` read by range requests; + `check` downloads the file whole, since it validates every byte. The + tools now read through `File::storage` and the format crate's `*_in` + functions; local output is unchanged. +- **`File::storage()`** (facade) returns the file's bytes from the + superblock on (the `as_bytes` view, cache image laid over) as a + `&(dyn Storage + Send + Sync)` for every backend, so code that parses a + file itself works on remote files too. +- **`VlResolver::element_in` / `string_element_in`** (format): the + `element`/`string_element` lookups over any `Storage`. + ### Range reads, milestone M2: raw data and `File::open_storage` (2026-09-26) - **`clawhdf5::File::open_storage(Arc)`** opens a file served by any `clawhdf5_format::storage::Storage` and gives the diff --git a/README.md b/README.md index 88ad61b..cfa82d4 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,12 @@ breaking change, are in [CHANGELOG.md](CHANGELOG.md). filtered top-k, never slower than unfiltered), and opt-in re-ranking and confidence rejection, which used to be reachable only through `ClawhdfBackend`. +**Remote files (unreleased)** +- New crate `clawhdf5-remote`: `open_url("http://…")` reads a file on an + HTTP server (or in S3/GCS/Azure, opt-in) by range requests through a + block cache, without downloading it; `h5rs` takes URLs with its `remote` + feature. See [Reading remote files](#reading-remote-files). + **Tooling** - CI now runs the h5py/netCDF4 interop suites for real (they had been skipping silently) and runs an aarch64 job for the NEON kernels. @@ -450,6 +456,50 @@ Each call changes the file in place (no rewrite) and syncs it. What it cannot change safely is refused before anything is written; see [known issues](docs/known-issues.md) for the limits. +### Reading remote files + +[`clawhdf5-remote`](crates/clawhdf5-remote/README.md) opens a file on an +HTTP server (or, with its `s3`/`gcs`/`azure` features, in an object store) +without downloading it: the read API is the same `clawhdf5::File`, and +only the bytes an operation needs are fetched, by `Range` requests through +a block cache (1 MiB blocks; opening fetches the first one). A file that +changes on the server while it is open is an error, never a mix of old and +new bytes. + +```rust +let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/tall.h5")?; +let values = file.dataset("/g2/dset2.1")?.read_f64()?; +``` + +To try it without a server of your own, the crate's test server serves a +directory with range support: + +```bash +cargo run -p clawhdf5-remote --example range_server -- crates/clawhdf5/tests/fixtures 127.0.0.1:8000 +# in another shell: list the file, read one dataset, print what it cost +cargo run -p clawhdf5-remote --example read_url -- http://127.0.0.1:8000/tall.h5 /g2/dset2.1 +``` + +```text +/g1 group +/g2 group +/g2/dset2.1 dataset [10] F32 +/g2/dset2.2 dataset [3, 5] F32 +/g1/g1.1 group +/g1/g1.2 group +/g1/g1.2/g1.2.1 group +/g1/g1.1/dset1.1.1 dataset [10, 10] I32 +/g1/g1.1/dset1.1.2 dataset [20] I32 +/g2/dset2.1: 10 values, first [1.0, 1.100000023841858, 1.2000000476837158, ...] +1 range requests (the one at open included), 9968 bytes fetched, 9968 bytes cached +``` + +(`tall.h5` is 9 968 bytes, so the first block holds all of it.) `h5rs` +built with `--features remote` takes the same URLs: +`h5rs ls -r http://127.0.0.1:8000/tall.h5`. Plain HTTP builds no C; +`https://` is the `https` feature (rustls with ring, which compiles C). +Limits are in [known issues](docs/known-issues.md). + ### Python `crates/clawhdf5-py` is a Python package (PyO3 + numpy) that reads HDF5 with @@ -678,7 +728,7 @@ let exported = backend.export_markdown("MEMORY.md")?; ## Crate Map ``` -clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests +clawhdf5 workspace (19 crates, ~86K lines of Rust in src/, ~104K with tests and benches; plus libaec-sys, an internal FFI bindings crate for the optional szip feature) │ @@ -690,7 +740,8 @@ clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests │ ├── clawhdf5 — High-level API │ ├── clawhdf5-netcdf4 — NetCDF-4 support │ ├── clawhdf5-accel — SIMD (AVX2, NEON incl. SDOT int8; AVX-512 behind `avx512`) -│ └── clawhdf5-gpu — GPU compute (wgpu, hand-written WGSL compute shaders) +│ ├── clawhdf5-gpu — GPU compute (wgpu, hand-written WGSL compute shaders) +│ └── clawhdf5-remote — Remote files: HTTP(S) range requests, object stores, block cache │ ├── Agent Memory │ ├── clawhdf5-agent — Memory engine (24.7K lines, 32 modules; chained-CRC WAL) @@ -705,6 +756,7 @@ clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests │ └── clawhdf5-wasm — Browser (WebAssembly, wasm-bindgen; read-only) │ └── Tooling + ├── clawhdf5-tools — h5rs: ls, dump, stat, diff, check └── clawhdf5-bench — Benchmark suite ``` diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1098922..2c7d1a4 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -5,8 +5,10 @@ change. Progress: M0 and M1 are done, and so is M2 (branch `feat/p3-m2-raw-data`): every read path of the format crate works through `Storage`, v2 B-trees, dense groups and raw data included, and `File::open_storage` gives the facade's read API over any `Storage` (see -`CHANGELOG.md`, "Range reads, milestone M2"). M3 (a remote backend with -its block cache) is next. Every count in §1–§2 was +`CHANGELOG.md`, "Range reads, milestone M2"). M3 is done on branch +`feat/p3-m3-remote`: the `clawhdf5-remote` crate (block cache, HTTP(S), +object stores) and URLs in `h5rs` (see the M3 status below). M4 (wasm) is +next. Every count in §1–§2 was taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given next to it. No timing numbers appear here on purpose: the machine was shared with other build jobs when this was written. @@ -458,6 +460,43 @@ fast path within benchmark noise. §2; the page size for paged files; the first block prefetched on open) and a request counter exposed for tests and users. - Python bindings: `clawhdf5.File("s3://…")` / `https://` through it. +- *Status 2026-09-26:* done on branch `feat/p3-m3-remote`, except the + Python bindings, with these choices: + - A new crate, `clawhdf5-remote`, instead of a `remote` feature of + `clawhdf5-io`: `open_url` returns a `clawhdf5::File`, and `clawhdf5-io` + sits below the facade. + - HTTP(S) through `ureq` (`HttpStorage`), not object_store's HTTP store: + that one pulls reqwest with aws-lc-rs (C), while plain HTTP through + ureq builds no C, so it is the default feature; `https` adds rustls + with ring. S3/GCS/Azure go through `object_store` (`ObjectStoreStorage`, + features `s3`/`gcs`/`azure`, opt-in because of aws-lc-rs); the + `object-store` feature alone (in-memory, local files, a store you + build) is pure Rust. object_store is async: the storage blocks on a + two-thread tokio runtime of its own, and refuses to run inside another + runtime. + - `BlockCache` (any `Storage`): 1 MiB blocks and a 64 MiB LRU budget by + default, the first block fetched at open (for HTTP by the request + that learns the length), the missing blocks of one read fetched as + runs of consecutive blocks in one parallel batch, per-block in-flight + deduplication across threads, and reads that miss more than half the + budget not kept. The page size of paged files is not used as the block + size yet. `CacheStats` and `HttpStats` count requests and bytes. + - The file is pinned at open by ETag (else Last-Modified, or the object's + version) and length; a change is an error, not mixed data. + - `h5rs` (feature `remote`) reads through the new `File::storage()`, the + file's view as a `Storage`, so its parsing works on remote files; its + `check` downloads the file whole. + - Measured with `crates/clawhdf5-remote/tests/http.rs` (tank, 2026-09-26, + `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus + CLAWHDF5_REMOTE_REPORT=1 cargo test --release -p clawhdf5-remote --test + http -- --nocapture corpus`), requests as the test server counted + them: the 621 corpus files that open read over HTTP exactly as through + `File::open`. Open + list (every group's entries, every dataset's shape + and type) of all of them: 640 requests, 55.5 MB of 254 MB; then reading + each file's largest dataset under 64 MiB: 96 more (171 MB in all). The + same work without a cache: 141 936 requests. Per file: A lists in 2 + requests (§2 predicted 2 blocks of 1 MiB), B in 1, C in 7 (its whole + 6.4 MB: 35 001 object headers spread over the file). **M4 — wasm lazy loading (1–2 weeks).** - `clawhdf5-wasm`: `openUrl(url) -> Promise` backed by `fetch` with a diff --git a/docs/known-issues.md b/docs/known-issues.md index 313bc8e..860e1eb 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -766,20 +766,22 @@ which is what libhdf5 itself writes. ## Range reads (`File::open_storage`) limits **Status:** open (added 2026-09-26, milestone M2 of -`docs/design/range-reads.md`). `File::open_storage` reads any -`clawhdf5_format::storage::Storage` through the whole read API, and every -format-crate read path works through `Storage::read_at`/`read_ranges`, but: +`docs/design/range-reads.md`; remote backends added by M3). `File::open_storage` +reads any `clawhdf5_format::storage::Storage` through the whole read API, +every format-crate read path works through `Storage::read_at`/`read_ranges`, +and `clawhdf5-remote` serves HTTP(S) and object-store files through a block +cache, but: -- **No remote backend and no block cache yet** (milestone M3). A `Storage` - is asked for each structure as the parsers need it, several times over - for some (an object header is re-read by each lookup through it): one - pass over the conformance corpus — open, list, every attribute, every - dataset once — is 176 092 `read_at` calls for 621 files, 92 489 of them - for the 35 001-group `h5stat_newgrat.h5` (2026-09-26, tank, - `crates/clawhdf5/tests/storage_equivalence.rs` with - `CLAWHDF5_STORAGE_CORPUS`). A backend over a network needs a cache in - front of it. `Storage::read_ranges` defaults to one `read_at` per range; - coalescing is the backend's job. +- **A `Storage` without a cache is asked for each structure as the parsers + need it**, several times over for some (an object header is re-read by + each lookup through it): one pass over the conformance corpus — open, + list, every attribute, every dataset once — is 176 092 `read_at` calls + for 621 files, 92 489 of them for the 35 001-group `h5stat_newgrat.h5` + (2026-09-26, tank, `crates/clawhdf5/tests/storage_equivalence.rs` with + `CLAWHDF5_STORAGE_CORPUS`). A backend of your own over a network needs a + cache in front of it: wrap it in `clawhdf5_remote::BlockCache`, as + `open_url` does. `Storage::read_ranges` defaults to one `read_at` per + range; coalescing is the backend's (or the cache's) job. - A group lookup by name in a version-1 (symbol-table) group lists the whole group (dense groups use their name index). Over a range backend that is one read per symbol-table node and name, per lookup. @@ -790,10 +792,12 @@ format-crate read path works through `Storage::read_at`/`read_ranges`, but: `read_*_zerocopy`) need the file in memory and answer `FormatError::ContiguousStorageRequired` otherwise; `File::as_bytes()` panics for such a file (`File::contiguous_bytes()` is the fallible form). - `LazyFile`, `MmapFile`, the Python and wasm bindings and `h5rs` still read - a whole file. + `LazyFile`, `MmapFile` and the Python and wasm bindings still read a + whole file (`h5rs` reads through `File::storage`, and takes URLs with its + `remote` feature). - The file's length is read once, at open: a growing file (SWMR) is not - followed (milestone M5). + followed (milestone M5). A remote file is pinned at open, so one that + grows is `RemoteError::FileChanged`. - Not new, but visible through the equivalence tests: a full read through the file's chunk cache (`read_raw_data_cached`, `read_raw_data_indexed`, and so `Dataset::read_*`) lists a damaged dataset's chunks in hash-map @@ -801,6 +805,38 @@ format-crate read path works through `Storage::read_at`/`read_ranges`, but: the next (`cve-2025-2310.h5`); the values of a dataset that reads are not affected. +## Remote files (`clawhdf5-remote`) limits + +**Status:** open (added 2026-09-26, milestone M3 of +`docs/design/range-reads.md`). + +- **Python and the browser cannot open URLs yet.** `clawhdf5.File` (PyO3) + parses through `File::as_bytes`, which a remote file does not have; the + wasm reader's `openUrl` is milestone M4. +- **The block size is fixed** (1 MiB unless `CacheConfig` says otherwise). + The design's policy of using a paged file's page size as the block size + is not implemented, and only the first block is read ahead. +- **Checked against local servers and one public one.** The tests use an + in-process HTTP/1.1 server and object_store's in-memory and local-file + stores. HTTPS was checked by hand against `raw.githubusercontent.com` + (2026-09-26, tank: `h5rs dump` of h5py's `vlen_string_dset.h5` by URL + equals the downloaded file's). The `s3`, `gcs` and `azure` backends are + built and their URL parsing tested, but they have not been run against a + real bucket. +- **A server with neither a strong ETag nor Last-Modified** can only be + checked by length, so a same-length replacement mid-read would go + unnoticed; `HttpOptions::require_validator` refuses such servers. A weak + ETag (`W/"…"`) cannot be sent as `If-Match`, so it counts as none. +- **Credentials:** HTTP takes extra headers (`HttpOptions::headers`, e.g. + `Authorization`); `h5rs` has no option for them. The cloud stores read + credentials from the environment only. +- Each `ObjectStoreStorage` owns a tokio runtime with two worker threads. +- `h5rs check` downloads a remote file whole (it validates every byte), and + a URL cannot carry a `FILE/OBJECT` suffix; `h5rs` uses the default cache + settings. +- The zero-copy methods and `File::as_bytes` are unavailable on a remote + file (see the range-read limits above). + ## `clawhdf5-wasm` (browser) limits **Status:** open (by design for now; added 2026-09-26). -- 2.54.0 From 4f5697fdd9ff99e2d9b4c00648b6361d166a2592 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:31:26 -0500 Subject: [PATCH 22/49] clawhdf5-remote: readers waiting on a failed fetch get its error A reader that waited for another reader's fetch of a block got "the fetch of this block failed" when that fetch failed, not why: a file replaced on the server while open was reported as FileChanged to one thread and as an anonymous failure to the others. The fetch's error is now handed to every reader waiting on it. Regression test: four threads read the same block from a slow backend whose fetches fail with a "changed while open" error; each gets that error (it failed for the waiters before this change). Also fixes the ignore-Range test, broken by the previous commit: the test server now counts a body before sending it, so "the refused body was not read" is checked as "refused at the first response". Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/cache.rs | 73 ++++++++++++++++++++++------ crates/clawhdf5-remote/tests/http.rs | 6 +-- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs index 233938c..a434814 100644 --- a/crates/clawhdf5-remote/src/cache.rs +++ b/crates/clawhdf5-remote/src/cache.rs @@ -98,7 +98,7 @@ type Block = Arc<[u8]>; /// A fetch in progress: the readers waiting for a block wait on this. struct Flight { - result: Mutex>>, + result: Mutex>>, done: Condvar, } @@ -110,7 +110,7 @@ impl Flight { }) } - fn finish(&self, r: Result) { + fn finish(&self, r: Result) { let mut slot = lock(&self.result); if slot.is_none() { *slot = Some(r); @@ -118,7 +118,7 @@ impl Flight { self.done.notify_all(); } - fn wait(&self) -> Result { + fn wait(&self) -> Result { let mut slot = lock(&self.result); loop { if let Some(r) = slot.as_ref() { @@ -173,10 +173,20 @@ pub struct BlockCache { } /// Fails every flight a fetch claimed and did not complete (an error or a -/// panic in the backend), so no reader waits forever. +/// panic in the backend), so no reader waits forever. The waiting readers +/// get the fetch's error. struct FlightGuard<'a, S> { cache: &'a BlockCache, flights: Vec<(u64, Arc)>, + error: Option, +} + +impl FlightGuard<'_, S> { + /// Record `e` for the readers waiting on this fetch; returns it. + fn fail(&mut self, e: FormatError) -> FormatError { + self.error = Some(e.clone()); + e + } } impl Drop for FlightGuard<'_, S> { @@ -191,8 +201,12 @@ impl Drop for FlightGuard<'_, S> { } } drop(st); + let error = self + .error + .take() + .unwrap_or_else(|| FormatError::Storage("the fetch of this block failed".into())); for (_, f) in &self.flights { - f.finish(Err("the fetch of this block failed".into())); + f.finish(Err(error.clone())); } } } @@ -348,6 +362,7 @@ impl BlockCache { let mut guard = FlightGuard { cache: self, flights: Vec::new(), + error: None, }; { let mut st = lock(&self.state); @@ -410,13 +425,16 @@ impl BlockCache { self.counters .requests .fetch_add(runs.len() as u64, Ordering::Relaxed); - let fetched = self.inner.read_ranges(&runs)?; + let fetched = match self.inner.read_ranges(&runs) { + Ok(f) => f, + Err(e) => return Err(guard.fail(e)), + }; if fetched.len() != runs.len() { - return Err(FormatError::Storage(format!( + return Err(guard.fail(FormatError::Storage(format!( "backend returned {} ranges for {} requested", fetched.len(), runs.len() - ))); + )))); } let mut got: Vec<(u64, Block)> = Vec::with_capacity(guard.flights.len()); for (run, bytes) in runs.iter().zip(&fetched) { @@ -424,12 +442,12 @@ impl BlockCache { .bytes_fetched .fetch_add(bytes.len() as u64, Ordering::Relaxed); if bytes.len() as u64 != run.end - run.start { - return Err(FormatError::Storage(format!( + return Err(guard.fail(FormatError::Storage(format!( "short read from the backend: {} of {} bytes at offset {}", bytes.len(), run.end - run.start, run.start - ))); + )))); } let bs = self.config.block_size; let mut i = run.start / bs; @@ -476,12 +494,7 @@ impl BlockCache { } for (i, f) in waits { - match f.wait() { - Ok(data) => { - have.insert(i, data); - } - Err(e) => return Err(FormatError::Storage(e)), - } + have.insert(i, f.wait()?); } Ok(have) } @@ -786,6 +799,34 @@ mod tests { } } + /// A slow backend whose fetches all fail with a telling error. + struct SlowFailing; + + impl Storage for SlowFailing { + fn read_at(&self, _: u64, _: usize) -> Result, FormatError> { + std::thread::sleep(std::time::Duration::from_millis(50)); + Err(FormatError::Storage("the file changed while open".into())) + } + fn len(&self) -> u64 { + 1 << 20 + } + } + + #[test] + fn readers_waiting_on_a_failed_fetch_get_its_error() { + let c = BlockCache::new(SlowFailing, small()); + std::thread::scope(|s| { + let hs: Vec<_> = (0..4) + .map(|_| s.spawn(|| c.read_at(0, 10).map(|b| b.len()))) + .collect(); + for h in hs { + let e = h.join().unwrap().unwrap_err().to_string(); + assert!(e.contains("changed while open"), "{e}"); + } + }); + assert!(c.stats().waits > 0); + } + #[test] fn concurrent_readers_never_fetch_a_block_twice() { let d = data(64 * 1024); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index b0a0e34..5a68072 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -301,11 +301,7 @@ fn a_server_that_ignores_range_is_refused_or_downloaded_when_allowed() { matches!(err, Error::Remote(RemoteError::RangeNotSupported(_))), "{err}" ); - assert!( - server.bytes() < bytes.len() as u64, - "refusing must not download the file ({} bytes read)", - server.bytes() - ); + assert_eq!(server.requests(), 1, "refused at the first response"); let mut opts = quick(); opts.http.allow_full_download = true; let storage = storage_for_url(&url, &opts).unwrap(); -- 2.54.0 From 5062b907bd37e9578699dd9b0041149fdd655b09 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:23:27 -0500 Subject: [PATCH 23/49] clawhdf5-remote tests: the server counts only requests for its files A local port scanner's GET / reached the test listeners and was counted, failing the exact request budgets (and consuming injected 503s). Requests for paths the server does not serve are now answered 404 without being counted, delayed or failed; the query string is not part of the path. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/tests/common/server.rs | 41 ++++++++++++------- crates/clawhdf5-remote/tests/http.rs | 18 ++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 4749f50..a060a9d 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -3,7 +3,10 @@ //! and Last-Modified with `If-Match` / `If-Unmodified-Since` (412), and //! switches to misbehave — ignore ranges (200 with the whole file), cut //! bodies short, answer 503, respond slowly, send no validators. It counts -//! requests and body bytes, and logs every range asked for. +//! requests and body bytes, and logs every range asked for — only for the +//! paths it serves: a request for any other path (a local port scanner's +//! `GET /`, say) is answered 404 and not counted, so request budgets in +//! tests stay exact. #![allow(dead_code)] @@ -45,7 +48,8 @@ pub struct Shared { pub fail_next: AtomicU32, /// Sleep this long before answering each request. pub delay_ms: AtomicU64, - /// Requests served (every status). + /// Requests for a served path (every status); requests for other + /// paths are not counted. pub requests: AtomicU64, /// Body bytes sent. pub bytes: AtomicU64, @@ -223,14 +227,31 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { headers.insert(k.trim().to_ascii_lowercase(), v.trim().to_string()); } } + // The query string (a presigned URL's signature, say) is not part + // of the file's name. + let path = path.split('?').next().unwrap_or("").to_string(); + let close = headers + .get("connection") + .is_some_and(|v| v.eq_ignore_ascii_case("close")); + let res = { + let files = s.files.read().unwrap(); + files + .get(&path) + .map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone())) + }; + let Some((data, etag, lm)) = res else { + // Not ours: not counted, not delayed, no failure injected. + write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?; + if close { + return Ok(()); + } + continue; + }; s.requests.fetch_add(1, Ordering::SeqCst); let delay = s.delay_ms.load(Ordering::SeqCst); if delay > 0 { std::thread::sleep(Duration::from_millis(delay)); } - let close = headers - .get("connection") - .is_some_and(|v| v.eq_ignore_ascii_case("close")); if s.fail_next .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) .is_ok() @@ -241,16 +262,6 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { )?; continue; } - let res = { - let files = s.files.read().unwrap(); - files - .get(&path) - .map(|r| (r.data.clone(), r.etag.clone(), r.last_modified.clone())) - }; - let Some((data, etag, lm)) = res else { - write!(out, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?; - continue; - }; let len = data.len() as u64; let mut validators = String::new(); if s.weak_etag.load(Ordering::SeqCst) { diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 5a68072..2c95cd1 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -526,3 +526,21 @@ fn bad_urls_and_statuses_are_clean_errors() { "{err}" ); } + +/// A request for a path the server does not serve (a local port scanner's +/// `GET /`) does not count against a test's request budget. +#[test] +fn requests_for_other_paths_are_not_counted() { + use std::io::{Read, Write}; + let server = Server::start(vec![("/m.h5".into(), multi_block_file())]); + let mut probe = std::net::TcpStream::connect(server.addr).unwrap(); + probe + .write_all(b"GET / HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n") + .unwrap(); + let mut answer = String::new(); + probe.read_to_string(&mut answer).unwrap(); + assert!(answer.starts_with("HTTP/1.1 404"), "{answer}"); + storage_for_url(&server.url("/m.h5"), &quick()).unwrap(); + assert_eq!(server.requests(), 1, "only the open counts"); + assert_eq!(server.log().len(), 1); +} -- 2.54.0 From e8aaf050be83990b6d9031a83e241b245feb8e0d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:24:48 -0500 Subject: [PATCH 24/49] clawhdf5-remote: no overflow on lengths near u64::MAX A server can claim any length in Content-Range. block_len computed start + block_size, which overflowed in the last blocks of a file claimed to be near u64::MAX (a panic in debug builds, a wrapped value in release); insert() multiplied block indices unchecked. The cache's block arithmetic is now saturating/checked, and a run that does not split into whole blocks is an error instead of an endless loop or a slice panic. The test server gains fake_total (claim a length, serve zeros past the data); a test reads the last bytes of such files and opens a file whose superblock EOF and root addresses sit near u64::MAX. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/cache.rs | 70 ++++++++++++++++--- crates/clawhdf5-remote/tests/common/server.rs | 25 ++++++- crates/clawhdf5-remote/tests/http.rs | 33 +++++++++ 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs index a434814..151cc84 100644 --- a/crates/clawhdf5-remote/src/cache.rs +++ b/crates/clawhdf5-remote/src/cache.rs @@ -305,9 +305,9 @@ impl BlockCache { let end = offset.saturating_add(bytes.len() as u64).min(self.len); let mut i = offset.div_ceil(bs); let mut st = lock(&self.state); - while i * bs < end { - let start = i * bs; - let block_end = (start + bs).min(self.len); + // Checked: a hostile server can claim a length near u64::MAX. + while let Some(start) = i.checked_mul(bs).filter(|&s| s < end) { + let block_end = start.saturating_add(bs).min(self.len); if block_end > end { break; } @@ -330,9 +330,14 @@ impl BlockCache { Some(offset / bs..(end - 1) / bs + 1) } + /// Length of block `i` (0 past the end of the file). Saturating: the + /// length may be anything a server claimed, up to `u64::MAX`. fn block_len(&self, i: u64) -> u64 { - let start = i * self.config.block_size; - (start + self.config.block_size).min(self.len) - start + let start = i.saturating_mul(self.config.block_size); + start + .saturating_add(self.config.block_size) + .min(self.len) + .saturating_sub(start) } fn keep(&self, st: &mut State, i: u64, data: Block) { @@ -453,8 +458,17 @@ impl BlockCache { let mut i = run.start / bs; let mut pos = 0usize; while pos < bytes.len() { - let n = self.block_len(i) as usize; - got.push((i, Arc::from(&bytes[pos..pos + n]))); + let n = usize::try_from(self.block_len(i)).unwrap_or(usize::MAX); + let Some(block) = pos.checked_add(n).and_then(|e| bytes.get(pos..e)) else { + return Err(guard.fail(FormatError::Storage(format!( + "a run at offset {} does not split into whole blocks", + run.start + )))); + }; + if n == 0 { + break; + } + got.push((i, Arc::from(block))); pos += n; i += 1; } @@ -512,7 +526,9 @@ impl BlockCache { } } runs.into_iter() - .map(|(a, b)| a * bs..(b * bs + self.block_len(b))) + .map(|(a, b)| { + a.saturating_mul(bs)..b.saturating_mul(bs).saturating_add(self.block_len(b)) + }) .collect() } @@ -863,4 +879,42 @@ mod tests { assert_eq!(n, blocks.len(), "a block was fetched twice: {fetched:?}"); assert!(c.stats().waits > 0, "readers should have shared fetches"); } + + /// A backend claiming any length (a hostile server's `Content-Range`) + /// and serving zeros. + struct Zeros(u64); + + impl Storage for Zeros { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let end = offset.saturating_add(len as u64).min(self.0); + Ok(Cow::Owned(vec![0; end.saturating_sub(offset) as usize])) + } + fn len(&self) -> u64 { + self.0 + } + } + + #[test] + fn lengths_near_u64_max_do_not_overflow() { + for len in [u64::MAX, u64::MAX - 1, u64::MAX - 1000, 1 << 63] { + let c = BlockCache::new(Zeros(len), small()); + for (off, n) in [ + (len - 100, 50), + (len - 10, 100), + (len - 1, 1), + (len - 3000, 3000), + (u64::MAX, 10), + ] { + let got = c.read_at(off, n).unwrap(); + assert_eq!(got.len() as u64, len.saturating_sub(off).min(n as u64)); + } + let got = c + .read_ranges(&[len - 5000..len, u64::MAX - 1..u64::MAX]) + .unwrap(); + assert_eq!(got[0].len(), 5000); + c.insert(len - 5, &[0; 5]); + c.insert(u64::MAX - 5, &[0; 5]); + c.prefetch(len - 1, 10).unwrap(); + } + } } diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index a060a9d..13f279a 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -48,6 +48,10 @@ pub struct Shared { pub fail_next: AtomicU32, /// Sleep this long before answering each request. pub delay_ms: AtomicU64, + /// When non-zero, claim the file is this long (in `Content-Range` and + /// when checking ranges) and serve zeros past its real end: a hostile + /// server lying about the length. + pub fake_total: AtomicU64, /// Requests for a served path (every status); requests for other /// paths are not counted. pub requests: AtomicU64, @@ -262,7 +266,8 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { )?; continue; } - let len = data.len() as u64; + let fake = s.fake_total.load(Ordering::SeqCst); + let len = if fake > 0 { fake } else { data.len() as u64 }; let mut validators = String::new(); if s.weak_etag.load(Ordering::SeqCst) { validators = format!("ETag: W/{etag}\r\n"); @@ -291,6 +296,7 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { .unwrap() .push((path.clone(), range.and_then(Result::ok))); } + let mut padded = Vec::new(); let (status, body, extra) = match range { Some(Err(())) => { write!( @@ -302,7 +308,7 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { } Some(Ok((a, b))) => ( "206 Partial Content", - &data[a as usize..=b as usize], + slice_or_zeros(&data, a, b, &mut padded), format!("Content-Range: bytes {a}-{b}/{len}\r\n"), ), None => ("200 OK", &data[..], String::new()), @@ -342,3 +348,18 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { } } } + +/// `data[a..=b]`, padded with zeros past its end (into `padded`). +fn slice_or_zeros<'a>(data: &'a [u8], a: u64, b: u64, padded: &'a mut Vec) -> &'a [u8] { + let real = data.len() as u64; + if b < real { + return &data[a as usize..=b as usize]; + } + let n = usize::try_from(b - a + 1).expect("range fits in memory"); + padded.resize(n, 0); + if a < real { + let have = (real - a) as usize; + padded[..have].copy_from_slice(&data[a as usize..]); + } + padded +} diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 2c95cd1..2955f54 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -544,3 +544,36 @@ fn requests_for_other_paths_are_not_counted() { assert_eq!(server.requests(), 1, "only the open counts"); assert_eq!(server.log().len(), 1); } + +/// A server claiming a length near `u64::MAX` (and serving zeros past the +/// real data), with a file whose addresses point at the end of that range: +/// clean errors or zeros, never an arithmetic overflow (a panic in debug). +#[test] +fn a_server_claiming_a_huge_length_does_not_overflow() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut bytes = + std::fs::read(root.join("../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5")) + .unwrap(); + // Superblock v0: end-of-file address, then the root symbol table + // entry's object header, B-tree and heap addresses. + bytes[0x28..0x30].copy_from_slice(&(u64::MAX - 1).to_le_bytes()); + for at in [0x40, 0x50, 0x58] { + bytes[at..at + 8].copy_from_slice(&0xFFFF_FFFF_FFFF_F000u64.to_le_bytes()); + } + let server = Server::start(vec![("/h.h5".into(), bytes)]); + for total in [u64::MAX, u64::MAX - 1, 1 << 62] { + server.shared.fake_total.store(total, Ordering::SeqCst); + let url = server.url("/h.h5"); + let storage = storage_for_url(&url, &quick()).unwrap(); + assert_eq!(clawhdf5_format::storage::Storage::len(&*storage), total); + for (off, n) in [(total - 100, 50), (total - 10, 100), (total - 1, 1)] { + let got = clawhdf5_format::storage::Storage::read_at(&*storage, off, n).unwrap(); + assert_eq!(got.len() as u64, (total - off).min(n as u64)); + assert!(got.iter().all(|&b| b == 0)); + } + if let Ok(f) = File::open_storage(storage) { + let _ = transcript(&f); + } + let _ = open_url_with(&url, &quick()).map(|f| transcript(&f)); + } +} -- 2.54.0 From 8df5b209a78f87cdf324baee12d2d3e086527ae7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:27:14 -0500 Subject: [PATCH 25/49] clawhdf5-remote, h5rs: never allocate a length the server only claims h5rs check URL read the whole file with one read_at(0, len), len being whatever Content-Range said. BlockCache listed every block index of the span and preallocated len bytes: a server claiming 2^62 bytes for a 10 KB file made h5rs abort (memory allocation of 35184372088832 bytes failed). - BlockCache: a read spanning more than the budget (or eight max_requests) is fetched piece by piece and not kept, its output growing only as data arrives; read_ranges falls back to that per range; prefetch is clamped to the budget. - New clawhdf5_remote::download(storage, max_bytes): refuses a claimed length above the limit (RemoteError::TooLarge) before any request, then reads in 64 MiB steps. New RemoteError::Backend for read errors. - h5rs check downloads through it, with --max-download N (default 1 GiB). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/cache.rs | 115 ++++++++++++++++++++++++-- crates/clawhdf5-remote/src/error.rs | 19 ++++- crates/clawhdf5-remote/src/lib.rs | 42 ++++++++++ crates/clawhdf5-remote/tests/http.rs | 27 ++++++ crates/clawhdf5-tools/README.md | 4 +- crates/clawhdf5-tools/src/check.rs | 12 ++- crates/clawhdf5-tools/src/h5.rs | 19 +++-- crates/clawhdf5-tools/tests/remote.rs | 21 +++++ 8 files changed, 239 insertions(+), 20 deletions(-) diff --git a/crates/clawhdf5-remote/src/cache.rs b/crates/clawhdf5-remote/src/cache.rs index 151cc84..2d4fa8e 100644 --- a/crates/clawhdf5-remote/src/cache.rs +++ b/crates/clawhdf5-remote/src/cache.rs @@ -25,7 +25,10 @@ //! A failed fetch fails every reader waiting for it, and is not cached. //! - **Large reads do not flush the cache.** A call whose missing blocks //! add up to more than half the budget is served without keeping them -//! (a big chunked read would otherwise evict all the metadata). +//! (a big chunked read would otherwise evict all the metadata). A read +//! spanning more than the budget (or eight `max_request`s, if more) is +//! fetched piece by piece, its output growing only as data arrives: +//! nothing is allocated for a length a server merely claims. //! - **Local storages pass through.** A backend that holds the whole file //! in memory ([`Storage::as_contiguous`]) is read directly. @@ -285,12 +288,46 @@ impl BlockCache { if self.inner.as_contiguous().is_some() { return Ok(()); } - let Some(blocks) = self.block_span(offset, len) else { + // Readahead beyond the budget would only evict itself. + let Some(blocks) = self.block_span(offset, len.min(self.config.capacity)) else { return Ok(()); }; self.blocks(&blocks.collect::>(), true).map(|_| ()) } + /// Blocks fetched per step of a read spanning more than this: the + /// larger of the budget and a full parallel batch of requests (eight + /// `max_request`s), 64 MiB by default. + fn piece_blocks(&self) -> u64 { + let bytes = self + .config + .capacity + .max(self.config.max_request.saturating_mul(8)); + (bytes / self.config.block_size).max(1) + } + + /// A read of more blocks than [`piece_blocks`](Self::piece_blocks): + /// fetched and copied one piece at a time, nothing kept. The block list + /// is never materialised and the output grows only as data arrives, so + /// a length a server merely claims (up to `u64::MAX`) costs nothing + /// until bytes actually come back. + fn read_large(&self, offset: u64, end: u64, span: Range) -> Result, FormatError> { + let bs = self.config.block_size; + let piece = self.piece_blocks(); + let mut out = Vec::new(); + let mut first = span.start; + while first < span.end { + let last = first.saturating_add(piece).min(span.end); + let wanted: Vec = (first..last).collect(); + let blocks = self.blocks(&wanted, false)?; + let a = offset.max(first.saturating_mul(bs)); + let b = end.min(last.saturating_mul(bs)); + self.assemble_into(&mut out, a, b, &blocks); + first = last; + } + Ok(out) + } + /// Put bytes already fetched (such as the first block, which a backend /// may get while it probes the file's length) into the cache: `bytes` /// are the file's bytes at `offset`. Only whole blocks (or the file's @@ -534,8 +571,20 @@ impl BlockCache { /// Copy `[offset, end)` out of `blocks`. fn assemble(&self, offset: u64, end: u64, blocks: &HashMap) -> Vec { + let mut out = Vec::with_capacity(usize::try_from(end - offset).unwrap_or(0)); + self.assemble_into(&mut out, offset, end, blocks); + out + } + + /// Append `[offset, end)` out of `blocks` to `out`. + fn assemble_into( + &self, + out: &mut Vec, + offset: u64, + end: u64, + blocks: &HashMap, + ) { let bs = self.config.block_size; - let mut out = Vec::with_capacity((end - offset) as usize); let mut pos = offset; while pos < end { let i = pos / bs; @@ -545,7 +594,6 @@ impl BlockCache { out.extend_from_slice(&block[from..to]); pos = i * bs + to as u64; } - out } } @@ -559,6 +607,9 @@ impl Storage for BlockCache { return Ok(Cow::Owned(Vec::new())); }; let end = offset.saturating_add(len as u64).min(self.len); + if span.end - span.start > self.piece_blocks() { + return Ok(Cow::Owned(self.read_large(offset, end, span)?)); + } let blocks = self.blocks(&span.collect::>(), true)?; Ok(Cow::Owned(self.assemble(offset, end, &blocks))) } @@ -574,7 +625,8 @@ impl Storage for BlockCache { self.counters .reads .fetch_add(ranges.len() as u64, Ordering::Relaxed); - let mut wanted = BTreeSet::new(); + let mut spans = Vec::with_capacity(ranges.len()); + let mut total = 0u64; for r in ranges { if r.end < r.start { return Err(FormatError::Storage( @@ -582,9 +634,30 @@ impl Storage for BlockCache { )); } if let Some(span) = self.block_span(r.start, r.end - r.start) { - wanted.extend(span); + total = total.saturating_add(span.end - span.start); + spans.push(span); } } + if total > self.piece_blocks() { + // Too many blocks for one batch: range by range, each in pieces. + return ranges + .iter() + .map(|r| { + let end = r.end.min(self.len); + match self.block_span(r.start, r.end - r.start) { + None => Ok(Cow::Owned(Vec::new())), + Some(span) if span.end - span.start > self.piece_blocks() => { + Ok(Cow::Owned(self.read_large(r.start, end, span)?)) + } + Some(span) => { + let blocks = self.blocks(&span.collect::>(), true)?; + Ok(Cow::Owned(self.assemble(r.start, end, &blocks))) + } + } + }) + .collect(); + } + let wanted: BTreeSet = spans.into_iter().flatten().collect(); let wanted: Vec = wanted.into_iter().collect(); let blocks = self.blocks(&wanted, true)?; Ok(ranges @@ -917,4 +990,34 @@ mod tests { c.prefetch(len - 1, 10).unwrap(); } } + + /// A backend claiming 2^62 bytes that holds only a few: a read of + /// "the whole file" fails at the first short piece instead of + /// allocating (or listing the blocks of) the claimed length. + struct Claims(Vec); + + impl Storage for Claims { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + self.0 + .as_slice() + .read_at(offset, len) + .map(|c| Cow::Owned(c.into_owned())) + } + fn len(&self) -> u64 { + 1 << 62 + } + } + + #[test] + fn a_huge_claimed_length_is_not_allocated() { + let c = BlockCache::new(Claims(data(5000)), CacheConfig::default()); + let e = c.read_at(0, usize::MAX).unwrap_err().to_string(); + assert!(e.contains("short read"), "{e}"); + let e = c + .read_ranges(&[0..u64::MAX, 10..20]) + .unwrap_err() + .to_string(); + assert!(e.contains("short read"), "{e}"); + c.prefetch(0, u64::MAX).unwrap_err(); + } } diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index d5d1feb..b2546b6 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -38,9 +38,19 @@ pub enum RemoteError { Transport(String), /// An error from the object store. ObjectStore(String), - /// Called in a way the backend cannot serve (for example a blocking - /// read from inside an async runtime). + /// Called in a way the backend cannot serve. Usage(String), + /// The file is larger than a download was allowed to be + /// ([`download`](crate::download)). + TooLarge { + /// The file's length, as the server reports it. + len: u64, + /// The limit. + limit: u64, + }, + /// A read through a backend failed; its error, as text (the form a + /// [`clawhdf5::File`] read reports it in). + Backend(String), } impl RemoteError { @@ -71,6 +81,11 @@ impl std::fmt::Display for RemoteError { RemoteError::Transport(s) => write!(f, "network error: {s}"), RemoteError::ObjectStore(s) => write!(f, "object store: {s}"), RemoteError::Usage(s) => write!(f, "{s}"), + RemoteError::TooLarge { len, limit } => write!( + f, + "the remote file is {len} bytes, more than the download limit of {limit} bytes" + ), + RemoteError::Backend(s) => write!(f, "{s}"), } } } diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index f233954..a441bab 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -16,6 +16,7 @@ //! `gcs`, `azure`) → a [`clawhdf5::File`] with the whole read API. //! - [`storage_for_url`] gives the cached storage itself, to open with //! [`clawhdf5::File::open_storage`] and to read its [`CacheStats`]. +//! - [`download`] reads a whole remote file into memory, up to a limit. //! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried //! with backoff), [`ObjectStoreStorage`] (any `object_store` store, //! feature `object-store`), and [`BlockCache`] over any @@ -151,6 +152,47 @@ pub fn cached(backend: Backend, options: &Options) -> Result Result, Error> { + let len = storage.len(); + if len > max_bytes { + return Err(RemoteError::TooLarge { + len, + limit: max_bytes, + } + .into()); + } + const STEP: u64 = 64 << 20; + let mut out = Vec::new(); + let mut pos = 0u64; + while pos < len { + let want = (len - pos).min(STEP); + let got = storage + .read_at(pos, want as usize) + .map_err(|e| RemoteError::Backend(e.to_string()))?; + if got.len() as u64 != want { + return Err(RemoteError::BadResponse(format!( + "{} bytes at offset {pos} instead of {want}", + got.len() + )) + .into()); + } + out.extend_from_slice(&got); + pos += want; + } + Ok(out) +} + /// Open the object at `path` of any `object_store` store (in memory, local /// files, or a cloud store you configured) through a block cache. #[cfg(feature = "object-store")] diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 2955f54..b4411b6 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -577,3 +577,30 @@ fn a_server_claiming_a_huge_length_does_not_overflow() { let _ = open_url_with(&url, &quick()).map(|f| transcript(&f)); } } + +/// `download` reads a whole file in bounded steps, and refuses a claimed +/// length beyond its limit before reading anything. +#[test] +fn download_is_bounded_by_its_limit_not_the_claimed_length() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let storage = storage_for_url(&url, &quick()).unwrap(); + let got = clawhdf5_remote::download(&*storage, clawhdf5_remote::DEFAULT_MAX_DOWNLOAD).unwrap(); + assert_eq!(got, bytes); + let e = clawhdf5_remote::download(&*storage, 1000).unwrap_err(); + assert!( + matches!(e, Error::Remote(RemoteError::TooLarge { limit: 1000, .. })), + "{e}" + ); + server.shared.fake_total.store(1 << 62, Ordering::SeqCst); + let storage = storage_for_url(&url, &quick()).unwrap(); + server.reset(); + let e = + clawhdf5_remote::download(&*storage, clawhdf5_remote::DEFAULT_MAX_DOWNLOAD).unwrap_err(); + assert!( + matches!(e, Error::Remote(RemoteError::TooLarge { len, .. }) if len == 1 << 62), + "{e}" + ); + assert_eq!(server.requests(), 0, "refused before reading"); +} diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 42d829d..fc120ad 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -40,7 +40,9 @@ h5rs diff local.h5 http://127.0.0.1:8000/file.h5 ``` A URL names the whole file (`FILE/OBJECT` suffixes are for local paths). -`check` validates every byte, so it downloads a remote file whole first. +`check` validates every byte, so it downloads a remote file whole first — +up to `--max-download N` bytes (default 1 GiB), refusing a longer file +before reading any of it. The output is the local file's (`tests/remote.rs` compares every subcommand). diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index e7d97e5..ea8e09c 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -28,7 +28,7 @@ use crate::h5::{Error, ErrorKind, H5, Kind}; use crate::info::{self, DsInfo}; pub const USAGE: &str = "\ -usage: h5rs check [--data] [-q] [--max-bytes N] FILE +usage: h5rs check [--data] [-q] [--max-bytes N] [--max-download N] FILE Validate FILE's structure: walk every object from the root group, parse every header message, verify the checksums of version 2+ structures @@ -45,6 +45,9 @@ problem is printed with the address of the structure involved. datasets and attributes into its global heap collection -q, --quiet print only the problems, not the summary --max-bytes N largest dataset read by --data (default 1 GiB) + --max-download N + largest remote (URL) file downloaded to check it + (default 1 GiB) Exit status: 0 no problems, 1 problems found, 2 usage error or file not found, 3 internal error."; @@ -124,6 +127,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let mut read_data = false; let mut quiet = false; let mut max_bytes = None; + let mut max_download = 1u64 << 30; let mut file = None; while let Some(a) = args.next() { match a.as_str() { @@ -133,6 +137,10 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { Some(n) => max_bytes = Some(n), None => return args.usage_error(out, "--max-bytes needs a number", USAGE), }, + "--max-download" => match args.number() { + Some(n) => max_download = n, + None => return args.usage_error(out, "--max-download needs a number", USAGE), + }, "-h" | "--help" => { writeln!(out.o, "{USAGE}")?; return Ok(0); @@ -150,7 +158,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let path = std::path::Path::new(&file); let mut h5 = if crate::h5::is_url(&file) { // check validates every byte, so a remote file is downloaded whole. - match H5::open_arg_whole(&file) { + match H5::open_arg_whole(&file, max_download) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs check: {e}")?; diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index c63ccb3..d840074 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -248,8 +248,10 @@ impl H5 { /// [`H5::open_arg`], with a remote file downloaded whole into memory /// first — for `check`, which validates every byte of the file anyway - /// and parses it as one slice ([`H5::data`]). - pub fn open_arg_whole(arg: &str) -> Result
{ + /// and parses it as one slice ([`H5::data`]). A remote file longer + /// than `max_download` bytes is refused before anything is read: its + /// length is only what the server claims. + pub fn open_arg_whole(arg: &str, max_download: u64) -> Result
{ let h5 = H5::open_arg(arg)?; if h5.file.contiguous_bytes().is_some() { return Ok(h5); @@ -259,12 +261,8 @@ impl H5 { let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) .map_err(|e| Error::new(format!("{arg}: {e}")))?; - let len = usize::try_from(storage.len()) - .map_err(|_| Error::new(format!("{arg}: too large to download")))?; - let bytes = storage - .read_at(0, len) - .map_err(|e| Error::new(format!("{arg}: {e}")))? - .into_owned(); + let bytes = clawhdf5_remote::download(&*storage, max_download) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; let size = bytes.len() as u64; let file = File::from_bytes(bytes).map_err(|e| { Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) @@ -272,7 +270,10 @@ impl H5 { Ok(H5::new(PathBuf::from(arg), file, size)) } #[cfg(not(feature = "remote"))] - unreachable!("open_arg refuses URLs without the remote feature") + { + let _ = max_download; + unreachable!("open_arg refuses URLs without the remote feature") + } } /// The file's bytes from the superblock on: what every address indexes. diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs index d14bd61..426d1f7 100644 --- a/crates/clawhdf5-tools/tests/remote.rs +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -98,3 +98,24 @@ fn url_errors_are_clean() { assert!(out.contains("`https` feature"), "{out}"); } } + +/// `check` downloads a remote file whole, but never trusts the length the +/// server claims: a server claiming 2^62 bytes for a small file is refused +/// before anything is allocated (it used to abort the process), and +/// `--max-download` caps real files too. +#[test] +fn check_refuses_a_remote_file_beyond_the_download_limit() { + use std::sync::atomic::Ordering; + let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5"); + let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]); + let url = server.url("/t.h5"); + let (out, rc) = h5rs(&["check", &url]); + assert_eq!(rc, 0, "{out}"); + let (out, rc) = h5rs(&["check", "--max-download", "1000", &url]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("download limit of 1000 bytes"), "{out}"); + server.shared.fake_total.store(1 << 62, Ordering::SeqCst); + let (out, rc) = h5rs(&["check", &url]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("more than the download limit"), "{out}"); +} -- 2.54.0 From c04e34620eda88821ccb052d9c4bbb230817bb81 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:30:02 -0500 Subject: [PATCH 26/49] clawhdf5-remote, h5rs: URLs' credentials are never shown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every RemoteError message and HttpStorage's Debug output held the URL as given, with any user:password@ and the query string — for a presigned S3/GCS/Azure URL, its signature or token. An application logging the error leaked the credential. - New clawhdf5_remote::redact_url: no userinfo, no fragment, query values replaced by REDACTED (plain key names kept). - HttpStorage formats every message with the redacted URL, and scrubs the URL's secret parts from errors of the HTTP client (whose texts can echo the URI); Debug shows the redacted URL. storage_for_url's and the object store URL errors are redacted too. HttpStorage::url() still returns the URL as given, documented as not for logging. - h5rs prints FILE arguments that are URLs redacted: in errors and in dump/stat/check/diff output. - The test server can force a status and send a wrong Content-Range. Tests: 404, 403 (at open and on a read), wrong Content-Range (at open and on a read), no range support, encoded body, ETag change, timeout, connection closed and bad scheme errors, Display and Debug, contain none of the secrets; h5rs likewise for every subcommand. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/error.rs | 140 ++++++++++++++++++ crates/clawhdf5-remote/src/http.rs | 84 +++++++---- crates/clawhdf5-remote/src/lib.rs | 12 +- crates/clawhdf5-remote/src/object.rs | 11 +- crates/clawhdf5-remote/tests/common/server.rs | 16 +- crates/clawhdf5-remote/tests/http.rs | 118 +++++++++++++++ crates/clawhdf5-tools/src/check.rs | 2 +- crates/clawhdf5-tools/src/diff.rs | 5 +- crates/clawhdf5-tools/src/dump.rs | 1 + crates/clawhdf5-tools/src/h5.rs | 40 ++++- crates/clawhdf5-tools/src/stat.rs | 2 +- crates/clawhdf5-tools/tests/remote.rs | 39 +++++ 12 files changed, 421 insertions(+), 49 deletions(-) diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index b2546b6..20494cd 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -2,8 +2,126 @@ use clawhdf5_format::error::FormatError; +/// `url` as it may be shown in an error, a `Debug` output or a log: no +/// userinfo (`user:password@`), no fragment, and the query string's values +/// replaced by `REDACTED` (a presigned S3/GCS/Azure URL carries its +/// signature or token there). Keys are kept when they look like plain +/// names, so a message still says which kind of URL it was. +/// +/// ``` +/// assert_eq!( +/// clawhdf5_remote::redact_url("https://me:pw@host/f.h5?X-Amz-Signature=abc&a=1#x"), +/// "https://host/f.h5?X-Amz-Signature=REDACTED&a=REDACTED" +/// ); +/// ``` +pub fn redact_url(url: &str) -> String { + let (scheme, rest) = match url.split_once("://") { + Some((s, r)) => (Some(s), r), + None => (None, url), + }; + let rest = rest.split('#').next().unwrap_or(""); + let (before_query, query) = match rest.split_once('?') { + Some((a, q)) => (a, Some(q)), + None => (rest, None), + }; + let mut out = String::with_capacity(url.len()); + if let Some(s) = scheme { + out.push_str(s); + out.push_str("://"); + let auth_end = before_query.find('/').unwrap_or(before_query.len()); + let (authority, path) = before_query.split_at(auth_end); + out.push_str( + authority + .rsplit_once('@') + .map_or(authority, |(_, host)| host), + ); + out.push_str(path); + } else { + out.push_str(before_query); + } + if let Some(q) = query { + out.push('?'); + let plain = |k: &str| { + !k.is_empty() + && k.len() <= 64 + && k.bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + }; + let parts: Vec = q + .split('&') + .map(|kv| { + let k = kv.split('=').next().unwrap_or(""); + if plain(k) { + format!("{k}=REDACTED") + } else { + "REDACTED".to_string() + } + }) + .collect(); + out.push_str(&parts.join("&")); + } + out +} + +/// Replaces the secret parts of one URL (its userinfo and query string, +/// and the URL itself) wherever they appear in a message — such as the +/// text of an error from the HTTP client. +#[derive(Debug, Clone)] +pub(crate) struct Redactor { + shown: String, + secrets: Vec<(String, String)>, +} + +impl Redactor { + pub(crate) fn new(url: &str) -> Redactor { + let shown = redact_url(url); + let mut secrets = vec![(url.to_string(), shown.clone())]; + let rest = url.split_once("://").map_or(url, |(_, r)| r); + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + if let Some((userinfo, _)) = authority.rsplit_once('@') + && !userinfo.is_empty() + { + secrets.push((format!("{userinfo}@"), String::new())); + secrets.push((userinfo.to_string(), "REDACTED".into())); + } + if let Some((_, q)) = rest.split('#').next().unwrap_or("").split_once('?') + && !q.is_empty() + { + let shown_q = shown.split_once('?').map_or("", |(_, q)| q).to_string(); + secrets.push((q.to_string(), shown_q)); + for kv in q.split('&') { + if let Some((_, v)) = kv.split_once('=') + && v.len() >= 4 + { + secrets.push((v.to_string(), "REDACTED".into())); + } + } + } + Redactor { shown, secrets } + } + + /// The URL, redacted. + pub(crate) fn shown(&self) -> &str { + &self.shown + } + + /// `msg` with every secret part of the URL replaced. + pub(crate) fn scrub(&self, msg: &str) -> String { + let mut m = msg.to_string(); + for (secret, with) in &self.secrets { + if m.contains(secret.as_str()) { + m = m.replace(secret.as_str(), with); + } + } + m + } +} + /// Why a remote file could not be opened or read. /// +/// No message carries a URL's credentials: URLs appear as +/// [`redact_url`] shows them. +/// /// Inside a [`clawhdf5::File`] read these arrive as /// `clawhdf5::Error::Format(FormatError::Storage(message))`, the message /// being this error's `Display`. @@ -54,6 +172,28 @@ pub enum RemoteError { } impl RemoteError { + /// The error with every secret part of `r`'s URL scrubbed from its text. + #[cfg_attr(not(any(feature = "http", feature = "object-store")), allow(dead_code))] + pub(crate) fn scrubbed(self, r: &Redactor) -> RemoteError { + let f = |s: String| r.scrub(&s); + match self { + RemoteError::InvalidUrl(s) => RemoteError::InvalidUrl(f(s)), + RemoteError::UnsupportedScheme(s) => RemoteError::UnsupportedScheme(f(s)), + RemoteError::RangeNotSupported(s) => RemoteError::RangeNotSupported(f(s)), + RemoteError::FileChanged(s) => RemoteError::FileChanged(f(s)), + RemoteError::Status { code, what } => RemoteError::Status { + code, + what: f(what), + }, + RemoteError::BadResponse(s) => RemoteError::BadResponse(f(s)), + RemoteError::Transport(s) => RemoteError::Transport(f(s)), + RemoteError::ObjectStore(s) => RemoteError::ObjectStore(f(s)), + RemoteError::Usage(s) => RemoteError::Usage(f(s)), + e @ RemoteError::TooLarge { .. } => e, + RemoteError::Backend(s) => RemoteError::Backend(f(s)), + } + } + /// Whether retrying the same request may succeed. #[cfg_attr(not(feature = "http"), allow(dead_code))] pub(crate) fn is_transient(&self) -> bool { diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index ce3d68f..14f221e 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -36,7 +36,7 @@ use std::time::Duration; use clawhdf5_format::error::FormatError; use clawhdf5_format::storage::Storage; -use crate::error::RemoteError; +use crate::error::{Redactor, RemoteError}; /// Settings of an [`HttpStorage`]. #[derive(Debug, Clone)] @@ -107,7 +107,10 @@ enum Validator { /// documentation](self). pub struct HttpStorage { agent: ureq::Agent, + /// The URL as given, credentials and all: only ever sent to the server. url: String, + /// Shows the URL without its credentials, in every message. + redactor: Redactor, len: u64, validator: Validator, options: HttpOptions, @@ -122,7 +125,7 @@ pub struct HttpStorage { impl std::fmt::Debug for HttpStorage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("HttpStorage") - .field("url", &self.url) + .field("url", &self.redactor.shown()) .field("len", &self.len) .field("validator", &self.validator) .field("full_download", &self.full.is_some()) @@ -181,15 +184,25 @@ impl HttpStorage { /// and the bytes that request fetched (the file's start), for a /// [`BlockCache`](crate::BlockCache) to keep. pub fn open(url: &str, options: HttpOptions) -> Result<(HttpStorage, Vec), RemoteError> { + let redactor = Redactor::new(url); + HttpStorage::open_inner(url, options, &redactor).map_err(|e| e.scrubbed(&redactor)) + } + + fn open_inner( + url: &str, + options: HttpOptions, + redactor: &Redactor, + ) -> Result<(HttpStorage, Vec), RemoteError> { + let shown = redactor.shown(); let lower = url.to_ascii_lowercase(); if lower.starts_with("https://") { if !cfg!(feature = "https") { return Err(RemoteError::UnsupportedScheme(format!( - "{url}: https:// needs the `https` feature of clawhdf5-remote" + "{shown}: https:// needs the `https` feature of clawhdf5-remote" ))); } } else if !lower.starts_with("http://") { - return Err(RemoteError::UnsupportedScheme(url.to_string())); + return Err(RemoteError::UnsupportedScheme(shown.to_string())); } let config = ureq::Agent::config_builder() .http_status_as_error(false) @@ -198,6 +211,7 @@ impl HttpStorage { let mut storage = HttpStorage { agent: ureq::Agent::new_with_config(config), url: url.to_string(), + redactor: redactor.clone(), len: 0, validator: Validator::None, options, @@ -216,14 +230,15 @@ impl HttpStorage { } if storage.options.require_validator && storage.validator == Validator::None { return Err(RemoteError::Usage(format!( - "{url}: the server sends neither a strong ETag nor Last-Modified, so a change \ + "{shown}: the server sends neither a strong ETag nor Last-Modified, so a change \ of the file could not be detected (HttpOptions::require_validator)" ))); } Ok((storage, bytes)) } - /// The URL. + /// The URL as given — with any credentials it carries, so do not log + /// it; [`redact_url`](crate::redact_url) gives a form that can be. pub fn url(&self) -> &str { &self.url } @@ -307,18 +322,20 @@ impl HttpStorage { let got = reader .take(cap.saturating_add(1)) .read_to_end(&mut buf) - .map_err(|e| RemoteError::Transport(format!("{}: reading the body: {e}", self.url))); + .map_err(|e| { + RemoteError::Transport(format!("{}: reading the body: {e}", self.redactor.shown())) + }); self.bytes.fetch_add(buf.len() as u64, Ordering::Relaxed); got?; match want { Some(n) if buf.len() as u64 != n => Err(RemoteError::BadResponse(format!( "{}: body of {} bytes, expected {n}", - self.url, + self.redactor.shown(), buf.len() ))), None if buf.len() as u64 > limit => Err(RemoteError::Usage(format!( "{}: the file is larger than HttpOptions::max_full_download ({limit} bytes)", - self.url + self.redactor.shown() ))), _ => Ok(buf), } @@ -332,7 +349,7 @@ impl HttpStorage { self.requests.fetch_add(1, Ordering::Relaxed); let resp = self.request(Some((0, n - 1))).call().map_err(transport)?; let status = resp.status().as_u16(); - check_identity(&self.url, &resp)?; + check_identity(self.redactor.shown(), &resp)?; let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) { (Some(e), _) if !e.starts_with("W/") => Validator::ETag(e.to_string()), (_, Some(t)) => Validator::LastModified(t.to_string()), @@ -341,21 +358,27 @@ impl HttpStorage { match status { 206 => { let cr = header(&resp, "content-range").ok_or_else(|| { - RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + RemoteError::BadResponse(format!( + "{}: 206 without Content-Range", + self.redactor.shown() + )) })?; let (a, b, total) = content_range(cr).ok_or_else(|| { - RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + RemoteError::BadResponse(format!( + "{}: bad Content-Range {cr:?}", + self.redactor.shown() + )) })?; let total = total.ok_or_else(|| { RemoteError::BadResponse(format!( "{}: the server does not report the file's length (Content-Range {cr:?})", - self.url + self.redactor.shown() )) })?; if a != 0 || b >= total || b > n - 1 { return Err(RemoteError::BadResponse(format!( "{}: asked for bytes 0-{}, got Content-Range {cr:?}", - self.url, + self.redactor.shown(), n - 1 ))); } @@ -367,14 +390,15 @@ impl HttpStorage { return Err(RemoteError::RangeNotSupported(format!( "{} answered a range request with the whole file (status 200); set \ HttpOptions::allow_full_download to download it", - self.url + self.redactor.shown() ))); } let want = header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); if want.is_some_and(|w: u64| w > self.options.max_full_download) { return Err(RemoteError::Usage(format!( "{}: the file is larger than HttpOptions::max_full_download ({} bytes)", - self.url, self.options.max_full_download + self.redactor.shown(), + self.options.max_full_download ))); } let bytes = self.body(resp, want, self.options.max_full_download)?; @@ -382,11 +406,11 @@ impl HttpStorage { } 416 => Err(RemoteError::Usage(format!( "{}: status 416 for the first bytes (an empty file?)", - self.url + self.redactor.shown() ))), code => Err(RemoteError::Status { code, - what: self.url.clone(), + what: self.redactor.shown().to_string(), }), } } @@ -398,14 +422,15 @@ impl HttpStorage { .request(Some((start, end - 1))) .call() .map_err(transport)?; - let changed = |why: String| RemoteError::FileChanged(format!("{}: {why}", self.url)); - check_identity(&self.url, &resp)?; + let changed = + |why: String| RemoteError::FileChanged(format!("{}: {why}", self.redactor.shown())); + check_identity(self.redactor.shown(), &resp)?; match resp.status().as_u16() { 206 => {} 200 => { return Err(RemoteError::RangeNotSupported(format!( "{} answered a range request with the whole file (status 200)", - self.url + self.redactor.shown() ))); } 412 => { @@ -417,7 +442,7 @@ impl HttpStorage { code => { return Err(RemoteError::Status { code, - what: format!("{} bytes {start}-{}", self.url, end - 1), + what: format!("{} bytes {start}-{}", self.redactor.shown(), end - 1), }); } } @@ -439,10 +464,16 @@ impl HttpStorage { Validator::None => {} } let cr = header(&resp, "content-range").ok_or_else(|| { - RemoteError::BadResponse(format!("{}: 206 without Content-Range", self.url)) + RemoteError::BadResponse(format!( + "{}: 206 without Content-Range", + self.redactor.shown() + )) })?; let (a, b, total) = content_range(cr).ok_or_else(|| { - RemoteError::BadResponse(format!("{}: bad Content-Range {cr:?}", self.url)) + RemoteError::BadResponse(format!( + "{}: bad Content-Range {cr:?}", + self.redactor.shown() + )) })?; if let Some(total) = total && total != self.len @@ -452,7 +483,7 @@ impl HttpStorage { if a != start || b != end - 1 { return Err(RemoteError::BadResponse(format!( "{}: asked for bytes {start}-{}, got Content-Range {cr:?}", - self.url, + self.redactor.shown(), end - 1 ))); } @@ -461,6 +492,7 @@ impl HttpStorage { fn fetch(&self, start: u64, end: u64) -> Result, RemoteError> { self.with_retries(|| self.fetch_once(start, end)) + .map_err(|e| e.scrubbed(&self.redactor)) } /// `[offset, offset + len)` clamped to the file, or `None` if empty. @@ -539,7 +571,7 @@ impl Storage for HttpStorage { if out.len() != ranges.len() { return Err(FormatError::Storage(format!( "{}: a parallel range read failed", - self.url + self.redactor.shown() ))); } Ok(out) diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index a441bab..cedbfe9 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -45,7 +45,7 @@ use clawhdf5::File; use clawhdf5_format::storage::Storage; pub use cache::{BlockCache, CacheConfig, CacheStats}; -pub use error::{Error, RemoteError}; +pub use error::{Error, RemoteError, redact_url}; #[cfg(feature = "http")] pub use http::{HttpOptions, HttpStats, HttpStorage}; #[cfg(feature = "object-store")] @@ -93,13 +93,13 @@ pub fn storage_for_url(url: &str, options: &Options) -> Result http_storage(url, options), "s3" | "s3a" | "gs" | "az" | "azure" | "abfs" | "abfss" | "adl" => { cloud_storage(url, &scheme, options) } - _ => Err(RemoteError::UnsupportedScheme(url.to_string()).into()), + _ => Err(RemoteError::UnsupportedScheme(redact_url(url)).into()), } } @@ -116,7 +116,8 @@ fn http_storage(url: &str, options: &Options) -> Result, Erro #[cfg(not(feature = "http"))] fn http_storage(url: &str, _options: &Options) -> Result, Error> { Err(RemoteError::UnsupportedScheme(format!( - "{url}: http(s):// needs the `http` feature of clawhdf5-remote" + "{}: http(s):// needs the `http` feature of clawhdf5-remote", + redact_url(url) )) .into()) } @@ -136,7 +137,8 @@ fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result "azure", }; Err(RemoteError::UnsupportedScheme(format!( - "{url}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote" + "{}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote", + redact_url(url) )) .into()) } diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index 6bafe8d..c8c5664 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -239,13 +239,14 @@ impl Storage for ObjectStoreStorage { /// `GOOGLE_*`, `AZURE_*`). #[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))] pub(crate) fn store_for_url(url: &str) -> Result<(Arc, Path), RemoteError> { + let redactor = crate::error::Redactor::new(url); let parsed = object_store::path::Path::parse( url.split_once("://") .and_then(|(_, rest)| rest.split_once('/')) .map(|(_, key)| key) .unwrap_or(""), ) - .map_err(|e| RemoteError::InvalidUrl(format!("{url}: {e}")))?; + .map_err(|e| RemoteError::InvalidUrl(format!("{}: {e}", crate::redact_url(url))))?; let scheme = url.split_once("://").map(|(s, _)| s.to_ascii_lowercase()); let store: Arc = match scheme.as_deref() { #[cfg(feature = "s3")] @@ -253,23 +254,23 @@ pub(crate) fn store_for_url(url: &str) -> Result<(Arc, Path), R object_store::aws::AmazonS3Builder::from_env() .with_url(url) .build() - .map_err(os_error)?, + .map_err(|e| os_error(e).scrubbed(&redactor))?, ), #[cfg(feature = "gcs")] Some("gs") => Arc::new( object_store::gcp::GoogleCloudStorageBuilder::from_env() .with_url(url) .build() - .map_err(os_error)?, + .map_err(|e| os_error(e).scrubbed(&redactor))?, ), #[cfg(feature = "azure")] Some("az" | "azure" | "abfs" | "abfss" | "adl") => Arc::new( object_store::azure::MicrosoftAzureBuilder::from_env() .with_url(url) .build() - .map_err(os_error)?, + .map_err(|e| os_error(e).scrubbed(&redactor))?, ), - _ => return Err(RemoteError::UnsupportedScheme(url.to_string())), + _ => return Err(RemoteError::UnsupportedScheme(crate::redact_url(url))), }; Ok((store, parsed)) } diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 13f279a..2cdd7c1 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -52,6 +52,11 @@ pub struct Shared { /// when checking ranges) and serve zeros past its real end: a hostile /// server lying about the length. pub fake_total: AtomicU64, + /// When non-zero, answer every request for a served file with this + /// status (and an empty body). + pub force_status: AtomicU32, + /// Answer ranges with a `Content-Range` one byte off. + pub wrong_range: AtomicBool, /// Requests for a served path (every status); requests for other /// paths are not counted. pub requests: AtomicU64, @@ -256,6 +261,11 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { if delay > 0 { std::thread::sleep(Duration::from_millis(delay)); } + let forced = s.force_status.load(Ordering::SeqCst); + if forced != 0 { + write!(out, "HTTP/1.1 {forced} Forced\r\nContent-Length: 0\r\n\r\n")?; + continue; + } if s.fail_next .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) .is_ok() @@ -309,7 +319,11 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { Some(Ok((a, b))) => ( "206 Partial Content", slice_or_zeros(&data, a, b, &mut padded), - format!("Content-Range: bytes {a}-{b}/{len}\r\n"), + if s.wrong_range.load(Ordering::SeqCst) { + format!("Content-Range: bytes {}-{}/{len}\r\n", a + 1, b + 1) + } else { + format!("Content-Range: bytes {a}-{b}/{len}\r\n") + }, ), None => ("200 OK", &data[..], String::new()), }; diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index b4411b6..c61ee6e 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -604,3 +604,121 @@ fn download_is_bounded_by_its_limit_not_the_claimed_length() { ); assert_eq!(server.requests(), 0, "refused before reading"); } + +/// A URL's credentials — userinfo, and the query string of a presigned +/// URL — never appear in an error message or a `Debug` output, whatever +/// failed. +#[test] +fn credentials_never_appear_in_errors_or_debug() { + const SECRETS: [&str; 4] = ["hunter2", "SECRETSIG", "AKIDSECRET", "user:"]; + fn clean(what: &str, text: &str) { + for s in SECRETS { + assert!(!text.contains(s), "{what}: {s} leaked in {text}"); + } + } + fn check_err(what: &str, r: Result) { + let Err(e) = r else { + panic!("{what}: expected an error") + }; + clean(what, &format!("{e}")); + clean(what, &format!("{e:?}")); + } + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let secret_url = |path: &str| { + format!( + "http://user:hunter2@{}{path}?X-Amz-Credential=AKIDSECRET&X-Amz-Signature=SECRETSIG", + server.addr + ) + }; + let url = secret_url("/m.h5"); + let mut opts = quick(); + opts.http.retries = 0; + + // Opening works through such a URL, and Debug shows it redacted. + let (http, _) = HttpStorage::open(&url, opts.http.clone()).unwrap(); + let debug = format!("{http:?}"); + clean("Debug", &debug); + assert!(debug.contains("X-Amz-Signature=REDACTED"), "{debug}"); + assert_eq!(http.url(), url, "the URL itself is kept for requests"); + + check_err("404", open_url_with(&secret_url("/missing.h5"), &opts)); + server.shared.force_status.store(403, Ordering::SeqCst); + check_err("403", open_url_with(&url, &opts)); + server.shared.force_status.store(0, Ordering::SeqCst); + server.shared.wrong_range.store(true, Ordering::SeqCst); + check_err("bad range at open", open_url_with(&url, &opts)); + server.shared.wrong_range.store(false, Ordering::SeqCst); + server.shared.ignore_range.store(true, Ordering::SeqCst); + check_err("no range support", open_url_with(&url, &opts)); + server.shared.ignore_range.store(false, Ordering::SeqCst); + server.shared.gzip_label.store(true, Ordering::SeqCst); + check_err("encoded body", open_url_with(&url, &opts)); + server.shared.gzip_label.store(false, Ordering::SeqCst); + + // Errors of reads after the open, through a File. + let f = open_url_with(&url, &opts).unwrap(); + server.shared.wrong_range.store(true, Ordering::SeqCst); + check_err( + "bad range", + f.dataset("big") + .map(|d| d.read_f64()) + .and_then(|r| r) + .map_err(Error::Hdf5), + ); + server.shared.wrong_range.store(false, Ordering::SeqCst); + let f = open_url_with(&url, &opts).unwrap(); + server.shared.force_status.store(403, Ordering::SeqCst); + check_err( + "403 on a read", + f.dataset("big") + .map(|d| d.read_f64()) + .and_then(|r| r) + .map_err(Error::Hdf5), + ); + server.shared.force_status.store(0, Ordering::SeqCst); + let f = open_url_with(&url, &opts).unwrap(); + let mut other = bytes.clone(); + let n = other.len(); + other[n / 2] ^= 0xff; + server.shared.put("/m.h5", other); + check_err( + "ETag change", + f.dataset("big") + .map(|d| d.read_f64()) + .and_then(|r| r) + .map_err(Error::Hdf5), + ); + // A timeout. + let mut slow = opts.clone(); + slow.http.timeout = Duration::from_millis(100); + server.shared.delay_ms.store(1000, Ordering::SeqCst); + check_err("timeout", open_url_with(&url, &slow)); + server.shared.delay_ms.store(0, Ordering::SeqCst); + // A connection closed unanswered, a bad scheme. + let port = { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + std::thread::spawn(move || { + for c in l.incoming() { + drop(c); + } + }); + port + }; + check_err( + "connection closed", + open_url_with( + &format!("http://user:hunter2@127.0.0.1:{port}/a.h5?X-Amz-Signature=SECRETSIG"), + &opts, + ), + ); + check_err( + "scheme", + open_url("ftp://user:hunter2@example.com/a.h5?X-Amz-Signature=SECRETSIG"), + ); + assert_eq!( + clawhdf5_remote::redact_url("https://me:pw@host:8/d/f.h5?X-Amz-Signature=abc&a=1#frag"), + "https://host:8/d/f.h5?X-Amz-Signature=REDACTED&a=REDACTED" + ); +} diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index ea8e09c..4a36554 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -204,7 +204,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } } if !quiet { - c.summary(&file, out)?; + c.summary(&crate::h5::shown(&file), out)?; } Ok(if c.panicked { 3 diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index af684c5..d02bb8c 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -196,7 +196,8 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { Err(_) => { writeln!( out.e, - "h5rs diff: object <{obj}> could not be found in <{f}>" + "h5rs diff: object <{obj}> could not be found in <{}>", + crate::h5::shown(f) )?; return Ok(2); } @@ -214,7 +215,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let entries = match collected { Ok(e) => e, Err(e) => { - writeln!(out.e, "h5rs diff: {f}: {e}")?; + writeln!(out.e, "h5rs diff: {}: {e}", crate::h5::shown(f))?; return Ok(2); } }; diff --git a/crates/clawhdf5-tools/src/dump.rs b/crates/clawhdf5-tools/src/dump.rs index 4b59f00..d7076e0 100644 --- a/crates/clawhdf5-tools/src/dump.rs +++ b/crates/clawhdf5-tools/src/dump.rs @@ -98,6 +98,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { problems: 0, paths: OnceCell::new(), }; + let file = crate::h5::shown(&file); let fname = std::path::Path::new(&file) .file_name() .map(|s| s.to_string_lossy().into_owned()) diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index d840074..c130f1f 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -229,20 +229,21 @@ impl H5 { if !is_url(arg) { return H5::open(Path::new(arg)); } + let name = shown(arg); #[cfg(feature = "remote")] { let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) - .map_err(|e| Error::new(format!("{arg}: {e}")))?; + .map_err(|e| Error::new(format!("{name}: {e}")))?; let size = storage.len(); let file = File::open_storage(storage).map_err(|e| { - Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + Error::new(format!("{name}: not an HDF5 file this tool can open: {e}")) })?; - Ok(H5::new(PathBuf::from(arg), file, size)) + Ok(H5::new(PathBuf::from(&name), file, size)) } #[cfg(not(feature = "remote"))] Err(Error::new(format!( - "{arg}: URLs need h5rs built with the `remote` feature" + "{name}: URLs need h5rs built with the `remote` feature" ))) } @@ -256,18 +257,19 @@ impl H5 { if h5.file.contiguous_bytes().is_some() { return Ok(h5); } + let name = shown(arg); #[cfg(feature = "remote")] { let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) - .map_err(|e| Error::new(format!("{arg}: {e}")))?; + .map_err(|e| Error::new(format!("{name}: {e}")))?; let bytes = clawhdf5_remote::download(&*storage, max_download) - .map_err(|e| Error::new(format!("{arg}: {e}")))?; + .map_err(|e| Error::new(format!("{name}: {e}")))?; let size = bytes.len() as u64; let file = File::from_bytes(bytes).map_err(|e| { - Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + Error::new(format!("{name}: not an HDF5 file this tool can open: {e}")) })?; - Ok(H5::new(PathBuf::from(arg), file, size)) + Ok(H5::new(PathBuf::from(&name), file, size)) } #[cfg(not(feature = "remote"))] { @@ -777,6 +779,28 @@ pub fn split_file_arg(arg: &str) -> (String, Option) { (arg.to_string(), None) } +/// A FILE argument as it may be printed: a URL without its credentials +/// (userinfo, query string values — a presigned URL's signature), a path +/// as given. +pub fn shown(arg: &str) -> String { + if !is_url(arg) { + return arg.to_string(); + } + #[cfg(feature = "remote")] + { + clawhdf5_remote::redact_url(arg) + } + #[cfg(not(feature = "remote"))] + { + let (scheme, rest) = arg.split_once("://").unwrap_or(("", arg)); + let rest = rest.split(['?', '#']).next().unwrap_or(""); + let host_end = rest.find('/').unwrap_or(rest.len()); + let (authority, path) = rest.split_at(host_end); + let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + format!("{scheme}://{host}{path}") + } +} + /// Whether a FILE argument is a URL (`scheme://...`) rather than a path. pub fn is_url(arg: &str) -> bool { arg.split_once("://").is_some_and(|(scheme, _)| { diff --git a/crates/clawhdf5-tools/src/stat.rs b/crates/clawhdf5-tools/src/stat.rs index 32e91fe..611fdf2 100644 --- a/crates/clawhdf5-tools/src/stat.rs +++ b/crates/clawhdf5-tools/src/stat.rs @@ -206,7 +206,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { if let Err(e) = walk { errors.push(e.to_string()); } - report(&h5, &file, &s, out)?; + report(&h5, &crate::h5::shown(&file), &s, out)?; for e in &errors { writeln!(out.e, "h5rs stat: {e}")?; } diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs index 426d1f7..0ae0b40 100644 --- a/crates/clawhdf5-tools/tests/remote.rs +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -119,3 +119,42 @@ fn check_refuses_a_remote_file_beyond_the_download_limit() { assert_eq!(rc, 2, "{out}"); assert!(out.contains("more than the download limit"), "{out}"); } + +/// A URL's credentials (userinfo, a presigned URL's query string) are not +/// printed: not in errors, not in the file name of the output. +#[test] +fn credentials_in_urls_are_not_printed() { + let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5"); + let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]); + let url = |path: &str| { + format!( + "http://user:hunter2@{}{path}?X-Amz-Signature=SECRETSIG", + server.addr + ) + }; + for args in [ + vec!["ls", "-r"], + vec!["dump"], + vec!["stat"], + vec!["check"], + vec!["check", "--max-download", "10"], + ] { + for path in ["/t.h5", "/missing.h5"] { + let u = url(path); + let mut a = args.clone(); + a.push(&u); + let (out, _) = h5rs(&a); + assert!( + !out.contains("hunter2") && !out.contains("SECRETSIG"), + "h5rs {}: {out}", + a.join(" ") + ); + } + } + let (out, rc) = h5rs(&["diff", tall.to_str().unwrap(), &url("/t.h5"), "/nope"]); + assert_eq!(rc, 2, "{out}"); + assert!( + !out.contains("hunter2") && !out.contains("SECRETSIG"), + "{out}" + ); +} -- 2.54.0 From 7d629f49e3fa6864dc6fcc527a3bc4a0087918d0 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:30:30 -0500 Subject: [PATCH 27/49] format: bound and batch every chunk fetch over Storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the full read split its chunk fetches into 64 MiB batches. The selection path, the indexed read and the parallel_read decoders fetched every chunk's stored bytes in one read_ranges call, each extent bounded only by the file length, so a crafted chunk index pointing many chunks at one large extent made File::open_storage hold chunks x extent bytes (3.3 GB from a 16.8 MB file) before the first decode error. - storage::for_each_extent_batch is now the one way raw-data reads fetch chunk bytes: batches of at most RAW_BATCH_BYTES (now pub), each decoded before the next is fetched. Used by the full, cached, indexed, selection and parallel_read paths; the sweep read uses read_extent per chunk. - ExtentReq carries each chunk's claimed extent (bounds-checked as before, same errors) and the prefix actually fetched: filters::stored_chunk_limit — the chunk size if unfiltered, else each applied filter's worst-case growth (n + n/4 + 4096 per codec; unbounded only for an application-registered codec). The in-memory path cuts the slice it decodes the same way, so both paths still agree. - tests/raw_fetch_bounds.rs: a crafted chunked_large.h5 (ten chunks all claiming 20 MiB at one padding blob) read through every path over a storage that records the largest single fetch; and 160 MiB of legitimate unfiltered chunks fetched batch by batch. Before: one 80 MiB fetch (selection) and one 160 MiB fetch; after: within the budget. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +- crates/clawhdf5-format/src/chunked_read.rs | 203 ++++++----- crates/clawhdf5-format/src/filter_registry.rs | 16 + crates/clawhdf5-format/src/filters.rs | 38 ++ crates/clawhdf5-format/src/parallel_read.rs | 238 +++++++------ crates/clawhdf5-format/src/partial_read.rs | 90 ++--- crates/clawhdf5-format/src/storage.rs | 162 ++++++--- .../clawhdf5-format/tests/raw_fetch_bounds.rs | 332 ++++++++++++++++++ crates/clawhdf5/src/reader.rs | 8 +- docs/design/range-reads.md | 8 +- 10 files changed, 825 insertions(+), 285 deletions(-) create mode 100644 crates/clawhdf5-format/tests/raw_fetch_bounds.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 661a2eb..21e857d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,9 +47,18 @@ `read_vl_strings_in`, `read_vl_bytes_in`), `AttributeMessage::read_vl_strings_in`, `provenance::verify_dataset_in`. - A chunked read first lists the chunks it needs, then fetches all their - stored bytes with **one `read_ranges` call** (per 64 MiB of stored - data), so a remote backend can coalesce and parallelise them, then - decodes as before (in parallel with the `parallel` feature). Chunks the + stored bytes with **one `read_ranges` call** per batch of at most 64 MiB + (`storage::RAW_BATCH_BYTES`), so a remote backend can coalesce and + parallelise them, then decodes each batch as before (in parallel with + the `parallel` feature) before fetching the next. Every path that reads + chunks — full, cached, indexed, sweep, selection and the + `parallel_read` decoders — goes through the same batching, and no chunk + fetches more of its stored bytes than its decoded size can need (the + chunk size if unfiltered; else each applied filter's worst-case growth, + generously: `n + n/4 + 4096` per codec, unbounded only for a codec the + application registered). A crafted chunk index that points every chunk + at one huge extent therefore costs a bounded fetch, not + `chunks x extent` bytes (`tests/raw_fetch_bounds.rs`). Chunks the file's chunk cache already holds are not fetched. A selection fetches only the chunks its bounding box overlaps; a contiguous selection only its runs (adjacent ones merged). A global-heap collection is read once diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 140cf37..fc35c38 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -19,7 +19,7 @@ use crate::filters::{DecodeScratch, decompress_chunk_exact_with}; #[cfg(feature = "std")] use crate::filters::{all_filters_skipped, decompress_chunk_exact}; use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks_in}; -use crate::storage::{ExtentBytes, Storage, Window, raw_batches, read_exact_at}; +use crate::storage::{ExtentBytes, ExtentReq, Storage, Window, for_each_extent_batch, read_extent}; #[cfg(feature = "std")] use std::sync::Arc; @@ -256,52 +256,17 @@ fn fill_from_chunks( cache: CacheUse<'_>, output: &mut [u8], ) -> Result<(), FormatError> { - let contiguous = file_data.as_contiguous().is_some(); let out = OutBuf::new(output); - let batches = raw_batches(chunks.len(), contiguous, |i| chunks[i].chunk_size as usize); - for batch in batches { - fill_batch( - file_data, - &chunks[batch], - pipeline, - placer, - chunk_total_bytes, - cache, - &out, - )?; - } - Ok(()) -} - -/// Whether a full read looks chunk `c` up in the cache (see -/// [`fill_from_chunks`]): a filtered chunk of a dataset the cache keeps. -#[cfg(feature = "std")] -fn uses_cache(cache: CacheUse<'_>, pipeline: Option<&FilterPipeline>, c: &ChunkInfo) -> bool { - matches!(cache, Some((_, _, true))) - && pipeline.is_some_and(|pl| !all_filters_skipped(pl, c.filter_mask)) -} - -/// [`fill_from_chunks`] for one batch of chunks. -#[allow(clippy::too_many_arguments)] -fn fill_batch( - file_data: &S, - chunks: &[ChunkInfo], - pipeline: Option<&FilterPipeline>, - placer: &ChunkPlacer, - chunk_total_bytes: usize, - cache: CacheUse<'_>, - out: &OutBuf<'_>, -) -> Result<(), FormatError> { - let rank = placer.rank; - let elem_size = placer.elem_size as u32; #[cfg(not(feature = "std"))] let _ = cache; + #[cfg(feature = "std")] + let rank = placer.rank; // Over a backend without the whole file in memory, the decoded chunks // the cache holds are taken now (so a chunk evicted before it is placed // is not left without bytes), and only the others are fetched. #[cfg(feature = "std")] - let hits: Vec>> = match cache { + let hits: Vec = match cache { Some((cache, key, true)) if file_data.as_contiguous().is_none() => chunks .iter() .map(|c| { @@ -314,25 +279,90 @@ fn fill_batch( .collect(), _ => Vec::new(), }; - let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { - Vec::new() - } else { - chunks - .iter() - .enumerate() - .map(|(i, c)| { - #[cfg(feature = "std")] - let wanted = hits.get(i).is_none_or(Option::is_none); - #[cfg(not(feature = "std"))] - let wanted = { - let _ = i; - true - }; - (c.address, c.chunk_size as usize, wanted) - }) - .collect() - }; - let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; + #[cfg(not(feature = "std"))] + let hits: Vec = Vec::new(); + let reqs: Vec = chunks + .iter() + .enumerate() + .map(|(i, c)| { + let wanted = hits.get(i).is_none_or(Option::is_none); + chunk_req(c, pipeline, chunk_total_bytes, wanted) + }) + .collect(); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + fill_batch( + &chunks[batch.clone()], + &reqs[batch.clone()], + hits.get(batch.clone()).unwrap_or_default(), + batch.start, + raw_bytes, + pipeline, + placer, + chunk_total_bytes, + cache, + &out, + ) + }) +} + +/// The extent of chunk `c`'s stored bytes to fetch (`wanted`) or only +/// bounds-check: its whole stored size is checked against the file, and at +/// most [`crate::filters::stored_chunk_limit`] of it is read and decoded. +pub(crate) fn chunk_req( + c: &ChunkInfo, + pipeline: Option<&FilterPipeline>, + chunk_bytes: usize, + wanted: bool, +) -> ExtentReq { + let len = c.chunk_size as usize; + ExtentReq { + addr: c.address, + len, + fetch: wanted.then(|| { + len.min(crate::filters::stored_chunk_limit( + pipeline, + c.filter_mask, + chunk_bytes, + )) + }), + } +} + +/// A decoded chunk a full read took from the cache. +#[cfg(feature = "std")] +type CacheHit = Option>; +#[cfg(not(feature = "std"))] +type CacheHit = Option; + +/// Whether a full read looks chunk `c` up in the cache (see +/// [`fill_from_chunks`]): a filtered chunk of a dataset the cache keeps. +#[cfg(feature = "std")] +fn uses_cache(cache: CacheUse<'_>, pipeline: Option<&FilterPipeline>, c: &ChunkInfo) -> bool { + matches!(cache, Some((_, _, true))) + && pipeline.is_some_and(|pl| !all_filters_skipped(pl, c.filter_mask)) +} + +/// [`fill_from_chunks`] for one batch of chunks: `chunks` (with their +/// extents `reqs` and, over a backend without the file in memory, the cache +/// hits taken for them) are chunks `first..` of the read, whose stored bytes +/// `raw_bytes` holds. +#[allow(clippy::too_many_arguments)] +fn fill_batch( + chunks: &[ChunkInfo], + reqs: &[ExtentReq], + hits: &[CacheHit], + first: usize, + raw_bytes: &ExtentBytes<'_>, + pipeline: Option<&FilterPipeline>, + placer: &ChunkPlacer, + chunk_total_bytes: usize, + cache: CacheUse<'_>, + out: &OutBuf<'_>, +) -> Result<(), FormatError> { + let rank = placer.rank; + let elem_size = placer.elem_size as u32; + #[cfg(not(feature = "std"))] + let _ = cache; // Decode chunk `i` and place it. Its callers below run it either on one // thread, or on several for chunks whose regions are pairwise disjoint, @@ -346,15 +376,14 @@ fn fill_batch( ))); } let offsets = &c.offsets[..rank]; - let size = c.chunk_size as usize; #[cfg(feature = "std")] if let Some(Some(hit)) = hits.get(i) { - raw_bytes.check(i, c.address, size)?; + raw_bytes.check(first + i, &reqs[i])?; // SAFETY: see above. unsafe { placer.place(hit, offsets, out) }; return Ok(()); } - let raw = raw_bytes.get(i, c.address, size)?; + let raw = raw_bytes.get(first + i, &reqs[i])?; let Some(pl) = pipeline else { // SAFETY: see above. unsafe { placer.place(raw, offsets, out) }; @@ -1913,9 +1942,9 @@ pub fn read_chunked_data_sweep_in( cached } else { // Decompress from file - let c_addr = to_usize(chunk_info.address)?; - let size = chunk_info.chunk_size as usize; - let raw_chunk = read_exact_at(file_data, c_addr as u64, size)?; + to_usize(chunk_info.address)?; + let req = chunk_req(chunk_info, pipeline, chunk_total_bytes, true); + let raw_chunk = read_extent(file_data, &req)?; let raw_chunk = &*raw_chunk; let dec = if let Some(pl) = pipeline { decompress_chunk_exact( @@ -2052,46 +2081,60 @@ pub fn read_chunked_data_indexed_in( let chunk_total_bytes = plan.chunk_total_bytes; // The decoded chunks the cache holds, and the stored bytes of the - // others: fetched in one batch when the file is not in memory. + // others: fetched batch by batch when the file is not in memory. let hits: Vec>> = plan .mappings .iter() .map(|m| cache.get_decompressed_in(addr, &m.coord)) .collect(); - let extents: Vec<(u64, usize, bool)> = plan + let reqs: Vec = plan .mappings .iter() .zip(&hits) - .map(|(m, hit)| (m.file_offset, m.file_size as usize, hit.is_none())) + .map(|(m, hit)| { + let len = m.file_size as usize; + ExtentReq { + addr: m.file_offset, + len, + fetch: hit.is_none().then(|| { + len.min(crate::filters::stored_chunk_limit( + pipeline, + m.filter_mask, + chunk_total_bytes, + )) + }), + } + }) .collect(); - let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; // Decompress chunks (using LRU cache where possible) let mut chunk_buffers: Vec> = Vec::with_capacity(plan.mappings.len()); - for (i, (m, hit)) in plan.mappings.iter().zip(hits).enumerate() { - let (coord, file_offset, file_size, filter_mask) = - (&m.coord, &m.file_offset, &m.file_size, &m.filter_mask); - if let Some(cached) = hit { - chunk_buffers.push(cached); - } else { - let raw_chunk = raw_bytes.get(i, *file_offset, *file_size as usize)?; + let mut hits = hits.into_iter(); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + for i in batch { + let m = &plan.mappings[i]; + if let Some(cached) = hits.next().flatten() { + chunk_buffers.push(cached); + continue; + } + let raw_chunk = raw_bytes.get(i, &reqs[i])?; let decompressed = if let Some(pl) = pipeline { decompress_chunk_exact( raw_chunk, pl, chunk_total_bytes, elem_size as u32, - *filter_mask, - coord, + m.filter_mask, + &m.coord, )? } else { raw_chunk.to_vec() }; let aligned = CacheAlignedBuffer::from_vec(decompressed); - let arc = cache.put_decompressed_aligned_in(addr, coord.clone(), aligned); - chunk_buffers.push(arc); + chunk_buffers.push(cache.put_decompressed_aligned_in(addr, m.coord.clone(), aligned)); } - } + Ok(()) + })?; // Assemble using pre-computed layout let mut output = vec![0u8; plan.output_bytes]; diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 7670a8b..9855c24 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -188,6 +188,22 @@ pub fn is_filter_available(id: u16) -> bool { } } +/// Whether chunks filtered with `id` may be decoded by a codec the +/// application registered (whose stored sizes this crate cannot bound). +pub(crate) fn may_be_registered(id: u16) -> bool { + if builtin_filter(id).is_some_and(|b| !b.is_shared()) { + return false; + } + #[cfg(feature = "std")] + { + registered(id).is_some() + } + #[cfg(not(feature = "std"))] + { + false + } +} + #[cfg(feature = "std")] mod custom { use super::FilterCodec; diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 6390aee..e247983 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -63,6 +63,44 @@ fn filter_output_bound(filter_id: u16, input: usize) -> usize { } } +/// Most stored bytes a chunk whose decoded size is `chunk_bytes` can need: +/// what a raw-data read fetches of (and decodes from) a chunk, whatever size +/// its index entry claims. A crafted index that points many chunks at huge +/// extents then costs no more than legitimate chunks would. +/// +/// An unfiltered chunk (no pipeline, or every filter skipped by +/// `filter_mask`) is the chunk itself: `chunk_bytes`, and bytes past them +/// were never used. A filtered chunk is bounded by each applied filter's +/// worst-case growth in the write direction: shuffle keeps the size, +/// Fletcher32 adds 4 bytes, and any other codec gets `n + n/4 + 4096` — a +/// deliberately generous bound (bzip2 grows 1000 random bytes by 252, more +/// than the decoders' own `n + n/8 + 64` output bound), since a legitimate +/// chunk cut short here would fail to read. A filter handled by a codec the +/// application registered is not ours to bound: such a chunk is limited +/// only by the file. +pub(crate) fn stored_chunk_limit( + pipeline: Option<&FilterPipeline>, + filter_mask: u32, + chunk_bytes: usize, +) -> usize { + let Some(pipeline) = pipeline else { + return chunk_bytes; + }; + let mut size = chunk_bytes; + for (i, filter) in pipeline.filters.iter().enumerate() { + if filter_skipped(filter_mask, i) { + continue; + } + size = match filter.filter_id { + FILTER_SHUFFLE => size, + FILTER_FLETCHER32 => size.saturating_add(4), + id if filter_registry::may_be_registered(id) => return usize::MAX, + _ => size.saturating_add(size / 4).saturating_add(4096), + }; + } + size +} + /// Whether bit `index` of a chunk's filter mask says filter `index` was /// skipped when the chunk was written. fn filter_skipped(filter_mask: u32, index: usize) -> bool { diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 75a4bd3..606173c 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -12,24 +12,21 @@ use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::decompress_chunk_exact; use crate::lane_partition::{self, LaneStats, PartitionStats}; -use crate::storage::{ExtentBytes, Storage}; +use crate::storage::{ExtentReq, Storage, for_each_extent_batch}; -/// The stored bytes of every chunk in `chunks`, fetched in one -/// [`Storage::read_ranges`] call when the file is not in memory (each -/// chunk's bounds error is reported when that chunk is decoded, as before). -fn fetch_all<'a, S: Storage + ?Sized>( - file_data: &'a S, +/// The extents of `chunks`' stored bytes (see +/// [`crate::chunked_read::chunk_req`]), fetched batch by batch with +/// [`for_each_extent_batch`] when the file is not in memory (each chunk's +/// bounds error is reported when that chunk is decoded, as before). +fn chunk_reqs( chunks: &[ChunkInfo], -) -> Result, FormatError> { - let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { - Vec::new() - } else { - chunks - .iter() - .map(|c| (c.address, c.chunk_size as usize, true)) - .collect() - }; - ExtentBytes::fetch(file_data, &extents) + pipeline: Option<&FilterPipeline>, + chunk_total_bytes: usize, +) -> Vec { + chunks + .iter() + .map(|c| crate::chunked_read::chunk_req(c, pipeline, chunk_total_bytes, true)) + .collect() } /// Threshold: only use parallel decompression when chunk count exceeds this. @@ -238,62 +235,75 @@ pub fn decompress_chunks_lane_partitioned_in( .unwrap_or(1) }); - let raw_bytes = fetch_all(file_data, chunks)?; - let assignments = lane_partition::partition_chunks(chunks.len(), lanes, seed); - let num_lanes = assignments.len(); - - // Each lane processes its assigned chunks and returns results + stats. - let lane_results: Result, LaneStats)>, FormatError> = assignments - .into_par_iter() - .map(|indices| { - let mut results = Vec::with_capacity(indices.len()); - let mut stats = LaneStats::default(); - - for &index in &indices { - let chunk_info = &chunks[index]; - let size = chunk_info.chunk_size as usize; - let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; - - let decompressed = decompress_chunk_exact( - raw_chunk, - pipeline, - chunk_total_bytes, - element_size, - chunk_info.filter_mask, - &chunk_info.offsets, - )?; - - stats.chunks_processed += 1; - stats.compressed_bytes += size as u64; - stats.decompressed_bytes += decompressed.len() as u64; - - results.push(DecompressedChunk { - index, - data: decompressed, - }); - } - - Ok((results, stats)) - }) - .collect(); - - let lane_results = lane_results?; - - // Aggregate stats - let mut partition_stats = PartitionStats::new(num_lanes); + let reqs = chunk_reqs(chunks, Some(pipeline), chunk_total_bytes); + let mut ordered: Vec> = Vec::with_capacity(chunks.len()); + let mut partition_stats = PartitionStats::new(0); partition_stats.total_chunks = chunks.len(); - for (lane_idx, (_, stats)) in lane_results.iter().enumerate() { - partition_stats.per_lane[lane_idx] = stats.clone(); - } + // Each batch of fetched chunks is partitioned into lanes and decoded + // before the next batch is fetched (with the file in memory there is + // one batch: all the chunks). + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + let assignments = lane_partition::partition_chunks(batch.len(), lanes, seed); + // Each lane processes its assigned chunks and returns results + stats. + let lane_results: Result, LaneStats)>, FormatError> = + assignments + .into_par_iter() + .map(|indices| { + let mut results = Vec::with_capacity(indices.len()); + let mut stats = LaneStats::default(); - // Flatten and sort by original index to restore order - let mut all_chunks: Vec = lane_results - .into_iter() - .flat_map(|(chunks, _)| chunks) - .collect(); - all_chunks.sort_by_key(|dc| dc.index); + for &local in &indices { + let index = batch.start + local; + let chunk_info = &chunks[index]; + let size = chunk_info.chunk_size as usize; + let raw_chunk = raw_bytes.get(index, &reqs[index])?; - let ordered = all_chunks.into_iter().map(|dc| dc.data).collect(); + let decompressed = decompress_chunk_exact( + raw_chunk, + pipeline, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + &chunk_info.offsets, + )?; + + stats.chunks_processed += 1; + stats.compressed_bytes += size as u64; + stats.decompressed_bytes += decompressed.len() as u64; + + results.push(DecompressedChunk { + index, + data: decompressed, + }); + } + + Ok((results, stats)) + }) + .collect(); + let lane_results = lane_results?; + + // Aggregate stats + if partition_stats.per_lane.len() < lane_results.len() { + partition_stats + .per_lane + .resize_with(lane_results.len(), LaneStats::default); + partition_stats.num_lanes = lane_results.len(); + } + for (lane, (_, stats)) in partition_stats.per_lane.iter_mut().zip(&lane_results) { + lane.chunks_processed += stats.chunks_processed; + lane.compressed_bytes += stats.compressed_bytes; + lane.decompressed_bytes += stats.decompressed_bytes; + } + + // Flatten and sort by original index to restore order + let mut all_chunks: Vec = lane_results + .into_iter() + .flat_map(|(chunks, _)| chunks) + .collect(); + all_chunks.sort_by_key(|dc| dc.index); + ordered.extend(all_chunks.into_iter().map(|dc| dc.data)); + Ok(()) + })?; Ok((ordered, partition_stats)) } @@ -325,33 +335,38 @@ pub fn decompress_chunks_parallel_in( ) -> Result>, FormatError> { use rayon::prelude::*; - let raw_bytes = fetch_all(file_data, chunks)?; - let results: Result, FormatError> = chunks - .par_iter() - .enumerate() - .map(|(index, chunk_info)| { - let size = chunk_info.chunk_size as usize; - let raw_chunk = raw_bytes.get(index, chunk_info.address, size)?; + let reqs = chunk_reqs(chunks, Some(pipeline), chunk_total_bytes); + let mut ordered: Vec> = Vec::with_capacity(chunks.len()); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + let results: Result, FormatError> = batch + .clone() + .into_par_iter() + .map(|index| { + let chunk_info = &chunks[index]; + let raw_chunk = raw_bytes.get(index, &reqs[index])?; - let decompressed = decompress_chunk_exact( - raw_chunk, - pipeline, - chunk_total_bytes, - element_size, - chunk_info.filter_mask, - &chunk_info.offsets, - )?; + let decompressed = decompress_chunk_exact( + raw_chunk, + pipeline, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + &chunk_info.offsets, + )?; - Ok(DecompressedChunk { - index, - data: decompressed, + Ok(DecompressedChunk { + index, + data: decompressed, + }) }) - }) - .collect(); + .collect(); - let mut result_vec = results?; - result_vec.sort_by_key(|dc| dc.index); - Ok(result_vec.into_iter().map(|dc| dc.data).collect()) + let mut result_vec = results?; + result_vec.sort_by_key(|dc| dc.index); + ordered.extend(result_vec.into_iter().map(|dc| dc.data)); + Ok(()) + })?; + Ok(ordered) } /// Decompress chunks sequentially (fallback when parallel is not warranted). @@ -373,26 +388,29 @@ pub fn decompress_chunks_sequential_in( chunk_total_bytes: usize, element_size: u32, ) -> Result>, FormatError> { - let raw_bytes = fetch_all(file_data, chunks)?; + let reqs = chunk_reqs(chunks, pipeline, chunk_total_bytes); let mut result = Vec::with_capacity(chunks.len()); - for (i, chunk_info) in chunks.iter().enumerate() { - let size = chunk_info.chunk_size as usize; - let raw_chunk = raw_bytes.get(i, chunk_info.address, size)?; + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + for i in batch { + let chunk_info = &chunks[i]; + let raw_chunk = raw_bytes.get(i, &reqs[i])?; - let decompressed = if let Some(pl) = pipeline { - decompress_chunk_exact( - raw_chunk, - pl, - chunk_total_bytes, - element_size, - chunk_info.filter_mask, - &chunk_info.offsets, - )? - } else { - raw_chunk.to_vec() - }; - result.push(decompressed); - } + let decompressed = if let Some(pl) = pipeline { + decompress_chunk_exact( + raw_chunk, + pl, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + &chunk_info.offsets, + )? + } else { + raw_chunk.to_vec() + }; + result.push(decompressed); + } + Ok(()) + })?; Ok(result) } diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index a9f630c..3361485 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -26,7 +26,7 @@ use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; use crate::filters::{all_filters_skipped, decompress_chunk_exact_with}; use crate::selection::Selection; -use crate::storage::{ExtentBytes, Storage}; +use crate::storage::{ExtentReq, Storage, for_each_extent_batch}; /// The smallest axis-aligned box containing every selected element, as /// `(start, extent)` per dimension. `None` when there is nothing to gain or @@ -373,50 +373,50 @@ pub fn read_selection_in( }) }) .collect(); - // Their stored bytes, in one batch when the file is not in memory. - let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() { - Vec::new() - } else { - wanted - .iter() - .map(|c| (c.address, c.chunk_size as usize, true)) - .collect() - }; - let raw_bytes = ExtentBytes::fetch(file_data, &extents)?; - // Chunks are decoded into this thread's reusable buffers. - crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { - for (i, chunk) in wanted.iter().enumerate() { - let origin = &chunk.offsets[..rank]; - usize::try_from(chunk.address) - .map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?; - let raw = raw_bytes.get(i, chunk.address, chunk.chunk_size as usize)?; - // Mirrors the full-read path: filter-mask bit i set means - // filter i was not applied to this chunk. - let data: &[u8] = match pipeline { - Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { - decompress_chunk_exact_with( - raw, - pl, - chunk_bytes, - elem_size as u32, - chunk.filter_mask, - &chunk.offsets[..rank], - scratch, - )? - } - _ => raw, - }; - copy_overlap( - data, - origin, - &chunk_shape, - &mut boxed, - &box_start, - &box_extent, - elem_size, - ); - } - Ok(()) + // Their stored bytes, batch by batch when the file is not in + // memory; each batch's chunks are decoded into this thread's + // reusable buffers before the next batch is fetched. + let reqs: Vec = wanted + .iter() + .map(|c| crate::chunked_read::chunk_req(c, pipeline, chunk_bytes, true)) + .collect(); + for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| { + crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> { + for i in batch { + let chunk = wanted[i]; + let origin = &chunk.offsets[..rank]; + usize::try_from(chunk.address).map_err(|_| { + FormatError::Overflow("chunk address exceeds usize".into()) + })?; + let raw = raw_bytes.get(i, &reqs[i])?; + // Mirrors the full-read path: filter-mask bit i set + // means filter i was not applied to this chunk. + let data: &[u8] = match pipeline { + Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { + decompress_chunk_exact_with( + raw, + pl, + chunk_bytes, + elem_size as u32, + chunk.filter_mask, + &chunk.offsets[..rank], + scratch, + )? + } + _ => raw, + }; + copy_overlap( + data, + origin, + &chunk_shape, + &mut boxed, + &box_start, + &box_extent, + elem_size, + ); + } + Ok(()) + }) })?; } _ => return Ok(None), diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 0dd0540..2b4b9f1 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -337,14 +337,57 @@ pub fn read_upto( } /// Most stored bytes fetched by one [`Storage::read_ranges`] call when a -/// read gathers many extents (a chunked dataset's chunks): a larger read is -/// fetched and decoded batch by batch, so a remote backend never holds more -/// than this much undecoded data per read. -pub(crate) const RAW_BATCH_BYTES: usize = 64 << 20; +/// read gathers many extents (a chunked dataset's chunks, a selection's +/// runs): a larger read is fetched and decoded batch by batch, so a backend +/// without the file in memory never holds more than this much undecoded +/// data per read (or one extent, when a single one is larger — and every +/// chunk's extent is bounded by what the chunk can need, see +/// [`crate::filters::stored_chunk_limit`]). +pub const RAW_BATCH_BYTES: usize = 64 << 20; -/// The stored bytes of a list of extents (chunks, contiguous runs), fetched -/// together: [`Storage::read_ranges`] is called once for all of them, so a -/// remote backend can coalesce and parallelise the requests. +/// One extent of a raw-data read: `len` bytes stored at `addr`, whose +/// bounds are checked against the file, of which the first `fetch` bytes +/// are read (`None`: only checked, not read — its bytes are not needed). +/// +/// `fetch` below `len` bounds what a crafted size field can make a read +/// fetch: a chunk never needs more of its stored bytes than its decoded +/// size allows, however large its index entry says it is. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ExtentReq { + pub addr: u64, + pub len: usize, + pub fetch: Option, +} + +impl ExtentReq { + /// How many bytes are read for this extent. + #[inline] + fn fetch_len(&self) -> usize { + self.fetch.map_or(0, |f| f.min(self.len)) + } +} + +/// One extent's bytes on their own (see [`ExtentReq`]): the whole extent's +/// bounds checked as [`read_exact_at`] checks them, and its first +/// `req.fetch` bytes read (none when `fetch` is `None`). +pub(crate) fn read_extent<'a, S: Storage + ?Sized>( + file: &'a S, + req: &ExtentReq, +) -> Result, FormatError> { + let start = usize::try_from(req.addr).unwrap_or(usize::MAX); + match start.checked_add(req.len) { + Some(end) if end <= len_usize(file) => read_exact_at(file, req.addr, req.fetch_len()), + _ => Err(FormatError::UnexpectedEof { + expected: start.saturating_add(req.len), + available: len_usize(file), + }), + } +} + +/// The stored bytes of one batch of extents (chunks, contiguous runs), +/// fetched together: [`Storage::read_ranges`] is called once for the batch, +/// so a remote backend can coalesce and parallelise the requests. See +/// [`for_each_extent_batch`], which is how every raw-data read gets them. /// /// With the whole file in memory nothing is fetched: [`Self::get`] slices /// it, as the slice readers did. Either way an extent that does not lie in @@ -356,8 +399,12 @@ pub(crate) const RAW_BATCH_BYTES: usize = 64 << 20; pub(crate) enum ExtentBytes<'a> { /// The whole file. Contiguous(&'a [u8]), - /// Each extent's bytes, or its bounds error. - Fetched(Vec>), + /// Each extent's bytes, or its bounds error; the first is extent + /// `base` of the read. + Fetched { + base: usize, + extents: Vec>, + }, } /// One extent of [`ExtentBytes::Fetched`]. @@ -371,33 +418,35 @@ pub(crate) enum Extent<'a> { } impl<'a> ExtentBytes<'a> { - /// Fetch `extents` (`(address, length, wanted)`): the bytes of those - /// `wanted`, and the bounds check of all of them. - pub(crate) fn fetch( + /// Fetch `reqs`, extents `base..base + reqs.len()` of the read: the + /// bytes of those wanted, and the bounds check of all of them. + fn fetch( file: &'a S, - extents: &[(u64, usize, bool)], + reqs: &[ExtentReq], + base: usize, ) -> Result { if let Some(all) = file.as_contiguous() { return Ok(ExtentBytes::Contiguous(all)); } let file_len = len_usize(file); let mut ranges = Vec::new(); - let mut out = Vec::with_capacity(extents.len()); + let mut out = Vec::with_capacity(reqs.len()); // Positions in `out` of the extents being read, in `ranges` order. let mut slots = Vec::new(); - for &(addr, len, wanted) in extents { - let checked = - crate::addr::to_usize(addr).and_then(|start| match start.checked_add(len) { + for req in reqs { + let checked = crate::addr::to_usize(req.addr).and_then(|start| { + match start.checked_add(req.len) { Some(end) if end <= file_len => Ok(()), _ => Err(FormatError::UnexpectedEof { - expected: start.saturating_add(len), + expected: start.saturating_add(req.len), available: file_len, }), - }); + } + }); match checked { - Ok(()) if wanted => { + Ok(()) if req.fetch.is_some() => { slots.push(out.len()); - ranges.push(addr..addr + len as u64); + ranges.push(req.addr..req.addr + req.fetch_len() as u64); out.push(Extent::NotFetched); } Ok(()) => out.push(Extent::NotFetched), @@ -418,41 +467,46 @@ impl<'a> ExtentBytes<'a> { out[slot] = Extent::Bytes(bytes); } } - Ok(ExtentBytes::Fetched(out)) + Ok(ExtentBytes::Fetched { base, extents: out }) } - /// Whether extent `i` (at `addr`, `len` bytes, as passed to - /// [`Self::fetch`]) lies in the file: its bounds error if not. - pub(crate) fn check(&self, i: usize, addr: u64, len: usize) -> Result<(), FormatError> { + /// Whether extent `i` of the read (`req`) lies in the file: its bounds + /// error if not. + pub(crate) fn check(&self, i: usize, req: &ExtentReq) -> Result<(), FormatError> { match self { - ExtentBytes::Contiguous(_) => self.get(i, addr, len).map(|_| ()), - ExtentBytes::Fetched(v) => match v.get(i) { - Some(Extent::Err(e)) => Err(e.clone()), - Some(_) => Ok(()), - None => Err(not_fetched()), - }, + ExtentBytes::Contiguous(_) => self.get(i, req).map(|_| ()), + ExtentBytes::Fetched { base, extents } => { + match i.checked_sub(*base).and_then(|j| extents.get(j)) { + Some(Extent::Err(e)) => Err(e.clone()), + Some(_) => Ok(()), + None => Err(not_fetched()), + } + } } } - /// Extent `i`'s bytes (at `addr`, `len` bytes, as passed to - /// [`Self::fetch`]). - pub(crate) fn get(&self, i: usize, addr: u64, len: usize) -> Result<&[u8], FormatError> { + /// Extent `i` of the read (`req`): its first `req.fetch` bytes, the + /// same whether the file is in memory or not. + pub(crate) fn get(&self, i: usize, req: &ExtentReq) -> Result<&[u8], FormatError> { match self { ExtentBytes::Contiguous(all) => { - let start = crate::addr::to_usize(addr)?; + let start = crate::addr::to_usize(req.addr)?; start - .checked_add(len) + .checked_add(req.len) .and_then(|end| all.get(start..end)) + .map(|b| &b[..req.fetch_len()]) .ok_or(FormatError::UnexpectedEof { - expected: start.saturating_add(len), + expected: start.saturating_add(req.len), available: <[u8]>::len(all), }) } - ExtentBytes::Fetched(v) => match v.get(i) { - Some(Extent::Bytes(b)) => Ok(b), - Some(Extent::Err(e)) => Err(e.clone()), - _ => Err(not_fetched()), - }, + ExtentBytes::Fetched { base, extents } => { + match i.checked_sub(*base).and_then(|j| extents.get(j)) { + Some(Extent::Bytes(b)) => Ok(b), + Some(Extent::Err(e)) => Err(e.clone()), + _ => Err(not_fetched()), + } + } } } } @@ -462,6 +516,30 @@ fn not_fetched() -> FormatError { FormatError::Storage("an extent that was not fetched was asked for".into()) } +/// The one way raw-data reads fetch stored bytes: `reqs` are split into +/// consecutive batches of at most [`RAW_BATCH_BYTES`] of fetched bytes (at +/// least one extent each — and no extent fetches more than its +/// [`ExtentReq::fetch`]), and for each batch in turn its bytes are fetched +/// with one [`Storage::read_ranges`] call and `f(batch, &bytes)` is called, +/// with `bytes` indexed by the extent's position in `reqs`. A batch's bytes +/// are dropped before the next batch is fetched, and an error from `f` +/// stops the read before anything more is fetched. +/// +/// With the whole file in memory there is nothing to fetch: one call, over +/// all of `reqs`, that slices the file. +pub(crate) fn for_each_extent_batch<'a, S: Storage + ?Sized>( + file: &'a S, + reqs: &[ExtentReq], + mut f: impl FnMut(Range, &ExtentBytes<'a>) -> Result<(), FormatError>, +) -> Result<(), FormatError> { + let contiguous = file.as_contiguous().is_some(); + for batch in raw_batches(reqs.len(), contiguous, |i| reqs[i].fetch_len()) { + let bytes = ExtentBytes::fetch(file, &reqs[batch.clone()], batch.start)?; + f(batch, &bytes)?; + } + Ok(()) +} + /// Split `n` extents, whose sizes `size(i)` gives, into consecutive batches /// of at most [`RAW_BATCH_BYTES`] (at least one extent each): the ranges of /// `0..n` to fetch together. With the whole file in memory (`contiguous`) diff --git a/crates/clawhdf5-format/tests/raw_fetch_bounds.rs b/crates/clawhdf5-format/tests/raw_fetch_bounds.rs new file mode 100644 index 0000000..8217323 --- /dev/null +++ b/crates/clawhdf5-format/tests/raw_fetch_bounds.rs @@ -0,0 +1,332 @@ +//! What a raw-data read fetches from a [`Storage`] without the file in +//! memory is bounded: by batch (at most `RAW_BATCH_BYTES` per +//! `read_ranges` call) and by chunk (never more of a chunk's stored bytes +//! than its decoded size can need), on every path that reads chunks — full, +//! cached, indexed, sweep, selection and the `parallel_read` decoders. +//! +//! A crafted chunk index can point every chunk at one huge extent. Slicing +//! an in-memory file costs nothing there, but a backend that fetches would +//! hold `chunks x extent` bytes before the first chunk failed to decode. + +use std::borrow::Cow; +use std::ops::Range; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +use clawhdf5_format::chunk_cache::ChunkCache; +use clawhdf5_format::chunked_read::{ + ChunkInfo, SweepContext, list_chunks, read_chunked_data_sweep_in, +}; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read::{ + read_raw_data_cached_in, read_raw_data_full_in, read_raw_data_indexed_in, + read_raw_data_selection_in, +}; +use clawhdf5_format::dataspace::Dataspace; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::filter_pipeline::FilterPipeline; +use clawhdf5_format::group_v2; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::selection::Selection; +use clawhdf5_format::storage::{RAW_BATCH_BYTES, Storage}; +use clawhdf5_format::superblock::Superblock; + +/// A read_at-only storage that records the most bytes one call fetched +/// (a `read_ranges` call counts all its ranges together) and the total. +struct PeakStorage { + data: Vec, + peak: AtomicU64, + total: AtomicU64, +} + +impl PeakStorage { + fn new(data: Vec) -> Self { + PeakStorage { + data, + peak: AtomicU64::new(0), + total: AtomicU64::new(0), + } + } + + fn reset(&self) { + self.peak.store(0, Relaxed); + self.total.store(0, Relaxed); + } + + fn served(&self, offset: u64, len: usize) -> Vec { + self.data + .as_slice() + .read_at(offset, len) + .unwrap() + .into_owned() + } +} + +impl Storage for PeakStorage { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let got = self.served(offset, len); + self.peak.fetch_max(got.len() as u64, Relaxed); + self.total.fetch_add(got.len() as u64, Relaxed); + Ok(Cow::Owned(got)) + } + + fn len(&self) -> u64 { + self.data.len() as u64 + } + + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + let got: Vec> = ranges + .iter() + .map(|r| self.served(r.start, (r.end - r.start) as usize)) + .collect(); + let bytes: u64 = got.iter().map(|g| g.len() as u64).sum(); + self.peak.fetch_max(bytes, Relaxed); + self.total.fetch_add(bytes, Relaxed); + Ok(got.into_iter().map(Cow::Owned).collect()) + } +} + +struct Chunked { + layout: DataLayout, + dataspace: Dataspace, + datatype: Datatype, + pipeline: Option, + os: u8, + ls: u8, +} + +/// The one chunked dataset of fixture `name`. +fn chunked(bytes: &[u8]) -> Chunked { + let sb = Superblock::parse(bytes, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + for child in group_v2::resolve_group_children(bytes, &sb, sb.root_group_address).unwrap() { + let header = + ObjectHeader::parse(bytes, child.object_header_address as usize, os, ls).unwrap(); + let msg = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| m.data.clone()) + }; + let Some(dl) = msg(MessageType::DataLayout) else { + continue; + }; + let layout = DataLayout::parse(&dl, os, ls).unwrap(); + if !matches!(layout, DataLayout::Chunked { .. }) { + continue; + } + return Chunked { + layout, + datatype: Datatype::parse(&msg(MessageType::Datatype).unwrap()) + .unwrap() + .0, + dataspace: Dataspace::parse(&msg(MessageType::Dataspace).unwrap(), ls).unwrap(), + pipeline: msg(MessageType::FilterPipeline).map(|p| FilterPipeline::parse(&p).unwrap()), + os, + ls, + }; + } + panic!("no chunked dataset"); +} + +/// Claimed stored size of every chunk in the crafted file. +const HUGE: u32 = 20 << 20; + +/// `chunked_large.h5` (1000 `i32` in ten gzip chunks, a v1 B-tree index) +/// with `HUGE` bytes of padding appended and every chunk's index entry +/// rewritten to claim `HUGE` stored bytes at the padding: ten chunks, 200 +/// MiB of extents, in a 20 MiB file. +fn crafted() -> (Vec, Chunked, Vec) { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut bytes = std::fs::read(dir.join("chunked_large.h5")).unwrap(); + let ds = chunked(&bytes); + let es = ds.datatype.type_size() as usize; + let (chunks, _) = list_chunks(&bytes, &ds.layout, &ds.dataspace, es, ds.os, ds.ls).unwrap(); + assert_eq!(chunks.len(), 10); + let blob = bytes.len() as u64; + for c in &chunks { + // v1 B-tree key (size, filter mask, offsets + 0) then the child + // address. + let mut pat = Vec::new(); + pat.extend_from_slice(&c.chunk_size.to_le_bytes()); + pat.extend_from_slice(&c.filter_mask.to_le_bytes()); + // The key holds one offset per dimension plus the element offset + // (0); `offsets` may or may not list that last one. + for d in 0..=ds.dataspace.dimensions.len() { + pat.extend_from_slice(&c.offsets.get(d).copied().unwrap_or(0).to_le_bytes()); + } + pat.extend_from_slice(&c.address.to_le_bytes()); + let at = bytes + .windows(pat.len()) + .position(|w| w == pat.as_slice()) + .expect("chunk key"); + bytes[at..at + 4].copy_from_slice(&HUGE.to_le_bytes()); + let a = at + pat.len() - 8; + bytes[a..a + 8].copy_from_slice(&blob.to_le_bytes()); + } + bytes.resize(bytes.len() + HUGE as usize, 0x5a); + let ds = chunked(&bytes); + let (chunks, _) = list_chunks(&bytes, &ds.layout, &ds.dataspace, es, ds.os, ds.ls).unwrap(); + assert!( + chunks + .iter() + .all(|c| c.chunk_size == HUGE && c.address == blob) + ); + (bytes, ds, chunks) +} + +/// Most a crafted chunk of the fixture may fetch: its decoded size (400 +/// bytes) grown by one codec, generously. +const CHUNK_LIMIT: u64 = 400 + 100 + 4096; + +#[test] +fn crafted_chunk_index_cannot_amplify_fetches() { + let (bytes, ds, chunks) = crafted(); + let st = PeakStorage::new(bytes.clone()); + let pl = ds.pipeline.as_ref(); + let (dl, sp, dt, os, ls) = (&ds.layout, &ds.dataspace, &ds.datatype, ds.os, ds.ls); + let check = + |what: &str, got: Result, FormatError>, want: Result, FormatError>| { + // Same outcome as slicing the whole file. + assert_eq!(got, want, "{what}"); + let (peak, total) = (st.peak.load(Relaxed), st.total.load(Relaxed)); + assert!( + peak <= RAW_BATCH_BYTES as u64, + "{what}: one fetch of {peak} bytes" + ); + // Every chunk's fetch is bounded by what it can need, whatever its + // index entry claims (plus the index and header reads). + assert!( + total <= chunks.len() as u64 * CHUNK_LIMIT + 64 * 1024, + "{what}: fetched {total} bytes" + ); + st.reset(); + }; + let slice: &[u8] = &bytes; + + let sel = Selection::Hyperslab { + start: vec![100], + stride: vec![1], + count: vec![400], + block: vec![1], + }; + check( + "selection", + read_raw_data_selection_in(&st, dl, sp, dt, pl, os, ls, &sel), + read_raw_data_selection_in(slice, dl, sp, dt, pl, os, ls, &sel), + ); + check( + "full", + read_raw_data_full_in(&st, dl, sp, dt, pl, os, ls), + read_raw_data_full_in(slice, dl, sp, dt, pl, os, ls), + ); + check( + "cached", + read_raw_data_cached_in(&st, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + read_raw_data_cached_in(slice, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + ); + check( + "indexed", + read_raw_data_indexed_in(&st, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + read_raw_data_indexed_in(slice, dl, sp, dt, pl, os, ls, &ChunkCache::new()), + ); + check( + "sweep", + read_chunked_data_sweep_in( + &st, + dl, + sp, + dt, + pl, + os, + ls, + &ChunkCache::new(), + &mut SweepContext::new(4, 2), + ), + read_chunked_data_sweep_in( + slice, + dl, + sp, + dt, + pl, + os, + ls, + &ChunkCache::new(), + &mut SweepContext::new(4, 2), + ), + ); + #[cfg(feature = "parallel")] + { + use clawhdf5_format::parallel_read::{ + decompress_chunks_lane_partitioned_in, decompress_chunks_parallel_in, + decompress_chunks_sequential_in, + }; + let pl = pl.unwrap(); + let flat = |r: Result>, FormatError>| r.map(|v| v.concat()); + check( + "parallel", + flat(decompress_chunks_parallel_in(&st, &chunks, pl, 400, 4)), + flat(decompress_chunks_parallel_in(slice, &chunks, pl, 400, 4)), + ); + check( + "sequential", + flat(decompress_chunks_sequential_in( + &st, + &chunks, + Some(pl), + 400, + 4, + )), + flat(decompress_chunks_sequential_in( + slice, + &chunks, + Some(pl), + 400, + 4, + )), + ); + check( + "lane partitioned", + flat( + decompress_chunks_lane_partitioned_in(&st, &chunks, pl, 400, 4, 7, Some(3)) + .map(|(v, _)| v), + ), + flat( + decompress_chunks_lane_partitioned_in(slice, &chunks, pl, 400, 4, 7, Some(3)) + .map(|(v, _)| v), + ), + ); + } +} + +/// Legitimately large chunks (unfiltered, 4 MiB each, 160 MiB in all) are +/// fetched batch by batch: no call holds more than the batch budget, and +/// the data is right. +#[cfg(feature = "parallel")] +#[test] +fn large_reads_are_fetched_in_batches() { + use clawhdf5_format::parallel_read::decompress_chunks_sequential_in; + const CHUNK: usize = 4 << 20; + let data: Vec = (0..2 * CHUNK).map(|i| (i % 251) as u8).collect(); + let chunks: Vec = (0..40u64) + .map(|i| ChunkInfo { + chunk_size: CHUNK as u32, + filter_mask: 0, + offsets: vec![i * CHUNK as u64], + address: (i % 2) * CHUNK as u64, + }) + .collect(); + let st = PeakStorage::new(data.clone()); + let got = decompress_chunks_sequential_in(&st, &chunks, None, CHUNK, 1).unwrap(); + assert_eq!(got.len(), 40); + for (i, c) in got.iter().enumerate() { + let at = (i % 2) * CHUNK; + assert!(c == &data[at..at + CHUNK], "chunk {i}"); + } + let peak = st.peak.load(Relaxed); + assert!(peak <= RAW_BATCH_BYTES as u64, "one fetch of {peak} bytes"); + assert_eq!(st.total.load(Relaxed), 40 * CHUNK as u64); +} diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 6bd7b35..0b1ac71 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -349,9 +349,11 @@ impl File { /// selections, variable-length data, virtual datasets). /// /// Every read goes through the storage's [`Storage::read_at`] and - /// [`Storage::read_ranges`] (a chunked read fetches all the chunks it - /// needs with one `read_ranges` call per 64 MiB), so nothing is read that - /// the operation does not need. A storage that has the whole file in + /// [`Storage::read_ranges`] (a chunked read fetches the chunks it needs + /// with one `read_ranges` call per batch of at most 64 MiB, decoding + /// each batch before the next, and never more of a chunk than its + /// decoded size can need), so nothing is read that the operation does + /// not need. A storage that has the whole file in /// memory ([`Storage::as_contiguous`]) is read as [`File::from_bytes`] /// reads its buffer. The storage holds the whole file: a user block is /// found and skipped, and bytes past the end of file the superblock diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1098922..c42c5d3 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -435,8 +435,12 @@ fast path within benchmark noise. (checked: identical conformance results; the bench gate below still has to be run on an idle machine). - Chunked reads fetch in batches of at most 64 MiB of stored bytes, one - `read_ranges` call each, so a remote read never holds more than that - undecoded; chunks already in the chunk cache are not fetched. + `read_ranges` call each, decoding each batch before fetching the next, + on every chunk-reading path (one helper, + `storage::for_each_extent_batch`); and no chunk fetches more than its + decoded size can need (`filters::stored_chunk_limit`), so a remote read + holds at most one batch undecoded even over a crafted chunk index; + chunks already in the chunk cache are not fetched. - The typed readers' zero-copy fast path became "read the contiguous bytes once": over a range storage a contiguous `read_f64` is one read, and a native contiguous selection reads only its runs. -- 2.54.0 From 680c90b3a8f19662b54b0ba235e05944a94c7a62 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:31:39 -0500 Subject: [PATCH 28/49] clawhdf5-remote: redirects are followed safely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ureq's defaults followed up to 10 redirects, including from https to plain http, and forwarded the custom HttpOptions::headers (X-Api-Key, Cookie, ...) to whatever host a redirect named — only Authorization was stripped. HttpStorage now follows redirects itself (ureq's max_redirects is 0): - at most HttpOptions::max_redirects per request (default 5; 0 refuses any redirect), then RemoteError::Redirect; - never from https to another scheme, nor to a non-http(s) URL; - once a redirect leaves the URL's origin (scheme, host, port), none of the custom headers is sent any more (Authorization included); - each hop counts as a request; errors show the target redacted. Tests: a redirect to another local port reads the right data and the target never sees X-Api-Key or Authorization (it did before); a same-origin redirect keeps them; a loop stops after 6 requests; 0 refuses; unit tests for target resolution, the https downgrade and origins. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/error.rs | 6 + crates/clawhdf5-remote/src/http.rs | 177 ++++++++++++++++-- crates/clawhdf5-remote/tests/common/server.rs | 37 +++- crates/clawhdf5-remote/tests/http.rs | 55 ++++++ 4 files changed, 260 insertions(+), 15 deletions(-) diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index 20494cd..6e3c551 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -158,6 +158,10 @@ pub enum RemoteError { ObjectStore(String), /// Called in a way the backend cannot serve. Usage(String), + /// A redirect that is not followed: from `https` to `http`, to + /// another scheme, or beyond + /// [`HttpOptions::max_redirects`](crate::HttpOptions). + Redirect(String), /// The file is larger than a download was allowed to be /// ([`download`](crate::download)). TooLarge { @@ -189,6 +193,7 @@ impl RemoteError { RemoteError::Transport(s) => RemoteError::Transport(f(s)), RemoteError::ObjectStore(s) => RemoteError::ObjectStore(f(s)), RemoteError::Usage(s) => RemoteError::Usage(f(s)), + RemoteError::Redirect(s) => RemoteError::Redirect(f(s)), e @ RemoteError::TooLarge { .. } => e, RemoteError::Backend(s) => RemoteError::Backend(f(s)), } @@ -221,6 +226,7 @@ impl std::fmt::Display for RemoteError { RemoteError::Transport(s) => write!(f, "network error: {s}"), RemoteError::ObjectStore(s) => write!(f, "object store: {s}"), RemoteError::Usage(s) => write!(f, "{s}"), + RemoteError::Redirect(s) => write!(f, "redirect refused: {s}"), RemoteError::TooLarge { len, limit } => write!( f, "the remote file is {len} bytes, more than the download limit of {limit} bytes" diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index 14f221e..9fd10c6 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -17,6 +17,10 @@ //! [`HttpOptions::allow_full_download`] is set: then the file is //! downloaded once, at open, and read from memory. //! +//! Redirects are followed (at most [`HttpOptions::max_redirects`]), but +//! never from `https` to `http`, and the custom +//! [`HttpOptions::headers`] are not sent to another origin. +//! //! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and //! a body shorter or longer than its `Content-Range` — are retried with //! exponential backoff. Responses are requested with @@ -36,7 +40,7 @@ use std::time::Duration; use clawhdf5_format::error::FormatError; use clawhdf5_format::storage::Storage; -use crate::error::{Redactor, RemoteError}; +use crate::error::{Redactor, RemoteError, redact_url}; /// Settings of an [`HttpStorage`]. #[derive(Debug, Clone)] @@ -63,9 +67,17 @@ pub struct HttpOptions { /// `Last-Modified`, since a change of the file could then go unnoticed /// (only its length is checked). pub require_validator: bool, - /// Extra headers sent with every request (for example - /// `Authorization`). + /// Extra headers sent with every request to the URL's own origin (for + /// example `Authorization` or `X-Api-Key`). They are never sent to + /// another origin a redirect leads to. pub headers: Vec<(String, String)>, + /// Redirects followed per request (0: none, a redirect is an error). + /// A redirect from `https` to plain `http` is always refused; once a + /// redirect leaves the URL's origin (scheme, host and port), the + /// [`headers`](Self::headers) are no longer sent. Every request of a + /// file follows the redirects again (the target is not remembered, as + /// a presigned target may expire). + pub max_redirects: u32, } impl Default for HttpOptions { @@ -80,6 +92,7 @@ impl Default for HttpOptions { max_full_download: 1 << 30, require_validator: false, headers: Vec::new(), + max_redirects: 5, } } } @@ -166,6 +179,64 @@ fn check_identity(url: &str, resp: &ureq::http::Response) -> Result< } } +/// Scheme, host (lowercase, no userinfo) and port of an absolute URL. +fn origin(url: &str) -> Option<(String, String, u16)> { + let (scheme, rest) = url.split_once("://")?; + let scheme = scheme.to_ascii_lowercase(); + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + let host_port = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + let default = match scheme.as_str() { + "http" => 80, + "https" => 443, + _ => return None, + }; + // "[v6]:port", "host:port", or either without a port. + let (host, port) = match host_port.rsplit_once(':') { + Some((h, p)) if !p.contains(']') => (h, p.parse().ok()?), + _ => (host_port, default), + }; + Some((scheme, host.to_ascii_lowercase(), port)) +} + +fn same_origin(a: &str, b: &str) -> bool { + matches!((origin(a), origin(b)), (Some(x), Some(y)) if x == y) +} + +/// The URL a redirect from `base` to `location` goes to, if it may be +/// followed: `http`/`https` only, and never from `https` to `http`. +fn redirect_target(base: &str, location: &str) -> Result { + let location = location.trim(); + let (scheme, rest) = base.split_once("://").unwrap_or(("http", base)); + let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, path) = rest.split_at(authority_end); + let target = if location.contains("://") { + location.to_string() + } else if let Some(r) = location.strip_prefix("//") { + format!("{scheme}://{r}") + } else if location.starts_with('/') { + format!("{scheme}://{authority}{location}") + } else { + let path = path.split(['?', '#']).next().unwrap_or(""); + let dir = path.rsplit_once('/').map_or("", |(d, _)| d); + format!("{scheme}://{authority}{dir}/{location}") + }; + let Some((to_scheme, _, _)) = origin(&target) else { + return Err(RemoteError::Redirect(format!( + "{} redirects to {}, which is not an http(s) URL", + redact_url(base), + redact_url(&target) + ))); + }; + if scheme.eq_ignore_ascii_case("https") && to_scheme != "https" { + return Err(RemoteError::Redirect(format!( + "{} redirects to {}: a downgrade from https is refused", + redact_url(base), + redact_url(&target) + ))); + } + Ok(target) +} + fn transport(e: ureq::Error) -> RemoteError { match e { ureq::Error::StatusCode(code) => RemoteError::Status { @@ -204,8 +275,10 @@ impl HttpStorage { } else if !lower.starts_with("http://") { return Err(RemoteError::UnsupportedScheme(shown.to_string())); } + // Redirects are followed by `call`, which applies our rules. let config = ureq::Agent::config_builder() .http_status_as_error(false) + .max_redirects(0) .timeout_global(Some(options.timeout)) .build(); let mut storage = HttpStorage { @@ -286,14 +359,15 @@ impl HttpStorage { } } + /// One `GET` of `url`. The options' custom headers are sent only to + /// the URL's own origin (`trusted`). fn request( &self, + url: &str, range: Option<(u64, u64)>, + trusted: bool, ) -> ureq::RequestBuilder { - let mut req = self - .agent - .get(&self.url) - .header("Accept-Encoding", "identity"); + let mut req = self.agent.get(url).header("Accept-Encoding", "identity"); if let Some((a, b)) = range { req = req.header("Range", format!("bytes={a}-{b}")); } @@ -302,12 +376,60 @@ impl HttpStorage { Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t), Validator::None => {} } - for (k, v) in &self.options.headers { - req = req.header(k, v); + if trusted { + for (k, v) in &self.options.headers { + req = req.header(k, v); + } } req } + /// Send a ranged `GET`, following redirects by the rules of + /// [`HttpOptions::max_redirects`]: at most that many, never from + /// `https` to anything else, and without the custom headers once the + /// chain has left the URL's origin. Each hop counts as a request. + fn call( + &self, + range: Option<(u64, u64)>, + ) -> Result, RemoteError> { + let mut url = self.url.clone(); + let mut trusted = true; + let mut hops = 0u32; + loop { + if hops > 0 { + self.requests.fetch_add(1, Ordering::Relaxed); + } + let resp = self + .request(&url, range, trusted) + .call() + .map_err(|e| transport(e).scrubbed(&Redactor::new(&url)))?; + if !matches!(resp.status().as_u16(), 301 | 302 | 303 | 307 | 308) { + return Ok(resp); + } + let status = resp.status().as_u16(); + let Some(location) = header(&resp, "location") else { + return Err(RemoteError::BadResponse(format!( + "{}: status {status} without a Location", + redact_url(&url) + ))); + }; + let next = redirect_target(&url, location)?; + if hops >= self.options.max_redirects { + return Err(RemoteError::Redirect(format!( + "{} redirects to {}: more than HttpOptions::max_redirects ({})", + redact_url(&url), + redact_url(&next), + self.options.max_redirects + ))); + } + if !same_origin(&next, &self.url) { + trusted = false; + } + url = next; + hops += 1; + } + } + /// Read a body of exactly `want` bytes (or up to `limit` when `want` is /// unknown). fn body( @@ -347,7 +469,7 @@ impl HttpStorage { fn probe(&self) -> Result<(u64, Validator, Vec, bool), RemoteError> { let n = self.options.first_request.max(1); self.requests.fetch_add(1, Ordering::Relaxed); - let resp = self.request(Some((0, n - 1))).call().map_err(transport)?; + let resp = self.call(Some((0, n - 1)))?; let status = resp.status().as_u16(); check_identity(self.redactor.shown(), &resp)?; let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) { @@ -418,10 +540,7 @@ impl HttpStorage { /// One request for `[start, end)` (inside the file, non-empty). fn fetch_once(&self, start: u64, end: u64) -> Result, RemoteError> { self.requests.fetch_add(1, Ordering::Relaxed); - let resp = self - .request(Some((start, end - 1))) - .call() - .map_err(transport)?; + let resp = self.call(Some((start, end - 1)))?; let changed = |why: String| RemoteError::FileChanged(format!("{}: {why}", self.redactor.shown())); check_identity(self.redactor.shown(), &resp)?; @@ -594,4 +713,34 @@ mod tests { assert_eq!(content_range("items 0-1/2"), None); assert_eq!(content_range("bytes */1000"), None); } + + #[test] + fn redirect_targets_and_origins() { + use super::{redirect_target, same_origin}; + let t = |b: &str, l: &str| redirect_target(b, l).map_err(|e| e.to_string()); + assert_eq!( + t("http://a:8/d/f.h5?x=1", "g.h5").unwrap(), + "http://a:8/d/g.h5" + ); + assert_eq!(t("http://a/d/f.h5", "/g.h5").unwrap(), "http://a/g.h5"); + assert_eq!(t("https://a/d/f.h5", "//b/g.h5").unwrap(), "https://b/g.h5"); + assert_eq!(t("http://a/f", "https://b/g").unwrap(), "https://b/g"); + let e = t("https://a/f.h5", "http://a/f.h5").unwrap_err(); + assert!(e.contains("downgrade"), "{e}"); + assert!(t("HTTPS://a/f.h5", "http://b/f.h5").is_err()); + assert!(t("https://a/f", "//b/g").is_ok()); + assert!(t("http://a/f", "ftp://b/g").is_err()); + let e = t("https://u:pw@a/f?sig=SECRET", "http://b/g?sig=OTHER").unwrap_err(); + assert!( + !e.contains("SECRET") && !e.contains("OTHER") && !e.contains("pw"), + "{e}" + ); + assert!(same_origin("http://a/x", "http://A:80/y")); + assert!(same_origin("https://u:p@a:443/x", "https://a/y")); + assert!(!same_origin("http://a/x", "https://a/x")); + assert!(!same_origin("http://a:1/x", "http://a:2/x")); + assert!(!same_origin("http://a/x", "http://b/x")); + assert!(same_origin("http://[::1]:8/x", "http://[::1]:8/y")); + assert!(!same_origin("http://[::1]/x", "http://[::1]:8/y")); + } } diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 2cdd7c1..1708084 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -57,6 +57,10 @@ pub struct Shared { pub force_status: AtomicU32, /// Answer ranges with a `Content-Range` one byte off. pub wrong_range: AtomicBool, + /// Path → `Location`: answered `302 Found` (counted as a request). + pub redirects: RwLock>, + /// (path, headers) of every counted request, header names lowercase. + pub seen: Mutex)>>, /// Requests for a served path (every status); requests for other /// paths are not counted. pub requests: AtomicU64, @@ -119,11 +123,31 @@ impl Server { self.shared.bytes.load(Ordering::SeqCst) } - /// Zero the counters and the log. + /// Zero the counters and the logs. pub fn reset(&self) { self.shared.requests.store(0, Ordering::SeqCst); self.shared.bytes.store(0, Ordering::SeqCst); self.shared.log.lock().unwrap().clear(); + self.shared.seen.lock().unwrap().clear(); + } + + /// Answer `path` with a redirect to `location`. + pub fn redirect(&self, path: &str, location: &str) { + self.shared + .redirects + .write() + .unwrap() + .insert(path.to_string(), location.to_string()); + } + + /// Whether any request the server counted carried header `name`. + pub fn saw_header(&self, name: &str) -> bool { + self.shared + .seen + .lock() + .unwrap() + .iter() + .any(|(_, h)| h.contains_key(name)) } /// The ranges asked for so far. @@ -242,6 +266,16 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { let close = headers .get("connection") .is_some_and(|v| v.eq_ignore_ascii_case("close")); + let redirect = s.redirects.read().unwrap().get(&path).cloned(); + if let Some(location) = redirect { + s.requests.fetch_add(1, Ordering::SeqCst); + s.seen.lock().unwrap().push((path.clone(), headers.clone())); + write!( + out, + "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n" + )?; + continue; + } let res = { let files = s.files.read().unwrap(); files @@ -257,6 +291,7 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { continue; }; s.requests.fetch_add(1, Ordering::SeqCst); + s.seen.lock().unwrap().push((path.clone(), headers.clone())); let delay = s.delay_ms.load(Ordering::SeqCst); if delay > 0 { std::thread::sleep(Duration::from_millis(delay)); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index c61ee6e..08119f3 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -722,3 +722,58 @@ fn credentials_never_appear_in_errors_or_debug() { "https://host:8/d/f.h5?X-Amz-Signature=REDACTED&a=REDACTED" ); } + +/// Redirects are followed within limits: custom credential headers reach +/// only the URL's own origin (not another port a redirect leads to), a +/// same-origin redirect keeps them, loops end at `max_redirects`, and +/// `max_redirects = 0` refuses any redirect. (The https→http downgrade +/// refusal is a unit test of `redirect_target`: no TLS server here.) +#[test] +fn redirects_are_followed_safely() { + let bytes = multi_block_file(); + let local = File::from_bytes(bytes.clone()).unwrap(); + let want = local.dataset("big").unwrap().read_f64().unwrap(); + let target = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let front = Server::start(vec![("/m.h5".into(), bytes.clone())]); + front.redirect("/r.h5", &target.url("/m.h5")); + front.redirect("/s.h5", "/m.h5"); + front.redirect("/loop.h5", "/loop.h5"); + let mut opts = quick(); + opts.http.headers = vec![ + ("X-Api-Key".into(), "sekrit".into()), + ("Authorization".into(), "Bearer tok".into()), + ]; + + // Cross-origin (another port): followed, the headers stay behind. + let f = open_url_with(&front.url("/r.h5"), &opts).unwrap(); + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); + assert!(front.saw_header("x-api-key"), "sent to the URL's origin"); + assert!(target.requests() > 1); + assert!( + !target.saw_header("x-api-key") && !target.saw_header("authorization"), + "credential headers forwarded across origins: {:?}", + target.shared.seen.lock().unwrap() + ); + + // Same origin: followed with the headers. + front.reset(); + let f = open_url_with(&front.url("/s.h5"), &opts).unwrap(); + assert_eq!(f.dataset("big").unwrap().read_f64().unwrap(), want); + let seen = front.shared.seen.lock().unwrap().clone(); + assert!( + seen.iter() + .filter(|(p, _)| p == "/m.h5") + .all(|(_, h)| h.get("x-api-key").map(String::as_str) == Some("sekrit")) + ); + + // A loop ends after max_redirects (5 by default): 6 requests. + front.reset(); + let e = open_url_with(&front.url("/loop.h5"), &opts).unwrap_err(); + assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}"); + assert_eq!(front.requests(), 6); + + // No redirects allowed. + opts.http.max_redirects = 0; + let e = open_url_with(&front.url("/s.h5"), &opts).unwrap_err(); + assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}"); +} -- 2.54.0 From 61e34927dce32be401a6a563608c1875b4b1582c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:32:15 -0500 Subject: [PATCH 29/49] clawhdf5-remote: a 200 covering the requested range is the whole file The first request asks for bytes=0-1048575. RFC 9110 lets a server answer 200 when the range covers the whole representation, so a file under 1 MiB on a server that does support ranges could be refused as 'does not support range requests'. A 200 whose Content-Length (or, without one, its body, read at most that far) is within the range asked for is now kept as the whole file and read from memory; a longer one is still refused unless allow_full_download is set. Test: a 9968-byte file served with 200 opens in one request with the transcript of File::open (it was refused before); with a 4096-byte first request it is still refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/http.rs | 32 +++++++++++++++++++++++----- crates/clawhdf5-remote/tests/http.rs | 23 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index 9fd10c6..67b5612 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -12,6 +12,9 @@ //! open is [`RemoteError::FileChanged`], never a mix of old and new bytes. //! (A server that sends neither validator cannot be checked beyond the //! length; [`HttpOptions::require_validator`] refuses such servers.) +//! - a `200` answer to the first request whose body is no longer than the +//! range asked for is the whole file (a server may answer so when the +//! range covers it): it is kept and read from memory. //! - a server that ignores `Range` and answers `200` with the whole file is //! refused with [`RemoteError::RangeNotSupported`], unless //! [`HttpOptions::allow_full_download`] is set: then the file is @@ -508,15 +511,34 @@ impl HttpStorage { Ok((total, validator, bytes, false)) } 200 => { - if !self.options.allow_full_download { - return Err(RemoteError::RangeNotSupported(format!( + // RFC 9110 lets a server answer 200 when the range covers + // the whole file: a body no longer than the range asked + // for is the whole file, ranges supported or not. + let want: Option = + header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); + let refused = || { + RemoteError::RangeNotSupported(format!( "{} answered a range request with the whole file (status 200); set \ HttpOptions::allow_full_download to download it", self.redactor.shown() - ))); + )) + }; + if !self.options.allow_full_download { + return match want { + Some(w) if w <= n => { + let bytes = self.body(resp, Some(w), 0)?; + Ok((bytes.len() as u64, validator, bytes, true)) + } + Some(_) => Err(refused()), + // No length: read at most the range asked for. + None => match self.body(resp, None, n) { + Ok(bytes) => Ok((bytes.len() as u64, validator, bytes, true)), + Err(RemoteError::Usage(_)) => Err(refused()), + Err(e) => Err(e), + }, + }; } - let want = header(&resp, "content-length").and_then(|v| v.trim().parse().ok()); - if want.is_some_and(|w: u64| w > self.options.max_full_download) { + if want.is_some_and(|w| w > self.options.max_full_download) { return Err(RemoteError::Usage(format!( "{}: the file is larger than HttpOptions::max_full_download ({} bytes)", self.redactor.shown(), diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 08119f3..679f977 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -777,3 +777,26 @@ fn redirects_are_followed_safely() { let e = open_url_with(&front.url("/s.h5"), &opts).unwrap_err(); assert!(matches!(e, Error::Remote(RemoteError::Redirect(_))), "{e}"); } + +/// A server may answer the first request (bytes 0 to 1 MiB - 1) with 200 +/// when that covers the whole file: a body no longer than the range asked +/// for is accepted as the whole file, in one request. +#[test] +fn a_200_covering_the_requested_range_is_the_whole_file() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let tall = root.join("../clawhdf5/tests/fixtures/tall.h5"); + let bytes = std::fs::read(&tall).unwrap(); + assert!(bytes.len() < 1 << 20); + let server = Server::start(vec![("/t.h5".into(), bytes.clone())]); + server.shared.ignore_range.store(true, Ordering::SeqCst); + let storage = storage_for_url(&server.url("/t.h5"), &quick()).unwrap(); + let f = File::open_storage(storage).unwrap(); + assert_eq!(f.contiguous_bytes(), Some(&bytes[..])); + assert_eq!(transcript(&f), transcript(&File::open(&tall).unwrap())); + assert_eq!(server.requests(), 1); + // Larger than the range asked for: still refused. + let mut opts = quick().http; + opts.first_request = 4096; + let e = HttpStorage::open(&server.url("/t.h5"), opts).unwrap_err(); + assert!(matches!(e, RemoteError::RangeNotSupported(_)), "{e}"); +} -- 2.54.0 From 30a1ed6b9c3be83d2e3ae982885452be0c0203d5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:34:31 -0500 Subject: [PATCH 30/49] clawhdf5-remote: request timeouts scale with the body timeout_global (60 s) covered a whole request, and a request can carry 8 MiB (max_request): below about 140 KB/s every block run timed out, was retried from scratch and failed, so a slow link could not read remote files at all. HttpOptions::timeout (now 30 s) bounds connecting and receiving the response headers; the body gets timeout + its size at the new HttpOptions::min_speed (16 KiB/s by default: 94 s for a 1 MiB block). A slow but moving link is not cut off; a stalled one still fails. (ureq has no idle timeout; its body timeout is a total budget.) The test server can throttle bodies and stall mid-body. Test: a 256 KiB block at 256 KiB/s reads with a 300 ms timeout (it failed before), and a body stalled for 20 s fails in under 5 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/http.rs | 32 +++++++++++++++-- crates/clawhdf5-remote/tests/common/server.rs | 32 ++++++++++++++--- crates/clawhdf5-remote/tests/http.rs | 34 +++++++++++++++++++ 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/crates/clawhdf5-remote/src/http.rs b/crates/clawhdf5-remote/src/http.rs index 67b5612..221ca6c 100644 --- a/crates/clawhdf5-remote/src/http.rs +++ b/crates/clawhdf5-remote/src/http.rs @@ -24,6 +24,11 @@ //! never from `https` to `http`, and the custom //! [`HttpOptions::headers`] are not sent to another origin. //! +//! Timeouts scale with the request: [`HttpOptions::timeout`] to connect +//! and to get the response headers, and for the body that plus its size at +//! [`HttpOptions::min_speed`] — a slow link is not cut off mid-block, a +//! stalled connection still is. +//! //! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and //! a body shorter or longer than its `Content-Range` — are retried with //! exponential backoff. Responses are requested with @@ -53,8 +58,15 @@ pub struct HttpOptions { pub retries: u32, /// Delay before the first retry; doubled for each further one. pub backoff: Duration, - /// Timeout of one request, from connecting to the end of the body. + /// Time allowed to connect, and then to receive the response headers. + /// The body gets this plus the time it takes at + /// [`min_speed`](Self::min_speed), so a request's budget grows with its + /// size: a slow but moving link is not cut off, a stalled one is. pub timeout: Duration, + /// Slowest transfer rate tolerated, in bytes per second: receiving a + /// body of `n` bytes may take `timeout + n / min_speed` (16 KiB/s by + /// default: 94 s for a 1 MiB block, 9 min for an 8 MiB request). + pub min_speed: u64, /// Requests of one `read_ranges` call in flight at once. pub max_parallel: usize, /// Bytes fetched by the first request, from offset 0 (the superblock and @@ -88,7 +100,8 @@ impl Default for HttpOptions { HttpOptions { retries: 3, backoff: Duration::from_millis(200), - timeout: Duration::from_secs(60), + timeout: Duration::from_secs(30), + min_speed: 16 << 10, max_parallel: 8, first_request: crate::cache::DEFAULT_BLOCK_SIZE, allow_full_download: false, @@ -282,7 +295,8 @@ impl HttpStorage { let config = ureq::Agent::config_builder() .http_status_as_error(false) .max_redirects(0) - .timeout_global(Some(options.timeout)) + .timeout_connect(Some(options.timeout)) + .timeout_recv_response(Some(options.timeout)) .build(); let mut storage = HttpStorage { agent: ureq::Agent::new_with_config(config), @@ -374,6 +388,18 @@ impl HttpStorage { if let Some((a, b)) = range { req = req.header("Range", format!("bytes={a}-{b}")); } + // The body's budget scales with what it may carry: the range, or + // a whole file the server may send instead. + let mut body = range.map_or(0, |(a, b)| b.saturating_sub(a).saturating_add(1)); + if self.options.allow_full_download { + body = body.max(self.options.max_full_download); + } + let secs = body as f64 / self.options.min_speed.max(1) as f64; + let body_timeout = self + .options + .timeout + .saturating_add(Duration::try_from_secs_f64(secs).unwrap_or(Duration::MAX)); + let mut req = req.config().timeout_recv_body(Some(body_timeout)).build(); match &self.validator { Validator::ETag(e) => req = req.header("If-Match", e), Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t), diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index 1708084..ec7c8ae 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -55,6 +55,11 @@ pub struct Shared { /// When non-zero, answer every request for a served file with this /// status (and an empty body). pub force_status: AtomicU32, + /// When non-zero, send bodies at about this many bytes per second. + pub throttle_bps: AtomicU64, + /// When non-zero, stop this many milliseconds after the headers and + /// half the body (a stalled connection). + pub stall_ms: AtomicU64, /// Answer ranges with a `Content-Range` one byte off. pub wrong_range: AtomicBool, /// Path → `Location`: answered `302 Found` (counted as a request). @@ -384,13 +389,32 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { } else { body }; - let mut response = head.into_bytes(); - response.extend_from_slice(body); // Counted before the client can have the bytes, so a test that // resets the counters after a read never sees them arrive late. s.bytes.fetch_add(body.len() as u64, Ordering::SeqCst); - out.write_all(&response)?; - out.flush()?; + let bps = s.throttle_bps.load(Ordering::SeqCst); + let stall = s.stall_ms.load(Ordering::SeqCst); + if bps > 0 || stall > 0 { + out.write_all(head.as_bytes())?; + let (first, rest) = body.split_at(if stall > 0 { body.len() / 2 } else { 0 }); + out.write_all(first)?; + out.flush()?; + if stall > 0 { + std::thread::sleep(Duration::from_millis(stall)); + } + for piece in rest.chunks(4096) { + out.write_all(piece)?; + out.flush()?; + if bps > 0 { + std::thread::sleep(Duration::from_micros(4096 * 1_000_000 / bps)); + } + } + } else { + let mut response = head.into_bytes(); + response.extend_from_slice(body); + out.write_all(&response)?; + out.flush()?; + } if truncate || close { let _ = out.shutdown(std::net::Shutdown::Both); return Ok(()); diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 679f977..2bf6da1 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -800,3 +800,37 @@ fn a_200_covering_the_requested_range_is_the_whole_file() { let e = HttpStorage::open(&server.url("/t.h5"), opts).unwrap_err(); assert!(matches!(e, RemoteError::RangeNotSupported(_)), "{e}"); } + +/// A slow link is not cut off: the body's time budget grows with its size +/// (`timeout` + size at `min_speed`), so a 256 KiB block at 256 KiB/s +/// (1 s) reads with a 300 ms `timeout`. A stalled body still fails, soon. +#[test] +fn slow_links_read_and_stalled_ones_fail() { + let bytes = multi_block_file(); + let server = Server::start(vec![("/m.h5".into(), bytes.clone())]); + let url = server.url("/m.h5"); + let mut opts = quick(); + opts.cache.block_size = 256 << 10; + opts.cache.max_request = 256 << 10; + opts.http.timeout = Duration::from_millis(300); + server + .shared + .throttle_bps + .store(256 << 10, Ordering::SeqCst); + let f = open_url_with(&url, &opts).unwrap(); + assert_eq!(f.root().groups().unwrap(), ["grp"]); + assert_eq!( + f.dataset("grp/small").unwrap().read_f64().unwrap(), + [1.0, 2.0, 3.0] + ); + server.shared.throttle_bps.store(0, Ordering::SeqCst); + + // Stalled mid-body for 20 s: a timeout well before that. + server.shared.stall_ms.store(20_000, Ordering::SeqCst); + opts.http.retries = 0; + opts.http.min_speed = 64 << 20; + let t = std::time::Instant::now(); + let e = open_url_with(&url, &opts).unwrap_err(); + assert!(matches!(e, Error::Remote(RemoteError::Transport(_))), "{e}"); + assert!(t.elapsed() < Duration::from_secs(5), "{:?}", t.elapsed()); +} -- 2.54.0 From b086dc3c2bba49fbfa562c8f4ce0ded537a454a5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:34:31 -0500 Subject: [PATCH 31/49] format: read a contiguous selection's runs merged across small gaps gather_storage merged only runs that touch, so a strided selection of a contiguous dataset over a Storage became one range (and one owned Vec) per element: a stride-2 read of 32M f32 through File::open_storage made 16,777,232 read_at calls, took 2.0 s and peaked at 2.09 GB. The selection is now walked twice. The first walk checks the runs and plans spans: runs in increasing order at most 4 KiB apart (GATHER_GAP_BYTES) are read as one span up to 8 MiB (GATHER_SPAN_BYTES; a longer run is split), so nothing is stored per run. The spans are fetched in RAW_BATCH_BYTES batches while the second walk copies each run out of its span. Same checks and errors as before. The same read is now 32 reads and 0.31 s (File::open: 0.08 s). contiguous_read_interop: every h5py-checked selection is also read through File::open_storage and must give libhdf5's bytes; a new test bounds the range reads of strided, blocked, column and point selections (stride 2: at most 1 data read; 563,200 before). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +- crates/clawhdf5-format/src/gather.rs | 283 +++++++++++++----- crates/clawhdf5-format/src/storage.rs | 2 +- .../clawhdf5/tests/contiguous_read_interop.rs | 85 ++++++ 4 files changed, 294 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21e857d..3c6a7b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,10 @@ `chunks x extent` bytes (`tests/raw_fetch_bounds.rs`). Chunks the file's chunk cache already holds are not fetched. A selection fetches only the chunks its bounding box overlaps; a contiguous selection only - its runs (adjacent ones merged). A global-heap collection is read once + its runs, merged into reads of up to 8 MiB across gaps of up to 4 KiB + (a stride-2 selection of 32M `f32` is 32 reads and 0.3 s over a + `CountingStorage`, where one read per element was 16.8M reads, 2.0 s + and 2.1 GB peak). A global-heap collection is read once per resolver and kept (within the resolver's 32 MiB budget). - Each extent's bounds error is the one the slice readers gave, reported when the read reaches that extent, so a damaged file fails with the diff --git a/crates/clawhdf5-format/src/gather.rs b/crates/clawhdf5-format/src/gather.rs index 2b8a4ab..08480f1 100644 --- a/crates/clawhdf5-format/src/gather.rs +++ b/crates/clawhdf5-format/src/gather.rs @@ -13,7 +13,7 @@ use alloc::{vec, vec::Vec}; use crate::data_read::NativeElement; use crate::error::FormatError; use crate::selection::Selection; -use crate::storage::Storage; +use crate::storage::{ExtentBytes, ExtentReq, Storage, raw_batches}; /// Row-major element strides of `dims` (the last dimension has stride 1). fn strides(dims: &[u64]) -> Vec { @@ -262,12 +262,88 @@ pub(crate) fn gather( Ok(out) } +/// Largest gap between two of a selection's runs that [`gather_storage`] +/// reads through rather than asking for the runs separately: skipping a +/// few KiB costs a remote backend far less than another request (and a +/// local one less than another call and allocation). +pub(crate) const GATHER_GAP_BYTES: usize = 4 << 10; + +/// Largest single read [`gather_storage`] makes of a selection's runs: runs +/// are merged into reads up to this size, and a longer run is split. +pub(crate) const GATHER_SPAN_BYTES: usize = 8 << 20; + +/// Call `emit(first_element, element_count)` for each run of a validated +/// hyperslab or point selection (in output order; see [`hyperslab_runs`]), +/// or the error for a hyperslab of the wrong rank or a point outside `dims` +/// (runs before that point have been emitted). +fn selection_runs( + dims: &[u64], + selection: &Selection, + emit: &mut dyn FnMut(u64, u64), +) -> Result<(), FormatError> { + match selection { + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let rank = dims.len(); + if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { + return Err(FormatError::SelectionOutOfBounds( + "hyperslab rank does not match dataset rank".into(), + )); + } + hyperslab_runs(dims, start, stride, count, block, emit); + } + Selection::Points(points) => { + let strides = strides(dims); + let mut coalesce = Coalesce { + start: 0, + len: 0, + emit, + }; + for p in points { + if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) { + return Err(FormatError::SelectionOutOfBounds( + "selection addresses elements outside the dataset".into(), + )); + } + let at = p + .iter() + .zip(&strides) + .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s))); + coalesce.push(at, 1); + } + coalesce.flush(); + } + Selection::None | Selection::All => {} + } + Ok(()) +} + +/// One read of [`gather_storage`]: bytes `[start, end)` of the dataset, +/// which hold the output's bytes up to `out_end` (from where the previous +/// span's end left off). +#[derive(Clone, Copy)] +struct Span { + start: usize, + end: usize, + out_end: usize, +} + /// [`gather`] of bytes (`T = u8`) from a dataset that is not in memory: the /// dataset's `src_len` bytes start at `base` in `file`, which must hold all -/// of them (the caller checks). The selection's runs are collected first, -/// adjacent ones merged, and fetched with one [`Storage::read_ranges`] call, -/// so only the selected bytes are read. Same checks and errors as -/// [`gather`]. +/// of them (the caller checks). Same checks and errors as [`gather`]. +/// +/// The selection's runs are walked twice. The first walk checks them and +/// plans the reads: runs in increasing order with at most +/// [`GATHER_GAP_BYTES`] between them are read as one span (the gap is read +/// and dropped), up to [`GATHER_SPAN_BYTES`] per span. So a strided +/// selection is a few large reads, not one per element, and nothing is +/// allocated per run. The spans are fetched batch by batch (one +/// [`Storage::read_ranges`] call per [`crate::storage::RAW_BATCH_BYTES`]) +/// while the second walk copies each run out of its span. pub(crate) fn gather_storage( file: &S, base: u64, @@ -297,11 +373,15 @@ pub(crate) fn gather_storage( } }; let out_bytes = crate::chunked_read::checked_byte_len(n_elements, elem_size)?; - // Runs as (byte offset in the dataset, byte length), in output order. - let mut runs: Vec<(usize, usize)> = Vec::new(); + let outside = || { + FormatError::SelectionOutOfBounds("selection addresses elements outside the dataset".into()) + }; + + // First walk: check every run and plan the spans. + let mut spans: Vec = Vec::new(); let mut total = 0usize; let mut failed = false; - let mut collect = |first: u64, n: u64| { + selection_runs(dims, selection, &mut |first: u64, n: u64| { if failed { return; } @@ -314,78 +394,123 @@ pub(crate) fn gather_storage( .and_then(|n| n.checked_mul(elem_size)), ) .and_then(|(at, len)| Some((at, len, at.checked_add(len)?))); - match range { - Some((at, len, end)) if end <= src_len && total + len <= out_bytes => { - match runs.last_mut() { - Some((a, l)) if *a + *l == at => *l += len, - _ => runs.push((at, len)), - } - total += len; - } - _ => failed = true, - } - }; - let mut bad_point = false; - match selection { - Selection::Hyperslab { - start, - stride, - count, - block, - } => { - let rank = dims.len(); - if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { - return Err(FormatError::SelectionOutOfBounds( - "hyperslab rank does not match dataset rank".into(), - )); - } - hyperslab_runs(dims, start, stride, count, block, &mut collect); - } - Selection::Points(points) => { - let strides = strides(dims); - let mut coalesce = Coalesce { - start: 0, - len: 0, - emit: &mut collect, - }; - for p in points { - if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) { - bad_point = true; - break; - } - let at = p - .iter() - .zip(&strides) - .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s))); - coalesce.push(at, 1); - } - coalesce.flush(); - } - Selection::None | Selection::All => {} - } - if failed || bad_point || total != out_bytes { - return Err(FormatError::SelectionOutOfBounds( - "selection addresses elements outside the dataset".into(), - )); - } - let ranges: Vec> = runs - .iter() - .map(|&(at, len)| base + at as u64..base + (at + len) as u64) - .collect(); - let fetched = file.read_ranges(&ranges)?; - if fetched.len() != ranges.len() { - return Err(FormatError::Storage( - "read_ranges returned the wrong number of ranges".into(), - )); - } - let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes); - for (bytes, &(_, len)) in fetched.iter().zip(&runs) { - let Some(b) = bytes.get(..len) else { - return Err(FormatError::Storage( - "short read inside the file (the storage shrank or the backend failed)".into(), - )); + let Some((mut at, mut len)) = range + .filter(|&(_, len, end)| end <= src_len && len <= out_bytes - total) + .map(|(at, len, _)| (at, len)) + else { + failed = true; + return; }; - out.extend_from_slice(b); + while len > 0 { + let room = match spans.last_mut() { + Some(s) + if at >= s.end + && at - s.end <= GATHER_GAP_BYTES + && at - s.start < GATHER_SPAN_BYTES => + { + let take = len.min(GATHER_SPAN_BYTES - (at - s.start)); + s.end = at + take; + s.out_end += take; + take + } + _ => { + let take = len.min(GATHER_SPAN_BYTES); + spans.push(Span { + start: at, + end: at + take, + out_end: total + take, + }); + take + } + }; + total += room; + at += room; + len -= room; + } + })?; + if failed || total != out_bytes { + return Err(outside()); + } + + // The spans' reads, and the batches they are fetched in. + let reqs: Vec = spans + .iter() + .map(|s| ExtentReq { + addr: base + s.start as u64, + len: s.end - s.start, + fetch: Some(s.end - s.start), + }) + .collect(); + let batches = raw_batches(reqs.len(), false, |i| reqs[i].len); + + // Second walk: copy each run out of its span, fetching each batch of + // spans when the walk reaches it (and dropping the previous one). + let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes); + let mut span = 0usize; + let mut batch = 0usize; + let mut fetched: Option> = None; + let mut error: Option = None; + selection_runs(dims, selection, &mut |first: u64, n: u64| { + if error.is_some() { + return; + } + // Checked by the first walk. + let mut at = first as usize * elem_size; + let mut len = n as usize * elem_size; + while len > 0 { + while spans.get(span).is_some_and(|s| s.out_end <= out.len()) { + span += 1; + } + if fetched.is_none() || span >= batches[batch].end { + fetched = None; + while batches.get(batch).is_some_and(|b| span >= b.end) { + batch += 1; + } + let (Some(b), Some(_)) = (batches.get(batch).cloned(), spans.get(span)) else { + // The second walk emitted more than the first. + error = Some(outside()); + return; + }; + match ExtentBytes::fetch(file, &reqs[b.clone()], b.start) { + Ok(f) => fetched = Some(f), + Err(e) => { + error = Some(e); + return; + } + } + } + let s = spans[span]; + let take = len.min(s.out_end - out.len()); + let bytes = match fetched + .as_ref() + .map(|f| f.get(span, &reqs[span])) + .unwrap_or_else(|| Err(outside())) + { + Ok(b) => b, + Err(e) => { + error = Some(e); + return; + } + }; + match at + .checked_sub(s.start) + .and_then(|o| bytes.get(o..o.checked_add(take)?)) + { + Some(b) => out.extend_from_slice(b), + None => { + error = Some(outside()); + return; + } + } + at += take; + len -= take; + } + })?; + if let Some(e) = error { + return Err(e); + } + if out.len() != out_bytes { + return Err(outside()); } Ok(out) } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 2b4b9f1..7425296 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -420,7 +420,7 @@ pub(crate) enum Extent<'a> { impl<'a> ExtentBytes<'a> { /// Fetch `reqs`, extents `base..base + reqs.len()` of the read: the /// bytes of those wanted, and the bounds check of all of them. - fn fetch( + pub(crate) fn fetch( file: &'a S, reqs: &[ExtentReq], base: usize, diff --git a/crates/clawhdf5/tests/contiguous_read_interop.rs b/crates/clawhdf5/tests/contiguous_read_interop.rs index 188856e..ed4d4fa 100644 --- a/crates/clawhdf5/tests/contiguous_read_interop.rs +++ b/crates/clawhdf5/tests/contiguous_read_interop.rs @@ -12,9 +12,11 @@ use std::path::Path; use std::process::Command; +use std::sync::Arc; use clawhdf5::File; use clawhdf5_format::selection::Selection; +use clawhdf5_format::storage::CountingStorage; fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) @@ -364,9 +366,31 @@ with h5py.File("{path}", "r") as f: )); let file = File::open(&path).unwrap(); + // The same file over a storage that serves only range reads: the + // selections read only their runs (see `strided_selections_over_storage_ + // read_few_ranges`) and must give the same bytes. + let remote = File::open_storage(Arc::new(CountingStorage::new( + std::fs::read(&path).unwrap(), + ))) + .unwrap(); for (k, (name, code, sel)) in cases.iter().enumerate() { let ds = file.dataset(name).unwrap(); let want_bytes = std::fs::read(dir.path().join(format!("sel_{k}.bin"))).unwrap(); + let rds = remote.dataset(name).unwrap(); + assert!( + rds.read_selection(sel).unwrap() == want_bytes, + "{name} {sel:?}: bytes over a range storage differ from libhdf5's" + ); + assert_eq!( + rds.read_f64_selection(sel).unwrap(), + ds.read_f64_selection(sel).unwrap(), + "{name} {sel:?}: f64 over a range storage" + ); + assert_eq!( + rds.read_i32_selection(sel).unwrap(), + ds.read_i32_selection(sel).unwrap(), + "{name} {sel:?}: i32 over a range storage" + ); let got_bytes = ds.read_selection(sel).unwrap(); assert!( got_bytes == want_bytes, @@ -401,3 +425,64 @@ with h5py.File("{path}", "r") as f: ); } } + +/// Over a storage without the file in memory, a selection of a contiguous +/// dataset reads its runs merged across small gaps: a strided selection is +/// a few large reads, not one per element (563 200 for the stride-2 case +/// before), and the values are libhdf5's. +#[test] +fn strided_selections_over_storage_read_few_ranges() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("contig.h5"); + write_file(&path); + let storage = Arc::new(CountingStorage::new(std::fs::read(&path).unwrap())); + let remote = File::open_storage(storage.clone()).unwrap(); + let local = File::open(&path).unwrap(); + // f4le_big is 1100 x 1024 f32: 4 KiB rows, 4.4 MB in all. + let name = "f4le_big"; + let every_100th: Vec> = (0..1100u64 * 1024) + .step_by(100) + .map(|i| vec![i / 1024, i % 1024]) + .collect(); + let backwards: Vec> = every_100th.iter().rev().take(50).cloned().collect(); + // (selection, most range reads it may take) + let cases: Vec<(Selection, u64)> = vec![ + // Every other element of every row: one read of the whole dataset. + (slab(&[(0, 1, 1100, 1), (0, 2, 512, 1)]), 1), + // Blocks of 3 every 7 on every other row: rows are 4 KiB apart, so + // one read per selected row at most. + (slab(&[(0, 2, 550, 1), (1, 7, 146, 3)]), 550), + // Every 100th element, in order: 400-byte gaps, one read. + (Selection::Points(every_100th), 1), + // Points going backwards are not merged. + (Selection::Points(backwards), 50), + // A column: 4 KiB apart, merged. + (slab(&[(0, 1, 1100, 1), (5, 1, 1, 1)]), 1), + ]; + let ds = remote.dataset(name).unwrap(); + for (sel, most) in cases { + storage.reset(); + let got = ds.read_f32_selection(&sel).unwrap(); + let reads = storage.reads(); + assert_eq!( + got, + local + .dataset(name) + .unwrap() + .read_f32_selection(&sel) + .unwrap(), + "{sel:?}" + ); + // A few reads of metadata besides the data. + assert!(reads <= most + 8, "{sel:?}: {reads} range reads"); + storage.reset(); + let bytes = ds.read_selection(&sel).unwrap(); + assert!( + storage.reads() <= most + 8, + "{sel:?}: {} reads", + storage.reads() + ); + assert_eq!(bytes.len(), got.len() * 4); + } +} -- 2.54.0 From c5b2afbc35598a3cbc9ddf8744ce8bfe37ffaf13 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:35:36 -0500 Subject: [PATCH 32/49] clawhdf5-remote: ObjectStoreStorage works from any thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It refused whenever Handle::try_current() was Ok, which is also the case inside spawn_blocking threads — so the workaround its own error message recommended failed the same way, and the backend could only be used from a bare std::thread in a tokio application. Reads are now spawned on the storage's own runtime and the caller waits on a channel: the future never runs on the caller's thread, so neither a spawn_blocking thread nor a current-thread runtime can deadlock or panic (a read inside a runtime blocks that thread, like any blocking call; the docs still recommend spawn_blocking there). Tests: a read in spawn_blocking of a multi-thread runtime and a read inside a current-thread runtime's task give File::open's values (both errors before). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 +-- crates/clawhdf5-remote/README.md | 10 ++--- crates/clawhdf5-remote/src/object.rs | 35 ++++++++------- crates/clawhdf5-remote/tests/object_store.rs | 45 +++++++++++++++++++- docs/design/range-reads.md | 7 +-- 5 files changed, 75 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69941c3..344afbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,9 +32,10 @@ backoff; bodies are asked for with `Accept-Encoding: identity` and an encoded one is refused. The ranges of one call are fetched in parallel. - **`ObjectStoreStorage`** reads one object of any `object_store` store, - pinned by ETag (else version or modification time) and size. It blocks - on a small tokio runtime it owns; called from inside another runtime it - refuses (`RemoteError::Usage`) instead of blocking a worker. + pinned by ETag (else version or modification time) and size. Each read + runs on a small tokio runtime the storage owns while the caller waits, + so it works from any thread, `spawn_blocking` and other runtimes + included. `open_object(store, path, options)` opens a file through a block cache. - Counted on the conformance corpus (tank, 2026-09-26, `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus CLAWHDF5_REMOTE_REPORT=1 diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md index b4a487d..c64c7c6 100644 --- a/crates/clawhdf5-remote/README.md +++ b/crates/clawhdf5-remote/README.md @@ -62,11 +62,11 @@ checks both). ## Object stores `object_store` is async; `Storage` is synchronous (parsing is CPU work). -`ObjectStoreStorage` owns a small tokio runtime (two worker threads) and -blocks the calling thread on it for each read, so it is read from ordinary -threads, several at once. From inside an async runtime it refuses -(`RemoteError::Usage`) rather than block a worker: read in -`tokio::task::spawn_blocking`. The object is pinned by its ETag +`ObjectStoreStorage` owns a small tokio runtime (two worker threads): each +read runs there while the calling thread waits, so it works from any +thread, several at once — including `tokio::task::spawn_blocking` and code +inside another runtime (where `spawn_blocking` is still the better place, +since a read blocks the thread it is called on). The object is pinned by its ETag (`If-Match`, and compared on every response), else its version or modification time, and its size. The ranges of one read are fetched concurrently (up to 8). diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index c8c5664..ecfb7ac 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -3,12 +3,11 @@ //! //! `object_store` is async and [`Storage`] is synchronous (parsing is CPU //! work; `docs/design/range-reads.md` §3 (a)). The storage owns a small -//! multi-threaded tokio runtime (two worker threads) and blocks the calling -//! thread on it for each read, so it can be used from ordinary threads — -//! several at once. Calling it from inside another tokio runtime would -//! block that runtime's worker, so it refuses with -//! [`RemoteError::Usage`]: from async code, read in -//! `tokio::task::spawn_blocking`. +//! multi-threaded tokio runtime (two worker threads): each read is spawned +//! on it and the calling thread waits for the result, so it can be used +//! from any thread — several at once, and from async code too. From async +//! code prefer `tokio::task::spawn_blocking` (a read blocks the thread it +//! is called on, which inside a runtime is one of its workers). //! //! The object is pinned when the storage is made: its size, and its ETag //! (sent as `If-Match` with every read, and compared with every response) @@ -105,19 +104,23 @@ impl ObjectStoreStorage { ) } - fn block_on( + /// Run `fut` on the storage's own runtime and wait for it. The future + /// never runs on the caller's thread, so the caller's context does not + /// matter: a plain thread, `spawn_blocking`, or even inside another + /// runtime (whose thread is then blocked for the duration of the read, + /// as by any blocking call, but nothing deadlocks or panics). + fn block_on( &self, - fut: impl std::future::Future>, + fut: impl std::future::Future> + Send + 'static, ) -> Result { - if tokio::runtime::Handle::try_current().is_ok() { - return Err(RemoteError::Usage( - "ObjectStoreStorage blocks on its own runtime and cannot be read from inside \ - an async runtime; read in tokio::task::spawn_blocking" - .into(), - )); - } let rt = self.runtime.as_ref().expect("runtime lives until drop"); - rt.block_on(fut) + let (tx, rx) = std::sync::mpsc::sync_channel(1); + rt.spawn(async move { + let _ = tx.send(fut.await); + }); + rx.recv().map_err(|_| { + RemoteError::ObjectStore("the object store task ended without a result".into()) + })? } fn options(&self, range: Range) -> GetOptions { diff --git a/crates/clawhdf5-remote/tests/object_store.rs b/crates/clawhdf5-remote/tests/object_store.rs index 0d2f5fa..a3208ba 100644 --- a/crates/clawhdf5-remote/tests/object_store.rs +++ b/crates/clawhdf5-remote/tests/object_store.rs @@ -109,10 +109,51 @@ fn missing_objects_and_async_callers_are_clean_errors() { assert!(ObjectStoreStorage::new(store.clone(), ObjectPath::from("nope.h5")).is_err()); put(store.as_ref(), "m.h5", multi_block_file()); let storage = ObjectStoreStorage::new(store, ObjectPath::from("m.h5")).unwrap(); - // From inside a runtime: refused, not a panic or a deadlock. + // From inside a current-thread runtime: works (the read runs on the + // storage's own runtime), no panic, no deadlock. let rt = tokio_rt(); let r = rt.block_on(async { storage.read_at(0, 10).map(|b| b.len()) }); - assert!(r.unwrap_err().to_string().contains("spawn_blocking")); + assert_eq!(r.unwrap(), 10); // Dropping the storage inside a runtime does not panic. rt.block_on(async move { drop(storage) }); } + +/// The advice for async callers works: a read in `spawn_blocking` of a +/// multi-threaded runtime (where `Handle::try_current` is Ok), and a read +/// straight inside a current-thread runtime's task. +#[test] +fn object_stores_read_from_spawn_blocking_and_inside_runtimes() { + let store: Arc = Arc::new(InMemory::new()); + let bytes = multi_block_file(); + put(store.as_ref(), "m.h5", bytes.clone()); + let want = File::from_bytes(bytes) + .unwrap() + .dataset("big") + .unwrap() + .read_f64() + .unwrap(); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .build() + .unwrap(); + let (s, w) = (store.clone(), want.clone()); + let got = rt + .block_on(async move { + tokio::task::spawn_blocking(move || { + assert!(tokio::runtime::Handle::try_current().is_ok()); + let (f, _) = + open_object(s, "m.h5", &Options::default()).map_err(|e| e.to_string())?; + let v = f.dataset("big").unwrap().read_f64().unwrap(); + Ok::<_, String>(v == w) + }) + .await + }) + .unwrap(); + assert_eq!(got, Ok(true)); + let rt = tokio_rt(); + let got = rt.block_on(async { + let (f, _) = open_object(store, "m.h5", &Options::default()).unwrap(); + f.dataset("big").unwrap().read_f64().unwrap() + }); + assert_eq!(got, want); +} diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 2c7d1a4..123dc3b 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -471,9 +471,10 @@ fast path within benchmark noise. with ring. S3/GCS/Azure go through `object_store` (`ObjectStoreStorage`, features `s3`/`gcs`/`azure`, opt-in because of aws-lc-rs); the `object-store` feature alone (in-memory, local files, a store you - build) is pure Rust. object_store is async: the storage blocks on a - two-thread tokio runtime of its own, and refuses to run inside another - runtime. + build) is pure Rust. object_store is async: each read runs on a + two-thread tokio runtime of the storage's own while the caller waits, + so the caller's context (a plain thread, `spawn_blocking`, another + runtime) does not matter. - `BlockCache` (any `Storage`): 1 MiB blocks and a 64 MiB LRU budget by default, the first block fetched at open (for HTTP by the request that learns the length), the missing blocks of one read fetched as -- 2.54.0 From ef480746daa4e6876caf8e0f2113d0123439dc43 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:36:35 -0500 Subject: [PATCH 33/49] clawhdf5-remote: a failed first fetch is Error::Remote cached() mapped an error of its open-time prefetch (a network error, a changed file) to Error::Hdf5(Format(Storage)), misclassifying it for callers that match on the variant. It is now Error::Remote (RemoteError::Backend with the backend's message). open_object and the s3/gs/az URLs fetch the first block of an ObjectStoreStorage directly, so their errors keep their kind (FileChanged, ObjectStore). Test: cached() over a backend whose reads fail gives Error::Remote(Backend) (Error::Hdf5 before). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/lib.rs | 24 +++++++++++++++++++----- crates/clawhdf5-remote/src/object.rs | 5 +++++ crates/clawhdf5-remote/tests/http.rs | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5-remote/src/lib.rs b/crates/clawhdf5-remote/src/lib.rs index cedbfe9..2835440 100644 --- a/crates/clawhdf5-remote/src/lib.rs +++ b/crates/clawhdf5-remote/src/lib.rs @@ -126,7 +126,7 @@ fn http_storage(url: &str, _options: &Options) -> Result, Err fn cloud_storage(url: &str, _scheme: &str, options: &Options) -> Result, Error> { let (store, path) = object::store_for_url(url)?; let storage = ObjectStoreStorage::new(store, path)?; - Ok(Arc::new(cached(Box::new(storage), options)?)) + Ok(Arc::new(object_cached(storage, options)?)) } #[cfg(not(any(feature = "s3", feature = "gcs", feature = "azure")))] @@ -144,13 +144,27 @@ fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result Result { let cache = BlockCache::new(backend, options.cache.clone()); let first = cache.config().block_size; cache .prefetch(0, first) - .map_err(|e| Error::Hdf5(clawhdf5::Error::Format(e)))?; + .map_err(|e| RemoteError::Backend(e.to_string()))?; + Ok(cache) +} + +/// [`cached`] for an [`ObjectStoreStorage`]: its first block is fetched +/// directly, so a failure keeps its kind (`FileChanged`, `ObjectStore`). +#[cfg(feature = "object-store")] +fn object_cached(storage: ObjectStoreStorage, options: &Options) -> Result { + // The block size BlockCache::new will use. + let block = options.cache.block_size.max(512); + let first = storage.fetch_first(block)?; + let cache = BlockCache::new(Box::new(storage) as Backend, options.cache.clone()); + cache.insert(0, &first); Ok(cache) } @@ -205,8 +219,8 @@ pub fn open_object( ) -> Result<(File, Arc), Error> { let path = object_store::path::Path::parse(path) .map_err(|e| RemoteError::InvalidUrl(format!("{path}: {e}")))?; - let storage = Arc::new(cached( - Box::new(ObjectStoreStorage::new(store, path)?), + let storage = Arc::new(object_cached( + ObjectStoreStorage::new(store, path)?, options, )?); let file = File::open_storage(storage.clone())?; diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index ecfb7ac..c0b7748 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -138,6 +138,11 @@ impl ObjectStoreStorage { o } + /// The object's first `n` bytes (fewer if it is shorter). + pub(crate) fn fetch_first(&self, n: u64) -> Result, RemoteError> { + Ok(self.fetch_all(&[0..n])?.pop().unwrap_or_default()) + } + fn fetch_all(&self, ranges: &[Range]) -> Result>, RemoteError> { let len = self.meta.size; let jobs: Vec<(usize, Range)> = ranges diff --git a/crates/clawhdf5-remote/tests/http.rs b/crates/clawhdf5-remote/tests/http.rs index 2bf6da1..a63fb72 100644 --- a/crates/clawhdf5-remote/tests/http.rs +++ b/crates/clawhdf5-remote/tests/http.rs @@ -834,3 +834,27 @@ fn slow_links_read_and_stalled_ones_fail() { assert!(matches!(e, Error::Remote(RemoteError::Transport(_))), "{e}"); assert!(t.elapsed() < Duration::from_secs(5), "{:?}", t.elapsed()); } + +/// `cached` reports a failed first fetch as a remote error, not as an +/// HDF5 format error. +#[test] +fn cached_reports_a_failed_prefetch_as_remote() { + use clawhdf5_format::error::FormatError; + use std::borrow::Cow; + struct Down; + impl clawhdf5_format::storage::Storage for Down { + fn read_at(&self, _: u64, _: usize) -> Result, FormatError> { + Err(RemoteError::Transport("connection refused".into()).into()) + } + fn len(&self) -> u64 { + 1 << 20 + } + } + let e = clawhdf5_remote::cached(Box::new(Down), &Options::default()) + .map(|_| ()) + .unwrap_err(); + assert!( + matches!(&e, Error::Remote(RemoteError::Backend(m)) if m.contains("connection refused")), + "{e:?}" + ); +} -- 2.54.0 From efb88f94e31198c7359d37097b2e90e6c5772188 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:36:35 -0500 Subject: [PATCH 34/49] h5rs: check URL opens the remote file once open_arg_whole opened and parsed the remote file through open_arg, then opened it again to download it, so every `h5rs check URL` probed the server twice. It now opens the storage once and downloads through the same block cache (whose first block the probe already filled). Test: check --data of a file within one block costs exactly one request (two before). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-tools/src/h5.rs | 11 +++++++---- crates/clawhdf5-tools/tests/remote.rs | 11 +++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index c130f1f..7cba7c3 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -253,13 +253,14 @@ impl H5 { /// than `max_download` bytes is refused before anything is read: its /// length is only what the server claims. pub fn open_arg_whole(arg: &str, max_download: u64) -> Result
{ - let h5 = H5::open_arg(arg)?; - if h5.file.contiguous_bytes().is_some() { - return Ok(h5); + if !is_url(arg) { + return H5::open_arg(arg); } let name = shown(arg); #[cfg(feature = "remote")] { + // One open (one probe of the server); the download then reads + // through the same cache, the first block already in it. let storage = clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) .map_err(|e| Error::new(format!("{name}: {e}")))?; @@ -274,7 +275,9 @@ impl H5 { #[cfg(not(feature = "remote"))] { let _ = max_download; - unreachable!("open_arg refuses URLs without the remote feature") + Err(Error::new(format!( + "{name}: URLs need h5rs built with the `remote` feature" + ))) } } diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs index 0ae0b40..de39814 100644 --- a/crates/clawhdf5-tools/tests/remote.rs +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -158,3 +158,14 @@ fn credentials_in_urls_are_not_printed() { "{out}" ); } + +/// `check URL` opens the file once: for a file within the first block, +/// one request in all (it probed the server twice before). +#[test] +fn check_url_probes_the_server_once() { + let tall = Path::new(env!("CARGO_MANIFEST_DIR")).join("../clawhdf5/tests/fixtures/tall.h5"); + let server = server::Server::start(vec![("/t.h5".into(), std::fs::read(&tall).unwrap())]); + let (out, rc) = h5rs(&["check", "--data", &server.url("/t.h5")]); + assert_eq!(rc, 0, "{out}"); + assert_eq!(server.requests(), 1, "{:?}", server.log()); +} -- 2.54.0 From 0e98ffc498e702afc34636b3f43df75a5d8b9c35 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:37:05 -0500 Subject: [PATCH 35/49] docs: remote files after the adversarial review CHANGELOG, the clawhdf5-remote and h5rs READMEs and the remote-files known issues: redirect rules, scaled timeouts (min_speed), URL redaction, claimed lengths never allocated (download, --max-download), a 200 for a small file accepted, and ObjectStoreStorage from any thread. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 19 ++++++++++++++++++- crates/clawhdf5-remote/README.md | 17 +++++++++++++++++ crates/clawhdf5-tools/README.md | 3 ++- docs/known-issues.md | 20 ++++++++++++++++---- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 344afbf..a281969 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,21 @@ `408`/`429`/`5xx` and cut-off bodies are retried with exponential backoff; bodies are asked for with `Accept-Encoding: identity` and an encoded one is refused. The ranges of one call are fetched in parallel. + A `200` answer to the first request whose body fits the range asked + for is taken as the whole file (a server may answer so for a small + file). Timeouts scale with the request: `HttpOptions::timeout` (30 s) + to connect and to get the headers, plus the body's size at + `HttpOptions::min_speed` (16 KiB/s), so a slow link is not cut off. + Redirects: at most `HttpOptions::max_redirects` (5), never from `https` + to `http`, and `HttpOptions::headers` are not sent to another origin. + No error or `Debug` output contains a URL's userinfo or query values + (`redact_url`; presigned URLs carry their signature there). + - **Hostile lengths**: the length a server claims is never used to + allocate. The cache's arithmetic is checked (a length near `u64::MAX` + used to overflow), a read spanning more than the budget is fetched + piece by piece with its output growing as data arrives, and + `download(storage, max_bytes)` reads a whole file only up to a limit + (`RemoteError::TooLarge` before any request otherwise). - **`ObjectStoreStorage`** reads one object of any `object_store` store, pinned by ETag (else version or modification time) and size. Each read runs on a small tokio runtime the storage owns while the caller waits, @@ -49,7 +64,9 @@ requests; the tests hold it to at most 3. - **`h5rs` takes URLs** with the new `remote` feature (`remote-https` for `https://`): `ls`, `dump`, `stat` and `diff` read by range requests; - `check` downloads the file whole, since it validates every byte. The + `check` downloads the file whole, since it validates every byte, up to + `--max-download N` (1 GiB by default). URLs are printed without their + credentials. The tools now read through `File::storage` and the format crate's `*_in` functions; local output is unchanged. - **`File::storage()`** (facade) returns the file's bytes from the diff --git a/crates/clawhdf5-remote/README.md b/crates/clawhdf5-remote/README.md index c64c7c6..cd663b4 100644 --- a/crates/clawhdf5-remote/README.md +++ b/crates/clawhdf5-remote/README.md @@ -38,10 +38,27 @@ fetched, by `Range` requests, through a block cache. refused (`RemoteError::RangeNotSupported`) without reading the body, unless `HttpOptions::allow_full_download` is set; then the file is downloaded once and read from memory. + A `200` whose body is no longer than the range asked for is the whole + (small) file, and is accepted. - **Retries**: connection failures, timeouts, `408`/`429`/`5xx` and bodies that end early are retried with exponential backoff (3 retries, from 200 ms). Bodies are requested with `Accept-Encoding: identity`; an encoded body is refused. +- **Timeouts** scale with the request: `HttpOptions::timeout` (30 s) to + connect and to receive the headers, and for the body that plus its size + at `HttpOptions::min_speed` (16 KiB/s) — a slow link is not cut off + mid-block, a stalled connection still fails. +- **Redirects** are followed up to `HttpOptions::max_redirects` (5; 0 + refuses them), never from `https` to `http`. Once a redirect leaves the + URL's origin (scheme, host, port), `HttpOptions::headers` (API keys, + `Authorization`, cookies) are no longer sent. +- **Credentials stay out of messages**: every error and `Debug` output + shows URLs through `redact_url` — no `user:password@`, query values + replaced by `REDACTED` (a presigned S3/GCS URL's signature lives there). +- **Claimed lengths are not trusted**: nothing is allocated for the length + a server reports; a read spanning more than the cache budget is fetched + in pieces as data arrives, and `download(&storage, max_bytes)` reads a + whole file only up to a limit (`DEFAULT_MAX_DOWNLOAD`, 1 GiB). The zero-copy methods of `clawhdf5` (`read_raw_ref`, `read_*_zerocopy`, `File::as_bytes`) borrow the whole file from memory, so they are errors diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index fc120ad..24ecb9c 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -42,7 +42,8 @@ h5rs diff local.h5 http://127.0.0.1:8000/file.h5 A URL names the whole file (`FILE/OBJECT` suffixes are for local paths). `check` validates every byte, so it downloads a remote file whole first — up to `--max-download N` bytes (default 1 GiB), refusing a longer file -before reading any of it. +before reading any of it. URLs are printed without their credentials +(userinfo, query string values). The output is the local file's (`tests/remote.rs` compares every subcommand). diff --git a/docs/known-issues.md b/docs/known-issues.md index 860e1eb..7275020 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -829,11 +829,23 @@ cache, but: ETag (`W/"…"`) cannot be sent as `If-Match`, so it counts as none. - **Credentials:** HTTP takes extra headers (`HttpOptions::headers`, e.g. `Authorization`); `h5rs` has no option for them. The cloud stores read - credentials from the environment only. + credentials from the environment only. Messages and `Debug` output show + URLs through `redact_url` (no userinfo, query values `REDACTED`); + `HttpStorage::url()` returns the URL as given and must not be logged. +- **Redirects:** at most `HttpOptions::max_redirects` (5) per request, + never from `https` to `http`, and the custom headers are not sent once a + redirect leaves the URL's origin. The redirect target is not remembered: + every request of a redirected file costs its hops again. +- **Timeouts:** ureq has no idle timeout, only a total one for the body, + so the body's budget is `timeout + size / min_speed` (30 s + 16 KiB/s + by default). A connection that stalls mid-body is detected only when + that budget runs out (94 s for a 1 MiB block, 9 min for an 8 MiB run). - Each `ObjectStoreStorage` owns a tokio runtime with two worker threads. -- `h5rs check` downloads a remote file whole (it validates every byte), and - a URL cannot carry a `FILE/OBJECT` suffix; `h5rs` uses the default cache - settings. + A read called from inside another runtime blocks that runtime's thread + for its duration (it works, but `spawn_blocking` is the better place). +- `h5rs check` downloads a remote file whole (it validates every byte), up + to `--max-download` (1 GiB by default), and a URL cannot carry a + `FILE/OBJECT` suffix; `h5rs` uses the default cache settings. - The zero-copy methods and `File::as_bytes` are unavailable on a remote file (see the range-read limits above). -- 2.54.0 From 67e72b30d772bb57832186336e6587e782947a4c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:37:47 -0500 Subject: [PATCH 36/49] clawhdf5-remote: clippy clean with every feature set checked_div in the test server's throttle, a slice for the single range of fetch_first, and dead-code allowances for the redaction helpers in a build with neither http nor a cloud store. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-remote/src/error.rs | 13 ++++++++++++- crates/clawhdf5-remote/src/object.rs | 5 ++++- crates/clawhdf5-remote/tests/common/server.rs | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-remote/src/error.rs b/crates/clawhdf5-remote/src/error.rs index 6e3c551..2e56bef 100644 --- a/crates/clawhdf5-remote/src/error.rs +++ b/crates/clawhdf5-remote/src/error.rs @@ -66,12 +66,20 @@ pub fn redact_url(url: &str) -> String { /// Replaces the secret parts of one URL (its userinfo and query string, /// and the URL itself) wherever they appear in a message — such as the /// text of an error from the HTTP client. +#[cfg_attr( + not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")), + allow(dead_code) +)] #[derive(Debug, Clone)] pub(crate) struct Redactor { shown: String, secrets: Vec<(String, String)>, } +#[cfg_attr( + not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")), + allow(dead_code) +)] impl Redactor { pub(crate) fn new(url: &str) -> Redactor { let shown = redact_url(url); @@ -177,7 +185,10 @@ pub enum RemoteError { impl RemoteError { /// The error with every secret part of `r`'s URL scrubbed from its text. - #[cfg_attr(not(any(feature = "http", feature = "object-store")), allow(dead_code))] + #[cfg_attr( + not(any(feature = "http", feature = "s3", feature = "gcs", feature = "azure")), + allow(dead_code) + )] pub(crate) fn scrubbed(self, r: &Redactor) -> RemoteError { let f = |s: String| r.scrub(&s); match self { diff --git a/crates/clawhdf5-remote/src/object.rs b/crates/clawhdf5-remote/src/object.rs index c0b7748..3ac9955 100644 --- a/crates/clawhdf5-remote/src/object.rs +++ b/crates/clawhdf5-remote/src/object.rs @@ -140,7 +140,10 @@ impl ObjectStoreStorage { /// The object's first `n` bytes (fewer if it is shorter). pub(crate) fn fetch_first(&self, n: u64) -> Result, RemoteError> { - Ok(self.fetch_all(&[0..n])?.pop().unwrap_or_default()) + Ok(self + .fetch_all(std::slice::from_ref(&(0..n)))? + .pop() + .unwrap_or_default()) } fn fetch_all(&self, ranges: &[Range]) -> Result>, RemoteError> { diff --git a/crates/clawhdf5-remote/tests/common/server.rs b/crates/clawhdf5-remote/tests/common/server.rs index ec7c8ae..cc27e6f 100644 --- a/crates/clawhdf5-remote/tests/common/server.rs +++ b/crates/clawhdf5-remote/tests/common/server.rs @@ -405,8 +405,8 @@ fn serve(conn: TcpStream, s: &Shared) -> std::io::Result<()> { for piece in rest.chunks(4096) { out.write_all(piece)?; out.flush()?; - if bps > 0 { - std::thread::sleep(Duration::from_micros(4096 * 1_000_000 / bps)); + if let Some(us) = (4096 * 1_000_000u64).checked_div(bps) { + std::thread::sleep(Duration::from_micros(us)); } } } else { -- 2.54.0 From 89e7977943776598945b80f50cc0de8f3c236892 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:41:10 -0500 Subject: [PATCH 37/49] clawhdf5: parse local files through the slice, and cache the contiguous view Since M2 the facade handed the metadata parsers its FileData view, so a local file ran the parsers monomorphised for FileData, whose Storage impl worked out contiguous() (patched and overlay checks, two range conversions) on every structure read. A metadata walk of h5stat_newgrat.h5 (35,001 groups: open, entries and attrs of each) was about 4.5% slower than at 8f59b2e. - FileData works out its contiguous slice once at open (a borrow of its own heap/mapped buffer, kept as a pointer; see the SAFETY notes). - with_bytes! hands the in-memory slice to the format parsers when the file has one (header parsing, attributes, group listings and lookups, path resolution, shared messages, VL decoding), so local files run the [u8] parsers as before; storage-backed files still get FileData. Provisional A/B on tank (load 5-11), best of 30, 5 alternating rounds: walk 26.07-26.37 ms at 8f59b2e, 27.24-28.03 ms before this commit, 26.52-26.98 ms after. Caching alone did not move it (27.02-27.47 ms); the dispatch did. File::open read_f32 on 32M f32 (contiguous, chunked, gzip) stays within noise of 8f59b2e. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/src/reader.rs | 188 ++++++++++++++++++++++++---------- 1 file changed, 135 insertions(+), 53 deletions(-) diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 0b1ac71..191c614 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -64,6 +64,24 @@ impl Backing { } } +/// Evaluate `$body` with `$d` bound to the bytes the `clawhdf5_format` +/// parsers read: the in-memory slice when the file has one (a `Vec`, an +/// mmap), so local files run the parsers monomorphised for `[u8]` — the +/// slice code, as before the range-read migration — and otherwise the +/// [`FileData`] itself, whose reads go to the storage. +macro_rules! with_bytes { + ($data:expr, |$d:ident| $body:expr) => {{ + let data: &FileData = $data; + match data.contiguous() { + Some($d) => $body, + None => { + let $d = data; + $body + } + } + }}; +} + /// The file's bytes, viewed from the superblock on and up to the end of /// file the superblock records. A file may start with a user block (the /// superblock at 512, 1024, …); every HDF5 address is relative to the @@ -93,8 +111,28 @@ struct FileData { /// then); every object lookup here fails with this error, and no /// metadata is read from the file's own, possibly stale, bytes. image_error: Option, + /// [`Self::contiguous`], worked out once at open: every structure a + /// parser reads asks for it, and the patched/overlay checks and range + /// conversions behind it cost a local metadata walk a few percent. + contiguous: Option, } +/// A borrow of the HDF5 data held by a [`FileData`]'s own `backing` or +/// `patched` buffer (see [`FileData::contiguous`]). +#[derive(Clone, Copy)] +struct WholeView { + ptr: *const u8, + len: usize, +} + +// SAFETY: a `WholeView` is only a borrow of bytes owned (through `backing` +// or `patched`) by the `FileData` holding it, which is `Send + Sync`: the +// bytes are never written after open, so sharing the pointer across +// threads is sharing a `&[u8]`. +unsafe impl Send for WholeView {} +// SAFETY: as above. +unsafe impl Sync for WholeView {} + impl FileData { /// Locate the superblock and parse it. A truncated file is refused, and /// bytes past the recorded end of file are not read, as in libhdf5. @@ -127,17 +165,17 @@ impl FileData { ImageView::Patched(p) => (Some(p), None), ImageView::Unloadable(e) => (None, Some(e)), }; - Ok(( - Self { - backing, - base: base as u64, - end: end as u64, - patched, - overlay: Vec::new(), - image_error, - }, - superblock, - )) + let mut data = Self { + backing, + base: base as u64, + end: end as u64, + patched, + overlay: Vec::new(), + image_error, + contiguous: None, + }; + data.contiguous = data.find_contiguous(); + Ok((data, superblock)) } /// [`Self::new`] for a [`Storage`] backend: the same checks, through @@ -152,7 +190,11 @@ impl FileData { patched: None, overlay: Vec::new(), image_error: None, + contiguous: None, }; + // Worked out again below, once the end of file and any cache image + // are known. + data.contiguous = data.find_contiguous(); let superblock = Superblock::parse_in(&data, 0)?; data.end = base + superblock.data_end(base, file_len)?; match superblock_ext::cache_image_state_in(&data, &superblock)? { @@ -167,13 +209,37 @@ impl FileData { .collect(); } } + data.contiguous = data.find_contiguous(); Ok((data, superblock)) } /// The HDF5 data as one slice, when the file is in memory (a `Vec`, an /// mmap, or a storage that holds it all and has no cache image to lay /// over it). + #[inline] fn contiguous(&self) -> Option<&[u8]> { + // SAFETY: `find_contiguous` borrowed these bytes from `backing` or + // `patched`, which this `FileData` owns and never changes after + // open. They live on the heap or in a mapping (a `Vec`'s buffer, an + // mmap, a private copy, or a buffer inside the `Arc`'d storage), so + // they stay put when the `FileData` moves, and they live as long as + // `self`. + self.contiguous + .map(|v| unsafe { core::slice::from_raw_parts(v.ptr, v.len) }) + } + + /// [`Self::contiguous`], worked out from `backing` and `patched`. + fn find_contiguous(&self) -> Option { + let bytes = self.compute_contiguous()?; + Some(WholeView { + ptr: bytes.as_ptr(), + len: bytes.len(), + }) + } + + /// The HDF5 data as one slice, from `backing` and `patched` (see + /// [`Self::contiguous`]). + fn compute_contiguous(&self) -> Option<&[u8]> { if let Some(p) = &self.patched { return p.get(usize::try_from(self.base).ok()?..usize::try_from(self.end).ok()?); } @@ -224,6 +290,7 @@ impl FileData { } impl Storage for FileData { + #[inline] fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { if let Some(all) = self.contiguous() { return all.read_at(offset, len); @@ -261,6 +328,7 @@ impl Storage for FileData { .collect()) } + #[inline] fn as_contiguous(&self) -> Option<&[u8]> { self.contiguous() } @@ -400,8 +468,11 @@ impl File { /// /// The path uses `/` separators (e.g., `"group1/values"`). pub fn dataset(&self, path: &str) -> Result, Error> { - let data = self.data.meta()?; - let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; + let addr = with_bytes!(self.data.meta()?, |d| group_v2::resolve_path_any_in( + d, + &self.superblock, + path + ))?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(path.to_string())); @@ -447,8 +518,11 @@ impl File { /// The path uses `/` separators (e.g., `"sensors"`). /// Use `"/"` or `""` for the root group. pub fn group(&self, path: &str) -> Result, Error> { - let data = self.data.meta()?; - let addr = group_v2::resolve_path_any_in(data, &self.superblock, path)?; + let addr = with_bytes!(self.data.meta()?, |d| group_v2::resolve_path_any_in( + d, + &self.superblock, + path + ))?; Ok(Group { file: self, address: addr, @@ -560,13 +634,13 @@ impl File { /// [`AttrValue::Raw`] attribute. Variable-length strings are resolved in /// this file's global heap; see [`Dataset::read_string`] for the values. pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result, Error> { - crate::vlen::decode_strings( - &self.data, + with_bytes!(&self.data, |d| crate::vlen::decode_strings( + d, datatype, raw, self.offset_size(), self.length_size(), - ) + )) } /// Like [`decode_strings`](Self::decode_strings) for variable-length @@ -577,13 +651,13 @@ impl File { datatype: &Datatype, raw: &[u8], ) -> Result>, Error> { - crate::vlen::decode_string_bytes( - self.data.meta()?, + with_bytes!(self.data.meta()?, |d| crate::vlen::decode_string_bytes( + d, datatype, raw, self.offset_size(), self.length_size(), - ) + )) } /// Decode the variable-length sequences in `raw`, a buffer of elements @@ -595,22 +669,22 @@ impl File { datatype: &Datatype, raw: &[u8], ) -> Result>, Error> { - crate::vlen::decode_vlen( - self.data.meta()?, + with_bytes!(self.data.meta()?, |d| crate::vlen::decode_vlen( + d, datatype, raw, self.offset_size(), self.length_size(), - ) + )) } fn parse_header(&self, address: u64) -> Result { - ObjectHeader::parse_in( - self.data.meta()?, + with_bytes!(self.data.meta()?, |d| ObjectHeader::parse_in( + d, address, self.superblock.offset_size, self.superblock.length_size, - ) + )) } fn offset_size(&self) -> u8 { @@ -688,8 +762,12 @@ impl<'f> Group<'f> { &self, ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.parse_header(self.address)?; - let data = &self.file.data; - read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) + with_bytes!(&self.file.data, |d| read_attrs( + d, + &hdr, + self.file.offset_size(), + self.file.length_size() + )) } /// Get a dataset within this group by name. @@ -719,14 +797,13 @@ impl<'f> Group<'f> { /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { let hdr = self.file.parse_header(self.address)?; - let data = &self.file.data; - read_attr( - data, + with_bytes!(&self.file.data, |d| read_attr( + d, &hdr, name, self.file.offset_size(), self.file.length_size(), - ) + )) } /// The object header address of the child called `name`: the entry of @@ -734,9 +811,13 @@ impl<'f> Group<'f> { /// name index rather than by listing the group (see /// [`group_v2::resolve_child`]). fn child_address(&self, name: &str) -> Result { - let data = self.file.data.meta()?; - group_v2::resolve_child_in(data, &self.file.superblock, self.address, name) - .map_err(Error::Format) + with_bytes!(self.file.data.meta()?, |d| group_v2::resolve_child_in( + d, + &self.file.superblock, + self.address, + name + )) + .map_err(Error::Format) } /// This group's children that can be opened, as `(name, object header @@ -757,9 +838,10 @@ impl<'f> Group<'f> { /// [`group_v2::resolve_group_children`]); dangling, external and /// user-defined links are left out. fn children(&self) -> Result, Error> { - let data = self.file.data.meta()?; - group_v2::resolve_group_children_in(data, &self.file.superblock, self.address) - .map_err(Error::Format) + with_bytes!(self.file.data.meta()?, |d| { + group_v2::resolve_group_children_in(d, &self.file.superblock, self.address) + }) + .map_err(Error::Format) } } @@ -1332,13 +1414,12 @@ impl<'f> Dataset<'f> { pub fn attrs_with_errors( &self, ) -> Result<(HashMap, Vec), Error> { - let data = &self.file.data; - read_attrs( - data, + with_bytes!(&self.file.data, |d| read_attrs( + d, &self.header, self.file.offset_size(), self.file.length_size(), - ) + )) } /// The attribute called `name`, or `None` if it has none by that name @@ -1346,14 +1427,13 @@ impl<'f> Dataset<'f> { /// that name, found without reading the other attributes when they are /// stored densely. pub fn attr(&self, name: &str) -> Result, Error> { - let data = &self.file.data; - read_attr( - data, + with_bytes!(&self.file.data, |d| read_attr( + d, &self.header, name, self.file.offset_size(), self.file.length_size(), - ) + )) } /// Verify this dataset's content against its stored provenance hash @@ -1393,12 +1473,14 @@ impl<'f> Dataset<'f> { .iter() .find(|m| m.msg_type == msg_type) .map(|msg| { - clawhdf5_format::shared_message::message_data_in( - &self.file.data, - msg, - self.file.offset_size(), - self.file.length_size(), - ) + with_bytes!(&self.file.data, |d| { + clawhdf5_format::shared_message::message_data_in( + d, + msg, + self.file.offset_size(), + self.file.length_size(), + ) + }) .map_err(Error::Format) }) .transpose() -- 2.54.0 From 6185874f9caaeb7edf61688d003e8a93cc517cc7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:43:00 -0500 Subject: [PATCH 38/49] format, clawhdf5: cut every Storage read to the range asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ExtentBytes and read_exact_at/read_upto rejected short results but passed longer-than-asked ones through, and FileData forwarded them too, so a Storage that broke read_at's contract by returning extra bytes had them decoded or returned as data (a contiguous dataset read gained 37 junk bytes). gather_storage alone trimmed. - storage::exact_len (new, pub): a read of len bytes as exactly len — cut when longer, an error when short. read_exact_at, read_upto and ExtentBytes (so chunk fetches and selection gathers) go through it. - FileData cuts a backend's answer to what it asked for before laying the cache image over it. - Tests: over a storage that appends 37 junk bytes to every read, every format-crate fixture reads exactly as from the slice (overlong_reads_are_cut_to_the_range_asked_for), and every facade fixture opens and reads through File::open_storage as through File::open (overlong_storage_reads_identically). Both failed before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 + crates/clawhdf5-format/src/storage.rs | 43 ++-- .../tests/storage_equivalence.rs | 195 +++++++++++------- crates/clawhdf5/src/reader.rs | 26 ++- crates/clawhdf5/tests/storage_equivalence.rs | 47 +++++ 5 files changed, 225 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c6a7b6..bbd2eb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,10 @@ - Each extent's bounds error is the one the slice readers gave, reported when the read reaches that extent, so a damaged file fails with the same error, in the same order, through either path. + - A backend that answers a read with more bytes than asked (breaking + `read_at`'s contract) never has the extra bytes used: every read is + cut to the range asked for (`storage::exact_len`), and a short answer + inside the file is an error. - **No behaviour change for in-memory and mapped files:** with `as_contiguous()` every path slices the file as before (checked below). - Tests (2026-09-26, tank): diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 7425296..482cfad 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -43,6 +43,9 @@ pub trait Storage { /// end of the storage (and empty when `offset` is at or past the end); /// a backend that cannot serve a range returns an error instead of a /// short read. + /// It is never longer than `len`; the parsers cut a longer result to + /// `len` (see [`exact_len`]) rather than read bytes from outside the + /// range. fn read_at(&self, offset: u64, len: usize) -> Result, FormatError>; /// Current length of the storage in bytes. @@ -218,13 +221,30 @@ pub fn read_exact_at( Some(end) if end <= file.len() => {} _ => return Err(eof()), } - let bytes = file.read_at(offset, len)?; - if bytes.len() < len { - // The storage shrank or the backend served a short read inside the - // file: never parse a partial structure. - return Err(short_read()); + // A short read (the storage shrank, or the backend served less inside + // the file) is an error: never parse a partial structure. + exact_len(file.read_at(offset, len)?, len) +} + +/// `bytes`, the result of asking a [`Storage`] for `len` bytes, as exactly +/// `len` bytes: a longer result (a backend that broke +/// [`Storage::read_at`]'s contract) is cut to `len`, so bytes from outside +/// the range asked for are never parsed or returned; a shorter one is an +/// error (the storage shrank, or the backend failed), never a partial +/// structure. +#[inline] +pub fn exact_len(bytes: Cow<'_, [u8]>, len: usize) -> Result, FormatError> { + match bytes.len().cmp(&len) { + core::cmp::Ordering::Equal => Ok(bytes), + core::cmp::Ordering::Less => Err(short_read()), + core::cmp::Ordering::Greater => Ok(match bytes { + Cow::Borrowed(b) => Cow::Borrowed(&b[..len]), + Cow::Owned(mut v) => { + v.truncate(len); + Cow::Owned(v) + } + }), } - Ok(bytes) } #[cold] @@ -329,11 +349,7 @@ pub fn read_upto( } let avail = file.len().saturating_sub(offset); let len = usize::try_from(avail).map_or(max, |a| a.min(max)); - let bytes = file.read_at(offset, len)?; - if bytes.len() < len { - return Err(short_read()); - } - Ok(bytes) + exact_len(file.read_at(offset, len)?, len) } /// Most stored bytes fetched by one [`Storage::read_ranges`] call when a @@ -461,10 +477,7 @@ impl<'a> ExtentBytes<'a> { )); } for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) { - if (bytes.len() as u64) < r.end - r.start { - return Err(short_read()); - } - out[slot] = Extent::Bytes(bytes); + out[slot] = Extent::Bytes(exact_len(bytes, (r.end - r.start) as usize)?); } } Ok(ExtentBytes::Fetched { base, extents: out }) diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index 71b0dc0..a25a79a 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -753,6 +753,73 @@ fn corpus_parses_identically_through_storage() { assert!(tally.files > 0); } +/// The whole read of every object reachable from the root group of `sb`'s +/// file — each dataset whole (fill-aware) and half of it through a +/// selection, and each group's listing — as one result to compare. +fn read_everything(file: &dyn Storage, sb: &Superblock) -> Result { + let (os, ls) = (sb.offset_size, sb.length_size); + let mut out = String::new(); + let mut queue = VecDeque::from([sb.root_group_address]); + let mut seen = HashSet::new(); + while let Some(addr) = queue.pop_front() { + if seen.len() > 200 || !seen.insert(addr) { + continue; + } + let header = ObjectHeader::parse_in(file, addr, os, ls)?; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .map(|m| message_data_with_sohm_in(file, m, os, ls)) + .transpose() + }; + if let (Some(dt), Some(ds), Some(dl)) = ( + find(MessageType::Datatype)?, + find(MessageType::Dataspace)?, + find(MessageType::DataLayout)?, + ) { + let dt = Datatype::parse(&dt)?.0; + let ds = Dataspace::parse(&ds, ls)?; + let dl = DataLayout::parse(&dl, os, ls)?; + let pl = find(MessageType::FilterPipeline)? + .map(|p| FilterPipeline::parse(&p)) + .transpose()?; + let data = read_full_with_fill_in( + &header.messages, + file, + &dl, + &ds, + dt.type_size() as usize, + os, + ls, + || read_raw_data_full_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls), + ); + out.push_str(&format!("{addr}: {data:?}\n")); + if let Some(&d0) = ds.dimensions.first() { + let rank = ds.dimensions.len(); + let sel = Selection::Hyperslab { + start: vec![0; rank], + stride: vec![1; rank], + count: std::iter::once(d0.div_ceil(2)) + .chain(ds.dimensions[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + let part = + read_raw_data_selection_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls, &sel); + out.push_str(&format!("{addr} half: {part:?}\n")); + } + } + let children = group_v2::resolve_group_children_in(file, sb, addr); + out.push_str(&format!("{addr} children: {children:?}\n")); + if let Ok(c) = children { + queue.extend(c.iter().map(|c| c.object_header_address)); + } + } + Ok(out) +} + /// A storage that misbehaves: fails its `fail_at`-th read (1-based, `0` /// never), and, with `short`, serves one byte less than asked for inside /// the file (a truncated response). @@ -808,78 +875,7 @@ fn misbehaving_storage_never_returns_wrong_data() { let Ok(sb) = Superblock::parse(hdf5, 0) else { continue; }; - // The whole read of every object, as one result to compare. - let everything = |file: &dyn Storage| -> Result { - let (os, ls) = (sb.offset_size, sb.length_size); - let mut out = String::new(); - let mut queue = VecDeque::from([sb.root_group_address]); - let mut seen = HashSet::new(); - while let Some(addr) = queue.pop_front() { - if seen.len() > 200 || !seen.insert(addr) { - continue; - } - let header = ObjectHeader::parse_in(file, addr, os, ls)?; - let find = |t: MessageType| { - header - .messages - .iter() - .find(|m| m.msg_type == t) - .map(|m| message_data_with_sohm_in(file, m, os, ls)) - .transpose() - }; - if let (Some(dt), Some(ds), Some(dl)) = ( - find(MessageType::Datatype)?, - find(MessageType::Dataspace)?, - find(MessageType::DataLayout)?, - ) { - let dt = Datatype::parse(&dt)?.0; - let ds = Dataspace::parse(&ds, ls)?; - let dl = DataLayout::parse(&dl, os, ls)?; - let pl = find(MessageType::FilterPipeline)? - .map(|p| FilterPipeline::parse(&p)) - .transpose()?; - let data = read_full_with_fill_in( - &header.messages, - file, - &dl, - &ds, - dt.type_size() as usize, - os, - ls, - || read_raw_data_full_in(file, &dl, &ds, &dt, pl.as_ref(), os, ls), - ); - out.push_str(&format!("{addr}: {data:?}\n")); - if let Some(&d0) = ds.dimensions.first() { - let rank = ds.dimensions.len(); - let sel = Selection::Hyperslab { - start: vec![0; rank], - stride: vec![1; rank], - count: std::iter::once(d0.div_ceil(2)) - .chain(ds.dimensions[1..].iter().copied()) - .collect(), - block: vec![1; rank], - }; - let part = read_raw_data_selection_in( - file, - &dl, - &ds, - &dt, - pl.as_ref(), - os, - ls, - &sel, - ); - out.push_str(&format!("{addr} half: {part:?}\n")); - } - } - let children = group_v2::resolve_group_children_in(file, &sb, addr); - out.push_str(&format!("{addr} children: {children:?}\n")); - if let Ok(c) = children { - queue.extend(c.iter().map(|c| c.object_header_address)); - } - } - Ok(out) - }; + let everything = |file: &dyn Storage| read_everything(file, &sb); let want = everything(&hdf5); let counting = CountingStorage::new(hdf5.to_vec()); assert_eq!( @@ -947,6 +943,61 @@ fn misbehaving_storage_never_returns_wrong_data() { ); } +/// A storage that breaks `read_at`'s contract the other way: every read +/// comes back with 37 bytes more than asked for (junk past the range). +struct Overlong { + data: Vec, +} + +impl Storage for Overlong { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let mut v = self.data.as_slice().read_at(offset, len)?.into_owned(); + v.extend(std::iter::repeat_n(0xa5, 37)); + Ok(std::borrow::Cow::Owned(v)) + } + + fn len(&self) -> u64 { + self.data.len() as u64 + } +} + +/// Bytes past the range asked for are never used: every read through a +/// storage that returns more than asked gives exactly the in-memory result. +#[test] +fn overlong_reads_are_cut_to_the_range_asked_for() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut files = Vec::new(); + hdf5_files(&dir, &mut files); + files.sort(); + let mut compared = 0; + for path in &files { + let Ok(bytes) = std::fs::read(path) else { + continue; + }; + let Ok((_, hdf5)) = split_user_block(&bytes) else { + continue; + }; + let Ok(sb) = Superblock::parse(hdf5, 0) else { + continue; + }; + let want = read_everything(&hdf5, &sb); + let got = read_everything( + &Overlong { + data: hdf5.to_vec(), + }, + &sb, + ); + assert_eq!( + format!("{got:?}"), + format!("{want:?}"), + "{}", + path.display() + ); + compared += 1; + } + assert!(compared > 40, "{compared}"); +} + /// A read_at-only storage that also counts `read_ranges` calls and ranges. struct BatchCounting { inner: CountingStorage, diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 191c614..a99373f 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -300,7 +300,7 @@ impl Storage for FileData { if len == 0 { return Ok(Cow::Borrowed(&[])); } - let bytes = self.remote()?.read_at(self.base + offset, len)?; + let bytes = cut_to(self.remote()?.read_at(self.base + offset, len)?, len); Ok(self.with_overlay(offset, bytes)) } @@ -323,8 +323,11 @@ impl Storage for FileData { let got = self.remote()?.read_ranges(&shifted)?; Ok(got .into_iter() - .zip(ranges) - .map(|(bytes, r)| self.with_overlay(r.start, bytes)) + .zip(ranges.iter().zip(&shifted)) + .map(|(bytes, (r, asked))| { + let bytes = cut_to(bytes, (asked.end - asked.start) as usize); + self.with_overlay(r.start, bytes) + }) .collect()) } @@ -334,6 +337,23 @@ impl Storage for FileData { } } +/// `bytes`, a backend's answer to a read of `len` bytes, without anything +/// past those `len` (a backend that returns more than asked breaks +/// [`Storage::read_at`]'s contract; the extra bytes are not the file's). A +/// short answer is passed on: the parsers' own checks refuse it. +fn cut_to(bytes: Cow<'_, [u8]>, len: usize) -> Cow<'_, [u8]> { + if bytes.len() <= len { + return bytes; + } + match bytes { + Cow::Borrowed(b) => Cow::Borrowed(&b[..len]), + Cow::Owned(mut v) => { + v.truncate(len); + Cow::Owned(v) + } + } +} + // --------------------------------------------------------------------------- // File // --------------------------------------------------------------------------- diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs index 7a2749b..71768d2 100644 --- a/crates/clawhdf5/tests/storage_equivalence.rs +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -432,3 +432,50 @@ fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() { "as_bytes over a range storage must not answer" ); } + +/// A storage that returns 37 junk bytes more than every read asked for. +struct Overlong(Vec); + +impl clawhdf5::Storage for Overlong { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let mut v = clawhdf5::Storage::read_at(self.0.as_slice(), offset, len)?.into_owned(); + v.extend(std::iter::repeat_n(0xa5, 37)); + Ok(std::borrow::Cow::Owned(v)) + } + + fn len(&self) -> u64 { + self.0.len() as u64 + } +} + +/// Bytes a misbehaving storage returns past the range asked for are never +/// read as the file's: every fixture reads through it as through +/// `File::open`. +#[test] +fn overlong_storage_reads_identically() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + hdf5_files(&root.join("tests/fixtures"), &mut files); + hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files); + files.sort(); + let mut compared = 0; + for path in &files { + let Ok(bytes) = std::fs::read(path) else { + continue; + }; + let Ok(local) = File::open(path) else { + continue; + }; + let mut remote = File::open_storage(Arc::new(Overlong(bytes))) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + remote.set_vds_resolver(sibling_resolver(path.parent().map(Path::to_path_buf))); + assert_eq!( + transcript(&local), + transcript(&remote), + "{}", + path.display() + ); + compared += 1; + } + assert!(compared >= 40, "{compared}"); +} -- 2.54.0 From 159e58855033d5a6335e40fd16c967a50f7d9152 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:23:59 -0500 Subject: [PATCH 39/49] filters: Fletcher-32 as libhdf5 computes it Our checksum reduced its sums with `% 65535`; libhdf5's H5_checksum_fletcher32 folds them with `(s & 0xffff) + (s >> 16)`, which leaves 0xffff where the modulo leaves 0. On about one chunk in 32768 libhdf5 refused the chunks we wrote and we refused the chunks it wrote. Every release since v2.1.0 is affected. clawhdf5_format::checksum::fletcher32 is a port of H5_checksum_fletcher32 and the filter's only implementation. Verification also accepts the byte-swapped form libhdf5 accepts (1.6.2 and earlier) and the `% 65535` form earlier releases wrote, so their files stay readable. The new interop test compares the checksum with libhdf5's own function (ctypes) on every 1- and 2-byte input and 40 000 random and fold-heavy inputs, and moves fold-case chunks between h5py and FileBuilder/FileEditor in both directions; with the old filters.rs the three file tests fail. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 26 ++ crates/clawhdf5-accel/src/lib.rs | 5 +- crates/clawhdf5-format/src/checksum.rs | 52 ++++ crates/clawhdf5-format/src/filters.rs | 66 +--- crates/clawhdf5/tests/fletcher32_interop.rs | 321 ++++++++++++++++++++ docs/known-issues.md | 35 +++ 6 files changed, 451 insertions(+), 54 deletions(-) create mode 100644 crates/clawhdf5/tests/fletcher32_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 551b079..964390d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ ## Unreleased +### Correctness: Fletcher-32 (2026-09-26) +- **Fletcher-32 checksums disagreed with libhdf5's on about one chunk in + 32768** (fixed 2026-09-26). **Every release is affected, v2.1.0 through + v2.7.0**, both directions: `FileBuilder`/`FileWriter` (`with_fletcher32`) + and, before release, `FileEditor` wrote chunks that h5py and libhdf5 + refuse ("filter returned failure during read"), and every reader + rejected valid libhdf5-written chunks with `Fletcher32Mismatch`. Our + checksum reduced its sums with `% 65535`; libhdf5's + `H5_checksum_fletcher32` uses the ones'-complement fold + `(s & 0xffff) + (s >> 16)`, which leaves 0xffff where the modulo leaves + 0, so the two differ whenever a sum is a non-zero multiple of 65535. + `clawhdf5_format::checksum::fletcher32` (new, public) is a port of + `H5_checksum_fletcher32` and the only implementation; the filter writes + and verifies with it, and, as libhdf5 does, also accepts a stored + checksum with the bytes of each 16-bit half swapped (libhdf5 1.6.2 and + earlier) and the `% 65535` form v2.7.0 and earlier wrote, so their files + stay readable. Tests: `crates/clawhdf5/tests/fletcher32_interop.rs` compares + it with libhdf5's own function (through ctypes) on every 1- and 2-byte + input and 40 000 random and fold-heavy ones, and has h5py read + fold-case chunks written by `FileBuilder` and `FileEditor` and us read + h5py's. Files written by earlier releases read with a fixed build; to + make one readable by libhdf5, rewrite its Fletcher-32 datasets with a + fixed build (see `docs/known-issues.md`). `clawhdf5_accel::checksum_fletcher32` is a + different, textbook Fletcher-32 (sums start at 0xffff) and is not used + for HDF5. + ### In-place editing: version-2 B-tree indexes, shrinking, dense attributes (2026-09-26) - **`FileEditor` adds, moves and resizes chunks of datasets with two or more unlimited dimensions** (version-2 B-tree chunk index, record types diff --git a/crates/clawhdf5-accel/src/lib.rs b/crates/clawhdf5-accel/src/lib.rs index 81a2f03..20d52a3 100644 --- a/crates/clawhdf5-accel/src/lib.rs +++ b/crates/clawhdf5-accel/src/lib.rs @@ -237,7 +237,10 @@ pub fn f16_to_f32_batch(input: &[u16], output: &mut [f32]) { convert::f16_to_f32_batch(input, output); } -/// Compute Fletcher-32 checksum. +/// Compute a textbook Fletcher-32 checksum (both sums start at 0xffff). +/// +/// This is not HDF5's checksum; the Fletcher-32 I/O filter uses +/// `clawhdf5_format::checksum::fletcher32`. pub fn checksum_fletcher32(data: &[u8]) -> u32 { checksum::checksum_fletcher32(data) } diff --git a/crates/clawhdf5-format/src/checksum.rs b/crates/clawhdf5-format/src/checksum.rs index 525689f..363d97f 100644 --- a/crates/clawhdf5-format/src/checksum.rs +++ b/crates/clawhdf5-format/src/checksum.rs @@ -14,6 +14,45 @@ pub fn jenkins_lookup3(data: &[u8]) -> u32 { hashlittle(data, 0) } +/// HDF5's Fletcher-32 checksum, as the Fletcher-32 I/O filter (filter id 3) +/// stores it after each chunk. +/// +/// A line-for-line port of `H5_checksum_fletcher32` (H5checksum.c, libhdf5 +/// 1.8 through 1.14): big-endian 16-bit words summed in blocks of 360, each +/// sum reduced after a block by the ones'-complement fold +/// `(s & 0xffff) + (s >> 16)` rather than `% 65535`, an odd trailing byte +/// taken as the high byte of a last word, and a final fold of both sums. +/// The fold and `% 65535` differ whenever a sum is a non-zero multiple of +/// 65535: the fold leaves 0xffff where the modulo gives 0, so the two +/// disagree on about one chunk in 32768 and libhdf5 rejects the other's +/// checksum. This must stay the only implementation. +pub fn fletcher32(data: &[u8]) -> u32 { + let mut sum1: u32 = 0; + let mut sum2: u32 = 0; + // 360 words keep both sums inside 32 bits between folds (the bound + // libhdf5 uses: after a fold sum1 < 0x10200, so sum2 stays below + // 360 * 361 / 2 * 0xffff + 360 * 0x10200 + 0x1fffe < 2^32). The adds wrap + // like the C unsigned arithmetic all the same. + let (words, odd) = data.as_chunks::<2>(); + for block in words.chunks(360) { + for w in block { + sum1 = sum1.wrapping_add((u32::from(w[0]) << 8) | u32::from(w[1])); + sum2 = sum2.wrapping_add(sum1); + } + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + } + if let [last] = odd { + sum1 = sum1.wrapping_add(u32::from(*last) << 8); + sum2 = sum2.wrapping_add(sum1); + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + } + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + (sum2 << 16) | sum1 +} + /// Compute CRC32 (IEEE / ISO 3309) over data. /// /// When the `fast-checksum` feature is enabled, this uses hardware CRC32 @@ -207,6 +246,19 @@ fn hashlittle(data: &[u8], initval: u32) -> u32 { mod tests { use super::*; + /// Values of libhdf5's `H5_checksum_fletcher32` (h5py 3.x's bundled + /// libhdf5, called through ctypes). The first three are sums that are + /// multiples of 65535, where `% 65535` gave 0 instead of 0xffff. + #[test] + fn fletcher32_matches_libhdf5() { + assert_eq!(fletcher32(&[0x00, 0x01, 0xff, 0xfe]), 0x0001_ffff); + assert_eq!(fletcher32(&[0xff; 720]), 0xffff_ffff); + assert_eq!(fletcher32(&[0xff; 721]), 0xff00_ff00); + assert_eq!(fletcher32(&[0xff; 1441]), 0xff00_ff00); + assert_eq!(fletcher32(&[]), 0); + assert_eq!(fletcher32(&[7]), 0x0700_0700); + } + #[test] fn empty_input() { // Empty input should return the initial state after no mixing diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 6390aee..47402a6 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -1678,56 +1678,6 @@ fn shuffle_compress_general(data: &[u8], n: usize, element_size: usize, result: } } -/// Compute HDF5 Fletcher32 checksum over data. -/// HDF5 uses a modified Fletcher32 that operates on 16-bit words. -/// -/// Optimized with wider accumulators: processes blocks of 360 words before -/// taking the modulo, reducing the number of expensive modulo operations. -/// (360 is the maximum block size that avoids u32 overflow for sum2.) -fn fletcher32_compute(data: &[u8]) -> u32 { - let mut sum1: u32 = 0; - let mut sum2: u32 = 0; - - // Process in blocks of 360 16-bit words (720 bytes) to delay modulo. - // Max sum1 before mod: 360 * 65535 = 23_592_600 < u32::MAX - // Max sum2 before mod: 360 * 23_592_600 ~ 8.5B > u32::MAX, but actual - // sum2 accumulates incrementally, so worst case is 360*360*65535/2 which - // fits in u64. We use u32 with block size 360 which is safe. - const BLOCK_WORDS: usize = 360; - const BLOCK_BYTES: usize = BLOCK_WORDS * 2; - - let mut offset = 0; - let len = data.len(); - - while offset + BLOCK_BYTES <= len { - let end = offset + BLOCK_BYTES; - let mut i = offset; - while i < end { - let val = ((data[i] as u32) << 8) | (data[i + 1] as u32); - sum1 += val; - sum2 += sum1; - i += 2; - } - sum1 %= 65535; - sum2 %= 65535; - offset = end; - } - - // Handle remaining bytes - while offset < len { - let val = if offset + 1 < len { - ((data[offset] as u32) << 8) | (data[offset + 1] as u32) - } else { - (data[offset] as u32) << 8 - }; - sum1 = (sum1 + val) % 65535; - sum2 = (sum2 + sum1) % 65535; - offset += 2; - } - - (sum2 << 16) | sum1 -} - /// Verify Fletcher32 checksum and strip it from the data. /// The last 4 bytes are the stored checksum. fn fletcher32_verify(data: &[u8]) -> Result, FormatError> { @@ -1749,8 +1699,18 @@ fn fletcher32_payload(data: &[u8]) -> Result { data[data.len() - 2], data[data.len() - 1], ]); - let computed = fletcher32_compute(payload); - if stored != computed { + let computed = crate::checksum::fletcher32(payload); + // libhdf5 also accepts the checksum with the bytes of each 16-bit half + // swapped, which is how 1.6.2 and earlier stored it + // (H5Z__filter_fletcher32's `reversed_fletcher`). + let reversed = ((computed & 0x00ff_00ff) << 8) | ((computed >> 8) & 0x00ff_00ff); + // clawhdf5 v2.7.0 and earlier reduced the sums `% 65535`, which gives 0 + // where libhdf5's fold gives 0xffff; accept that form too, so that files + // those releases wrote can still be read (and rewritten for libhdf5). + // It differs from `computed` only in a half that is 0xffff. + let half = |h: u32| if h == 0xffff { 0 } else { h }; + let legacy = (half(computed >> 16) << 16) | half(computed & 0xffff); + if stored != computed && stored != reversed && stored != legacy { return Err(FormatError::Fletcher32Mismatch { expected: stored, computed, @@ -1761,7 +1721,7 @@ fn fletcher32_payload(data: &[u8]) -> Result { /// Append Fletcher32 checksum to data. fn fletcher32_append(data: &[u8]) -> Result, FormatError> { - let checksum = fletcher32_compute(data); + let checksum = crate::checksum::fletcher32(data); let mut result = data.to_vec(); result.extend_from_slice(&checksum.to_le_bytes()); Ok(result) diff --git a/crates/clawhdf5/tests/fletcher32_interop.rs b/crates/clawhdf5/tests/fletcher32_interop.rs new file mode 100644 index 0000000..b6bf6b0 --- /dev/null +++ b/crates/clawhdf5/tests/fletcher32_interop.rs @@ -0,0 +1,321 @@ +//! Fletcher-32 against libhdf5. +//! +//! libhdf5's `H5_checksum_fletcher32` reduces its sums with the +//! ones'-complement fold `(s & 0xffff) + (s >> 16)`, which leaves 0xffff +//! where `% 65535` leaves 0. Our checksum once used `% 65535`, so on about +//! one chunk in 32768 (a sum that is a non-zero multiple of 65535) libhdf5 +//! rejected the chunks we wrote and we rejected the chunks it wrote. +//! +//! - The checksum is compared with libhdf5's own `H5_checksum_fletcher32`, +//! called through ctypes from the library h5py loads, over every one-byte +//! and two-byte input and a large corpus of random and fold-heavy inputs. +//! - Chunks engineered to hit the fold are written by `FileBuilder` and by +//! `FileEditor` and read by h5py, and written by h5py and read by us. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{File, FileBuilder, FileEditor}; +use clawhdf5_format::checksum::fletcher32; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn have_h5py() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .is_ok_and(|o| o.status.success()); + if !ok { + assert!( + std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + } + ok +} + +fn run_python(script: &str, args: &[&str]) -> String { + let out = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "python failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn tmp(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("clawhdf5_fletcher32_{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir.join(name) +} + +/// The checksum our code computed before it was fixed: each sum reduced +/// `% 65535`. Only used to show that the test data hits the disagreement. +fn fletcher32_mod(data: &[u8]) -> u32 { + let (mut s1, mut s2) = (0u64, 0u64); + for w in data.chunks(2) { + let v = (u64::from(w[0]) << 8) | w.get(1).map_or(0, |&b| u64::from(b)); + s1 = (s1 + v) % 65535; + s2 = (s2 + s1) % 65535; + } + ((s2 as u32) << 16) | s1 as u32 +} + +/// Splitmix64, so the data is the same on every run. +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } +} + +/// Writes every case of the file `argv[1]` (u32 LE length + bytes) back to +/// `argv[2]` as libhdf5's checksum of each, u32 LE. +const LIBHDF5_CHECKSUMS: &str = r#" +import ctypes, glob, os, struct, sys +import h5py +cands = glob.glob(os.path.join(os.path.dirname(h5py.__file__), os.pardir, 'h5py.libs', 'libhdf5-*.so*')) +cands += glob.glob(os.path.join(os.path.dirname(h5py.__file__), '.dylibs', 'libhdf5*.dylib')) +if cands: + lib = ctypes.CDLL(cands[0]) +else: + # A system h5py links the system libhdf5, already loaded. + import h5py.h5 + lib = ctypes.CDLL(h5py.h5.__file__) +f = lib.H5_checksum_fletcher32 +f.restype = ctypes.c_uint32 +f.argtypes = [ctypes.c_char_p, ctypes.c_size_t] +data = open(sys.argv[1], 'rb').read() +out = bytearray() +i = 0 +while i < len(data): + (n,) = struct.unpack_from('> = Vec::new(); + // Every one-byte input (the odd-length path alone) and every one-word + // input (65535 = 0xffff is the smallest fold). + cases.extend((0..=255u8).map(|b| vec![b])); + cases.extend((0..=u16::MAX).map(|w| w.to_be_bytes().to_vec())); + let mut rng = Rng(0x5eed_f1e7); + // Words drawn from values that make multiples of 65535 frequent, at + // lengths around the 360-word block boundaries, odd and even. + const FOLDY: [u16; 6] = [0, 1, 0xfffe, 0xffff, 0x8000, 0x7fff]; + for _ in 0..40_000 { + let len = match rng.next() % 4 { + 0 => (rng.next() % 16) as usize, + 1 => 718 + (rng.next() % 6) as usize, + 2 => 1438 + (rng.next() % 6) as usize, + _ => (rng.next() % 3000) as usize, + }; + let foldy = rng.next().is_multiple_of(2); + let mut v = Vec::with_capacity(len + 1); + while v.len() < len { + let w = if foldy { + FOLDY[(rng.next() % 6) as usize] + } else { + rng.next() as u16 + }; + v.extend_from_slice(&w.to_be_bytes()); + } + v.truncate(len); + cases.push(v); + } + // Long runs of 0xff: sums are multiples of 65535 at every block. + for len in [720, 721, 1440, 1441, 7200, 65536, 65537] { + cases.push(vec![0xff; len]); + } + let mut blob = Vec::new(); + for c in &cases { + blob.extend_from_slice(&(c.len() as u32).to_le_bytes()); + blob.extend_from_slice(c); + } + let input = tmp("cases.bin"); + let output = tmp("sums.bin"); + std::fs::write(&input, &blob).unwrap(); + run_python( + LIBHDF5_CHECKSUMS, + &[input.to_str().unwrap(), output.to_str().unwrap()], + ); + let sums = std::fs::read(&output).unwrap(); + assert_eq!(sums.len(), cases.len() * 4); + let mut folds = 0; + for (c, s) in cases.iter().zip(sums.as_chunks::<4>().0) { + let want = u32::from_le_bytes(*s); + assert_eq!( + fletcher32(c), + want, + "checksum of {} bytes {:02x?}...", + c.len(), + &c[..c.len().min(16)] + ); + if fletcher32_mod(c) != want { + folds += 1; + } + } + // The corpus must exercise the case `% 65535` got wrong. + assert!(folds > 500, "only {folds} fold cases"); +} + +const CHUNK: usize = 8; + +/// `n` chunks of `CHUNK` bytes, each one a chunk on which the old +/// `% 65535` checksum and libhdf5's differ (sum1, sum2 or both a non-zero +/// multiple of 65535), with an ordinary chunk between them. +fn fold_chunks(n: usize) -> Vec { + let mut rng = Rng(42); + let mut out = Vec::new(); + let mut found = 0; + while found < n { + // Build a chunk whose sum1 is a multiple of 65535 half the time, + // otherwise search at random for a sum2 fold. + let mut c: Vec = (0..CHUNK).map(|_| rng.next() as u8).collect(); + if found % 2 == 0 { + let words: u64 = c[..CHUNK - 2] + .chunks(2) + .map(|w| (u64::from(w[0]) << 8) | u64::from(w[1])) + .sum(); + let last = ((65535 - words % 65535) % 65535) as u16; + c[CHUNK - 2..].copy_from_slice(&last.to_be_bytes()); + } + if fletcher32(&c) != fletcher32_mod(&c) { + out.extend_from_slice(&c); + out.extend((0..CHUNK).map(|i| i as u8 + 1)); + found += 1; + } + } + out +} + +#[test] +fn h5py_reads_fold_case_chunks_we_write() { + if !have_h5py() { + return; + } + let data = fold_chunks(32); + // FileBuilder. + let built = tmp("built.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("d") + .with_u8_data(&data) + .with_chunks(&[CHUNK as u64]) + .with_fletcher32(); + b.write(&built).unwrap(); + // FileEditor, into a dataset h5py created. + let edited = tmp("edited.h5"); + run_python( + "import sys, h5py, numpy as np\n\ + with h5py.File(sys.argv[1], 'w') as f:\n\ + \x20 f.create_dataset('d', data=np.zeros(int(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)", + &[edited.to_str().unwrap(), &data.len().to_string()], + ); + FileEditor::open(&edited) + .unwrap() + .write_all("d", &data) + .unwrap(); + for path in [&built, &edited] { + let got = run_python( + "import sys, h5py\n\ + with h5py.File(sys.argv[1], 'r') as f:\n\ + \x20 assert f['d'].fletcher32\n\ + \x20 print(f['d'][:].tobytes().hex())", + &[path.to_str().unwrap()], + ); + assert_eq!(got, hex(&data), "{}", path.display()); + } +} + +#[test] +fn we_read_fold_case_chunks_h5py_writes() { + if !have_h5py() { + return; + } + let data = fold_chunks(32); + let path = tmp("h5py.h5"); + run_python( + "import sys, h5py, numpy as np\n\ + with h5py.File(sys.argv[1], 'w') as f:\n\ + \x20 f.create_dataset('d', data=np.frombuffer(bytes.fromhex(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)", + &[path.to_str().unwrap(), &hex(&data)], + ); + let file = File::open(&path).unwrap(); + let ds = file.dataset("d").unwrap(); + assert_eq!( + ds.read_selection(&clawhdf5_format::selection::Selection::All) + .unwrap(), + data + ); +} + +/// A checksum stored with the bytes of each 16-bit half swapped, as +/// libhdf5 1.6.2 and earlier wrote it, is accepted as libhdf5 accepts it; +/// so is the `% 65535` form clawhdf5 v2.7.0 and earlier wrote, so that +/// their files stay readable. +#[test] +fn legacy_checksums_are_accepted() { + use clawhdf5_format::filter_pipeline::{FILTER_FLETCHER32, FilterDescription, FilterPipeline}; + let payload = [1u8, 2, 3, 4, 5]; + let sum = fletcher32(&payload); + let swapped = ((sum & 0x00ff_00ff) << 8) | ((sum >> 8) & 0x00ff_00ff); + assert_ne!(sum, swapped); + let pipeline = FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: FILTER_FLETCHER32, + name: None, + client_data: vec![], + flags: 0, + }], + }; + for stored in [sum, swapped] { + let mut chunk = payload.to_vec(); + chunk.extend_from_slice(&stored.to_le_bytes()); + let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1) + .unwrap(); + assert_eq!(out, payload); + } + // Our old checksum of a fold-case chunk. + let fold = fold_chunks(1); + let fold = &fold[..CHUNK]; + let old = fletcher32_mod(fold); + assert_ne!(old, fletcher32(fold)); + let mut chunk = fold.to_vec(); + chunk.extend_from_slice(&old.to_le_bytes()); + let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, CHUNK, 1).unwrap(); + assert_eq!(out, fold); + let mut chunk = payload.to_vec(); + chunk.extend_from_slice(&(sum ^ 1).to_le_bytes()); + assert!( + clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1).is_err() + ); +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 1b05553..25d0960 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,41 @@ deleting it. --- +## Fletcher-32 checksums disagreed with libhdf5 on about 1 chunk in 32768 + +**Status:** fixed 2026-09-26, after v2.7.0. **Every release (v2.1.0 to +v2.7.0) is affected**, in both directions. + +Our Fletcher-32 reduced its two running sums with `% 65535`; libhdf5's +`H5_checksum_fletcher32` (H5checksum.c) folds them with +`(s & 0xffff) + (s >> 16)`. Both are arithmetic mod 65535, but where a sum +is a non-zero multiple of 65535 the fold leaves 0xffff and the modulo 0, so +the checksums differ — for random data about one chunk in 32768 (each of +the two sums hits it with probability about 1/65535). Found by the review +of the editor work: a random-edit fuzzer with gzip + Fletcher-32 hit it on +13 of about 100 seeds. + +- Chunks we wrote (`FileBuilder`/`FileWriter` `with_fletcher32`, and the + unreleased `FileEditor`) with such a sum are refused by h5py and libhdf5: + "filter returned failure during read". h5py writing `[1, 0xfffe]` as + big-endian `u2` stores checksum `0x0001ffff`; we computed `0x00010000`. +- Chunks libhdf5 wrote with such a sum were refused by every reader here + with `Fletcher32Mismatch`; the data itself was never wrong. + +**Fix:** `clawhdf5_format::checksum::fletcher32`, a port of +`H5_checksum_fletcher32`, used by the filter for writing and verifying. It +also accepts a checksum whose 16-bit halves are byte-swapped, as libhdf5 +does for files from 1.6.2 and earlier, and the `% 65535` form clawhdf5 +v2.7.0 and earlier wrote (the two differ only in a half that is 0xffff). +**Test:** +`crates/clawhdf5/tests/fletcher32_interop.rs` (libhdf5's own function +through ctypes on every 1- and 2-byte input plus 40 000 random and +fold-heavy inputs; h5py reads fold-case chunks from `FileBuilder` and +`FileEditor`; we read h5py's). **Existing data:** a Fletcher-32 dataset +written by v2.7.0 or earlier may hold chunks libhdf5 cannot read; a fixed +build reads them. Rewrite such datasets with a fixed build (read, then +write them again) before handing the file to libhdf5 or h5py. + ## LZF/Blosc chunks written with a stale filter mask **Status:** fixed 2026-09-26, before any release (the LZF and Blosc writers -- 2.54.0 From 930921e8cb992263825b3e9572d9820c0bc76067 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:27:00 -0500 Subject: [PATCH 40/49] edit: shrink by visiting the chunks that exist prune_plan stored one Vec for every chunk coordinate of the region a shrink cuts off, existing or not, so a sparse dataset exhausted memory (about 62 bytes per coordinate; (4, 2e7) with chunks (1, 1) took 2.5 GB, larger extents never finished). It now places each existing chunk in H5D__chunk_prune_by_extent's walk (its pass, then its coordinates) and sorts, which gives the same chunks, order and actions in memory and time proportional to the chunks that exist. A unit test checks the plan against the full walk (kept as the test's reference) for 3000 random extents and chunk subsets. The interop test shrinks a (4, 10^12) dataset with chunks (1, 1) and 9 chunks (v1 and v2 B-tree): 0.56 s and 43 MB peak; the old code aborted on allocation under an 8 GB limit. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +- .../tests/edit_coverage_interop.rs | 62 +++++ crates/clawhdf5/src/edit/mod.rs | 246 +++++++++++++----- 3 files changed, 245 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 964390d..6ae2b02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,10 @@ partial edge chunk outside the extent is overwritten with the fill value, so it reads as fill after a later growth. Growth under early allocation now allocates and fills the new chunks (`H5D__chunk_allocate`), which an - implicit index needs. Shrinking was `Error::Unsupported`. + implicit index needs. Shrinking was `Error::Unsupported`. Only the + chunks that exist are visited (placed in libhdf5's order), so shrinking + a sparse dataset costs memory and time in its chunks, not in the + coordinates cut off (a 2 x 10^12-coordinate shrink takes 0.6 s). - **`FileEditor::set_attr` handles dense attribute storage and creation order**: objects that track (and index) attribute creation order; the move to dense storage when an object reaches its compact limit (or an diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 7fc309a..fff256c 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1566,3 +1566,65 @@ fn resize_clawhdf5_written_datasets() { } check_tools(&path, true); } + +/// Shrinking a huge, sparse dataset costs time and memory in the chunks +/// that exist, not in the chunk coordinates cut off. The editor once stored +/// every coordinate of the cut-off region (about 62 bytes each), so this +/// 2 x 10^12-coordinate shrink ran out of memory. +#[test] +fn shrinking_a_huge_sparse_dataset_is_bounded() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + const N: u64 = 1_000_000_000_000; + for libver in ["earliest", "v110"] { + let path = dir.path().join(format!("sparse_{libver}.h5")); + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'w', libver=({libver:?}, 'latest')) as f:\n\ + \x20 d = f.create_dataset('b', shape=(4, {N}), maxshape=(None, None), chunks=(1, 1), dtype=' = got + .as_chunks::<4>() + .0 + .iter() + .map(|c| i32::from_le_bytes(*c)) + .collect(); + assert_eq!(got, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0], "{libver}"); + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 336fbb8..e1d301a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -1330,16 +1330,31 @@ enum Prune { Remove, } -/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks -/// from `old` to `new`, in its order. -fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { +/// The chunks of `existing` (keyed by scaled coordinates) that +/// `H5D__chunk_prune_by_extent` visits when the extent shrinks from `old` +/// to `new`, in its order, with what it does to each. +/// +/// libhdf5 walks every chunk coordinate of the region cut off, one pass per +/// shrunk dimension `op` in order: the coordinates with `scaled[op]` at or +/// past the new extent's chunk and every earlier shrunk dimension below its +/// own, row-major. It looks each one up in the index and stores nothing, +/// so a sparse dataset costs it time, not memory. Only chunks that exist +/// can be acted on, so this places each existing chunk in that walk (its +/// pass, then its coordinates) and sorts: the same chunks in the same +/// order, in memory and time proportional to the chunks that exist. +fn prune_plan<'c>( + old: &[u64], + new: &[u64], + cd: &[u64], + existing: &'c HashMap, ChunkInfo>, +) -> Vec<(&'c [u64], &'c ChunkInfo, Prune)> { let rank = new.len(); - let mut out = Vec::new(); - if old.contains(&0) { - return out; + if old.contains(&0) || cd.contains(&0) { + return Vec::new(); } - let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); - let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + // The last chunk index still (partly) inside the new extent; -1 when + // the dimension shrank to nothing. let max_fill: Vec = (0..rank) .map(|d| { if new[d] == 0 { @@ -1350,60 +1365,25 @@ fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { }) .collect(); let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); - let fill_dim: Vec = (0..rank) - .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) - .collect(); - for op in 0..rank { - if !shrunk[op] { - continue; - } - let mut scaled = vec![0u64; rank]; - scaled[op] = min_mod[op]; - let mut outside: Vec = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); - let mut n_out = outside.iter().filter(|&&o| o).count(); - loop { - if n_out == 0 { - out.push((scaled.clone(), Prune::Fill)); + let mut out: Vec<(usize, &[u64], &ChunkInfo, Prune)> = existing + .iter() + .filter(|(s, _)| s.len() == rank && s.iter().zip(&max_mod).all(|(c, m)| c <= m)) + .filter_map(|(s, info)| { + // The first shrunk dimension whose pass reaches this chunk; the + // passes before it covered only coordinates below their minimum. + let op = (0..rank).find(|&d| new[d] < old[d] && s[d] >= min_mod[d])?; + // Chunks with any coordinate past the last partly kept chunk go; + // the others lose the part outside the new extent. + let what = if (0..rank).all(|u| s[u] as i64 <= max_fill[u]) { + Prune::Fill } else { - out.push((scaled.clone(), Prune::Remove)); - } - let mut carry = true; - for i in (0..rank).rev() { - scaled[i] += 1; - if scaled[i] > max_mod[i] { - if i == op { - scaled[i] = min_mod[i]; - if outside[i] && fill_dim[i] { - outside[i] = false; - n_out -= 1; - } - } else { - scaled[i] = 0; - if outside[i] && max_fill[i] >= 0 { - outside[i] = false; - n_out -= 1; - } - } - } else { - if !outside[i] && scaled[i] as i64 > max_fill[i] { - outside[i] = true; - n_out += 1; - } - carry = false; - break; - } - } - if carry { - break; - } - } - if min_mod[op] == 0 { - // Every chunk was visited (the dimension shrank to nothing). - break; - } - max_mod[op] = min_mod[op] - 1; - } - out + Prune::Remove + }; + Some((op, s.as_slice(), info, what)) + }) + .collect(); + out.sort_unstable_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1))); + out.into_iter().map(|(_, s, i, w)| (s, i, w)).collect() } /// The chunk work of a resize: under early allocation, allocate and fill @@ -1468,12 +1448,9 @@ fn resize_chunks( })?; } if shrink && !existing.is_empty() { - for (scaled, what) in prune_plan(old, new, &cd) { - let Some(info) = existing.get(&scaled) else { - continue; - }; + for (scaled, info, what) in prune_plan(old, new, &cd, &existing) { match what { - Prune::Remove => ce.remove(img, &scaled, info)?, + Prune::Remove => ce.remove(img, scaled, info)?, Prune::Fill => { let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?; // Keep [0, count) in each dimension; fill the rest. @@ -1495,7 +1472,7 @@ fn resize_chunks( buf[at..at + es].copy_from_slice(&t.fill); } } - store_chunk(img, &mut ce, Some(info), &scaled, buf)?; + store_chunk(img, &mut ce, Some(info), scaled, buf)?; } } } @@ -1610,6 +1587,141 @@ fn decode_chunk( #[cfg(test)] mod tests { + use super::{Prune, prune_plan}; + use clawhdf5_format::chunked_read::ChunkInfo; + use std::collections::HashMap; + + /// `H5D__chunk_prune_by_extent`'s walk over every chunk coordinate of the + /// region cut off, stored (the implementation before `prune_plan` + /// visited only existing chunks). + fn prune_walk(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { + let rank = new.len(); + let mut out = Vec::new(); + if old.contains(&0) { + return out; + } + let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); + let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_fill: Vec = (0..rank) + .map(|d| { + if new[d] == 0 { + -1 + } else { + ((new[d].min(old[d]) - 1) / cd[d]) as i64 + } + }) + .collect(); + let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); + let fill_dim: Vec = (0..rank) + .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) + .collect(); + for op in 0..rank { + if !shrunk[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min_mod[op]; + let mut outside: Vec = + (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); + let mut n_out = outside.iter().filter(|&&o| o).count(); + loop { + if n_out == 0 { + out.push((scaled.clone(), Prune::Fill)); + } else { + out.push((scaled.clone(), Prune::Remove)); + } + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max_mod[i] { + if i == op { + scaled[i] = min_mod[i]; + if outside[i] && fill_dim[i] { + outside[i] = false; + n_out -= 1; + } + } else { + scaled[i] = 0; + if outside[i] && max_fill[i] >= 0 { + outside[i] = false; + n_out -= 1; + } + } + } else { + if !outside[i] && scaled[i] as i64 > max_fill[i] { + outside[i] = true; + n_out += 1; + } + carry = false; + break; + } + } + if carry { + break; + } + } + if min_mod[op] == 0 { + // Every chunk was visited (the dimension shrank to nothing). + break; + } + max_mod[op] = min_mod[op] - 1; + } + out + } + + /// `prune_plan` over a random subset of chunks gives the chunks of the + /// full walk that exist, in its order and with its actions. + #[test] + fn prune_plan_follows_the_full_walk() { + let mut x: u64 = 0x1234_5678; + let mut rnd = |n: u64| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x % n + }; + for _ in 0..3000 { + let rank = 1 + rnd(3) as usize; + let cd: Vec = (0..rank).map(|_| 1 + rnd(4)).collect(); + let old: Vec = (0..rank).map(|_| 1 + rnd(14)).collect(); + let new: Vec = old + .iter() + .map(|&o| if rnd(2) == 0 { o } else { rnd(o + 1) }) + .collect(); + let grid: Vec = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect(); + let mut existing = HashMap::new(); + let total: u64 = grid.iter().product(); + for flat in 0..total { + if rnd(3) == 0 { + continue; + } + let mut r = flat; + let mut s = vec![0; rank]; + for d in (0..rank).rev() { + s[d] = r % grid[d]; + r /= grid[d]; + } + let info = ChunkInfo { + chunk_size: 1, + filter_mask: 0, + offsets: s.iter().zip(&cd).map(|(a, b)| a * b).collect(), + address: flat, + }; + existing.insert(s, info); + } + let want: Vec<(Vec, bool)> = prune_walk(&old, &new, &cd) + .into_iter() + .filter(|(s, _)| existing.contains_key(s)) + .map(|(s, w)| (s, matches!(w, Prune::Fill))) + .collect(); + let got: Vec<(Vec, bool)> = prune_plan(&old, &new, &cd, &existing) + .into_iter() + .map(|(s, _, w)| (s.to_vec(), matches!(w, Prune::Fill))) + .collect(); + assert_eq!(got, want, "old {old:?} new {new:?} chunks {cd:?}"); + } + } + use std::cell::Cell; use std::path::Path; -- 2.54.0 From a69c5be8b2bd6d55ffbce4c33e37bd5907edaf44 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:29:31 -0500 Subject: [PATCH 41/49] format: read object header chunks from a queue, one buffer at a time The version-1 chunk walk nested continuation chunks depth-first and kept every enclosing chunk's buffer alive, up to 65 536 chunks. With storage that hands out owned buffers (CountingStorage, the Storage trait, remote storage) a crafted chain of chunks nested in each other read and held the square of the file's size (a 192 KB file read 768 MB). Chunks are now read from a FIFO queue of (address, length) pairs in the order their continuation messages are found, as H5O_protect does and as the editor's header walker already did, each buffer released before the next read. In both header versions a chunk starting at an address seen before (cycle) is refused, and so are chunks adding up to more than the file, which bounds a header's reads by the file's size. Overlap itself is allowed: libhdf5 reads cve-2025-7067.h5, whose continuation chunk overlaps chunk 0 (refusing overlap cost that conformance file). Tests: the nested chain is refused having read at most the file (it read n^2 bytes before); a 3000-chunk chain reads each chunk once; a chunk's messages follow the whole previous chunk (they were inserted at the continuation message); an overlapping continuation chunk is read. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +- crates/clawhdf5-format/src/object_header.rs | 287 ++++++++++++++------ 2 files changed, 218 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae2b02..1949400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,9 +87,15 @@ header whose continuation chunks chain more than 32 deep (a header that gains a chunk per attribute added when full, as libhdf5 and the editor grow it) was refused with `NestingDepthExceeded`; version-2 headers - stopped at 256 chunks. Chunks are now followed without recursion, in the - same order; a chunk address seen twice (a cycle) or more than 65 536 - chunks are refused. + stopped at 256 chunks. Chunks are now read one at a time from a queue, + in the order their continuation messages are found (libhdf5's + `H5O_protect` order, which the editor already used; a version-1 + chunk's messages used to be inserted at its continuation message), each + buffer released before the next is read; a chunk address seen twice (a + cycle), chunks adding up to more than the file (a crafted chain of + chunks nested in each other made storage with owned buffers read and + hold the square of the file's size), or more than 65 536 chunks are + refused, so a header's chunks read at most the file's size. - Tests: `crates/clawhdf5-tools/tests/edit_coverage_interop.rs` (h5py `earliest`/`v110`/`latest` and clawhdf5-written files; structure comparisons with libhdf5 for version-2 B-trees, shrink on every index, diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 383d553..ab652cb 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -232,12 +232,14 @@ impl ObjectHeader { /// end of the chunk, or leftover bytes too few for a message header (a /// "gap", which only version 2 allows). /// - /// Continuation chunks are followed depth-first, as met, with an - /// explicit stack: a chain of continuation chunks as long as libhdf5 - /// writes (each new chunk holding the next continuation message, one - /// per attribute added to a full header) is read without recursion. - /// A chunk address seen twice is a cycle and refused, as is a header of - /// more than [`MAX_V1_CHUNKS`] chunks. + /// Continuation chunks are read in the order their messages are found, + /// as `H5O_protect` loads them (so the messages keep libhdf5's order): + /// a queue of (address, length) pairs, each chunk read, parsed and + /// released before the next, so only one chunk buffer is alive at a + /// time whatever the storage. Every chunk must start at a new address + /// (else a cycle), and the chunks together may be no larger than the + /// file, so the bytes read stay within the file's size; a header of + /// more than [`MAX_V1_CHUNKS`] chunks is refused. fn parse_v1_chunk( file: &S, offset: u64, @@ -246,74 +248,67 @@ impl ObjectHeader { length_size: u8, messages: &mut Vec, ) -> Result { - let mut stack = vec![(read_exact_at(file, offset, length)?, 0usize)]; - let mut seen = BTreeSet::new(); - seen.insert(offset); - let mut count = 0usize; - - while let Some((chunk, pos)) = stack.last_mut() { - let data: &[u8] = chunk; + let mut spans = ChunkSpans::new(file.len(), offset, length)?; + let mut queue: Vec<(u64, usize)> = vec![(offset, length)]; + let mut chunk0_count = 0usize; + let mut next = 0usize; + while let Some(&(chunk_offset, chunk_length)) = queue.get(next) { + let chunk = read_exact_at(file, chunk_offset, chunk_length)?; + let data: &[u8] = &chunk; let end = data.len(); - if *pos >= end { - stack.pop(); - continue; - } - if end - *pos < V1_MSG_HEADER_SIZE { - return Err(FormatError::InvalidObjectHeader( - "gap found in early version of file format", - )); - } - let p = *pos; - let msg_type_raw = LittleEndian::read_u16(&data[p..p + 2]); - let msg_data_size = LittleEndian::read_u16(&data[p + 2..p + 4]) as usize; - let msg_flags = data[p + 4]; - // reserved(3) at p+5..p+8 - let p = p + V1_MSG_HEADER_SIZE; - - if !msg_data_size.is_multiple_of(8) { - return Err(FormatError::InvalidObjectHeader("message not aligned")); - } - if msg_data_size > end - p { - return Err(FormatError::InvalidObjectHeader( - "message size exceeds buffer end", - )); - } - let body = &data[p..p + msg_data_size]; - check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; - let msg_type = MessageType::from_u16(msg_type_raw); - if msg_type != MessageType::Nil { - messages.push(HeaderMessage { - msg_type, - size: msg_data_size, - flags: msg_flags, - creation_order: None, - data: body.to_vec(), - }); - } - // Follow continuations (v1 continuation chunks are just raw - // messages, no signature); check_message has checked the body. - let cont = if msg_type == MessageType::ObjectHeaderContinuation { - Some(( - read_offset(body, 0, offset_size)?, - to_usize(read_offset(body, offset_size as usize, length_size)?)?, - )) - } else { - None - }; - *pos = p + msg_data_size; - // Only the first chunk's messages are held to the prefix count. - if stack.len() == 1 { - count += 1; - } - if let Some((cont_offset, cont_length)) = cont { - if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { - return Err(FormatError::NestingDepthExceeded); + let mut pos = 0usize; + let mut count = 0usize; + while pos < end { + if end - pos < V1_MSG_HEADER_SIZE { + return Err(FormatError::InvalidObjectHeader( + "gap found in early version of file format", + )); } - stack.push((read_exact_at(file, cont_offset, cont_length)?, 0)); - } - } + let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); + let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; + let msg_flags = data[pos + 4]; + // reserved(3) at pos+5..pos+8 + pos += V1_MSG_HEADER_SIZE; - Ok(count) + if !msg_data_size.is_multiple_of(8) { + return Err(FormatError::InvalidObjectHeader("message not aligned")); + } + if msg_data_size > end - pos { + return Err(FormatError::InvalidObjectHeader( + "message size exceeds buffer end", + )); + } + let body = &data[pos..pos + msg_data_size]; + check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; + count += 1; + let msg_type = MessageType::from_u16(msg_type_raw); + if msg_type != MessageType::Nil { + messages.push(HeaderMessage { + msg_type, + size: msg_data_size, + flags: msg_flags, + creation_order: None, + data: body.to_vec(), + }); + } + // Queue continuations (v1 continuation chunks are just raw + // messages, no signature); check_message has checked the body. + if msg_type == MessageType::ObjectHeaderContinuation { + let cont_offset = read_offset(body, 0, offset_size)?; + let cont_length = + to_usize(read_offset(body, offset_size as usize, length_size)?)?; + spans.add(cont_offset, cont_length)?; + queue.push((cont_offset, cont_length)); + } + pos += msg_data_size; + } + // Only the first chunk's messages are held to the prefix count. + if next == 0 { + chunk0_count = count; + } + next += 1; + } + Ok(chunk0_count) } fn parse_v2( @@ -436,15 +431,14 @@ impl ObjectHeader { &mut continuations, )?; - // Follow continuations. A chunk address seen twice is a cycle in - // malformed data; a valid header can have many chunks (libhdf5 adds - // one whenever a message no longer fits), up to the same bound as a + // Follow continuations, one chunk buffer at a time. A chunk address + // seen twice is a cycle in malformed data, and the chunks may add up + // to no more than the file; a valid header can have many chunks (libhdf5 adds one + // whenever a message no longer fits), up to the same bound as a // version-1 header. - let mut seen = BTreeSet::new(); + let mut spans = ChunkSpans::new(file.len(), base as u64, chunk0_msg_end.saturating_add(4))?; while let Some((cont_offset, cont_length)) = continuations.pop() { - if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { - return Err(FormatError::NestingDepthExceeded); - } + spans.add(cont_offset as u64, cont_length)?; Self::parse_v2_continuation( file, cont_offset as u64, @@ -607,6 +601,45 @@ const V2_PREFIX_MAX: usize = 34; /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3). const V1_MSG_HEADER_SIZE: usize = 8; +/// The chunks of one object header read so far. A chunk starting where +/// another did is a cycle. Chunks of a valid header do not overlap, so +/// together they are no larger than the file; a header whose chunks add up +/// to more is refused, which bounds what its chunks can make a reader read +/// (a crafted chain of chunks each nested in the last would otherwise read +/// the file over and over). Overlap itself is not refused: libhdf5 reads +/// such headers (`cve-2025-7067.h5` has one). +struct ChunkSpans { + starts: BTreeSet, + /// Bytes of the chunks so far, and the most they may add up to. + total: u64, + budget: u64, +} + +impl ChunkSpans { + fn new(file_len: u64, start: u64, len: usize) -> Result { + let mut s = Self { + starts: BTreeSet::new(), + total: 0, + budget: file_len, + }; + s.add(start, len)?; + Ok(s) + } + + fn add(&mut self, start: u64, len: usize) -> Result<(), FormatError> { + if !self.starts.insert(start) || self.starts.len() > MAX_V1_CHUNKS { + return Err(FormatError::NestingDepthExceeded); + } + self.total = self.total.saturating_add(len as u64); + if self.total > self.budget { + return Err(FormatError::InvalidObjectHeader( + "object header chunks larger than the file", + )); + } + Ok(()) + } +} + /// Most chunks a version-1 object header may have (malformed-data guard; /// libhdf5 has no limit, and a header that gains one continuation chunk per /// attribute added can have many). @@ -972,6 +1005,104 @@ mod tests { assert_eq!(spaces, (0..200).map(|k| k as u8).collect::>()); } + /// A crafted version-1 header whose continuation chunks nest: each + /// chunk's continuation message points at the rest of that chunk. Read + /// depth-first with every enclosing chunk kept alive, from storage that + /// hands out owned buffers, it read n^2 bytes and held them all at once + /// (a 192 KB file read 768 MB). Chunks adding up to more than the file + /// are refused, and the bytes read stay within the file's size. + #[test] + fn nested_v1_continuation_chunks_are_bounded() { + use crate::storage::CountingStorage; + let n = 2000u64; + let a = 64u64; + let cont = |addr: u64, len: u64| { + let mut m = vec![0x10, 0, 16, 0, 0, 0, 0, 0]; + m.extend_from_slice(&addr.to_le_bytes()); + m.extend_from_slice(&len.to_le_bytes()); + m + }; + // Prefix: version 1, one message, reference count 1, 24 bytes. + let mut buf = vec![1, 0, 1, 0, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0]; + buf.extend_from_slice(&cont(a, 24 * n)); + buf.resize(a as usize, 0); + for k in 0..n { + if k + 1 < n { + buf.extend_from_slice(&cont(a + 24 * (k + 1), 24 * (n - k - 1))); + } else { + buf.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]); + buf.extend_from_slice(&[0; 16]); + } + } + let len = buf.len() as u64; + let s = CountingStorage::new(buf); + assert!(matches!( + ObjectHeader::parse_in(&s, 0, 8, 8), + Err(FormatError::InvalidObjectHeader( + "object header chunks larger than the file" + )) + )); + assert!( + s.bytes_read() <= 2 * len, + "read {} of {len}", + s.bytes_read() + ); + } + + /// libhdf5 reads a continuation chunk that overlaps the chunk holding + /// its message (`cve-2025-7067.h5` has one), and so does this reader. + #[test] + fn overlapping_v1_continuation_chunk_is_read() { + // Chunk 0 (at 16): continuation (24 bytes), then a NIL message at + // 40; the continuation chunk is that NIL message's 8-byte header. + let mut cont = 40u64.to_le_bytes().to_vec(); + cont.extend_from_slice(&8u64.to_le_bytes()); + let data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0000, &[][..], 0)], 8, 8); + let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); + assert_eq!(hdr.messages.len(), 1); + } + + /// A valid chain over owned-buffer storage reads each chunk once. + #[test] + fn long_v1_chain_reads_each_chunk_once() { + use crate::storage::CountingStorage; + let data = v1_chain(3000, false); + let len = data.len() as u64; + let s = CountingStorage::new(data); + let hdr = ObjectHeader::parse_in(&s, 0, 8, 8).unwrap(); + assert_eq!( + hdr.messages + .iter() + .filter(|m| m.msg_type == MessageType::Dataspace) + .count(), + 3000 + ); + assert!(s.bytes_read() <= len, "read {} of {len}", s.bytes_read()); + } + + /// Continuation chunks are read in the order their messages are found + /// (libhdf5's `H5O_protect`), so a chunk's messages follow every + /// message of the chunk before, not the continuation message. + #[test] + fn v1_continuation_messages_keep_libhdf5_order() { + // Chunk 0: continuation to A, dataspace [1]; A: dataspace [2]. + let a = 64u64; + let mut cont = a.to_le_bytes().to_vec(); + cont.extend_from_slice(&16u64.to_le_bytes()); + let mut data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0001, &[1; 8][..], 0)], 8, 8); + data.resize(a as usize, 0); + data.extend_from_slice(&[1, 0, 8, 0, 0, 0, 0, 0]); + data.extend_from_slice(&[2; 8]); + let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); + let spaces: Vec = hdr + .messages + .iter() + .filter(|m| m.msg_type == MessageType::Dataspace) + .map(|m| m.data[0]) + .collect(); + assert_eq!(spaces, [1, 2]); + } + #[test] fn v1_continuation_cycles_are_refused() { let data = v1_chain(5, true); -- 2.54.0 From c2ae7846c93581a9698934a994a64b96daeca9a7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:30:33 -0500 Subject: [PATCH 42/49] docs: label the h5repack sizes in the append-waste figures 306 104 and 49 930 are h5repack of the editor's file; the text read as if they were h5repack of libhdf5's, which measures 305 954 and 50 188. Both are now given, from measure_append_waste rerun on 2026-09-26 (file sizes unchanged). Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/known-issues.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/known-issues.md b/docs/known-issues.md index 25d0960..a178d47 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -98,10 +98,12 @@ Measured 2026-09-26 on tank with `cargo test -p clawhdf5-tools --test edit_interop -- --ignored --nocapture measure_append_waste` (one editor for the whole workload; file sizes are deterministic): 1000 appends of 100 `f8` values to a 1-D dataset with 1024-element chunks give 810 504 bytes -unfiltered, as libhdf5's file, and 306 780 bytes with gzip (307 210 before -reuse; libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values -with 4096-element gzip chunks give 79 829 bytes (119 684 before reuse) -against libhdf5's 50 292 (`h5repack`: 49 930): the chunk being appended to +unfiltered, as libhdf5's file (`h5repack` of either: 810 360), and 306 780 +bytes with gzip (307 210 before reuse; libhdf5's file: 306 058; `h5repack` +of the editor's file: 306 104, of libhdf5's: 305 954); 2000 appends of 10 +values with 4096-element gzip chunks give 79 829 bytes (119 684 before +reuse) against libhdf5's 50 292 (`h5repack` of the editor's file: 49 930, +of libhdf5's: 50 188): the chunk being appended to is followed by new index blocks and moves each time it grows, and the space it leaves is too small for its next, larger version. -- 2.54.0 From 8236b0e30ac965dccb1657fdaf78703b38304b04 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:50:23 -0500 Subject: [PATCH 43/49] edit: skip heap blocks too small for an attribute, as libhdf5 does An attribute needing a heap block larger than the next one was refused ("skipping blocks too small for an object", "a first object too large for the starting block"); once an object's move to dense storage was refused it refused every new attribute, so 24% of set_attr calls in the review's random workload failed. Following H5HF__hdr_update_iter, H5HF__man_iblock_root_create/_double and H5HF__hdr_skip_blocks, the smaller blocks are now skipped: the iterator moves past them and they become an indirect free section with a first row section (serialized, class 1, as H5HF__sect_indirect_serialize writes it) and ghost normal rows, added as returned space so it merges with a range skipped just before it (H5HF__sect_indirect_merge_row). Later objects that best-fit a row section get a block created there (H5HF__man_iblock_alloc_row / H5HF__sect_indirect_reduce_row: from the start or end of the range, or from its middle, which splits it, with libhdf5's span bookkeeping). Heaps with such sections, as libhdf5 writes them, are now read too (they were refused at open). dense_skipped_blocks_match_libhdf5 drives every path (merge, split, end, last entry, row wrap) on earliest/v110/latest files against libhdf5 doing the same edits one session each; heaps, free sections and index B-trees are equal after every phase. The refusal test now checks the skip against libhdf5 and keeps a real refusal (last object in a block); clawhdf5-written heaps get 1-4 KiB attributes too. The three tests fail on the previous fheap.rs. Random workload refusals: 24% -> 2.2%, all the documented last-object-in-a-block case. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 18 +- .../tests/edit_coverage_interop.rs | 190 +++++- crates/clawhdf5/src/edit/fheap.rs | 609 +++++++++++++++--- crates/clawhdf5/src/edit/mod.rs | 3 +- docs/known-issues.md | 20 +- 5 files changed, 730 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1949400..6b264f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,10 +63,20 @@ insertion. The heap is changed as `H5HF` changes it — best-fit free sections from its free-space manager (kept as libhdf5 keeps `FSHD`/ `FSSE`), new direct blocks through the root indirect block (created, - doubled), huge objects through the huge-object B-tree (deleted with the - last huge object), removed objects' space merged back — with libhdf5's - statistics: after the same attribute workload the heap, its free space - and both index B-trees equal libhdf5's. Attributes are encoded as libhdf5 + doubled), blocks too small for an attribute skipped as libhdf5 skips + them (`H5HF__hdr_skip_blocks`: an indirect free section with its row + sections, serialized as libhdf5 serializes them, merged with the range + skipped just before it, and later attributes given skipped blocks from + either end or the middle of a range, which splits it), huge objects + through the huge-object B-tree (deleted with the last huge object), + removed objects' space merged back — with libhdf5's statistics: after + the same attribute workload the heap, its free space and both index + B-trees equal libhdf5's (`dense_skipped_blocks_match_libhdf5` covers + every way of skipping, with libhdf5 doing one edit per session as the + editor does). In a random attribute workload (1-4 KiB attributes among + small ones) 24% of `set_attr` calls were refused before skipping was + implemented; 2.2% are now, all replacements of the last attribute in a + heap block. Attributes are encoded as libhdf5 encodes them for a file h5py opens `r+` (message version 1, 3 for non-ASCII names; simple dataspaces with their maximum dimensions). Still refused: see `docs/known-issues.md`. diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index fff256c..c2da7cf 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1230,10 +1230,11 @@ fn dense_attributes_on_clawhdf5_files() { let mut ed = FileEditor::open(&path).unwrap(); for i in 0..30u64 { for (k, o) in ["/", "d"].iter().enumerate() { - // Small attributes: a larger one needs a heap block bigger than - // the next (see dense_attribute_refusals_change_nothing). + // Strings up to 300 bytes and of 5000 (huge objects), and + // every tenth one of 1-4 KiB (a heap block larger than the + // next: blocks skipped). let v = match AV::make(i, k as u64 + 7) { - AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()), + AV::Str(s) if i % 10 == 3 => AV::Str(s.repeat(3000 / s.len().max(1) + 1)), v => v, }; ed.set_attr(o, &format!("n{i}"), &v.value()) @@ -1323,10 +1324,12 @@ fn check_root_and_attrs(path: &Path, want: &[AttrOp]) { assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); } -/// What the editor refuses in dense storage — an object larger than the -/// next heap block (libhdf5 would skip blocks and record their space as -/// free, which this editor does not do) — is `Error::Unsupported`, and the -/// file is left byte for byte as it was. +/// An attribute that needs a heap block larger than the next one: libhdf5 +/// skips the smaller blocks (`H5HF__hdr_skip_blocks`) and records them as +/// an indirect free section, and so does the editor — the heap and its +/// free space come out as libhdf5's. Replacing that attribute, the only +/// object in its block, with one of another size is still refused (libhdf5 +/// frees the block), and the refusal leaves the file byte-identical. #[test] fn dense_attribute_refusals_change_nothing() { if !tools_ok() { @@ -1334,16 +1337,29 @@ fn dense_attribute_refusals_change_nothing() { } let dir = tmpdir(); let path = dir.path().join("dense_refuse.h5"); - py(&format!( - "import h5py, numpy as np\n\ - with h5py.File({p:?}, 'w', libver='v110') as f:\n\ - \x20 g = f.create_group('g')\n\ - \x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n", - p = path.to_str().unwrap() - )); + let twin = dir.path().join("dense_refuse_h5py.h5"); + for p in [&path, &twin] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n", + p = p.to_str().unwrap() + )); + } + let big = AV::Str("x".repeat(2000)); + let mut ed = FileEditor::open(&path).unwrap(); + ed.set_attr("g", "big", &big.value()).unwrap(); + drop(ed); + py_r_plus( + &twin, + &[format!("\x20 f['g'].attrs.create('big', {})\n", big.py())], + ); + assert_eq!(dense_info(&path, "g"), dense_info(&twin, "g")); + check_tools(&path, true); let before = std::fs::read(&path).unwrap(); let mut ed = FileEditor::open(&path).unwrap(); - unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000)))); + unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("y".repeat(1500)))); drop(ed); assert!( std::fs::read(&path).unwrap() == before, @@ -1353,13 +1369,155 @@ fn dense_attribute_refusals_change_nothing() { py(&format!( "import h5py\n\ with h5py.File({p:?}, 'r+') as f:\n\ - \x20 f['g'].attrs['big'] = 'x' * 2000\n\ + \x20 f['g'].attrs['big'] = 'y' * 1500\n\ \x20 assert len(f['g'].attrs) == 13\n", p = path.to_str().unwrap() )); check_tools(&path, true); } +/// Attributes of every size up to the heap's 4 KiB managed limit, in an +/// order that makes libhdf5 skip heap blocks in every way it does: a first +/// attribute too large for the heap's starting block (at the move to dense +/// storage), a block larger than the rest of the current row, a root +/// indirect block doubled past rows, two skipped ranges merged; later small +/// attributes go into the skipped blocks (from either end of a skipped +/// range, and from its middle, which splits it). libhdf5 does the same +/// operations, one file session each (as the editor re-reads the heap for +/// each edit); after every phase the heaps, their free sections (single, +/// row and indirect) and index B-trees must be libhdf5's. +#[test] +fn dense_skipped_blocks_match_libhdf5() { + if !tools_ok() { + return; + } + for (libver, h5dump) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] { + let dir = tmpdir(); + let a = dir.path().join("skip_h5py.h5"); + let b = dir.path().join("skip_edit.h5"); + for p in [&a, &b] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \ + f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + for (k, o) in objs.iter().enumerate() { + want.push(( + o.to_string(), + "c0".into(), + AV::Str("s".repeat(1200 + 700 * k)), + )); + for i in 1..8i64 { + want.push((o.to_string(), format!("c{i}"), AV::Ints(vec![i]))); + } + } + for i in 0..8i64 { + want.push(("h".into(), format!("c{i}"), AV::Ints(vec![i]))); + } + // "h" starts with small attributes, so its heap has a root direct + // block and then a one-row root indirect block when the first + // large attribute comes: the rest of that row and the rows the + // doubling adds are skipped as two ranges, which merge; the next + // attributes take blocks from the start, the end and the middle + // (splitting it) of the merged range, and use up whole rows. + let h_phases: [&[usize]; 4] = [ + &[0; 20], + &[4060, 1500, 0, 3000, 0, 1500, 0, 600], + &[1000, 0, 700, 3000, 0, 0, 1500, 0, 900, 0, 0, 0, 1800], + &[0, 800, 0, 0, 0, 2000, 0, 0, 0, 0, 0, 0, 3500, 0, 0, 0, 0, 0], + ]; + // Sizes (string lengths, or 0 for a small int array) in phases. + let phases: [&[usize]; 4] = [ + &[0, 3000, 0, 700, 3900, 0, 150], + &[2500, 90, 0, 3500, 1800, 60, 3990, 0], + &[0, 0, 40, 300, 0, 900, 20, 0, 0, 500, 0, 30, 1000, 0, 0, 200], + &[0, 250, 0, 40, 600, 0, 0, 1100, 80, 0, 0, 350, 0, 0, 0, 10], + ]; + let mut n = 0; + let value = |len: usize, n: usize, k: usize| { + if len == 0 { + AV::Ints((0..1 + (n + k) as i64 % 7).collect()) + } else { + AV::Str( + (0..len + 11 * k) + .map(|j| (b'a' + ((j + n) % 26) as u8) as char) + .collect(), + ) + } + }; + for (ph, sizes) in phases.iter().enumerate() { + let mut ops: Vec = Vec::new(); + for &len in *sizes { + for (k, o) in objs.iter().enumerate() { + ops.push((o.to_string(), format!("n{n}"), value(len, n, k))); + } + n += 1; + } + for &len in h_phases[ph] { + ops.push(("h".into(), format!("n{n}"), value(len, n, 0))); + n += 1; + } + // libhdf5: one session per operation. + let lines: Vec = ops + .iter() + .map(|(o, nm, v)| { + format!( + "with h5py.File({p:?}, 'r+') as f: f[{o:?}].attrs.create({nm:?}, {})\n", + v.py(), + p = a.to_str().unwrap() + ) + }) + .collect(); + let sp = a.with_extension(format!("ops{ph}.py")); + std::fs::write(&sp, format!("import h5py, numpy as np\n{}", lines.concat())).unwrap(); + let out = Command::new(python()).arg(&sp).output().unwrap(); + assert!( + out.status.success(), + "h5py workload failed:\n{}", + text(&out) + ); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, nm, v) in &ops { + ed.set_attr(o, nm, &v.value()) + .unwrap_or_else(|e| panic!("{libver} phase {ph}: set {o}/{nm}: {e}")); + set_want(&mut want, o, nm, v.clone()); + } + drop(ed); + for o in objs.iter().chain(&["h"]) { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{libver}: dense storage of {o} differs from libhdf5's after phase {ph}" + ); + } + check_tools(&b, h5dump); + check_attr_values(&b, &want); + } + // libhdf5 goes on with the editor's file. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['g', 't', 'd', 'h']:\n\ + \x20 for i in range(6): f[o].attrs[f'late{{i}}'] = 'z' * (i * 700 + 5)\n\ + \x20 del f[o].attrs['c1']\n", + p = b.to_str().unwrap() + )); + want.retain(|(_, nm, _)| nm != "c1"); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + } +} + /// Space one edit frees is reused by later edits of the same editor: the /// chunks a shrink removes are where the chunks of the following growth /// go, so the file does not grow; with a new editor per edit (nothing to diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs index 03e6b1e..dfd3af5 100644 --- a/crates/clawhdf5/src/edit/fheap.rs +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -3,23 +3,29 @@ //! best-fitting free section the heap's free-space manager records (a new //! direct block when none fits: the root direct block of an empty heap, the //! next block of the root indirect block — created from the root direct -//! block, and doubled, as needed), huge objects (larger than the heap's +//! block, and doubled, as needed — with the blocks too small for the object +//! skipped and kept as free space for later objects), huge objects (larger than the heap's //! managed maximum) into their own file space tracked by the huge-object //! version-2 B-tree; removed objects return their space to the free-space //! manager, merged with adjacent free space. //! //! The heap's free-space manager (`FSHD` header, `FSSE` section info) is -//! kept exactly as libhdf5 keeps it: sections sorted by size then offset, +//! kept exactly as libhdf5 keeps it: single sections and the indirect +//! sections of skipped blocks (with their first and normal row sections), +//! sections sorted by size then offset, //! the header's counts and sizes, section info moved when its size changes, //! the manager deleted when it tracks nothing. Header statistics (managed //! space, allocated space, free space, allocation iterator, object counts) //! follow libhdf5's arithmetic. //! +//! The heap is read afresh for each edit, so the editor does what libhdf5 +//! does with one edit per file session (libhdf5 keeps an in-memory span per +//! indirect section that it recomputes when it reloads the section info). +//! //! What libhdf5 would do differently is refused ([`Error::Unsupported`], //! before anything is written): I/O filters on the heap, child indirect -//! blocks, skipped blocks (a first object too large for the next block), -//! free sections other than "single" ones, freeing a whole direct block, -//! directly addressed huge objects, tiny objects. +//! blocks (and free space in them), freeing a whole direct block, directly +//! addressed huge objects, tiny objects. use std::collections::{BTreeMap, BTreeSet}; @@ -67,8 +73,129 @@ const FS_NCLASSES: u16 = 4; const HUGE_BT2_TYPE: u8 = 1; const HUGE_BT2_NODE: u32 = 512; -/// The heap's free-space manager: only "single" sections (free space inside -/// a direct block), keyed by heap offset. +/// The heap's doubling table as its free sections need it: the block size +/// of each direct row, the direct block overhead, the width and the size +/// of a heap offset. +#[derive(Clone, Debug)] +struct Geo { + width: usize, + /// Block size of rows 0 .. max direct rows. + rblock: Vec, + /// `H5HF_MAN_ABS_DIRECT_OVERHEAD`. + ov: u64, + heap_off_size: usize, +} + +impl Geo { + /// Free space in a block of `row` (a row section's size). + fn rfree(&self, row: usize) -> u64 { + self.rblock[row] - self.ov + } + + /// Heap offset of `row`'s first block in the root indirect block. + fn row_off(&self, row: usize) -> u64 { + (0..row).map(|r| self.rblock[r] * self.width as u64).sum() + } + + /// `H5HF__dtable_span_size`. + fn span(&self, row: usize, col: usize, n: usize) -> u64 { + let w = self.width; + let end = row * w + col + n - 1; + let (end_row, end_col) = (end / w, end % w); + if row == end_row { + return self.rblock[row] * (end_col - col + 1) as u64; + } + let mut r = row; + let mut acc = 0; + if col > 0 { + acc = self.rblock[r] * (w - col) as u64; + r += 1; + } + while r < end_row { + acc += self.rblock[r] * w as u64; + r += 1; + } + acc + self.rblock[r] * (end_col + 1) as u64 + } +} + +/// A row section (`H5HF_FSPACE_SECT_FIRST_ROW` / `NORMAL_ROW`): `n` +/// unallocated direct blocks of one row of the root indirect block, from +/// column `col`; its size is one block's free space. +#[derive(Clone, Debug)] +struct RowSect { + addr: u64, + row: usize, + col: usize, + n: usize, +} + +/// An indirect section (`H5HF_FSPACE_SECT_INDIRECT`): `n` unallocated +/// entries of the root indirect block from (`row`, `col`) — blocks skipped +/// over to allocate a larger one (`H5HF__hdr_skip_blocks`). It is never in +/// the free-space manager itself; its row sections are, the first as the +/// serialized "first row" (which stands for the indirect section on disk), +/// the others as unserialized "ghost" normal rows. `span` is kept as +/// libhdf5 keeps it (reducing a row subtracts the row section's size, not +/// the block's), since merging compares it. +#[derive(Clone, Debug)] +struct Ind { + addr: u64, + row: usize, + col: usize, + n: usize, + span: u64, + rows: Vec, +} + +impl Ind { + /// `H5HF__sect_indirect_new` + `H5HF__sect_indirect_init_rows` for the + /// root indirect block (direct rows only). + fn new(geo: &Geo, row: usize, col: usize, n: usize) -> Self { + let w = geo.width; + let end = row * w + col + n - 1; + let (end_row, end_col) = (end / w, end % w); + let addr = geo.row_off(row) + geo.rblock[row] * col as u64; + let mut rows = Vec::new(); + let mut entries = if row == end_row { + end_col - col + 1 + } else { + w - col + }; + let mut row_col = col; + let mut off = addr; + for u in row..=end_row { + rows.push(RowSect { + addr: off, + row: u, + col: row_col, + n: entries, + }); + off += entries as u64 * geo.rblock[u]; + entries = if u + 1 < end_row { w } else { end_col + 1 }; + row_col = 0; + } + Self { + addr, + row, + col, + n, + span: geo.span(row, col, n), + rows, + } + } +} + +/// A free section `find` chose. +enum Found { + Single(u64, u64), + /// Row `.1` of indirect section `.0`. + Row(usize, usize), +} + +/// The heap's free-space manager: "single" sections (free space inside a +/// direct block) keyed by heap offset, and the indirect sections (with +/// their row sections) of skipped direct blocks. struct FreeSpace { addr: u64, max_sect_addr: u16, @@ -80,6 +207,8 @@ struct FreeSpace { sect_size: u64, alloc_sect_size: u64, sects: BTreeMap, + inds: Vec, + geo: Geo, } impl FreeSpace { @@ -87,7 +216,7 @@ impl FreeSpace { 6 + 4 * ls as usize + 8 + ls as usize + os as usize + 2 * ls as usize + 4 } - fn open(img: &Image<'_>, addr: u64) -> Result { + fn open(img: &Image<'_>, addr: u64, geo: &Geo) -> Result { let (os, ls) = (img.os, img.ls); let len = Self::hdr_len(os, ls); let d = img.read(addr, len)?; @@ -120,9 +249,6 @@ impl FreeSpace { let sect_addr = next(os as usize); let sect_size = next(l); let alloc_sect_size = next(l); - if ghost != 0 || tot_count != serial { - return Err(unsupported("free space in row or indirect sections")); - } let mut fs = Self { addr, max_sect_addr, @@ -134,8 +260,13 @@ impl FreeSpace { sect_size, alloc_sect_size, sects: BTreeMap::new(), + inds: Vec::new(), + geo: geo.clone(), }; if serial == 0 { + if tot_count != 0 || ghost != 0 || tot_space != 0 { + return Err(bad("free-space counts disagree")); + } return Ok(fs); } if sect_addr == undef(os) || sect_size < 9 + os as u64 || sect_size > alloc_sect_size { @@ -154,9 +285,10 @@ impl FreeSpace { let cnt_w = enc_size(serial); let len_w = enc_size(max_sect_size); let off_w = (usize::from(max_sect_addr)).div_ceil(8); + let ind_w = geo.heap_off_size + 6; let mut q = 5 + os as usize; let mut seen = 0u64; - let mut total = 0u64; + let mut addrs = BTreeSet::new(); while seen < serial { if q + cnt_w + len_w > n - 4 { return Err(bad("section info ends early")); @@ -165,8 +297,8 @@ impl FreeSpace { q += cnt_w; let size = le(&s[q..q + len_w]); q += len_w; - if count == 0 || size == 0 { - return Err(bad("empty section size node")); + if count == 0 || size == 0 || count > serial - seen { + return Err(bad("bad section size node")); } for _ in 0..count { if q + off_w + 1 > n - 4 { @@ -175,51 +307,141 @@ impl FreeSpace { let off = le(&s[q..q + off_w]); let class = s[q + off_w]; q += off_w + 1; - if class != 0 { - return Err(unsupported("free space in row or indirect sections")); + match class { + 0 => { + if fs.sects.insert(off, size).is_some() { + return Err(bad("duplicate free section")); + } + } + 1 => { + // H5HF__sect_indirect_deserialize. + if q + ind_w > n - 4 { + return Err(bad("section info ends early")); + } + let iblock_off = le(&s[q..q + geo.heap_off_size]); + let b = &s[q + geo.heap_off_size..]; + let row = usize::from(u16::from_le_bytes([b[0], b[1]])); + let col = usize::from(u16::from_le_bytes([b[2], b[3]])); + let cnt = usize::from(u16::from_le_bytes([b[4], b[5]])); + q += ind_w; + if iblock_off != 0 { + return Err(unsupported("free space in child indirect blocks")); + } + if cnt == 0 || col >= geo.width { + return Err(bad("bad indirect free section")); + } + let end_row = (row * geo.width + col + cnt - 1) / geo.width; + if end_row >= geo.rblock.len() { + return Err(unsupported("free space in child indirect blocks")); + } + let ind = Ind::new(geo, row, col, cnt); + if ind.addr != off || geo.rfree(row) != size { + return Err(bad("bad indirect free section")); + } + fs.inds.push(ind); + } + _ => return Err(bad("free section of an unknown class")), } - if fs.sects.insert(off, size).is_some() { + if !addrs.insert(off) { return Err(bad("duplicate free section")); } seen += 1; - total += size; } } - if total != tot_space { - return Err(bad("free-space total disagrees with its sections")); + let (space, total, ser, gh) = fs.totals(); + if (space, total, ser, gh) != (tot_space, tot_count, serial, ghost) { + return Err(bad("free-space totals disagree with its sections")); } Ok(fs) } + /// Every row section: (address, size, indirect section, row). + fn rows(&self) -> impl Iterator + '_ { + self.inds.iter().enumerate().flat_map(move |(i, ind)| { + ind.rows + .iter() + .enumerate() + .map(move |(r, rs)| (rs.addr, self.geo.rfree(rs.row), i, r)) + }) + } + + /// (total space, sections, serialized sections, ghost sections). + fn totals(&self) -> (u64, u64, u64, u64) { + let nrows: u64 = self.inds.iter().map(|i| i.rows.len() as u64).sum(); + let space = self.sects.values().sum::() + self.rows().map(|r| r.1).sum::(); + let singles = self.sects.len() as u64; + let firsts = self.inds.len() as u64; + (space, singles + nrows, singles + firsts, nrows - firsts) + } + + fn is_empty(&self) -> bool { + self.sects.is_empty() && self.inds.is_empty() + } + + /// The offset of the section nearest `addr` below it (or above it) in + /// the merge list — singles and first rows, by offset — when that + /// section is a first row, the only kind a first row merges with. + fn merge_neighbour(&self, addr: u64, below: bool) -> Option { + let firsts = self.inds.iter().map(|i| (i.addr, true)); + let singles = self.sects.keys().map(|&o| (o, false)); + let all = firsts.chain(singles); + let pick = if below { + all.filter(|e| e.0 < addr).max_by_key(|e| e.0) + } else { + all.filter(|e| e.0 > addr).min_by_key(|e| e.0) + }; + pick.and_then(|(a, first)| first.then_some(a)) + } + /// Best fit (`H5FS__sect_find_node`): the smallest section of at least - /// `size` bytes, the lowest offset among equals. - fn find(&self, size: u64) -> Option<(u64, u64)> { - self.sects + /// `size` bytes, the lowest offset among equals, singles and row + /// sections alike. + fn find(&self, size: u64) -> Option { + let single = self + .sects .iter() .filter(|&(_, &s)| s >= size) - .min_by_key(|&(&o, &s)| (s, o)) - .map(|(&o, &s)| (o, s)) + .map(|(&o, &s)| ((s, o), Found::Single(o, s))); + let rows = self + .rows() + .filter(|r| r.1 >= size) + .map(|(a, s, i, r)| ((s, a), Found::Row(i, r))); + single.chain(rows).min_by_key(|(k, _)| *k).map(|(_, f)| f) + } + + /// The serialized sections, by size then offset: (size, offset, + /// indirect section if a first row). + fn serial(&self) -> Vec<(u64, u64, Option)> { + let mut v: Vec<(u64, u64, Option)> = + self.sects.iter().map(|(&o, &s)| (s, o, None)).collect(); + v.extend( + self.inds + .iter() + .enumerate() + .map(|(i, ind)| (self.geo.rfree(ind.row), ind.addr, Some(i))), + ); + v.sort_unstable(); + v } /// The serialized section info's size (`H5FS__sect_serialize_size`). fn needed(&self, os: u8) -> u64 { - let n = self.sects.len() as u64; let prefix = 4 + 1 + u64::from(os) + 4; + let serial = self.serial(); + let n = serial.len() as u64; if n == 0 { return prefix; } - let sizes: BTreeSet = self.sects.values().copied().collect(); + let sizes: BTreeSet = serial.iter().map(|s| s.0).collect(); prefix + sizes.len() as u64 * (enc_size(n) + enc_size(self.max_sect_size)) as u64 + n * (u64::from(self.max_sect_addr).div_ceil(8) + 1) + + self.inds.len() as u64 * (self.geo.heap_off_size as u64 + 6) } fn serialize(&self, os: u8) -> Vec { - let n = self.sects.len() as u64; - let mut by_size: BTreeMap> = BTreeMap::new(); - for (&o, &s) in &self.sects { - by_size.entry(s).or_default().push(o); - } + let serial = self.serial(); + let n = serial.len() as u64; let cnt_w = enc_size(n); let len_w = enc_size(self.max_sect_size); let off_w = usize::from(self.max_sect_addr).div_ceil(8); @@ -229,13 +451,28 @@ impl FreeSpace { let mut a = vec![0u8; os as usize]; put_uint(&mut a, self.addr, os); d.extend_from_slice(&a); - for (size, offs) in by_size { - d.extend_from_slice(&(offs.len() as u64).to_le_bytes()[..cnt_w]); + let mut i = 0; + while i < serial.len() { + let size = serial[i].0; + let same = serial[i..].iter().take_while(|s| s.0 == size).count(); + d.extend_from_slice(&(same as u64).to_le_bytes()[..cnt_w]); d.extend_from_slice(&size.to_le_bytes()[..len_w]); - for o in offs { + for &(_, o, ind) in &serial[i..i + same] { d.extend_from_slice(&o.to_le_bytes()[..off_w]); - d.push(0); + match ind { + None => d.push(0), + Some(k) => { + // H5HF__sect_indirect_serialize (root block: offset 0). + let ind = &self.inds[k]; + d.push(1); + d.extend_from_slice(&vec![0u8; self.geo.heap_off_size]); + for v in [ind.row, ind.col, ind.n] { + d.extend_from_slice(&(v as u16).to_le_bytes()); + } + } + } } + i += same; } d } @@ -244,13 +481,12 @@ impl FreeSpace { let (os, ls) = (img.os, img.ls); let len = Self::hdr_len(os, ls); let l = ls as usize; - let n = self.sects.len() as u64; - let tot: u64 = self.sects.values().sum(); + let (tot, n, serial, ghost) = self.totals(); let mut d = Vec::with_capacity(len); d.extend_from_slice(b"FSHD"); d.push(0); d.push(FS_CLIENT_FHEAP); - for v in [tot, n, n, 0] { + for v in [tot, n, serial, ghost] { d.extend_from_slice(&v.to_le_bytes()[..l]); } for v in [self.nclasses, self.shrink, self.expand, self.max_sect_addr] { @@ -440,7 +676,7 @@ impl Heap { } } if fs_addr != undef(os) { - let fs = FreeSpace::open(img, fs_addr)?; + let fs = FreeSpace::open(img, fs_addr, &h.geo(os))?; if fs.max_sect_addr != max_index { return Err(bad("free-space manager does not match the heap")); } @@ -678,10 +914,23 @@ impl Heap { return Err(unsupported("tiny objects")); } let (off, size) = match self.fs.as_ref().and_then(|fs| fs.find(n)) { - Some((o, s)) => { + Some(Found::Single(o, s)) => { self.fs.as_mut().expect("found above").sects.remove(&o); (o, s) } + // H5HF__man_iblock_alloc_row: a skipped block, created now. + Some(Found::Row(i, r)) => { + let entry = self.alloc_row(i, r); + self.fs_dirty = true; + let w = usize::from(self.width); + let (row, col) = (entry / w, entry % w); + let bsize = self.row_size(row); + let block_off = self.row_off(row) + bsize * col as u64; + let a = self.dblock_create(img, block_off, bsize)?; + self.ents[entry] = a; + self.iblock_dirty = true; + (block_off + self.overhead(os), bsize - self.overhead(os)) + } None => self.dblock_new(img, n)?, }; // H5HF__sect_single_reduce: the object goes at the section's start. @@ -705,7 +954,16 @@ impl Heap { /// `H5FS_ADD_RETURNED_SPACE`), creating the free-space manager if the /// heap has none. fn fs_add(&mut self, img: &mut Image<'_>, off: u64, size: u64) -> Result<(), Error> { + self.fs_mut(img)?.sects.insert(off, size); + self.fs_dirty = true; + Ok(()) + } + + /// The free-space manager, created if the heap has none + /// (`H5HF__space_start`). + fn fs_mut(&mut self, img: &mut Image<'_>) -> Result<&mut FreeSpace, Error> { if self.fs.is_none() { + let geo = self.geo(img.os); let addr = img.alloc(FreeSpace::hdr_len(img.os, img.ls) as u64)?; self.fs = Some(FreeSpace { addr, @@ -718,18 +976,160 @@ impl Heap { sect_size: 0, alloc_sect_size: 0, sects: BTreeMap::new(), + inds: Vec::new(), + geo, }); self.fs_addr = addr; } - self.fs - .as_mut() - .expect("created above") - .sects - .insert(off, size); + Ok(self.fs.as_mut().expect("created above")) + } + + /// The doubling table's direct rows, for the free sections. + fn geo(&self, os: u8) -> Geo { + Geo { + width: usize::from(self.width), + rblock: (0..self.max_direct_rows) + .map(|r| self.row_size(r)) + .collect(), + ov: self.overhead(os), + heap_off_size: self.heap_off_size, + } + } + + /// `H5HF__hdr_skip_blocks`: `n` entries of the root indirect block + /// from `start` are skipped — the iterator moves past them and they + /// become an indirect free section (`H5HF__sect_indirect_add`), whose + /// first row is added as returned space, so it merges with an indirect + /// section just below it (`H5FS__sect_merge`). + fn skip_blocks(&mut self, img: &mut Image<'_>, start: usize, n: usize) -> Result<(), Error> { + let w = usize::from(self.width); + let (row, col) = (start / w, start % w); + if (start + n - 1) / w >= self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + let geo = self.geo(img.os); + self.man_iter_off += geo.span(row, col, n); + let iter = self.man_iter_off; + let fs = self.fs_mut(img)?; + let mut cur = Ind::new(&geo, row, col, n); + loop { + let mut modified = false; + // The nearest sections below and above in the merge list + // (singles and first rows, by offset). + let below = fs.merge_neighbour(cur.addr, true); + let above = fs.merge_neighbour(cur.addr, false); + if let Some(a) = below + && let Some(k) = fs.inds.iter().position(|x| x.addr == a) + && fs.inds[k].addr + fs.inds[k].span == cur.addr + { + let mut lower = fs.inds.remove(k); + if cur.addr >= iter { + return Err(unsupported("free space past the heap's end")); + } + merge_ind(&mut lower, cur, w); + cur = lower; + modified = true; + } + if let Some(a) = above + && let Some(k) = fs.inds.iter().position(|x| x.addr == a) + && cur.addr + cur.span == a + { + let upper = fs.inds.remove(k); + if upper.addr >= iter { + return Err(unsupported("free space past the heap's end")); + } + merge_ind(&mut cur, upper, w); + modified = true; + } + if !modified { + break; + } + } + // H5HF__sect_row_can_shrink: a section past the iterator would + // shrink the heap; skipped blocks never are. + if cur.addr >= iter { + return Err(unsupported("free space past the heap's end")); + } + fs.inds.push(cur); self.fs_dirty = true; Ok(()) } + /// `H5HF__sect_row_reduce` with `H5HF__sect_indirect_reduce_row`: take + /// one block out of row `r` of indirect section `i`; returns the root + /// indirect block entry to create it at. + fn alloc_row(&mut self, i: usize, r: usize) -> usize { + let w = usize::from(self.width); + let fs = self.fs.as_mut().expect("section found"); + let geo = fs.geo.clone(); + let ind = &mut fs.inds[i]; + let rs = ind.rows[r].clone(); + let row_start = rs.row * w + rs.col; + let row_end = row_start + rs.n - 1; + let start = ind.row * w + ind.col; + let end = start + ind.n - 1; + let (start_row, end_row) = (ind.row, end / w); + let from_start = !(row_end == end && start_row != end_row); + let entry = if from_start { row_start } else { row_end }; + ind.span -= geo.rfree(rs.row); + let mut peer = None; + if ind.n > 1 { + if entry == start { + ind.addr += geo.rblock[ind.row]; + ind.col += 1; + if ind.col == w { + ind.row += 1; + ind.col = 0; + // The row's last block: the row goes (below). + } + ind.n -= 1; + } else if entry == end { + ind.n -= 1; + } else { + // Split: the rows before this one become a peer section. + let peer_n = entry - start; + let peer_rows = rs.row - start_row; + let rest = ind.rows.split_off(peer_rows); + let p = Ind { + addr: ind.addr, + row: ind.row, + col: ind.col, + n: peer_n, + span: rs.addr - ind.addr, + rows: std::mem::replace(&mut ind.rows, rest), + }; + ind.addr = rs.addr + geo.rblock[rs.row]; + ind.span -= p.span; + ind.row = rs.row; + ind.col = rs.col + 1; + ind.n -= peer_n + 1; + peer = Some(p); + } + } else { + ind.n -= 1; + } + // The row section itself. + let ri = if peer.is_some() { 0 } else { r }; + let row = &mut ind.rows[ri]; + if row.n == 1 { + ind.rows.remove(ri); + } else { + if from_start { + row.addr += geo.rblock[row.row]; + row.col += 1; + } + row.n -= 1; + } + let gone = ind.rows.is_empty(); + if gone { + fs.inds.remove(i); + } + if let Some(p) = peer { + fs.inds.push(p); + } + entry + } + /// `H5HF__man_dblock_new`: a direct block for an object of `request` /// bytes; returns its free section (not in the free-space manager). fn dblock_new(&mut self, img: &mut Image<'_>, request: u64) -> Result<(u64, u64), Error> { @@ -753,30 +1153,16 @@ impl Heap { self.total_man_free += self.start_block_size - self.overhead(os); return Ok((self.overhead(os), self.start_block_size - self.overhead(os))); } - if self.root == undef(os) { - return Err(unsupported( - "a first object too large for the starting block", - )); - } - // H5HF__hdr_update_iter. - if self.root_rows == 0 { - self.root_create(img, min)?; - } - let (mut row, mut col) = self.iter_pos(); - let min_row = self.size_to_row(min); - if min_row > row && row < usize::from(self.root_rows) { - return Err(unsupported("skipping blocks too small for an object")); - } - while row >= usize::from(self.root_rows) { - self.root_double(img, min)?; - (row, col) = self.iter_pos(); - } + // H5HF__hdr_update_iter, then a block at the iterator. + self.update_iter(img, min)?; + let (row, col) = self.iter_pos(); if row >= self.max_direct_rows { return Err(unsupported("child indirect blocks")); } let size = self.row_size(row); if min > size { - return Err(unsupported("skipping blocks too small for an object")); + // libhdf5: "skipping direct block sizes not supported". + return Err(unsupported("a direct block smaller than the object")); } self.man_iter_off += size; let entry = row * usize::from(self.width) + col; @@ -787,6 +1173,37 @@ impl Heap { Ok((block_off + self.overhead(os), size - self.overhead(os))) } + /// `H5HF__hdr_update_iter`: make the iterator point at a block of at + /// least `min` bytes, creating the root indirect block, skipping + /// smaller blocks and doubling the root indirect block as needed. + fn update_iter(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + if self.root_rows == 0 { + return self.root_create(img, min); + } + let w = usize::from(self.width); + let min_row = self.size_to_row(min); + let (mut row, col) = self.iter_pos(); + let nrows = usize::from(self.root_rows); + if min_row > row && row < nrows { + let entry = row * w + col; + let skip = if min_row >= nrows { + nrows * w - entry + } else { + min_row * w - entry + }; + self.skip_blocks(img, entry, skip)?; + row = self.iter_pos().0; + } + while row >= usize::from(self.root_rows) { + self.root_double(img, min)?; + row = self.iter_pos().0; + } + if row >= self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + Ok(()) + } + /// The allocation iterator's (row, column) in the root indirect block. fn iter_pos(&self) -> (usize, usize) { if self.man_iter_off >= self.man_size { @@ -827,8 +1244,9 @@ impl Heap { Ok(a) } - /// `H5HF__man_iblock_root_create`: the root direct block becomes entry - /// 0 of a new root indirect block. + /// `H5HF__man_iblock_root_create`: a root indirect block, with the + /// root direct block (if any) as entry 0; when the block needed is + /// larger than the starting size, the smaller blocks are skipped. fn root_create(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { let os = img.os; let mut nrows = if self.start_root_rows == 0 { @@ -846,9 +1264,6 @@ impl Heap { if nrows > self.max_direct_rows { return Err(unsupported("child indirect blocks")); } - if min > self.start_block_size { - return Err(unsupported("skipping blocks too small for an object")); - } let have_direct = self.root != undef(os); let rows = u16::try_from(nrows).map_err(|_| bad("too many rows"))?; let a = img.alloc(self.iblock_len(rows, os) as u64)?; @@ -860,43 +1275,57 @@ impl Heap { self.man_iter_off = 0; } self.iblock_dirty = true; - self.root_rows = rows; - self.root = a; let w = u64::from(self.width); let ov = self.overhead(os); let mut acc: u64 = (0..nrows).map(|u| (self.row_size(u) - ov) * w).sum(); if have_direct { acc -= self.row_size(0) - ov; } + // The header points at the new block before the skipped blocks + // are added (the free sections need the rows in place). + self.root_rows = rows; + self.root = a; + if min > self.start_block_size { + let first = usize::from(have_direct); + self.skip_blocks(img, first, (nrows - 1) * usize::from(self.width) - first)?; + } self.man_size = self.row_off(nrows); self.total_man_free += acc; Ok(()) } /// `H5HF__man_iblock_root_double`: the root indirect block gets twice - /// its rows, at a new address. + /// its rows (at least enough for a block of `min` bytes, the smaller + /// blocks before it skipped), at a new address. fn root_double(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { let os = img.os; + let w = usize::from(self.width); let old = usize::from(self.root_rows); - let (row, _) = self.iter_pos(); - let next_size = self.row_size(row.min(self.max_root_rows - 1)); + let (next_row, next_col) = self.iter_pos(); + let next_entry = next_row * w + next_col; + let next_size = self.row_size(next_row.min(self.max_root_rows - 1)); + let (mut min_nrows, mut new_next_entry, mut skip) = (0, 0, false); if old < self.max_direct_rows && min > next_size { - return Err(unsupported("skipping blocks too small for an object")); + skip = true; + min_nrows = 1 + self.size_to_row(min); + new_next_entry = (min_nrows - 1) * w; } - let new = (2 * old).min(self.max_root_rows); + let new = min_nrows.max((2 * old).min(self.max_root_rows)); if new > self.max_direct_rows || new == old { return Err(unsupported("child indirect blocks")); } img.free(self.root, self.iblock_len(self.root_rows, os) as u64); let rows = u16::try_from(new).map_err(|_| bad("too many rows"))?; let a = img.alloc(self.iblock_len(rows, os) as u64)?; - let w = usize::from(self.width); self.ents.resize(new * w, undef(os)); + self.root = a; + self.iblock_dirty = true; + if skip { + self.skip_blocks(img, next_entry, new_next_entry - next_entry)?; + } let ov = self.overhead(os); let acc: u64 = (old * w..new * w).map(|u| self.row_size(u / w) - ov).sum(); self.root_rows = rows; - self.root = a; - self.iblock_dirty = true; self.man_size = 2 * self.row_off(new - 1); self.total_man_free += acc; Ok(()) @@ -1091,7 +1520,7 @@ impl Heap { self.fs_dirty = false; if let Some(mut fs) = self.fs.take() { let hdr_len = FreeSpace::hdr_len(os, img.ls) as u64; - if fs.sects.is_empty() { + if fs.is_empty() { // H5HF__space_close: a manager that tracks nothing is // deleted. img.free(fs.addr, hdr_len); @@ -1124,6 +1553,24 @@ impl Heap { } } +/// `H5HF__sect_indirect_merge_row` for two sections of the root indirect +/// block, `b` right after `a`: `b`'s rows join `a` (its first row joins +/// `a`'s last row when they share a row). +fn merge_ind(a: &mut Ind, mut b: Ind, w: usize) { + let end_row1 = (a.row * w + a.col + a.n - 1) / w; + if !b.rows.is_empty() { + if end_row1 == b.row { + let first = b.rows.remove(0); + if let Some(last) = a.rows.last_mut() { + last.n += first.n; + } + } + a.rows.append(&mut b.rows); + } + a.n += b.n; + a.span += b.span; +} + /// The ID in a huge-object B-tree record (type 1: address, length, ID). fn huge_rec_id(r: &[u8], os: u8, ls: u8) -> u64 { get_uint(&r[os as usize + ls as usize..], ls) diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index e1d301a..a5a3c1b 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -958,8 +958,7 @@ impl FileEditor { /// /// [`Error::Unsupported`] for shared attribute messages, heaps this /// editor cannot extend the way libhdf5 would (child indirect blocks, - /// skipped blocks, free space other than within direct blocks, freeing - /// a whole block), and version-1 object headers asked for an attribute + /// freeing a whole block), and version-1 object headers asked for an attribute /// larger than a header message holds. pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { if name.is_empty() { diff --git a/docs/known-issues.md b/docs/known-issues.md index a178d47..2539f63 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -74,13 +74,19 @@ reuse were added). `clawhdf5::FileEditor` refuses, with an optional filter only when its own build lacks it, which none does for these; - attributes in dense storage when the heap cannot take them the way - libhdf5 would: an attribute that needs a heap block larger than the next - one (libhdf5 skips blocks and records them as free space — in practice an - attribute of roughly 1 to 4 KiB going into a young heap), a heap with I/O - filters or child indirect blocks (more than about 512 KiB of attributes), - free space the heap tracks outside direct blocks, replacing the last - attribute left in a heap block by one of another size (libhdf5 frees the - block), directly addressed huge objects; and shared attribute messages; + libhdf5 would: replacing the last attribute left in a heap block by one + of another size (libhdf5 frees the block), a heap with I/O filters or + child indirect blocks (more than about 512 KiB of attributes), free + space in child indirect blocks, directly addressed huge objects; and + shared attribute messages. Measured 2026-09-26 on tank with the review's + random-edit harness (120 runs of 150 random edits, `earliest`/`v110`/ + `latest`, about 5600 `set_attr` calls of 8 bytes to 6 KiB): 2.2% of + `set_attr` calls are refused, every one the last-attribute-in-a-block + replacement; before blocks could be skipped (an attribute needing a heap + block larger than the next one — any attribute of about 1 KiB or more + once a heap has started, or at the move to dense storage), 24% were, + since an object whose move to dense storage was refused kept refusing + every new attribute; - version-1 object headers asked for an attribute larger than a header message (they have no dense storage); - partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external -- 2.54.0 From 75444950f3cd0dd30ac313df0ed9a06e4d4c4e91 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 19:04:56 -0500 Subject: [PATCH 44/49] clawhdf5: storage harness compares errors, not just failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The facade equivalence harness turned every data-read error into "Err", so it could not see File::open_storage failing differently from File::open (a Storage or ContiguousStorageRequired error where the mmap path gives a decode error, say). - value() keeps the whole error. The only allowance is for a line on which File::open itself varies between opens — the chunk cache lists a damaged dataset's chunks in hash-map order, so which failing chunk a full read reports varies (cve-2025-2310.h5, the one corpus file where this shows): both sides must fail there, and a fresh File::open (up to 64) must reproduce the storage's exact error. Open errors were already compared in full; they still agree. - The storage transcript may not contain ContiguousStorageRequired. - More selections: a strided hyperslab (every third row) through read_f64_selection, and out-of-order points through read_selection and read_i64_selection. - harness_compares_errors_not_just_failures checks the harness itself: two different errors are different values, and a difference File::open does not produce is reported. With full errors the harness passes on the 61 fixtures and on the corpus (701 files, 621 open). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 +- crates/clawhdf5/tests/storage_equivalence.rs | 136 ++++++++++++++++--- 2 files changed, 126 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbd2eb0..c74077f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,7 +98,15 @@ that open) through `File::open` and through `File::open_storage` over `CountingStorage`: the tree, every attribute (all, and each by name), every dataset's shape, types and values (all bytes, `f64`, `f32`, - `i64`, a hyperslab, strings, VL sequences) must be identical, and are. + `i64`, a box hyperslab, a strided one, out-of-order points, strings, VL + sequences) must be identical, and are — errors included, in full (open + errors and every read's). The one allowance is a line on which + `File::open` itself varies between two opens (the chunk cache lists a + damaged dataset's chunks in hash-map order, so which failing chunk a + full read reports varies: `cve-2025-2310.h5`), and then only if a fresh + `File::open` reproduces the storage's error. No storage read may answer + `ContiguousStorageRequired`. A storage that returns more bytes than + asked reads every fixture identically too. It also counts what one pass — open, list, read every attribute and every dataset once — asks of a storage with no cache: 176 092 `read_at` calls and 208 MB for the 621 corpus files (254 MB of files); the most diff --git a/crates/clawhdf5/tests/storage_equivalence.rs b/crates/clawhdf5/tests/storage_equivalence.rs index 71768d2..9a5ace8 100644 --- a/crates/clawhdf5/tests/storage_equivalence.rs +++ b/crates/clawhdf5/tests/storage_equivalence.rs @@ -47,15 +47,15 @@ fn digest(v: &T) -> String { format!("{}…[{} bytes, fnv {h:016x}]", &s[..80], s.len()) } -/// A data read's result: its value, or just `Err` — a full read goes +/// A data read's result: its value, or its error in full (the two paths +/// must fail the same way, not just both fail). One case is known to vary +/// between two `File`s and is allowed for in [`check`]: a full read goes /// through the file's chunk cache, which lists a damaged dataset's chunks -/// in hash-map order, so which failing chunk it reports varies from one -/// `File` to the next (the format crate's harness compares these errors on -/// the uncached path). -fn value(r: &Result) -> String { +/// in hash-map order, so which failing chunk it reports varies. +fn value(r: &Result) -> String { match r { Ok(v) => digest(v), - Err(_) => "Err".into(), + Err(e) => format!("Err({e:?})"), } } @@ -173,6 +173,43 @@ fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) { value(&ds.read_selection(&sel)) ) .unwrap(); + // Every third row (a strided hyperslab), and a few points out + // of order: the last element, the first, one in the middle. + let strided = Selection::Hyperslab { + start: vec![0; rank], + stride: std::iter::once(3) + .chain(std::iter::repeat_n(1, rank - 1)) + .collect(), + count: std::iter::once(d0.div_ceil(3)) + .chain(shape[1..].iter().copied()) + .collect(), + block: vec![1; rank], + }; + writeln!( + out, + "{path} f64 strided {}", + value(&ds.read_f64_selection(&strided)) + ) + .unwrap(); + if shape.iter().all(|&d| d > 0) { + let points = Selection::Points(vec![ + shape.iter().map(|&d| d - 1).collect(), + vec![0; rank], + shape.iter().map(|&d| d / 2).collect(), + ]); + writeln!( + out, + "{path} bytes points {}", + value(&ds.read_selection(&points)) + ) + .unwrap(); + writeln!( + out, + "{path} i64 points {}", + value(&ds.read_i64_selection(&points)) + ) + .unwrap(); + } } } match &raw_dt { @@ -296,20 +333,16 @@ fn check(path: &Path, totals: &mut Totals) { assert_eq!(local.user_block_size(), remote.user_block_size(), "{name}"); let want = transcript(&local); let got = transcript(&remote); + // Every read path works over a storage without the file in memory. + assert!( + !got.contains("ContiguousStorageRequired"), + "{name}: a read needed the file in memory" + ); if want != got { - let first = want - .lines() - .zip(got.lines()) - .find(|(w, g)| w != g) - .map(|(w, g)| format!("\n local: {w}\n storage: {g}")) - .unwrap_or_else(|| { - format!( - "\n {} vs {} lines", - want.lines().count(), - got.lines().count() - ) - }); - panic!("{name}: File::open_storage differs from File::open{first}"); + let first = unexplained_difference(path, &want, &got); + if let Some(first) = first { + panic!("{name}: File::open_storage differs from File::open{first}"); + } } totals.reads += storage.reads(); totals.bytes += storage.bytes_read(); @@ -334,6 +367,40 @@ fn check(path: &Path, totals: &mut Totals) { .push((reads, read_bytes, bytes.len() as u64, name)); } +/// Why the storage transcript `got` differs from the `File::open` one +/// `want`, or `None` when every line that differs is one `File::open` can +/// give too: a line that varies between two `File`s (the chunk cache's +/// hash-map order picks which failing chunk a damaged dataset's full read +/// reports) and whose storage value some fresh `File::open` reproduces. +/// Nothing else is allowed to differ. +fn unexplained_difference(path: &Path, want: &str, got: &str) -> Option { + let (want, got): (Vec<&str>, Vec<&str>) = (want.lines().collect(), got.lines().collect()); + if want.len() != got.len() { + return Some(format!("\n {} vs {} lines", want.len(), got.len())); + } + let mut open: Vec = (0..want.len()).filter(|&i| want[i] != got[i]).collect(); + // Only reads that fail on both sides may vary. + if let Some(&i) = open + .iter() + .find(|&&i| !(want[i].contains(" Err(") && got[i].contains(" Err("))) + { + return Some(format!("\n local: {}\n storage: {}", want[i], got[i])); + } + for _ in 0..64 { + let again = transcript(&File::open(path).unwrap()); + let again: Vec<&str> = again.lines().collect(); + open.retain(|&i| again.get(i) != Some(&got[i])); + if open.is_empty() { + return None; + } + } + let i = open[0]; + Some(format!( + "\n local: {}\n storage: {}\n (no File::open of 64 gave the storage's result)", + want[i], got[i] + )) +} + fn report(what: &str, totals: &mut Totals) { eprintln!( "{what}: {} files ({} open, {} bytes); comparison: {} read_at calls, {} bytes; \ @@ -479,3 +546,34 @@ fn overlong_storage_reads_identically() { } assert!(compared >= 40, "{compared}"); } + +/// The harness tells failures apart: two different errors are two +/// different transcripts, and only a difference `File::open` itself +/// produces between two opens is let through. +#[test] +fn harness_compares_errors_not_just_failures() { + let a: Result<(), FormatError> = Err(FormatError::ContiguousStorageRequired("x")); + let b: Result<(), FormatError> = Err(FormatError::Storage("x".into())); + assert_ne!(value(&a), value(&b)); + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../clawhdf5-format/tests/fixtures/chunked_2d.h5"); + let want = transcript(&File::open(&path).unwrap()); + assert_eq!(unexplained_difference(&path, &want, &want), None); + // A read that fails differently through the storage. + let line = want.lines().position(|l| l.contains(" all ")).unwrap(); + let (mut w, mut g): (Vec, Vec) = ( + want.lines().map(String::from).collect(), + want.lines().map(String::from).collect(), + ); + w[line] = format!( + "{} Err(Format(DataSizeMismatch))", + &w[line][..w[line].find(" all ").unwrap() + 4] + ); + g[line] = format!( + "{} Err(Format(Storage(\"injected\")))", + &g[line][..g[line].find(" all ").unwrap() + 4] + ); + assert!(unexplained_difference(&path, &w.join("\n"), &g.join("\n")).is_some()); + // A value against an error is never let through. + assert!(unexplained_difference(&path, &want, &g.join("\n")).is_some()); +} -- 2.54.0 From ea0508aaa5a6f20a8261ffbb056a46767549cb58 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 19:05:45 -0500 Subject: [PATCH 45/49] format: no truncating u64 -> usize casts in gather_storage and ExtentBytes check-32bit-casts.sh flagged two casts added by the previous commits; both values are bounded (checked by gather_storage's first walk, and built from a usize fetch length), so they go through addr::saturating_usize. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/gather.rs | 6 +++--- crates/clawhdf5-format/src/storage.rs | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/src/gather.rs b/crates/clawhdf5-format/src/gather.rs index 08480f1..5f73dbd 100644 --- a/crates/clawhdf5-format/src/gather.rs +++ b/crates/clawhdf5-format/src/gather.rs @@ -454,9 +454,9 @@ pub(crate) fn gather_storage( if error.is_some() { return; } - // Checked by the first walk. - let mut at = first as usize * elem_size; - let mut len = n as usize * elem_size; + // Checked by the first walk (these cannot saturate or wrap). + let mut at = crate::addr::saturating_usize(first).wrapping_mul(elem_size); + let mut len = crate::addr::saturating_usize(n).wrapping_mul(elem_size); while len > 0 { while spans.get(span).is_some_and(|s| s.out_end <= out.len()) { span += 1; diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 482cfad..08720ab 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -477,7 +477,8 @@ impl<'a> ExtentBytes<'a> { )); } for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) { - out[slot] = Extent::Bytes(exact_len(bytes, (r.end - r.start) as usize)?); + let len = crate::addr::saturating_usize(r.end - r.start); + out[slot] = Extent::Bytes(exact_len(bytes, len)?); } } Ok(ExtentBytes::Fetched { base, extents: out }) -- 2.54.0 From f37e7ae3263277319dba4bc39be5397194eb00c3 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 19:33:03 -0500 Subject: [PATCH 46/49] format: bound ZFP chunk fetches at 4x, not n + n/4 The M2 fetch bound gave every codec n + n/4 + 4096 stored bytes, but ZFP's fixed-rate mode stores up to 64 bits per value, doubling 4-byte types: 60 of the 2205 zfp_interop datasets (rate 64, f4/i4) failed with 'compressed stream ends early' once M2 and ZFP were merged. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 98cf270..28070d4 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -72,7 +72,8 @@ fn filter_output_bound(filter_id: u16, input: usize) -> usize { /// `filter_mask`) is the chunk itself: `chunk_bytes`, and bytes past them /// were never used. A filtered chunk is bounded by each applied filter's /// worst-case growth in the write direction: shuffle keeps the size, -/// Fletcher32 adds 4 bytes, and any other codec gets `n + n/4 + 4096` — a +/// Fletcher32 adds 4 bytes, ZFP gets `4n + 4096` (its fixed-rate mode stores +/// up to 64 bits per value), and any other codec gets `n + n/4 + 4096` — a /// deliberately generous bound (bzip2 grows 1000 random bytes by 252, more /// than the decoders' own `n + n/8 + 64` output bound), since a legitimate /// chunk cut short here would fail to read. A filter handled by a codec the @@ -95,6 +96,11 @@ pub(crate) fn stored_chunk_limit( FILTER_SHUFFLE => size, FILTER_FLETCHER32 => size.saturating_add(4), id if filter_registry::may_be_registered(id) => return usize::MAX, + // ZFP's fixed-rate mode stores up to 64 bits per value, so a + // 4-byte type doubles, and precision/accuracy modes add group + // test bits per bit plane on top: 4x plus a header is still a + // bound, where `n + n/4` cut rate-64 chunks short. + crate::filter_pipeline::FILTER_ZFP => size.saturating_mul(4).saturating_add(4096), _ => size.saturating_add(size / 4).saturating_add(4096), }; } -- 2.54.0 From 011e0dbb966fc4e9c002477e1f688e79781f84b5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 19:34:27 -0500 Subject: [PATCH 47/49] docs: conformance report after range-read M2/M3 and editor coverage (600 of 697 ok) Co-Authored-By: Claude Opus 5.5 (1M context) --- CONFORMANCE.md | 4 ++-- conformance/baseline.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 3a1f0ac..ff5b581 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -13,8 +13,8 @@ fatal. This file is generated by `conformance/run.sh`; do not edit it by hand. | | | |---|---| -| date | 2026-09-26 20:06 UTC | -| clawhdf5 commit | `8fadb9f4242a35323262701328d380806d379140` | +| date | 2026-09-27 00:34 UTC | +| clawhdf5 commit | `f37e7ae3263277319dba4bc39be5397194eb00c3` | | machine | `tank`: AMD Ryzen 7 7800X3D 8-Core Processor, 16 CPUs, 61 GiB, Linux 7.0.0-34-generic x86_64 | | command | `conformance/run.sh --no-fetch --update-baseline` | | rustc | rustc 1.98.1 (48a229cea 2026-09-01) | diff --git a/conformance/baseline.json b/conformance/baseline.json index 420a75b..0db7eb3 100644 --- a/conformance/baseline.json +++ b/conformance/baseline.json @@ -1,7 +1,7 @@ { "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. Regenerate with `conformance/run.sh --update-baseline` after an intended change.", - "commit": "8fadb9f4242a35323262701328d380806d379140", - "date": "2026-09-26 20:06 UTC", + "commit": "f37e7ae3263277319dba4bc39be5397194eb00c3", + "date": "2026-09-27 00:34 UTC", "reference": "h5py 3.16.0 / HDF5 2.0.0", "files": 697, "ok": 600, -- 2.54.0 From 4313917b4dd1d5c76c285e9d01151a21ba6613b3 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 20:30:16 -0500 Subject: [PATCH 48/49] format: object headers without per-header allocations for their chunk list Reading continuation chunks from a queue (7e5e920, a69c5be) allocated a queue Vec and a BTreeSet of chunk starts for every header, and inlined the per-chunk message loop into the generic parser: ObjectHeader::parse over 401 version-1 headers went from 24.8 to 45.7 us. ChunkSpans now keeps the first 8 chunks in an inline array (cycle check by scan) and is also the read queue; only a header of more chunks allocates (a boxed spill list and start set). The message loop of one version-1 chunk is its own non-generic function. Same checks as before: any number of chunks up to 65,536, cycles refused, chunks bounded by the file size, one chunk buffer alive at a time, libhdf5 message order, overlap allowed. The cycle test now also covers spilled chunk lists. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 + crates/clawhdf5-format/src/object_header.rs | 204 ++++++++++++++------ 2 files changed, 149 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1a92a..4556f91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -295,6 +295,14 @@ chunks nested in each other made storage with owned buffers read and hold the square of the file's size), or more than 65 536 chunks are refused, so a header's chunks read at most the file's size. + The first version allocated a queue and a set of chunk starts for every + header and made `ObjectHeader::parse` of 401 small headers 1.8x slower + (`local_metadata_bench`: 45.7 vs 24.8 µs on `main` `8f59b2e`). The first + 8 chunks now live in an inline array (cycle check by scan; only a longer + header allocates), and the per-chunk message loop is its own function + instead of being inlined into the generic parser. Provisional (tank load + 3–4; Criterion, separate binaries, 2 alternating rounds): 24.96–25.12 vs + 24.85–25.15 µs on `main`. - Tests: `crates/clawhdf5-tools/tests/edit_coverage_interop.rs` (h5py `earliest`/`v110`/`latest` and clawhdf5-written files; structure comparisons with libhdf5 for version-2 B-trees, shrink on every index, diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index ab652cb..d421ea0 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -1,7 +1,7 @@ //! HDF5 Object Header parsing (v1 and v2). #[cfg(not(feature = "std"))] -use alloc::{collections::BTreeSet, vec, vec::Vec}; +use alloc::{boxed::Box, collections::BTreeSet, vec::Vec}; #[cfg(feature = "std")] use std::collections::BTreeSet; @@ -248,60 +248,14 @@ impl ObjectHeader { length_size: u8, messages: &mut Vec, ) -> Result { + // The chunks found so far are also the queue of chunks to read. let mut spans = ChunkSpans::new(file.len(), offset, length)?; - let mut queue: Vec<(u64, usize)> = vec![(offset, length)]; let mut chunk0_count = 0usize; let mut next = 0usize; - while let Some(&(chunk_offset, chunk_length)) = queue.get(next) { + while let Some((chunk_offset, chunk_length)) = spans.get(next) { let chunk = read_exact_at(file, chunk_offset, chunk_length)?; - let data: &[u8] = &chunk; - let end = data.len(); - let mut pos = 0usize; - let mut count = 0usize; - while pos < end { - if end - pos < V1_MSG_HEADER_SIZE { - return Err(FormatError::InvalidObjectHeader( - "gap found in early version of file format", - )); - } - let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); - let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; - let msg_flags = data[pos + 4]; - // reserved(3) at pos+5..pos+8 - pos += V1_MSG_HEADER_SIZE; - - if !msg_data_size.is_multiple_of(8) { - return Err(FormatError::InvalidObjectHeader("message not aligned")); - } - if msg_data_size > end - pos { - return Err(FormatError::InvalidObjectHeader( - "message size exceeds buffer end", - )); - } - let body = &data[pos..pos + msg_data_size]; - check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; - count += 1; - let msg_type = MessageType::from_u16(msg_type_raw); - if msg_type != MessageType::Nil { - messages.push(HeaderMessage { - msg_type, - size: msg_data_size, - flags: msg_flags, - creation_order: None, - data: body.to_vec(), - }); - } - // Queue continuations (v1 continuation chunks are just raw - // messages, no signature); check_message has checked the body. - if msg_type == MessageType::ObjectHeaderContinuation { - let cont_offset = read_offset(body, 0, offset_size)?; - let cont_length = - to_usize(read_offset(body, offset_size as usize, length_size)?)?; - spans.add(cont_offset, cont_length)?; - queue.push((cont_offset, cont_length)); - } - pos += msg_data_size; - } + let count = + Self::parse_v1_messages(&chunk, offset_size, length_size, messages, &mut spans)?; // Only the first chunk's messages are held to the prefix count. if next == 0 { chunk0_count = count; @@ -311,6 +265,65 @@ impl ObjectHeader { Ok(chunk0_count) } + /// The messages of one version-1 chunk: each checked and appended to + /// `messages` (NIL ones dropped), each continuation added to `spans`. + /// Returns how many messages (NIL ones included) the chunk holds. + #[inline(never)] + fn parse_v1_messages( + data: &[u8], + offset_size: u8, + length_size: u8, + messages: &mut Vec, + spans: &mut ChunkSpans, + ) -> Result { + let end = data.len(); + let mut pos = 0usize; + let mut count = 0usize; + while pos < end { + if end - pos < V1_MSG_HEADER_SIZE { + return Err(FormatError::InvalidObjectHeader( + "gap found in early version of file format", + )); + } + let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); + let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; + let msg_flags = data[pos + 4]; + // reserved(3) at pos+5..pos+8 + pos += V1_MSG_HEADER_SIZE; + + if !msg_data_size.is_multiple_of(8) { + return Err(FormatError::InvalidObjectHeader("message not aligned")); + } + if msg_data_size > end - pos { + return Err(FormatError::InvalidObjectHeader( + "message size exceeds buffer end", + )); + } + let body = &data[pos..pos + msg_data_size]; + check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; + count += 1; + let msg_type = MessageType::from_u16(msg_type_raw); + if msg_type != MessageType::Nil { + messages.push(HeaderMessage { + msg_type, + size: msg_data_size, + flags: msg_flags, + creation_order: None, + data: body.to_vec(), + }); + } + // Queue continuations (v1 continuation chunks are just raw + // messages, no signature); check_message has checked the body. + if msg_type == MessageType::ObjectHeaderContinuation { + let cont_offset = read_offset(body, 0, offset_size)?; + let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?; + spans.add(cont_offset, cont_length)?; + } + pos += msg_data_size; + } + Ok(count) + } + fn parse_v2( file: &S, offset: u64, @@ -601,24 +614,47 @@ const V2_PREFIX_MAX: usize = 34; /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3). const V1_MSG_HEADER_SIZE: usize = 8; -/// The chunks of one object header read so far. A chunk starting where -/// another did is a cycle. Chunks of a valid header do not overlap, so +/// The chunks of one object header read so far, in the order they were +/// found (which is the order version-1 chunks are read in). A chunk starting +/// where another did is a cycle. Chunks of a valid header do not overlap, so /// together they are no larger than the file; a header whose chunks add up /// to more is refused, which bounds what its chunks can make a reader read /// (a crafted chain of chunks each nested in the last would otherwise read /// the file over and over). Overlap itself is not refused: libhdf5 reads /// such headers (`cve-2025-7067.h5` has one). +/// +/// Almost every header has at most a few chunks, and this runs once per +/// header, so the first [`INLINE_CHUNKS`] live in an inline array and are +/// checked for cycles by a scan; only a longer header allocates (the rest +/// of the list, and a set of starts). Allocating a queue and a set for +/// every header made parsing 401 small headers 1.8x slower. struct ChunkSpans { - starts: BTreeSet, + inline: [(u64, usize); INLINE_CHUNKS], + /// Chunks after the first [`INLINE_CHUNKS`], and every chunk start. + spill: Option>, + /// How many chunks there are. + len: usize, /// Bytes of the chunks so far, and the most they may add up to. total: u64, budget: u64, } +/// The chunks of a [`ChunkSpans`] beyond its inline ones. +struct SpilledSpans { + chunks: Vec<(u64, usize)>, + starts: BTreeSet, +} + +/// How many chunks [`ChunkSpans`] holds without allocating. +const INLINE_CHUNKS: usize = 8; + impl ChunkSpans { + #[inline] fn new(file_len: u64, start: u64, len: usize) -> Result { let mut s = Self { - starts: BTreeSet::new(), + inline: [(0, 0); INLINE_CHUNKS], + spill: None, + len: 0, total: 0, budget: file_len, }; @@ -626,11 +662,19 @@ impl ChunkSpans { Ok(s) } + /// Record the chunk `len` bytes at `start`. + #[inline] fn add(&mut self, start: u64, len: usize) -> Result<(), FormatError> { - if !self.starts.insert(start) || self.starts.len() > MAX_V1_CHUNKS { - return Err(FormatError::NestingDepthExceeded); - } self.total = self.total.saturating_add(len as u64); + if self.len < INLINE_CHUNKS { + if self.inline[..self.len].iter().any(|&(s, _)| s == start) { + return Err(FormatError::NestingDepthExceeded); + } + self.inline[self.len] = (start, len); + } else { + self.add_spilled(start, len)?; + } + self.len += 1; if self.total > self.budget { return Err(FormatError::InvalidObjectHeader( "object header chunks larger than the file", @@ -638,6 +682,33 @@ impl ChunkSpans { } Ok(()) } + + #[cold] + #[inline(never)] + fn add_spilled(&mut self, start: u64, len: usize) -> Result<(), FormatError> { + let inline = &self.inline; + let spill = self.spill.get_or_insert_with(|| { + Box::new(SpilledSpans { + chunks: Vec::new(), + starts: inline.iter().map(|&(s, _)| s).collect(), + }) + }); + if !spill.starts.insert(start) || self.len >= MAX_V1_CHUNKS { + return Err(FormatError::NestingDepthExceeded); + } + spill.chunks.push((start, len)); + Ok(()) + } + + /// The `i`th chunk recorded. + #[inline] + fn get(&self, i: usize) -> Option<(u64, usize)> { + if i < INLINE_CHUNKS { + (i < self.len).then(|| self.inline[i]) + } else { + self.spill.as_ref()?.chunks.get(i - INLINE_CHUNKS).copied() + } + } } /// Most chunks a version-1 object header may have (malformed-data guard; @@ -1105,11 +1176,18 @@ mod tests { #[test] fn v1_continuation_cycles_are_refused() { - let data = v1_chain(5, true); - assert!(matches!( - ObjectHeader::parse(&data, 0, 8, 8), - Err(FormatError::NestingDepthExceeded) - )); + // Within the inline chunk list, and past it (the cycle returns to + // an inline chunk once the list has spilled). + for n in [5, 7, 8, 9, 40] { + let data = v1_chain(n, true); + assert!( + matches!( + ObjectHeader::parse(&data, 0, 8, 8), + Err(FormatError::NestingDepthExceeded) + ), + "{n} chunks" + ); + } } #[test] -- 2.54.0 From ef428d756c96ab4a5b1ef76475474571ca5648bc Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 22:28:22 -0500 Subject: [PATCH 49/49] format: slice entry points for the facade's hot *_in calls (local listing back to main's speed) Since the M2 merge the facade handed in-memory files to the generic `*_in` parsers as `&[u8]` (`with_bytes!`), which instantiates them in the facade crate, where the format crate's private helpers do not inline without LTO: listing a 400-group v1 file through `File::open` was 7-10% slower than main. `ObjectHeader::parse_in`, `group_v2::{resolve_child_in, resolve_group_children_in, resolve_path_any_in}` and `attribute::{extract_attributes_tolerant_in, find_attribute_in}` now pass a storage with `as_contiguous()` to their non-generic slice entry point, compiled once in the format crate; other storages reach the same generic core as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 26 ++++++++++ crates/clawhdf5-format/src/attribute.rs | 39 +++++++++++++-- crates/clawhdf5-format/src/group_v2.rs | 55 ++++++++++++++++++--- crates/clawhdf5-format/src/object_header.rs | 31 +++++++++++- crates/clawhdf5-format/src/storage.rs | 12 +++++ 5 files changed, 151 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4556f91..664a831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,6 +149,32 @@ inside the file is an error. - **No behaviour change for in-memory and mapped files:** with `as_contiguous()` every path slices the file as before (checked below). +- **Speed on local files** (provisional: tank was shared with other jobs, + load 4–16 during the runs; Criterion `local_metadata_bench`, `main` + `8f59b2e` and this branch as separate binaries, 8 alternating rounds, + best round of each). Listing the 400-group version-1 file through the + facade (`File::open`, mmap) had become 7–10% slower than `main` (8.95–9.04 + vs 8.14–8.36 ms), from the M2 merge on (bisected over the merges: `main` + 8.17–8.27, `2893b6c` 8.69–8.76, `93e2d5f`/`7447dce`/`4313917` the same). + The parsers were unchanged: the facade now called the generic `*_in` + entry points with the slice (`with_bytes!`), which instantiates each + parser in the facade crate, where the format crate's private helpers do + not inline (no LTO); `main` called the `&[u8]` wrappers, compiled in the + format crate. Routing just `resolve_child`/`resolve_group_children` to + the wrappers took the listing from 8.66–8.70 to 8.33–8.46 ms. Fixed in + the format crate: `ObjectHeader::parse_in`, `group_v2::{resolve_child_in, + resolve_group_children_in, resolve_path_any_in}` and + `attribute::{extract_attributes_tolerant_in, find_attribute_in}` hand a + storage with `as_contiguous()` to their non-generic slice entry point, + compiled once in the format crate; everything else goes to the same + generic core as before, so `File::open_storage` is unchanged. Best + rounds, `main` / before / after: facade listing 8.49 / 9.07 / 8.68 ms + (+2.2% on `main`, was +6.9%); `ObjectHeader::parse` ×401 24.91 / 25.90 / + 25.51 µs; symbol-table nodes 1.99 / 1.94 / 1.94 µs; group B-tree walk + 360.6 / 365.0 / 361.1 ns. The B-tree walk's `btree_v1.rs` is identical to + `main` and it calls only format-crate code; its earlier +5–10% (323–355 + vs 353–363 ns) was run-to-run layout noise (at `2893b6c` it measured + 357–374 ns against `main`'s 355–371 in the same rounds). - Tests (2026-09-26, tank): - `clawhdf5-format/tests/storage_equivalence.rs` now also reads every dataset — whole, fill-aware, through a chunk cache (twice) and the diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index c8f69b5..8887bbb 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -470,16 +470,31 @@ pub fn extract_attributes_tolerant( offset_size: u8, length_size: u8, ) -> Result<(Vec, Vec), FormatError> { - extract_attributes_tolerant_in(file_data, header, offset_size, length_size) + extract_attributes_tolerant_core(file_data, header, offset_size, length_size) } /// [`extract_attributes_tolerant`] over any [`Storage`] (see -/// [`extract_attributes_full_in`] for dense storage). +/// [`extract_attributes_full_in`] for dense storage). One with the whole +/// file in memory is read as the slice, by code compiled in this crate (see +/// [`crate::storage`], "Slice entry points"). +#[inline] pub fn extract_attributes_tolerant_in( file_data: &S, header: &ObjectHeader, offset_size: u8, length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + match file_data.as_contiguous() { + Some(all) => extract_attributes_tolerant(all, header, offset_size, length_size), + None => extract_attributes_tolerant_core(file_data, header, offset_size, length_size), + } +} + +fn extract_attributes_tolerant_core( + file_data: &S, + header: &ObjectHeader, + offset_size: u8, + length_size: u8, ) -> Result<(Vec, Vec), FormatError> { let mut errors = Vec::new(); let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| { @@ -561,18 +576,34 @@ pub fn find_attribute_in_file( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - find_attribute_in(file_data, header, name, offset_size, length_size) + find_attribute_core(file_data, header, name, offset_size, length_size) } /// [`find_attribute_in_file`] over any [`Storage`] (see /// [`extract_attributes_full_in`] for dense storage, whose name index still -/// needs the whole file in memory). +/// needs the whole file in memory). One with the whole file in memory is +/// read as the slice, by code compiled in this crate (see +/// [`crate::storage`], "Slice entry points"). +#[inline] pub fn find_attribute_in( file_data: &S, header: &ObjectHeader, name: &str, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + match file_data.as_contiguous() { + Some(all) => find_attribute_in_file(all, header, name, offset_size, length_size), + None => find_attribute_core(file_data, header, name, offset_size, length_size), + } +} + +fn find_attribute_core( + file_data: &S, + header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let attr_info = find_attribute_info(header, offset_size)?; let dense = attr_info diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index 9dda843..dbc1f08 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -368,15 +368,30 @@ pub fn resolve_child( group_address: u64, name: &str, ) -> Result { - resolve_child_in(file_data, superblock, group_address, name) + resolve_child_core(file_data, superblock, group_address, name) } -/// [`resolve_child`] over any [`Storage`]. +/// [`resolve_child`] over any [`Storage`]. One with the whole file in memory +/// is read as the slice, by code compiled in this crate (see +/// [`crate::storage`], "Slice entry points"). +#[inline] pub fn resolve_child_in( file_data: &S, superblock: &Superblock, group_address: u64, name: &str, +) -> Result { + match file_data.as_contiguous() { + Some(all) => resolve_child(all, superblock, group_address, name), + None => resolve_child_core(file_data, superblock, group_address, name), + } +} + +fn resolve_child_core( + file_data: &S, + superblock: &Superblock, + group_address: u64, + name: &str, ) -> Result { let os = superblock.offset_size; let ls = superblock.length_size; @@ -453,14 +468,28 @@ pub fn resolve_path_any( superblock: &Superblock, path: &str, ) -> Result { - resolve_path_any_in(file_data, superblock, path) + resolve_path_any_core(file_data, superblock, path) } -/// [`resolve_path_any`] over any [`Storage`]. +/// [`resolve_path_any`] over any [`Storage`]. One with the whole file in memory +/// is read as the slice, by code compiled in this crate (see +/// [`crate::storage`], "Slice entry points"). +#[inline] pub fn resolve_path_any_in( file_data: &S, superblock: &Superblock, path: &str, +) -> Result { + match file_data.as_contiguous() { + Some(all) => resolve_path_any(all, superblock, path), + None => resolve_path_any_core(file_data, superblock, path), + } +} + +fn resolve_path_any_core( + file_data: &S, + superblock: &Superblock, + path: &str, ) -> Result { resolve_path_following_links( file_data, @@ -513,14 +542,28 @@ pub fn resolve_group_children( superblock: &Superblock, group_address: u64, ) -> Result, FormatError> { - resolve_group_children_in(file_data, superblock, group_address) + resolve_group_children_core(file_data, superblock, group_address) } -/// [`resolve_group_children`] over any [`Storage`]. +/// [`resolve_group_children`] over any [`Storage`]. One with the whole file in memory +/// is read as the slice, by code compiled in this crate (see +/// [`crate::storage`], "Slice entry points"). +#[inline] pub fn resolve_group_children_in( file_data: &S, superblock: &Superblock, group_address: u64, +) -> Result, FormatError> { + match file_data.as_contiguous() { + Some(all) => resolve_group_children(all, superblock, group_address), + None => resolve_group_children_core(file_data, superblock, group_address), + } +} + +fn resolve_group_children_core( + file_data: &S, + superblock: &Superblock, + group_address: u64, ) -> Result, FormatError> { let os = superblock.offset_size; let ls = superblock.length_size; diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index d421ea0..afcb9c5 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -116,25 +116,52 @@ impl ObjectHeader { /// Parse an object header at the given offset in the data buffer. /// /// `offset_size` and `length_size` come from the superblock. + #[inline] pub fn parse( data: &[u8], offset: usize, offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in(data, offset as u64, offset_size, length_size) + Self::parse_slice(data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]. /// /// Reads the prefix (at most [`V2_PREFIX_MAX`] bytes, signature /// included), then each chunk as one bounded read, continuation chunks - /// included. + /// included. A storage with the whole file in memory is parsed as its + /// slice, by code compiled in this crate (see + /// [`crate::storage`], "Slice entry points"). + #[inline] pub fn parse_in( file: &S, offset: u64, offset_size: u8, length_size: u8, + ) -> Result { + match file.as_contiguous() { + Some(all) => Self::parse_slice(all, offset, offset_size, length_size), + None => Self::parse_storage(file, offset, offset_size, length_size), + } + } + + /// [`Self::parse_storage`] for the slice, compiled in this crate: the + /// one copy [`Self::parse`] and [`Self::parse_in`] (in memory) call. + fn parse_slice( + data: &[u8], + offset: u64, + offset_size: u8, + length_size: u8, + ) -> Result { + Self::parse_storage(data, offset, offset_size, length_size) + } + + fn parse_storage( + file: &S, + offset: u64, + offset_size: u8, + length_size: u8, ) -> Result { // The longest prefix of either version, in one read. It holds the // whole prefix or ends at the end of the file, so its bounds checks diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 08720ab..d53bdc7 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -20,6 +20,18 @@ //! still works (`S = dyn Storage`), and a remote backend pays one indirect //! call per structure read. //! +//! # Slice entry points +//! +//! A generic core is instantiated in the crate that calls it, so a +//! downstream crate calling `parse_in::<[u8]>` gets its own copy of the +//! parser, compiled without this crate's private helpers inlined (there is +//! no cross-crate inlining of non-`#[inline]` functions without LTO): a +//! metadata walk through the facade ran about 6% slower that way than +//! through the `&[u8]` wrappers. The `*_in` entry points on the facade's hot +//! paths (object headers, group listing and lookup, attributes) therefore +//! check [`Storage::as_contiguous`] first and hand an in-memory file to +//! their non-generic `&[u8]` wrapper, compiled here; both run the one core. +//! //! The trait is synchronous and `no_std`: parsing is CPU work, and a remote //! backend bridges to its own I/O. -- 2.54.0