From 6248b411f0530e8762fd2b4278ee918968e2e990 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:43:49 -0500 Subject: [PATCH 01/48] format: one checked conversion from file address to index addr::to_usize turns a 64-bit file address or length into a slice index, failing with FormatError::Overflow where it does not fit usize (32-bit targets such as wasm32) instead of truncating like an `as usize` cast. Callers are converted in the following commits. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/addr.rs | 59 ++++++++++++++++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 1 + 2 files changed, 60 insertions(+) create mode 100644 crates/clawhdf5-format/src/addr.rs diff --git a/crates/clawhdf5-format/src/addr.rs b/crates/clawhdf5-format/src/addr.rs new file mode 100644 index 0000000..ec25e3d --- /dev/null +++ b/crates/clawhdf5-format/src/addr.rs @@ -0,0 +1,59 @@ +//! File address and length → in-memory index conversion. +//! +//! HDF5 addresses and lengths are 64-bit; the file is parsed through a +//! `&[u8]` indexed by `usize`. On a 64-bit target every `u64` fits, but on a +//! 32-bit one (`wasm32`, `i686`, `thumbv7em`) an address past `usize::MAX` +//! used to be truncated by an `as usize` cast — silently pointing at another +//! part of the file — or to panic. [`to_usize`] is the one conversion the +//! parsers use instead: such an address is a clean +//! [`FormatError::Overflow`]. It cannot be inside the data anyway: no slice +//! is longer than `isize::MAX` bytes. + +#[cfg(not(feature = "std"))] +use alloc::format; + +use crate::error::FormatError; + +/// A file address, offset or length from the file as a `usize` index. +/// +/// Fails with [`FormatError::Overflow`] when the value does not fit this +/// platform's `usize` (only possible on targets narrower than 64 bits). +#[inline] +pub fn to_usize(value: u64) -> Result { + usize::try_from(value).map_err(|_| too_large(value)) +} + +#[cold] +#[inline(never)] +fn too_large(value: u64) -> FormatError { + FormatError::Overflow(format!( + "file address or length {value:#x} exceeds this platform's address space" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn values_that_fit_convert_exactly() { + assert_eq!(to_usize(0), Ok(0)); + assert_eq!(to_usize(0x1234), Ok(0x1234)); + assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX)); + } + + #[test] + fn values_past_usize_max_are_an_error_not_truncated() { + // Only reachable where usize is narrower than u64; on a 64-bit host + // every u64 fits, which the first branch checks instead. + if let Some(past) = (usize::MAX as u64).checked_add(1) { + let err = to_usize(past).unwrap_err(); + assert!(matches!(err, FormatError::Overflow(_)), "{err:?}"); + // The value an `as usize` cast would have produced is not returned. + assert!(to_usize(u64::MAX).is_err()); + assert!(to_usize(past + 0x10).is_err()); + } else { + assert_eq!(to_usize(u64::MAX), Ok(u64::MAX as usize)); + } + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index a197c9f..2737c9b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -57,6 +57,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; +pub mod addr; pub mod attribute; pub mod attribute_info; pub mod btree_v1; -- 2.54.0 From 6a4707d7916f7cb2a99e8293983ac422c40334ac Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:43:18 -0500 Subject: [PATCH 02/48] format: add the Storage trait; make the error enums non-exhaustive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Range-read milestone M1, first step (docs/design/range-reads.md §3(a)): a synchronous, no_std read interface with u64 offsets, read_at returning Cow<[u8]>, read_ranges, len and an as_contiguous fast path. Implemented for [u8], Vec, &T, Box and Arc; slices serve borrowed bytes. read_exact_at reproduces the parsers' UnexpectedEof bounds error exactly, so converted modules keep their error values. FormatError gains Storage(String) and ContiguousStorageRequired; it and the facade Error are now #[non_exhaustive] (breaking for exhaustive matches, noted in the changelog; the Python bindings' match gets a wildcard arm). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 14 + crates/clawhdf5-format/src/error.rs | 21 ++ crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/storage.rs | 357 ++++++++++++++++++++++++++ crates/clawhdf5-py/src/lib.rs | 1 + crates/clawhdf5/src/error.rs | 5 + 6 files changed, 399 insertions(+) create mode 100644 crates/clawhdf5-format/src/storage.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f4316..4c6d6e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +### Range reads, milestone M1: the `Storage` trait (2026-09-26) +- **Breaking: `clawhdf5_format::error::FormatError` and `clawhdf5::Error` + are now `#[non_exhaustive]`.** An exhaustive `match` on either needs a + wildcard arm. `FormatError` has two new variants: `Storage(String)` (a + storage backend failed to serve a read) and + `ContiguousStorageRequired(&'static str)` (an operation not yet converted + to range reads was asked of a backend without the whole file in memory). +- New `clawhdf5_format::storage::Storage`, the synchronous, `no_std` read + interface of `docs/design/range-reads.md` option (a): `read_at(offset: + u64, len) -> Cow<[u8]>`, `read_ranges`, `len()` and an `as_contiguous()` + fast path; implemented for `[u8]`, `Vec`, and references, `Box`es + and (with `std`) `Arc`s of a `Storage`. Slices and `Vec`s serve borrowed + bytes, so parsing an in-memory file costs no copy. + ### Chunked full reads (2026-09-26) - **Chunks are decoded straight into the output, into reused buffers.** A full read of a chunked dataset faulted in about three times its size in diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index 7f9e3d2..449ff13 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -12,7 +12,11 @@ use std::string::String; use core::fmt; /// Errors that can occur when parsing HDF5 binary format structures. +/// +/// Non-exhaustive: new failure modes (new storage backends, new file +/// features) add variants, so a `match` needs a wildcard arm. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum FormatError { /// The HDF5 magic signature was not found at any valid offset. SignatureNotFound, @@ -243,6 +247,13 @@ pub enum FormatError { /// A metadata cache image block libhdf5 refuses to load (the reason is /// libhdf5's own error text). InvalidCacheImage(&'static str), + /// The [`Storage`](crate::storage::Storage) backend failed to serve a + /// read (an I/O or network error, or a short read inside the file). + Storage(String), + /// The operation still needs the whole file as one slice and the + /// [`Storage`](crate::storage::Storage) backend has no contiguous view + /// (`as_contiguous()` is `None`); the text names the operation. + ContiguousStorageRequired(&'static str), } impl fmt::Display for FormatError { @@ -529,6 +540,16 @@ impl fmt::Display for FormatError { FormatError::InvalidCacheImage(why) => { write!(f, "invalid metadata cache image: {why}") } + FormatError::Storage(why) => { + write!(f, "storage read failed: {why}") + } + FormatError::ContiguousStorageRequired(what) => { + write!( + f, + "{what} needs the whole file in memory, which this storage backend does \ + not provide" + ) + } } } } diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index a197c9f..23a8bdd 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -120,6 +120,7 @@ pub mod property_list; pub mod selection; pub mod shared_message; pub mod signature; +pub mod storage; pub mod superblock; pub mod superblock_ext; pub mod symbol_table; diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs new file mode 100644 index 0000000..76f1436 --- /dev/null +++ b/crates/clawhdf5-format/src/storage.rs @@ -0,0 +1,357 @@ +//! Where the parsers read the file from: the [`Storage`] trait. +//! +//! Every parser used to take the whole file as one `&[u8]`. [`Storage`] is +//! the abstraction that replaces it (see `docs/design/range-reads.md`, +//! option (a)): a parser asks for the bytes it needs, `[offset, offset + +//! len)`, with 64-bit offsets, and gets them back as a [`Cow`] — borrowed +//! when the backend holds the file in memory (a `Vec`, an mmap), owned when +//! it had to fetch them (a range request, a block cache). +//! +//! `impl Storage for [u8]` serves the in-memory case with no copy, and +//! [`Storage::as_contiguous`] lets a hot loop borrow the whole file at once +//! when the backend has it. Modules are converted one at a time: a converted +//! parser has an `*_in(file: &dyn Storage, ..)` core and keeps its old +//! `&[u8]` signature as a thin wrapper, so callers do not change. +//! +//! The trait is synchronous and `no_std`: parsing is CPU work, and a remote +//! backend bridges to its own I/O. + +#[cfg(not(feature = "std"))] +use alloc::{borrow::Cow, boxed::Box, vec::Vec}; +#[cfg(feature = "std")] +use std::{borrow::Cow, boxed::Box, vec::Vec}; + +use core::ops::Range; + +use crate::error::FormatError; + +/// A random-access source of file bytes. +/// +/// Offsets are relative to the start of the HDF5 data (the superblock), like +/// every address in the file. +pub trait Storage { + /// Bytes `[offset, offset + len)`. + /// + /// The result is shorter than `len` only when the range runs past the + /// end of the storage (and empty when `offset` is at or past the end); + /// a backend that cannot serve a range returns an error instead of a + /// short read. + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError>; + + /// Current length of the storage in bytes. + fn len(&self) -> u64; + + /// Whether the storage holds no bytes. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Several reads at once, in the order given. Backends that talk to a + /// remote store coalesce and parallelise these; the default reads them + /// one by one with [`Storage::read_at`]. + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + ranges + .iter() + .map(|r| { + let len = usize::try_from(r.end.saturating_sub(r.start)).map_err(|_| { + FormatError::Overflow("read range longer than the address space".into()) + })?; + self.read_at(r.start, len) + }) + .collect() + } + + /// The whole storage as one slice, when the backend has it in memory + /// (a `Vec`, an mmap). Hot loops use this to keep their zero-copy path; + /// `None` means every byte has to go through [`Storage::read_at`]. + fn as_contiguous(&self) -> Option<&[u8]> { + None + } +} + +impl Storage for [u8] { + #[inline] + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let n = self.len(); + let start = usize::try_from(offset).map_or(n, |o| o.min(n)); + let end = start.saturating_add(len).min(n); + Ok(Cow::Borrowed(&self[start..end])) + } + + #[inline] + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + + #[inline] + fn as_contiguous(&self) -> Option<&[u8]> { + Some(self) + } +} + +impl Storage for Vec { + #[inline] + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + self.as_slice().read_at(offset, len) + } + + #[inline] + fn len(&self) -> u64 { + Vec::len(self) as u64 + } + + #[inline] + fn as_contiguous(&self) -> Option<&[u8]> { + Some(self.as_slice()) + } +} + +impl Storage for &T { + #[inline] + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + (**self).read_at(offset, len) + } + + #[inline] + fn len(&self) -> u64 { + (**self).len() + } + + #[inline] + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + (**self).read_ranges(ranges) + } + + #[inline] + fn as_contiguous(&self) -> Option<&[u8]> { + (**self).as_contiguous() + } +} + +impl Storage for Box { + #[inline] + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + (**self).read_at(offset, len) + } + + #[inline] + fn len(&self) -> u64 { + (**self).len() + } + + #[inline] + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + (**self).read_ranges(ranges) + } + + #[inline] + fn as_contiguous(&self) -> Option<&[u8]> { + (**self).as_contiguous() + } +} + +#[cfg(feature = "std")] +impl Storage for std::sync::Arc { + #[inline] + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + (**self).read_at(offset, len) + } + + #[inline] + fn len(&self) -> u64 { + (**self).len() + } + + #[inline] + fn read_ranges(&self, ranges: &[Range]) -> Result>, FormatError> { + (**self).read_ranges(ranges) + } + + #[inline] + fn as_contiguous(&self) -> Option<&[u8]> { + (**self).as_contiguous() + } +} + +/// `storage.len()` as the `usize` the parsers' end-of-file errors report +/// (saturating on targets where the file is larger than the address space). +#[inline] +pub(crate) fn len_usize(file: &dyn Storage) -> usize { + usize::try_from(file.len()).unwrap_or(usize::MAX) +} + +/// Bytes `[offset, offset + len)`, all of them. +/// +/// A range that runs past the end of the storage is +/// [`FormatError::UnexpectedEof`] with `expected = offset + len` and +/// `available = storage length` — the error the `&[u8]` parsers give for +/// the same bounds check (`offset + len > file_data.len()`). +#[inline] +pub fn read_exact_at( + file: &dyn Storage, + offset: u64, + len: usize, +) -> Result, FormatError> { + let eof = || FormatError::UnexpectedEof { + expected: usize::try_from(offset) + .unwrap_or(usize::MAX) + .saturating_add(len), + available: len_usize(file), + }; + match offset.checked_add(len as u64) { + Some(end) if end <= file.len() => {} + _ => return Err(eof()), + } + let bytes = file.read_at(offset, len)?; + if bytes.len() < len { + // The storage shrank or the backend served a short read inside the + // file: never parse a partial structure. + return Err(eof()); + } + Ok(bytes) +} + +/// Up to `max` bytes from `offset` on: fewer only at the end of the +/// storage. For structures whose size is only known once their prefix has +/// been parsed and whose parsers bound-check what they are given. +#[inline] +pub fn read_upto(file: &dyn Storage, offset: u64, max: usize) -> Result, FormatError> { + let avail = file.len().saturating_sub(offset); + let len = usize::try_from(avail).map_or(max, |a| a.min(max)); + let bytes = file.read_at(offset, len)?; + if bytes.len() < len { + return Err(FormatError::Storage( + "short read inside the file (the storage shrank or the backend failed)".into(), + )); + } + Ok(bytes) +} + +/// Borrow the whole file for a code path that has not been converted to +/// [`Storage`] yet. On a backend without a contiguous view this is the +/// clean [`FormatError::ContiguousStorageRequired`] error, never a guess. +#[inline] +pub fn require_contiguous<'a>( + file: &'a dyn Storage, + what: &'static str, +) -> Result<&'a [u8], FormatError> { + file.as_contiguous() + .ok_or(FormatError::ContiguousStorageRequired(what)) +} + +/// A [`Storage`] over an in-memory buffer that serves every byte through +/// [`Storage::read_at`] (its [`Storage::as_contiguous`] is `None`, so no +/// parser can take the whole-slice shortcut), copies what it serves (as a +/// remote backend would), and counts the reads and bytes. +/// +/// It is the equivalence harness of the range-read migration: parsing a +/// file through it must give exactly what parsing the `&[u8]` gives, and +/// the counters are the request counts a cacheless range reader would make. +#[derive(Debug)] +pub struct CountingStorage { + data: Vec, + reads: portable_atomic::AtomicU64, + bytes: portable_atomic::AtomicU64, +} + +impl CountingStorage { + /// Serve `data` (the file from the superblock on). + pub fn new(data: Vec) -> Self { + CountingStorage { + data, + reads: portable_atomic::AtomicU64::new(0), + bytes: portable_atomic::AtomicU64::new(0), + } + } + + /// Number of `read_at` calls served so far. + pub fn reads(&self) -> u64 { + self.reads.load(portable_atomic::Ordering::Relaxed) + } + + /// Number of bytes served so far. + pub fn bytes_read(&self) -> u64 { + self.bytes.load(portable_atomic::Ordering::Relaxed) + } + + /// Reset both counters. + pub fn reset(&self) { + self.reads.store(0, portable_atomic::Ordering::Relaxed); + self.bytes.store(0, portable_atomic::Ordering::Relaxed); + } +} + +impl Storage for CountingStorage { + fn read_at(&self, offset: u64, len: usize) -> Result, FormatError> { + let got = self.data.as_slice().read_at(offset, len)?; + self.reads.fetch_add(1, portable_atomic::Ordering::Relaxed); + self.bytes + .fetch_add(got.len() as u64, portable_atomic::Ordering::Relaxed); + Ok(Cow::Owned(got.into_owned())) + } + + fn len(&self) -> u64 { + self.data.len() as u64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slice_reads_are_borrowed_and_clamped() { + let data: Vec = (0u8..10).collect(); + let s: &[u8] = &data; + let dynamic: &dyn Storage = &s; + assert_eq!(dynamic.len(), 10); + let r = dynamic.read_at(2, 3).unwrap(); + assert!(matches!(r, Cow::Borrowed(_))); + assert_eq!(&*r, &[2, 3, 4]); + assert_eq!(&*dynamic.read_at(8, 5).unwrap(), &[8, 9]); + assert!(dynamic.read_at(10, 5).unwrap().is_empty()); + assert!(dynamic.read_at(u64::MAX, 5).unwrap().is_empty()); + assert_eq!(dynamic.as_contiguous(), Some(&data[..])); + let v: &dyn Storage = &data; + assert_eq!(v.as_contiguous(), Some(&data[..])); + } + + #[test] + fn read_exact_matches_slice_bounds_errors() { + let data = [0u8; 10]; + let s: &[u8] = &data; + assert_eq!(&*read_exact_at(&s, 4, 6).unwrap(), &[0; 6]); + assert_eq!( + read_exact_at(&s, 4, 7).unwrap_err(), + FormatError::UnexpectedEof { + expected: 11, + available: 10 + } + ); + assert!(read_exact_at(&s, u64::MAX, 1).is_err()); + assert_eq!(read_upto(&s, 7, 100).unwrap().len(), 3); + assert_eq!(read_upto(&s, 70, 100).unwrap().len(), 0); + } + + #[test] + fn counting_storage_counts_and_hides_the_slice() { + let c = CountingStorage::new((0u8..10).collect()); + assert!(c.as_contiguous().is_none()); + let r = c.read_at(3, 4).unwrap(); + assert!(matches!(r, Cow::Owned(_))); + assert_eq!(&*r, &[3, 4, 5, 6]); + c.read_at(8, 4).unwrap(); + assert_eq!((c.reads(), c.bytes_read()), (2, 6)); + c.reset(); + assert_eq!((c.reads(), c.bytes_read()), (0, 0)); + } + + #[test] + fn read_ranges_default_loops() { + let data: Vec = (0u8..10).collect(); + let s: &[u8] = &data; + let got = s.read_ranges(&[1..3, 5..9]).unwrap(); + assert_eq!(&*got[0], &[1, 2]); + assert_eq!(&*got[1], &[5, 6, 7, 8]); + } +} diff --git a/crates/clawhdf5-py/src/lib.rs b/crates/clawhdf5-py/src/lib.rs index e3b5619..0d3873c 100644 --- a/crates/clawhdf5-py/src/lib.rs +++ b/crates/clawhdf5-py/src/lib.rs @@ -82,6 +82,7 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { | Error::ZeroCopyUnaligned { .. } => { PyErr::new::(e.to_string()) } + _ => PyErr::new::(e.to_string()), } } diff --git a/crates/clawhdf5/src/error.rs b/crates/clawhdf5/src/error.rs index de34355..08aabbd 100644 --- a/crates/clawhdf5/src/error.rs +++ b/crates/clawhdf5/src/error.rs @@ -6,7 +6,12 @@ use clawhdf5_format::error::FormatError; use clawhdf5_format::message_type::MessageType; /// Errors that can occur when using the high-level API. +/// +/// Non-exhaustive: new failure modes add variants, so a `match` needs a +/// wildcard arm. Failures of a [`Storage`](clawhdf5_format::storage::Storage) +/// backend arrive as `Error::Format(FormatError::Storage(..))`. #[derive(Debug)] +#[non_exhaustive] pub enum Error { /// I/O error from the filesystem. Io(std::io::Error), -- 2.54.0 From 512a6a753fc3d83218c16f0af3859a802e7185c7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:44:52 -0500 Subject: [PATCH 03/48] format: parse the superblock over Storage Superblock::parse_in(&dyn Storage, offset) reads one bounded window of 128 bytes (the largest superblock is 100) and runs the existing version parsers on it; parse and refresh_eof keep their &[u8] signatures as wrappers. No behaviour change: on a file longer than the window no bounds check can fail, and on a shorter one the window is the whole file. New test: every version and truncation parses to the same result through a read_at-only CountingStorage, in one read. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/superblock.rs | 65 +++++++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/crates/clawhdf5-format/src/superblock.rs b/crates/clawhdf5-format/src/superblock.rs index a565321..318c945 100644 --- a/crates/clawhdf5-format/src/superblock.rs +++ b/crates/clawhdf5-format/src/superblock.rs @@ -7,6 +7,11 @@ use byteorder::{ByteOrder, LittleEndian}; use crate::error::FormatError; use crate::signature::HDF5_SIGNATURE; +use crate::storage::{Storage, read_upto}; + +/// Bytes read to parse a superblock: more than the largest one (version 1 +/// with 8-byte offsets and lengths, 100 bytes). +const SUPERBLOCK_READ_LEN: usize = 128; /// Parsed HDF5 superblock (all versions). #[derive(Debug, Clone, PartialEq, Eq)] @@ -161,7 +166,16 @@ impl Superblock { file_data: &[u8], signature_offset: usize, ) -> Result { - let refreshed = Superblock::parse(file_data, signature_offset)?; + self.refresh_eof_in(&file_data, signature_offset as u64) + } + + /// [`Self::refresh_eof`] over any [`Storage`]. + pub fn refresh_eof_in( + &mut self, + file: &dyn Storage, + signature_offset: u64, + ) -> Result { + let refreshed = Superblock::parse_in(file, signature_offset)?; self.eof_address = refreshed.eof_address; self.consistency_flags = refreshed.consistency_flags; Ok(self.eof_address) @@ -219,15 +233,20 @@ impl Superblock { /// [`FormatError::UserBlockNotStripped`] because the addresses in the /// returned superblock would otherwise be applied to the wrong bytes. pub fn parse(data: &[u8], signature_offset: usize) -> Result { + Self::parse_in(&data, signature_offset as u64) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the first + /// [`SUPERBLOCK_READ_LEN`] bytes (fewer when the file is shorter, which + /// is then refused with the same end-of-file errors as a short slice). + pub fn parse_in(file: &dyn Storage, signature_offset: u64) -> Result { if signature_offset != 0 { - return Err(FormatError::UserBlockNotStripped(signature_offset as u64)); + return Err(FormatError::UserBlockNotStripped(signature_offset)); } - let d = data - .get(signature_offset..) - .ok_or(FormatError::UnexpectedEof { - expected: signature_offset + 1, - available: data.len(), - })?; + // Every bounds check below needs at most 100 bytes, so on a longer + // file none of them can fail and the window's length does not show. + let window = read_upto(file, 0, SUPERBLOCK_READ_LEN)?; + let d: &[u8] = &window; ensure_len(d, 9)?; // signature(8) + version(1) // Verify signature @@ -894,4 +913,34 @@ mod tests { assert_eq!(parsed.version, 3); assert_eq!(parsed.page_size, None); } + + /// Through a storage that serves only `read_at`, every version parses + /// to the same superblock, and every truncation to the same error, as + /// from a slice — in one read. + #[test] + fn parse_in_matches_slice_parse() { + use crate::storage::CountingStorage; + let mut files = vec![ + build_v0_bytes(8), + build_v0_bytes(4), + build_v1_bytes(8), + build_v1_bytes(4), + build_v2_bytes(8, 2), + build_v2_bytes(4, 3), + ]; + for f in files.clone() { + let mut long = f.clone(); + long.resize(4096, 0xAB); + files.push(long); + for cut in [0, 5, 9, 13, 20, 30, f.len() - 1] { + files.push(f[..cut.min(f.len())].to_vec()); + } + } + for f in files { + let want = Superblock::parse(&f, 0); + let storage = CountingStorage::new(f.clone()); + assert_eq!(Superblock::parse_in(&storage, 0), want, "{} bytes", f.len()); + assert_eq!(storage.reads(), 1); + } + } } -- 2.54.0 From cd828725c7da7f8566c9b34f60ac5dcd286b20c3 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:46:51 -0500 Subject: [PATCH 04/48] format: parse object headers over Storage ObjectHeader::parse_in(&dyn Storage, u64, ..) reads the signature, the prefix (a window of at most 34 bytes for version 2) and then each chunk, continuation chunks included, as one bounded read; the message loops run unchanged on the chunk with chunk-relative positions. parse keeps its &[u8] signature as a wrapper. Bounds errors are reported as before, with absolute positions and the file's length. New test: headers of both versions, with times, phase-change values, creation order and a continuation chunk, and every truncation of each, parse identically through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/object_header.rs | 163 +++++++++++++++----- 1 file changed, 128 insertions(+), 35 deletions(-) diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 1fc8696..d0f36db 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -7,6 +7,7 @@ use byteorder::{ByteOrder, LittleEndian}; use crate::error::FormatError; use crate::message_type::MessageType; +use crate::storage::{Storage, len_usize, read_exact_at, read_upto}; /// OHDR signature for v2 object headers. const OHDR_SIGNATURE: [u8; 4] = *b"OHDR"; @@ -118,32 +119,45 @@ impl ObjectHeader { offset_size: u8, length_size: u8, ) -> Result { - ensure_len(data, offset, 4)?; - if data[offset..offset + 4] == OHDR_SIGNATURE { - Self::parse_v2(data, offset, offset_size, length_size) + Self::parse_in(&data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]. + /// + /// Reads the signature, the prefix (at most [`V2_PREFIX_MAX`] bytes), + /// then each chunk as one bounded read, continuation chunks included. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + length_size: u8, + ) -> Result { + let sig = read_exact_at(file, offset, 4)?; + if *sig == OHDR_SIGNATURE { + Self::parse_v2(file, offset, offset_size, length_size) } else { - Self::parse_v1(data, offset, offset_size, length_size) + Self::parse_v1(file, offset, offset_size, length_size) } } fn parse_v1( - data: &[u8], - offset: usize, + file: &dyn Storage, + offset: u64, offset_size: u8, length_size: u8, ) -> Result { // version(1) + reserved(1) + num_messages(2) + ref_count(4) + header_size(4) = 12 // then pad to 8-byte alignment from start of header - ensure_len(data, offset, 12)?; + let prefix = read_exact_at(file, offset, 12)?; - let version = data[offset]; + let version = prefix[0]; if version != 1 { return Err(FormatError::InvalidObjectHeaderVersion(version)); } - let num_messages = LittleEndian::read_u16(&data[offset + 2..offset + 4]) as usize; - let reference_count = LittleEndian::read_u32(&data[offset + 4..offset + 8]); - let header_data_size = LittleEndian::read_u32(&data[offset + 8..offset + 12]) as usize; + let num_messages = LittleEndian::read_u16(&prefix[2..4]) as usize; + let reference_count = LittleEndian::read_u32(&prefix[4..8]); + let header_data_size = LittleEndian::read_u32(&prefix[8..12]) as usize; // libhdf5 (H5O__prefix_deserialize): a header with messages needs room // for at least one message header, and one without has an empty chunk. @@ -161,14 +175,13 @@ impl ObjectHeader { .checked_add(12 + padding) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: data.len(), + available: len_usize(file), })?; - ensure_len(data, msg_start, header_data_size)?; - + // parse_v1_chunk reads the chunk, with the bounds check that was here. let mut messages = Vec::new(); let chunk0_count = Self::parse_v1_chunk( - data, + file, msg_start, header_data_size, offset_size, @@ -209,8 +222,8 @@ impl ObjectHeader { /// "gap", which only version 2 allows). #[allow(clippy::too_many_arguments)] fn parse_v1_chunk( - data: &[u8], - offset: usize, + file: &dyn Storage, + offset: u64, length: usize, offset_size: u8, length_size: u8, @@ -220,9 +233,10 @@ impl ObjectHeader { if depth_remaining == 0 { return Err(FormatError::NestingDepthExceeded); } - ensure_len(data, offset, length)?; - let end = offset + length; - let mut pos = offset; + let chunk = read_exact_at(file, offset, length)?; + let data: &[u8] = &chunk; + let end = length; + let mut pos = 0usize; let mut count = 0usize; while pos < end { @@ -267,8 +281,8 @@ impl ObjectHeader { let cont_offset = read_offset(body, 0, offset_size)? as usize; let cont_length = read_offset(body, offset_size as usize, length_size)? as usize; Self::parse_v1_chunk( - data, - cont_offset, + file, + cont_offset as u64, cont_length, offset_size, length_size, @@ -282,11 +296,31 @@ impl ObjectHeader { } fn parse_v2( - data: &[u8], - offset: usize, + file: &dyn Storage, + offset: u64, offset_size: u8, length_size: u8, ) -> Result { + // The prefix, read as one window. The window holds the whole prefix + // or ends at the end of the file, so a position past the window is + // past the end of the file: `ensure_len` checks positions relative + // to the header against it and reports them as the whole-file check + // did, with absolute positions and the file's length. + let window = read_upto(file, offset, V2_PREFIX_MAX)?; + let data: &[u8] = &window; + let file_len = len_usize(file); + let base = usize::try_from(offset).unwrap_or(usize::MAX); + let abs = |rel: usize| base.saturating_add(rel); + let ensure_len = |_: &[u8], rel: usize, needed: usize| -> Result<(), FormatError> { + match rel.checked_add(needed) { + Some(end) if end <= data.len() => Ok(()), + _ => Err(FormatError::UnexpectedEof { + expected: abs(rel).saturating_add(needed), + available: file_len, + }), + } + }; + let offset = 0usize; // signature(4) + version(1) + flags(1) = 6 ensure_len(data, offset, 6)?; @@ -351,15 +385,20 @@ impl ObjectHeader { } let chunk0_msg_start = pos; - let chunk0_msg_end = pos + let chunk0_msg_end = abs(pos) .checked_add(chunk0_size) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: data.len(), - })?; + available: file_len, + })? + - base; + + // The whole first chunk, prefix to checksum, in one read (its + // bounds check is the one on the checksum's 4 bytes). + let chunk0 = read_exact_at(file, base as u64, chunk0_msg_end.saturating_add(4))?; + let data: &[u8] = &chunk0; // Validate checksum: from OHDR signature through all messages (before checksum) - ensure_len(data, chunk0_msg_end, 4)?; #[cfg(feature = "checksum")] { let stored = LittleEndian::read_u32(&data[chunk0_msg_end..chunk0_msg_end + 4]); @@ -394,8 +433,8 @@ impl ObjectHeader { } cont_remaining -= 1; Self::parse_v2_continuation( - data, - cont_offset, + file, + cont_offset as u64, cont_length, has_creation_order, offset_size, @@ -495,8 +534,8 @@ impl ObjectHeader { #[allow(clippy::too_many_arguments)] fn parse_v2_continuation( - data: &[u8], - offset: usize, + file: &dyn Storage, + offset: u64, length: usize, has_creation_order: bool, offset_size: u8, @@ -505,7 +544,9 @@ impl ObjectHeader { continuations: &mut Vec<(usize, usize)>, ) -> Result<(), FormatError> { // OCHK signature(4) + messages + checksum(4) - ensure_len(data, offset, length)?; + let chunk = read_exact_at(file, offset, length)?; + let data: &[u8] = &chunk; + let offset = 0usize; if length < 8 { return Err(FormatError::UnexpectedEof { expected: 8, @@ -546,6 +587,10 @@ impl ObjectHeader { } } +/// Longest version-2 object header prefix: signature(4) + version(1) + +/// flags(1) + times(16) + attribute phase change(4) + chunk-0 size(8). +const V2_PREFIX_MAX: usize = 34; + /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3). const V1_MSG_HEADER_SIZE: usize = 8; @@ -754,13 +799,13 @@ mod tests { let mut msg_bytes = Vec::new(); for (mtype, mdata, mflags) in messages { // v1 message sizes are multiples of 8 (the data is zero-padded). - let padded = mdata.len().div_ceil(8) * 8; + let padded = <[u8]>::len(mdata).div_ceil(8) * 8; msg_bytes.extend_from_slice(&mtype.to_le_bytes()); // type(2) msg_bytes.extend_from_slice(&(padded as u16).to_le_bytes()); // size(2) msg_bytes.push(*mflags); // flags(1) msg_bytes.extend_from_slice(&[0u8; 3]); // reserved(3) msg_bytes.extend_from_slice(mdata); // data - msg_bytes.resize(msg_bytes.len() + padded - mdata.len(), 0); + msg_bytes.resize(msg_bytes.len() + padded - <[u8]>::len(mdata), 0); } let mut buf = Vec::new(); @@ -1280,4 +1325,52 @@ mod tests { let err = ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(); assert!(matches!(err, FormatError::UnexpectedEof { .. })); } + + /// Every header, and every truncation of it, parses to the same result + /// (or the same error) through a `read_at`-only storage as from a slice; + /// a header in one chunk takes three reads (signature, prefix, chunk). + #[test] + fn parse_in_matches_slice_parse() { + use crate::storage::CountingStorage; + let mut headers = vec![ + build_v1_header(&[], 8, 8), + build_v1_header(&[(0x0001, &[1, 2, 3], 0), (0x0003, &[9; 8], 0)], 8, 8), + build_v2_header(0x00, &[(0x01, &[42], 0)], None), + build_v2_header(0x03, &[(0x01, &[1, 2], 0), (0x03, &[3], 0)], None), + build_v2_header(0x24, &[(0x01, &[1], 0)], Some((1, 2, 3, 4))), + build_v2_header(0x35, &[(0x01, &[1], 0)], Some((5, 6, 7, 8))), + ]; + // A v2 header with a continuation chunk at 256. + let mut ochk = OCHK_SIGNATURE.to_vec(); + ochk.extend_from_slice(&[0x03, 2, 0, 0, 0xDE, 0xAD]); + let sum = crate::checksum::jenkins_lookup3(&ochk); + ochk.extend_from_slice(&sum.to_le_bytes()); + let mut cont = 256u64.to_le_bytes().to_vec(); + cont.extend_from_slice(&(ochk.len() as u64).to_le_bytes()); + let main = build_v2_header(0x00, &[(0x01, &[42], 0), (0x10, &cont, 0)], None); + let mut with_cont = vec![0u8; 256 + ochk.len()]; + with_cont[..main.len()].copy_from_slice(&main); + with_cont[256..].copy_from_slice(&ochk); + headers.push(with_cont); + + for h in headers { + for at in [0usize, 3] { + for cut in 0..=h.len() { + let mut f = vec![0u8; at]; + f.extend_from_slice(&h[..cut]); + if at == 0 && cut == h.len() { + f.resize(f.len() + 64, 0); + } + let want = ObjectHeader::parse(&f, at, 8, 8); + let storage = CountingStorage::new(f.clone()); + let got = ObjectHeader::parse_in(&storage, at as u64, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "at {at}, cut {cut}"); + } + } + } + let one_chunk = build_v2_header(0x00, &[(0x01, &[42], 0)], None); + let storage = CountingStorage::new(one_chunk); + ObjectHeader::parse_in(&storage, 0, 8, 8).unwrap(); + assert_eq!(storage.reads(), 3); + } } -- 2.54.0 From 0d908facd3c0d0af5ec4d7a1e7b5dc3d145ee7f1 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:47:59 -0500 Subject: [PATCH 05/48] format: read the superblock extension and cache image over Storage read_superblock_extension_in, cache_image_state_in, CacheImage::decode_in and CacheImage::block_in take &dyn Storage (whose length is the end of file); the image block is one bounded read. The &[u8] functions are wrappers; applying an image in place still needs the bytes in memory. New test: extension messages, a cache image and a corrupt one decode to the same results through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/superblock_ext.rs | 104 ++++++++++++++++--- 1 file changed, 92 insertions(+), 12 deletions(-) diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index e1d1cf3..40efd5c 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -21,13 +21,14 @@ //! file is never copied whole. #[cfg(not(feature = "std"))] -use alloc::{collections::BTreeSet, vec::Vec}; +use alloc::{borrow::Cow, collections::BTreeSet, vec::Vec}; #[cfg(feature = "std")] -use std::collections::BTreeSet; +use std::{borrow::Cow, collections::BTreeSet}; use crate::error::FormatError; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; +use crate::storage::{Storage, read_exact_at}; use crate::superblock::Superblock; /// Message type of the File Space Info message. @@ -169,6 +170,15 @@ impl<'a> Cursor<'a> { pub fn read_superblock_extension( data: &[u8], sb: &Superblock, +) -> Result, FormatError> { + read_superblock_extension_in(&data, sb) +} + +/// [`read_superblock_extension`] over any [`Storage`]; its length is the +/// end of file. +pub fn read_superblock_extension_in( + file: &dyn Storage, + sb: &Superblock, ) -> Result, FormatError> { let os = sb.offset_size; let ls = sb.length_size; @@ -181,8 +191,8 @@ pub fn read_superblock_extension( return Ok(None); }; let addr = usize::try_from(addr).map_err(|_| ext_err("address out of range"))?; - let header = ObjectHeader::parse(data, addr, os, ls)?; - let eoa = data.len() as u64; + let header = ObjectHeader::parse_in(file, addr as u64, os, ls)?; + let eoa = file.len(); let mut ext = SuperblockExtension::default(); for msg in &header.messages { @@ -340,12 +350,21 @@ impl CacheImage { data: &[u8], location: CacheImageLocation, sb: &Superblock, + ) -> Result { + Self::decode_in(&data, location, sb) + } + + /// [`Self::decode`] over any [`Storage`]: one read of the image block. + pub fn decode_in( + file: &dyn Storage, + location: CacheImageLocation, + sb: &Superblock, ) -> Result { let (offset_size, length_size) = (sb.offset_size, sb.length_size); let bad = FormatError::InvalidCacheImage; - let block = image_block(data, location)?; - let eoa = data.len() as u64; - let mut c = Cursor::new(block, bad(RAN_OFF)); + let block = image_block_in(file, location)?; + let eoa = file.len(); + let mut c = Cursor::new(&block, bad(RAN_OFF)); // Header: signature, version, flags, image data length, entry count. if c.take(4)? != MDCI_SIGNATURE { @@ -463,6 +482,11 @@ impl CacheImage { image_block(data, self.location) } + /// [`Self::block`] over any [`Storage`]. + pub fn block_in<'a>(&self, file: &'a dyn Storage) -> Result, FormatError> { + image_block_in(file, self.location) + } + /// Write every entry over `dst`, the file's bytes from the superblock /// on (as long as the `data` the image was decoded from), taking the /// entries from `block` (the image block, see [`Self::block`]). `block` @@ -483,13 +507,28 @@ impl CacheImage { } fn image_block(data: &[u8], location: CacheImageLocation) -> Result<&[u8], FormatError> { + let (start, len) = image_block_range(data.len() as u64, location)?; + Ok(&data[start as usize..start as usize + len]) +} + +fn image_block_in( + file: &dyn Storage, + location: CacheImageLocation, +) -> Result, FormatError> { + let (start, len) = image_block_range(file.len(), location)?; + read_exact_at(file, start, len) +} + +/// Where the image block is, checked against a file of `file_len` bytes. +fn image_block_range(file_len: u64, location: CacheImageLocation) -> Result<(u64, usize), FormatError> { let bad = FormatError::InvalidCacheImage; let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?; let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?; start .checked_add(len) - .and_then(|end| data.get(start..end)) - .ok_or(bad("image block extends past the end of the file")) + .filter(|&end| end as u64 <= file_len) + .ok_or(bad("image block extends past the end of the file"))?; + Ok((start as u64, len)) } /// What an opener must do before reading a file's metadata: check the @@ -498,11 +537,19 @@ fn image_block(data: &[u8], location: CacheImageLocation) -> Result<&[u8], Forma /// ([`CacheImage::decode`]). `data` is the file from the superblock on, up /// to its recorded end of file. pub fn cache_image_state(data: &[u8], sb: &Superblock) -> Result { - match read_superblock_extension(data, sb)? { + cache_image_state_in(&data, sb) +} + +/// [`cache_image_state`] over any [`Storage`]. +pub fn cache_image_state_in( + file: &dyn Storage, + sb: &Superblock, +) -> Result { + match read_superblock_extension_in(file, sb)? { Some(SuperblockExtension { cache_image: Some(location), .. - }) => Ok(match CacheImage::decode(data, location, sb) { + }) => Ok(match CacheImage::decode_in(file, location, sb) { Ok(image) => CacheImageState::Loaded(image), Err(e) => CacheImageState::Unloadable(e), }), @@ -566,7 +613,7 @@ mod tests { /// holds the given messages, padded to `len` bytes. fn file_with_ext(messages: &[(u16, &[u8])], len: usize) -> Vec { let mut body = Vec::new(); - for (t, d) in messages { + for &(t, d) in messages { let padded = d.len().div_ceil(8) * 8; body.extend_from_slice(&t.to_le_bytes()); body.extend_from_slice(&(padded as u16).to_le_bytes()); @@ -809,4 +856,37 @@ mod tests { // An entry cannot be its own parent. assert!(load(image_with_deps(&[(40, b"C", 1, Some(40))])).is_err()); } + + /// The extension and cache image decode identically through a + /// `read_at`-only storage, errors included. + #[test] + fn storage_parse_matches_slice_parse() { + use crate::storage::CountingStorage; + let img = image(&[(16, b"HEADER"), (40, b"NODE")]); + let mut with_image = file_with_ext(&[(MSG_MDCI, &mdci(256, img.len() as u64))], 256); + with_image.extend_from_slice(&img); + let mut bad_image = with_image.clone(); + bad_image[256] = b'X'; + let files = [ + file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256), + file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(256, false, 0))], 256), + file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192), + file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565), + file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 60), + with_image, + bad_image, + ]; + for f in files { + let storage = CountingStorage::new(f.clone()); + let sb = sb_v2(48); + assert_eq!( + read_superblock_extension_in(&storage, &sb), + read_superblock_extension(&f, &sb) + ); + assert_eq!(cache_image_state_in(&storage, &sb), cache_image_state(&f, &sb)); + if let Ok(CacheImageState::Loaded(image)) = cache_image_state(&f, &sb) { + assert_eq!(&*image.block_in(&storage).unwrap(), image.block(&f).unwrap()); + } + } + } } -- 2.54.0 From 6a9bb02f37c0b9c5b2fef7dc84285383ffa038fe Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:49:05 -0500 Subject: [PATCH 06/48] format: read local heaps over Storage LocalHeap::parse_in reads the header in one bounded read, validate_free_list_in reads each free block's two lengths, and read_string_in reads from the string to the end of the data segment once and looks for the terminator there. The &[u8] methods are wrappers. New test: a heap without free space, with a valid free block and with a free block overrunning the segment, cut at every length, parse, validate and read strings identically through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/local_heap.rs | 108 ++++++++++++++++------- 1 file changed, 78 insertions(+), 30 deletions(-) diff --git a/crates/clawhdf5-format/src/local_heap.rs b/crates/clawhdf5-format/src/local_heap.rs index 39e9b26..a4e3480 100644 --- a/crates/clawhdf5-format/src/local_heap.rs +++ b/crates/clawhdf5-format/src/local_heap.rs @@ -4,6 +4,7 @@ use alloc::string::String; use crate::error::FormatError; +use crate::storage::{Storage, len_usize, read_exact_at}; /// Parsed HDF5 Local Heap header. #[derive(Debug, Clone)] @@ -16,21 +17,6 @@ pub struct LocalHeap { pub data_segment_address: u64, } -/// Checks that `[offset, offset + needed)` fits within `data`, guarding the -/// addition against `usize` overflow from a crafted near-`usize::MAX` offset. -fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { - if offset - .checked_add(needed) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: offset.saturating_add(needed), - available: data.len(), - }); - } - Ok(()) -} - fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { let s = size as usize; if pos.checked_add(s).is_none_or(|end| end > data.len()) { @@ -57,12 +43,24 @@ impl LocalHeap { offset: usize, offset_size: u8, length_size: u8, + ) -> Result { + Self::parse_in(&file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the header. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + length_size: u8, ) -> Result { // signature(4) + version(1) + reserved(3) = 8, then length_size*2 + offset_size let ls = length_size as usize; let os = offset_size as usize; let total = 8 + ls * 2 + os; - ensure_len(file_data, offset, total)?; + let header = read_exact_at(file, offset, total)?; + let file_data: &[u8] = &header; + let offset = 0usize; if &file_data[offset..offset + 4] != b"HEAP" { return Err(FormatError::InvalidLocalHeapSignature); @@ -99,6 +97,16 @@ impl LocalHeap { /// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the /// undefined address) is accepted as "no free list" too. pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> { + self.validate_free_list_in(&file_data, length_size) + } + + /// [`Self::validate_free_list`] over any [`Storage`]: two small reads + /// per free block. + pub fn validate_free_list_in( + &self, + file: &dyn Storage, + length_size: u8, + ) -> Result<(), FormatError> { const FREE_NULL: u64 = 1; let ls = length_size as usize; let undefined = if ls >= 8 { @@ -123,11 +131,12 @@ impl LocalHeap { .and_then(|a| usize::try_from(a).ok()) .ok_or(FormatError::InvalidLocalHeapFreeList)?; let block_offset = next; - next = read_offset(file_data, at, length_size)?; + next = read_offset(&read_exact_at(file, at as u64, ls)?, 0, length_size)?; if next == 0 { return Err(FormatError::InvalidLocalHeapFreeList); } - let block_size = read_offset(file_data, at + ls, length_size)?; + let block_size = + read_offset(&read_exact_at(file, (at + ls) as u64, ls)?, 0, length_size)?; if block_offset .checked_add(block_size) .is_none_or(|end| end > size) @@ -140,6 +149,17 @@ impl LocalHeap { /// Read a null-terminated string from the heap's data segment at the given byte offset. pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result { + self.read_string_in(&file_data, string_offset) + } + + /// [`Self::read_string`] over any [`Storage`]: one read, from the + /// string to the end of the data segment. + pub fn read_string_in( + &self, + file: &dyn Storage, + string_offset: u64, + ) -> Result { + let file_len = len_usize(file); let seg_addr = self.data_segment_address as usize; let str_start = seg_addr @@ -153,28 +173,24 @@ impl LocalHeap { "local heap seg_addr + data_segment_size overflow".into(), ))?; - if str_start >= file_data.len() || str_start >= seg_end { + if str_start >= file_len || str_start >= seg_end { return Err(FormatError::UnexpectedEof { expected: str_start + 1, - available: file_data.len(), + available: file_len, }); } // Find null terminator - let search_end = seg_end.min(file_data.len()); - let mut end = str_start; - while end < search_end && file_data[end] != 0 { - end += 1; - } - - if end >= search_end { + let search_end = seg_end.min(file_len); + let rest = read_exact_at(file, str_start as u64, search_end - str_start)?; + let Some(len) = rest.iter().position(|&b| b == 0) else { return Err(FormatError::UnexpectedEof { - expected: end + 1, + expected: search_end + 1, available: search_end, }); - } + }; - let s = core::str::from_utf8(&file_data[str_start..end]) + let s = core::str::from_utf8(&rest[..len]) .map_err(|_| FormatError::InvalidLocalHeapSignature)?; Ok(String::from(s)) } @@ -345,4 +361,36 @@ mod tests { let err = LocalHeap::parse(&file, 0, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLocalHeapVersion(1)); } + + /// Header, free list and strings read identically through a + /// `read_at`-only storage, for every truncation of the file. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + let plain = build_heap_file(0, 64, &["", "alpha", "beta"], 8, 8); + // A free block of 16 bytes at segment offset 12, ending the list. + let mut free = build_heap_file(0, 64, &["", "alpha", "beta", &"x".repeat(20)], 8, 8); + free[16..24].copy_from_slice(&12u64.to_le_bytes()); + free[64 + 12..64 + 20].copy_from_slice(&1u64.to_le_bytes()); + free[64 + 20..64 + 28].copy_from_slice(&16u64.to_le_bytes()); + let mut bad_free = free.clone(); + bad_free[64 + 20..64 + 28].copy_from_slice(&99u64.to_le_bytes()); + for full in [plain, free, bad_free] { + for cut in 0..=full.len() { + let f = &full[..cut]; + let storage = CountingStorage::new(f.to_vec()); + let want = LocalHeap::parse(f, 0, 8, 8); + let got = LocalHeap::parse_in(&storage, 0, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + let Ok(heap) = want else { continue }; + assert_eq!( + heap.validate_free_list_in(&storage, 8), + heap.validate_free_list(f, 8) + ); + for off in [0u64, 1, 2, 6, 7, 11, 100] { + assert_eq!(heap.read_string_in(&storage, off), heap.read_string(f, off)); + } + } + } + } } -- 2.54.0 From aab7ea9e8f23e45b3ad340c6a35974f45bbaef80 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:49:05 -0500 Subject: [PATCH 07/48] format: rustfmt the Storage conversions Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/object_header.rs | 16 ++++++++++------ crates/clawhdf5-format/src/storage.rs | 6 +++++- crates/clawhdf5-format/src/superblock_ext.rs | 15 ++++++++++++--- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index d0f36db..1fa4b41 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -385,13 +385,13 @@ impl ObjectHeader { } let chunk0_msg_start = pos; - let chunk0_msg_end = abs(pos) - .checked_add(chunk0_size) - .ok_or(FormatError::UnexpectedEof { + let Some(chunk0_abs_end) = abs(pos).checked_add(chunk0_size) else { + return Err(FormatError::UnexpectedEof { expected: usize::MAX, available: file_len, - })? - - base; + }); + }; + let chunk0_msg_end = chunk0_abs_end - base; // The whole first chunk, prefix to checksum, in one read (its // bounds check is the one on the checksum's 4 bytes). @@ -1364,7 +1364,11 @@ mod tests { let want = ObjectHeader::parse(&f, at, 8, 8); let storage = CountingStorage::new(f.clone()); let got = ObjectHeader::parse_in(&storage, at as u64, 8, 8); - assert_eq!(format!("{got:?}"), format!("{want:?}"), "at {at}, cut {cut}"); + assert_eq!( + format!("{got:?}"), + format!("{want:?}"), + "at {at}, cut {cut}" + ); } } } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 76f1436..fb19f32 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -215,7 +215,11 @@ pub fn read_exact_at( /// storage. For structures whose size is only known once their prefix has /// been parsed and whose parsers bound-check what they are given. #[inline] -pub fn read_upto(file: &dyn Storage, offset: u64, max: usize) -> Result, FormatError> { +pub fn read_upto( + file: &dyn Storage, + offset: u64, + max: usize, +) -> Result, FormatError> { let avail = file.len().saturating_sub(offset); let len = usize::try_from(avail).map_or(max, |a| a.min(max)); let bytes = file.read_at(offset, len)?; diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index 40efd5c..1a684f6 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -520,7 +520,10 @@ fn image_block_in( } /// Where the image block is, checked against a file of `file_len` bytes. -fn image_block_range(file_len: u64, location: CacheImageLocation) -> Result<(u64, usize), FormatError> { +fn image_block_range( + file_len: u64, + location: CacheImageLocation, +) -> Result<(u64, usize), FormatError> { let bad = FormatError::InvalidCacheImage; let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?; let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?; @@ -883,9 +886,15 @@ mod tests { read_superblock_extension_in(&storage, &sb), read_superblock_extension(&f, &sb) ); - assert_eq!(cache_image_state_in(&storage, &sb), cache_image_state(&f, &sb)); + assert_eq!( + cache_image_state_in(&storage, &sb), + cache_image_state(&f, &sb) + ); if let Ok(CacheImageState::Loaded(image)) = cache_image_state(&f, &sb) { - assert_eq!(&*image.block_in(&storage).unwrap(), image.block(&f).unwrap()); + assert_eq!( + &*image.block_in(&storage).unwrap(), + image.block(&f).unwrap() + ); } } } -- 2.54.0 From 06625b7470c72ccac3ff764577975d26e587aece Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:50:25 -0500 Subject: [PATCH 08/48] format: read global heap collections over Storage GlobalHeapCollection::parse_in / parse_index_in read the header, check the collection against the end of the file, and read the collection in one bounded read; objects are indexed in it with file offsets, as before. The &[u8] functions are wrappers. New test: collections with 4- and 8-byte lengths, one whose size runs past the file and one whose object overruns it, at two offsets and cut at every length, give identical results through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/global_heap.rs | 133 ++++++++++++++++++---- 1 file changed, 109 insertions(+), 24 deletions(-) diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index 474c7fd..c6de363 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -1,9 +1,12 @@ //! HDF5 Global Heap collection parsing. #[cfg(not(feature = "std"))] -use alloc::{format, string::String, vec::Vec}; +use alloc::{borrow::Cow, format, string::String, vec::Vec}; +#[cfg(feature = "std")] +use std::borrow::Cow; use crate::error::FormatError; +use crate::storage::{Storage, len_usize, read_exact_at}; /// Magic signature for global heap collections. const GCOL_SIGNATURE: [u8; 4] = *b"GCOL"; @@ -28,19 +31,20 @@ pub struct GlobalHeapObject { pub data: Vec, } -fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { +/// Checks that `[offset, offset + needed)` ends by `data_len`. +fn ensure_len(data_len: usize, offset: usize, needed: usize) -> Result<(), FormatError> { match offset.checked_add(needed) { - Some(end) if end <= data.len() => Ok(()), + Some(end) if end <= data_len => Ok(()), _ => Err(FormatError::UnexpectedEof { expected: offset.saturating_add(needed), - available: data.len(), + available: data_len, }), } } fn read_length(data: &[u8], offset: usize, length_size: u8) -> Result { let s = length_size as usize; - ensure_len(data, offset, s)?; + ensure_len(data.len(), offset, s)?; let slice = &data[offset..offset + s]; Ok(match length_size { 2 => u16::from_le_bytes([slice[0], slice[1]]) as u64, @@ -95,7 +99,17 @@ impl GlobalHeapCollection { offset: usize, length_size: u8, ) -> Result { - let index = Self::parse_index(file_data, offset, length_size)?; + Self::parse_in(&file_data, offset as u64, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the header, one of + /// the collection. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + length_size: u8, + ) -> Result { + let (bytes, base, index) = Self::read_collection(file, offset, length_size)?; Ok(GlobalHeapCollection { collection_size: index.collection_size, objects: index @@ -104,7 +118,7 @@ impl GlobalHeapCollection { .map(|o| GlobalHeapObject { index: o.index, reference_count: o.reference_count, - data: file_data[o.offset..o.offset + o.size].to_vec(), + data: bytes[o.offset - base..o.offset - base + o.size].to_vec(), }) .collect(), }) @@ -122,43 +136,72 @@ impl GlobalHeapCollection { offset: usize, length_size: u8, ) -> Result { + Self::parse_index_in(&file_data, offset as u64, length_size) + } + + /// [`Self::parse_index`] over any [`Storage`]: one read of the header, + /// one of the collection. The object offsets are file offsets. + pub fn parse_index_in( + file: &dyn Storage, + offset: u64, + length_size: u8, + ) -> Result { + Ok(Self::read_collection(file, offset, length_size)?.2) + } + + /// Read the collection at `offset` and index its objects: the + /// collection's bytes, its offset as a `usize`, and the index (with + /// file offsets). + fn read_collection( + file: &dyn Storage, + offset: u64, + length_size: u8, + ) -> Result<(Cow<'_, [u8]>, usize, GlobalHeapIndex), FormatError> { + let file_len = len_usize(file); // signature(4) + version(1) + reserved(3) + collection_size(length_size), // padded to a multiple of 8 as libhdf5 lays it out (`H5HG_SIZEOF_HDR`). // With 8-byte lengths the padding is 0; with 4-byte lengths it is 4, // and reading without it put every object 4 bytes early. let header_size = pad8(8 + length_size as usize); - ensure_len(file_data, offset, header_size)?; + let header = read_exact_at(file, offset, header_size)?; + let offset = usize::try_from(offset).map_err(|_| FormatError::UnexpectedEof { + expected: usize::MAX, + available: file_len, + })?; - if file_data[offset..offset + 4] != GCOL_SIGNATURE { + if header[..4] != GCOL_SIGNATURE { return Err(FormatError::InvalidGlobalHeapSignature); } - let version = file_data[offset + 4]; + let version = header[4]; if version != 1 { return Err(FormatError::InvalidGlobalHeapVersion(version)); } - let collection_size = read_length(file_data, offset + 8, length_size)?; + let collection_size = read_length(&header, 8, length_size)?; let collection_end = usize::try_from(collection_size) .ok() .and_then(|size| offset.checked_add(size)) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: file_len, })?; - if collection_end > file_data.len() { + if collection_end > file_len { return Err(FormatError::UnexpectedEof { expected: collection_end, - available: file_data.len(), + available: file_len, }); } + let collection = read_exact_at(file, offset as u64, collection_end - offset)?; + // Positions below are file offsets; `file_data(p)` is the byte at `p`. + let file_data = |p: usize| collection[p - offset]; let mut pos = offset + header_size; let mut objects = Vec::new(); // Parse objects until we hit index 0 (free space) or run out of space while pos + 2 <= collection_end { - let object_index = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); + let object_index = u16::from_le_bytes([file_data(pos), file_data(pos + 1)]); if object_index == 0 { // Free space marker — done @@ -168,11 +211,12 @@ impl GlobalHeapCollection { // object_index(2) + reference_count(2) + reserved(4) + // object_size(length_size), padded to 8 (`H5HG_SIZEOF_OBJHDR`). let obj_header_size = pad8(8 + length_size as usize); - ensure_len(&file_data[..collection_end], pos, obj_header_size)?; + ensure_len(collection_end, pos, obj_header_size)?; - let reference_count = u16::from_le_bytes([file_data[pos + 2], file_data[pos + 3]]); - let object_size = usize::try_from(read_length(file_data, pos + 8, length_size)?) - .map_err(|_| FormatError::Overflow("global heap object size".into()))?; + let reference_count = u16::from_le_bytes([file_data(pos + 2), file_data(pos + 3)]); + let object_size = + usize::try_from(read_length(&collection[pos - offset..], 8, length_size)?) + .map_err(|_| FormatError::Overflow("global heap object size".into()))?; pos += obj_header_size; if pos @@ -197,10 +241,11 @@ impl GlobalHeapCollection { pos = pos.saturating_add(pad8(object_size)); } - Ok(GlobalHeapIndex { + let index = GlobalHeapIndex { collection_size, objects, - }) + }; + Ok((collection, offset, index)) } /// Get an object by its index. @@ -226,7 +271,7 @@ mod tests { let mut obj_size_total = 0usize; for (_, _, data) in objects { let obj_header = pad8(8 + ls); - obj_size_total += obj_header + pad8(data.len()); + obj_size_total += obj_header + pad8(<[u8]>::len(data)); } // Free space marker (2 bytes for index 0) obj_size_total += 2; @@ -258,8 +303,8 @@ mod tests { buf.resize(buf.len() + (pad8(8 + ls) - (8 + ls)), 0); buf.extend_from_slice(data); // Pad to 8 bytes - let padded = pad8(data.len()); - buf.resize(buf.len() + (padded - data.len()), 0); + let padded = pad8(<[u8]>::len(data)); + buf.resize(buf.len() + (padded - <[u8]>::len(data)), 0); } // Free space marker @@ -327,4 +372,44 @@ mod tests { assert_eq!(coll.objects.len(), 1); assert_eq!(coll.objects[0].data, b"test"); } + + /// Collections, and every truncation of them, index and parse + /// identically through a `read_at`-only storage: two reads each. + #[test] + fn storage_parse_matches_slice_parse() { + use crate::storage::CountingStorage; + let objs: &[(u16, u16, &[u8])] = &[(1, 1, b"hello"), (2, 3, b"a longer object")]; + for ls in [4u8, 8] { + let coll = build_collection(objs, ls); + let mut corrupt = coll.clone(); + corrupt[8] = 200; // collection size past the end of the file + let mut overrun = coll.clone(); + let size_at = pad8(8 + ls as usize) + 8; + overrun[size_at] = 250; // first object runs past the collection + for full in [coll, corrupt, overrun] { + for at in [0usize, 5] { + for cut in 0..=full.len() { + let mut f = vec![0u8; at]; + f.extend_from_slice(&full[..cut]); + let storage = CountingStorage::new(f.clone()); + let want = GlobalHeapCollection::parse(&f, at, ls); + let got = GlobalHeapCollection::parse_in(&storage, at as u64, ls); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + let want = GlobalHeapCollection::parse_index(&f, at, ls); + let got = GlobalHeapCollection::parse_index_in(&storage, at as u64, ls); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + } + } + } + } + let storage = CountingStorage::new(build_collection(objs, 8)); + assert_eq!( + GlobalHeapCollection::parse_in(&storage, 0, 8) + .unwrap() + .objects + .len(), + 2 + ); + assert_eq!(storage.reads(), 2); + } } -- 2.54.0 From bcf3ae4856dd2ef0cfdd9a47acceb6de4d70c6fb Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:51:47 -0500 Subject: [PATCH 09/48] format: read symbol table nodes over Storage SymbolTableNode::parse_in reads the node's 8-byte header, checks the entries against the end of the file as before, and reads them in one bounded read. parse is a wrapper. New test: nodes with 4- and 8-byte offsets, valid and with a bad version, at two offsets and cut at every length, parse identically through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/symbol_table.rs | 68 ++++++++++++++++------ 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/crates/clawhdf5-format/src/symbol_table.rs b/crates/clawhdf5-format/src/symbol_table.rs index 9809c75..8fb4453 100644 --- a/crates/clawhdf5-format/src/symbol_table.rs +++ b/crates/clawhdf5-format/src/symbol_table.rs @@ -4,6 +4,7 @@ use alloc::vec::Vec; use crate::error::FormatError; +use crate::storage::{Storage, len_usize, read_exact_at}; /// Symbol Table message (type 0x0011) found in v1 group object headers. #[derive(Debug, Clone, PartialEq)] @@ -79,48 +80,53 @@ impl SymbolTableNode { offset: usize, offset_size: u8, ) -> Result { - // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8 - if offset - .checked_add(8) - .is_none_or(|end| end > file_data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: offset.saturating_add(8), - available: file_data.len(), - }); - } + Self::parse_in(&file_data, offset as u64, offset_size) + } - if &file_data[offset..offset + 4] != b"SNOD" { + /// [`Self::parse`] over any [`Storage`]: one read of the node's header, + /// one of its entries. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + ) -> Result { + let file_len = len_usize(file); + // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8 + let header = read_exact_at(file, offset, 8)?; + + if &header[..4] != b"SNOD" { return Err(FormatError::InvalidSymbolTableNodeSignature); } - let version = file_data[offset + 4]; + let version = header[4]; if version != 1 { return Err(FormatError::InvalidSymbolTableNodeVersion(version)); } - let num_symbols = - u16::from_le_bytes([file_data[offset + 6], file_data[offset + 7]]) as usize; + let num_symbols = u16::from_le_bytes([header[6], header[7]]) as usize; let os = offset_size as usize; // Each entry: link_name_offset(os) + obj_hdr_addr(os) + cache_type(4) + reserved(4) + scratch(16) let entry_size = os + os + 4 + 4 + 16; - let entries_start = offset + 8; + // `offset + 8` fits: the header's read checked it. + let entries_start = offset as usize + 8; let needed = entries_start.checked_add(num_symbols * entry_size).ok_or( FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: file_len, }, )?; - if needed > file_data.len() { + if needed > file_len { return Err(FormatError::UnexpectedEof { expected: needed, - available: file_data.len(), + available: file_len, }); } + let body = read_exact_at(file, entries_start as u64, num_symbols * entry_size)?; + let file_data: &[u8] = &body; let mut entries = Vec::with_capacity(num_symbols); - let mut pos = entries_start; + let mut pos = 0usize; for _ in 0..num_symbols { let link_name_offset = read_offset(file_data, pos, offset_size)?; pos += os; @@ -256,4 +262,28 @@ mod tests { let result = SymbolTableNode::parse(&data, usize::MAX / 2, 8); assert!(result.is_err()); } + + /// Nodes, cut at every length and at an offset, parse identically + /// through a `read_at`-only storage. + #[test] + fn storage_parse_matches_slice_parse() { + use crate::storage::CountingStorage; + for os in [4u8, 8] { + let node = build_snod(&[(0, 0x100, 0), (8, 0x200, 1), (16, 0x300, 2)], os); + let mut bad = node.clone(); + bad[4] = 2; + for full in [node, bad] { + for at in [0usize, 7] { + for cut in 0..=full.len() { + let mut f = vec![0u8; at]; + f.extend_from_slice(&full[..cut]); + let storage = CountingStorage::new(f.clone()); + let want = SymbolTableNode::parse(&f, at, os); + let got = SymbolTableNode::parse_in(&storage, at as u64, os); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + } + } + } + } + } } -- 2.54.0 From 24cbf12f16b0696287db48ffb5b0878919dec465 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:51:47 -0500 Subject: [PATCH 10/48] format: read group B-tree (v1) nodes over Storage BTreeV1Node::parse_in reads a node's header and then its keys and children, two bounded reads; collect_symbol_table_nodes_in walks the tree over any Storage. The &[u8] functions are wrappers. New test: nodes with siblings and 4- and 8-byte offsets cut at every length, and a two-level tree with truncated leaves, give identical results through a read_at-only CountingStorage (six reads for the three nodes). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/btree_v1.rs | 90 ++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/crates/clawhdf5-format/src/btree_v1.rs b/crates/clawhdf5-format/src/btree_v1.rs index b652dcc..3378131 100644 --- a/crates/clawhdf5-format/src/btree_v1.rs +++ b/crates/clawhdf5-format/src/btree_v1.rs @@ -4,6 +4,7 @@ use alloc::vec::Vec; use crate::error::FormatError; +use crate::storage::{Storage, read_exact_at}; /// A parsed B-tree v1 node. #[derive(Debug, Clone)] @@ -74,13 +75,28 @@ impl BTreeV1Node { file_data: &[u8], offset: usize, offset_size: u8, + length_size: u8, + ) -> Result { + Self::parse_in(&file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the node's header, + /// one of its keys and children. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, _length_size: u8, ) -> Result { // signature(4) + node_type(1) + node_level(1) + entries_used(2) = 8 // + left_sibling(offset_size) + right_sibling(offset_size) let os = offset_size as usize; let header_size = 8 + os * 2; - ensure_len(file_data, offset, header_size)?; + let header = read_exact_at(file, offset, header_size)?; + let file_data: &[u8] = &header; + // The header's read checked that `offset + header_size` fits. + let body_start = offset + header_size as u64; + let offset = 0usize; if &file_data[offset..offset + 4] != b"TREE" { return Err(FormatError::InvalidBTreeSignature); @@ -102,14 +118,15 @@ impl BTreeV1Node { } else { Some(read_offset(file_data, pos, offset_size)?) }; - pos += os; // For type 0: keys are offset_size bytes, children are offset_size bytes // Layout: key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N] let eu = entries_used as usize; let key_size = os; // For type 0, key = offset_size let needed = eu * (key_size + os) + key_size; // eu children + (eu+1) keys - ensure_len(file_data, pos, needed)?; + let body = read_exact_at(file, body_start, needed)?; + let file_data: &[u8] = &body; + let mut pos = 0usize; let mut keys = Vec::with_capacity(eu + 1); let mut children = Vec::with_capacity(eu); @@ -150,11 +167,21 @@ pub fn collect_symbol_table_nodes( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - collect_symbol_table_nodes_inner(file_data, btree_address, offset_size, length_size, 0) + collect_symbol_table_nodes_in(&file_data, btree_address, offset_size, length_size) +} + +/// [`collect_symbol_table_nodes`] over any [`Storage`]: two reads per node. +pub fn collect_symbol_table_nodes_in( + file: &dyn Storage, + btree_address: u64, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + collect_symbol_table_nodes_inner(file, btree_address, offset_size, length_size, 0) } fn collect_symbol_table_nodes_inner( - file_data: &[u8], + file: &dyn Storage, btree_address: u64, offset_size: u8, length_size: u8, @@ -164,7 +191,12 @@ fn collect_symbol_table_nodes_inner( return Err(FormatError::NestingDepthExceeded); } - let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?; + let node = BTreeV1Node::parse_in( + file, + btree_address as usize as u64, + offset_size, + length_size, + )?; if node.node_type != 0 { return Err(FormatError::InvalidBTreeNodeType(node.node_type)); @@ -178,7 +210,7 @@ fn collect_symbol_table_nodes_inner( let mut result = Vec::new(); for &child_addr in &node.children { let child_snods = collect_symbol_table_nodes_inner( - file_data, + file, child_addr, offset_size, length_size, @@ -317,4 +349,48 @@ mod tests { assert_eq!(node.entries_used, 1); assert_eq!(node.children, vec![0x50]); } + + /// Nodes and trees, cut at every length, parse identically through a + /// `read_at`-only storage. + #[test] + fn storage_parse_matches_slice_parse() { + use crate::storage::CountingStorage; + let nodes = [ + build_btree_node(0, 0, &[0, 5, 10], &[0x100, 0x200], None, None, 8), + build_btree_node(0, 0, &[0, 5], &[0x100], Some(0x40), Some(0x80), 4), + build_btree_node(1, 2, &[0, 5], &[0x100], None, Some(0x80), 8), + ]; + for (n, node) in nodes.iter().enumerate() { + let os = if n == 1 { 4 } else { 8 }; + for cut in 0..=node.len() { + let f = &node[..cut]; + let storage = CountingStorage::new(f.to_vec()); + let want = BTreeV1Node::parse(f, 0, os, 8); + let got = BTreeV1Node::parse_in(&storage, 0, os, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + } + } + let leaf1 = build_btree_node(0, 0, &[0, 5], &[0xA00], None, None, 8); + let leaf2 = build_btree_node(0, 0, &[5, 10], &[0xB00], None, None, 8); + let internal = build_btree_node(0, 1, &[0, 5, 10], &[0, 256], None, None, 8); + let mut file = vec![0u8; 512 + internal.len()]; + file[..leaf1.len()].copy_from_slice(&leaf1); + file[256..256 + leaf2.len()].copy_from_slice(&leaf2); + file[512..].copy_from_slice(&internal); + for cut in [file.len(), 300, 260, 100, 10] { + let mut f = file.clone(); + if cut < 512 { + // Truncate the leaves, keep the root. + f[cut..512].fill(0); + } + let storage = CountingStorage::new(f.clone()); + assert_eq!( + collect_symbol_table_nodes_in(&storage, 512, 8, 8), + collect_symbol_table_nodes(&f, 512, 8, 8) + ); + } + let storage = CountingStorage::new(file); + collect_symbol_table_nodes_in(&storage, 512, 8, 8).unwrap(); + assert_eq!(storage.reads(), 6); + } } -- 2.54.0 From a0160730f26643203402a3ba95aa6a635d6ee0d9 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:54:48 -0500 Subject: [PATCH 11/48] format: read fractal heaps over Storage FractalHeapHeader::parse_in reads the header as one window (a second, longer one when it holds an I/O filter pipeline); read_managed_object_in reads direct blocks, indirect blocks (one window up to the last child entry) and huge objects with bounded reads. The &[u8] methods are wrappers. A huge object indexed by the huge-object v2 B-tree, which is not converted yet, is a clean ContiguousStorageRequired error on a backend without the whole file in memory (after the "no index" check, so the error order is unchanged). storage::Window (crate-internal) reads a window of a structure and reports bounds failures exactly as the whole-file ensure_len did, and a short read inside the file is now a Storage error rather than an EOF. New tests: headers (with and without a filter pipeline) cut at every length, and managed objects in a direct root and through an indirect root, huge objects with direct IDs and tiny objects, give identical results through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/fractal_heap.rs | 228 ++++++++++++++++++--- crates/clawhdf5-format/src/storage.rs | 47 ++++- 2 files changed, 242 insertions(+), 33 deletions(-) diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index 9bc4ba4..1c6a77c 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -9,6 +9,7 @@ use byteorder::{ByteOrder, LittleEndian}; use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; +use crate::storage::{Storage, Window, len_usize, read_exact_at, require_contiguous}; /// Parsed fractal heap header (signature "FRHP"). #[derive(Debug, Clone)] @@ -138,6 +139,36 @@ impl FractalHeapHeader { offset_size: u8, length_size: u8, ) -> Result { + Self::parse_in(&file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the header (two + /// when it holds an I/O filter pipeline). + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + length_size: u8, + ) -> Result { + // Every field up to the checksum, without and with the filter + // information; the window holds all of it (or ends at the end of + // the file), so its bounds checks are the whole-file ones. + let (os, ls) = (usize::from(offset_size), usize::from(length_size)); + let unfiltered_len = 26 + 12 * ls + 3 * os; + let mut w = Window::read(file, offset, unfiltered_len)?; + if w.bytes.len() == unfiltered_len { + let filter_len = usize::from(u16::from_le_bytes([w.bytes[7], w.bytes[8]])); + if filter_len > 0 { + w = Window::read(file, offset, unfiltered_len + ls + 4 + filter_len)?; + } + } + let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed); + let read_offset = |_: &[u8], pos: usize, size: u8| { + w.ensure(pos, usize::from(size))?; + read_offset(&w.bytes, pos, size) + }; + let file_data: &[u8] = &w.bytes; + let offset = 0usize; ensure_len(file_data, offset, 5)?; if &file_data[offset..offset + 4] != b"FRHP" { return Err(FormatError::InvalidFractalHeapSignature); @@ -148,9 +179,6 @@ impl FractalHeapHeader { return Err(FormatError::InvalidFractalHeapVersion(version)); } - let os = offset_size as usize; - let ls = length_size as usize; - let mut pos = offset + 5; ensure_len(file_data, pos, 2)?; let heap_id_length = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); @@ -353,6 +381,18 @@ impl FractalHeapHeader { file_data: &[u8], id_bytes: &[u8], offset_size: u8, + ) -> Result, FormatError> { + self.read_managed_object_in(&file_data, id_bytes, offset_size) + } + + /// [`Self::read_managed_object`] over any [`Storage`]. A huge object + /// found through the huge-object v2 B-tree still needs the whole file + /// in memory ([`FormatError::ContiguousStorageRequired`] otherwise). + pub fn read_managed_object_in( + &self, + file_data: &dyn Storage, + id_bytes: &[u8], + offset_size: u8, ) -> Result, FormatError> { let Some(&first) = id_bytes.first() else { return Err(FormatError::UnexpectedEof { @@ -383,7 +423,7 @@ impl FractalHeapHeader { } /// Read a huge object (heap ID type 1). - fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result, FormatError> { + fn read_huge_object(&self, file: &dyn Storage, id: &[u8]) -> Result, FormatError> { let os = usize::from(self.offset_size); let ls = usize::from(self.length_size); // (address, stored length, filter mask, decoded length); the last two @@ -414,18 +454,17 @@ impl FractalHeapHeader { let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8); ensure_len(id, 1, key_len)?; let key = le_uint(&id[1..1 + key_len]); - self.find_huge_record(file_data, key)? + self.find_huge_record(file, key)? }; let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?; let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?; - ensure_len(file_data, start, len)?; - let stored = &file_data[start..start + len]; + let stored = read_exact_at(file, start as u64, len)?; match &self.filter_pipeline { - None => Ok(stored.to_vec()), + None => Ok(stored.into_owned()), Some(pipeline) => { let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?; - let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?; + let out = crate::filters::decompress_chunk_masked(&stored, pipeline, mem, 1, mask)?; if out.len() != mem { return Err(heap_error("filtered huge object decoded to the wrong size")); } @@ -438,7 +477,7 @@ impl FractalHeapHeader { /// (address, stored length, filter mask, decoded length). fn find_huge_record( &self, - file_data: &[u8], + file: &dyn Storage, key: u64, ) -> Result<(u64, u64, u32, u64), FormatError> { if is_undefined(self.huge_btree_address, self.offset_size) { @@ -446,6 +485,8 @@ impl FractalHeapHeader { "huge object ID but the heap has no huge-object index", )); } + // The v2 B-tree is read from a slice until it is converted. + let file_data = require_contiguous(file, "a huge fractal-heap object's B-tree")?; let hdr = BTreeV2Header::parse( file_data, self.huge_btree_address as usize, @@ -515,7 +556,7 @@ impl FractalHeapHeader { /// Read a managed object (heap ID type 0). fn read_heap_managed( &self, - file_data: &[u8], + file_data: &dyn Storage, id_bytes: &[u8], offset_size: u8, ) -> Result, FormatError> { @@ -565,7 +606,7 @@ impl FractalHeapHeader { /// through its filter pipeline, so the block is decoded first. fn read_from_direct_block( &self, - file_data: &[u8], + file: &dyn Storage, block: DirectBlock, target_offset: u64, length: usize, @@ -581,9 +622,9 @@ impl FractalHeapHeader { let stored_len = usize::try_from(block.filtered_size) .map_err(|_| heap_error("direct block size"))?; let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?; - ensure_len(file_data, block.addr, stored_len)?; + let stored = read_exact_at(file, block.addr as u64, stored_len)?; let decoded = crate::filters::decompress_chunk_masked( - &file_data[block.addr..block.addr + stored_len], + &stored, pipeline, size, 1, @@ -597,17 +638,16 @@ impl FractalHeapHeader { .checked_add(local_offset) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, - available: file_data.len(), + available: len_usize(file), })?; - ensure_len(file_data, pos, length)?; - Ok(file_data[pos..pos + length].to_vec()) + Ok(read_exact_at(file, pos as u64, length)?.into_owned()) } /// Read an object by traversing an indirect block to find the right direct block. #[allow(clippy::too_many_arguments)] fn read_from_indirect_block( &self, - file_data: &[u8], + file: &dyn Storage, iblock_addr: usize, nrows: u16, iblock_heap_offset: u64, @@ -621,16 +661,8 @@ impl FractalHeapHeader { "fractal heap: maximum recursion depth exceeded".into(), )); } - // Parse indirect block header - ensure_len(file_data, iblock_addr, 4)?; - if &file_data[iblock_addr..iblock_addr + 4] != b"FHIB" { - return Err(FormatError::InvalidFractalHeapSignature); - } - let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8); let iblock_header = 5 + offset_size as usize + block_offset_bytes; - let mut pos = iblock_addr + iblock_header; - let tw = self.table_width as u64; let nrows_usize = nrows as usize; let mut current_heap_offset = iblock_heap_offset; @@ -640,6 +672,38 @@ impl FractalHeapHeader { let start_indirect = self.max_direct_rows(); let max_direct_rows = nrows_usize.min(start_indirect); + // The block up to its last child entry, in one window: every + // position read below lies inside it, so its bounds checks are the + // whole-file ones. + let direct_entry = usize::from(offset_size) + + if self.filter_pipeline.is_some() { + usize::from(self.length_size) + 4 + } else { + 0 + }; + let entries = + |rows: usize, entry: usize| rows.saturating_mul(tw as usize).saturating_mul(entry); + let block_len = iblock_header + .saturating_add(entries(max_direct_rows, direct_entry)) + .saturating_add(entries( + nrows_usize.saturating_sub(start_indirect), + usize::from(offset_size), + )); + let w = Window::read(file, iblock_addr as u64, block_len)?; + let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed); + let read_offset = |_: &[u8], pos: usize, size: u8| { + w.ensure(pos, usize::from(size))?; + read_offset(&w.bytes, pos, size) + }; + let file_data: &[u8] = &w.bytes; + + // Parse indirect block header + ensure_len(file_data, 0, 4)?; + if &file_data[..4] != b"FHIB" { + return Err(FormatError::InvalidFractalHeapSignature); + } + let mut pos = iblock_header; + for row in 0..max_direct_rows { let block_size = self.block_size_for_row(row); @@ -671,7 +735,7 @@ impl FractalHeapHeader { && target_offset < block_end { return self.read_from_direct_block( - file_data, + file, DirectBlock { addr: child_addr as usize, size: block_size, @@ -704,7 +768,7 @@ impl FractalHeapHeader { && target_offset < block_end { return self.read_from_indirect_block( - file_data, + file, child_addr as usize, child_nrows, current_heap_offset, @@ -720,7 +784,7 @@ impl FractalHeapHeader { Err(FormatError::UnexpectedEof { expected: target_offset as usize + length, - available: file_data.len(), + available: len_usize(file), }) } @@ -1024,4 +1088,110 @@ mod tests { let id = [0x40u8, 0, 0, 0, 0, 0, 0]; assert!(hdr.read_managed_object(&file_data, &id, 8).is_err()); } + + /// Headers, and managed (in a direct root and through an indirect + /// root), huge and tiny objects read identically through a + /// `read_at`-only storage, for every truncation of the file. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + let (mut file, header_end) = build_simple_heap(8, 8); + // An indirect root block at 600: row 0 holds the direct block at + // 256, then three undefined blocks. + file[600..604].copy_from_slice(b"FHIB"); + let mut at = 600 + 5 + 8 + 2; + for addr in [256u64, u64::MAX, u64::MAX, u64::MAX] { + file[at..at + 8].copy_from_slice(&addr.to_le_bytes()); + at += 8; + } + file[900..905].copy_from_slice(b"huge!"); + let managed_id = |offset: u64, len: u64| { + let payload = offset | (len << 16); + let mut id = vec![0u8]; + id.extend_from_slice(&payload.to_le_bytes()[..6]); + id + }; + let mut huge = vec![0x10u8]; + huge.extend_from_slice(&900u64.to_le_bytes()); + huge.extend_from_slice(&5u64.to_le_bytes()); + let ids = [ + managed_id(15, 13), + managed_id(15, 200), + managed_id(130, 4), + huge, + vec![0x22, b'a', b'b', b'c', 0, 0, 0], + ]; + let mut cuts: Vec = (0..=header_end + 1).collect(); + cuts.extend([256, 260, 271, 280, 600, 610, 620, 640, 900, 903, file.len()]); + for cut in cuts { + let f = &file[..cut]; + let storage = CountingStorage::new(f.to_vec()); + let want = FractalHeapHeader::parse(f, 0, 8, 8); + let got = FractalHeapHeader::parse_in(&storage, 0, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "cut {cut}"); + let Ok(direct) = want else { continue }; + let mut indirect = direct.clone(); + indirect.root_block_address = 600; + indirect.current_rows_in_root_indirect_block = 1; + let mut huge_ids = direct.clone(); + huge_ids.heap_id_length = 17; + for hdr in [&direct, &indirect, &huge_ids] { + for id in &ids { + assert_eq!( + hdr.read_managed_object_in(&storage, id, 8), + hdr.read_managed_object(f, id, 8), + "cut {cut}" + ); + } + } + } + } + + /// A huge object found through the huge-object B-tree needs the whole + /// file in memory until the B-tree reader is converted: a clean error + /// on other storage. + #[test] + fn huge_object_btree_needs_contiguous_storage() { + use crate::storage::CountingStorage; + let (file, _) = build_simple_heap(8, 8); + let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap(); + hdr.huge_btree_address = 700; + let storage = CountingStorage::new(file); + assert_eq!( + hdr.read_managed_object_in(&storage, &[0x10, 1, 0, 0, 0, 0, 0], 8), + Err(FormatError::ContiguousStorageRequired( + "a huge fractal-heap object's B-tree" + )) + ); + } + + /// A header with an I/O filter pipeline (read in a second, longer + /// window) parses identically through a `read_at`-only storage, for + /// every truncation. + #[test] + fn filtered_header_parses_identically_through_storage() { + use crate::storage::CountingStorage; + let (simple, header_end) = build_simple_heap(8, 8); + let pipeline = [2u8, 1, 1, 0, 0, 0, 1, 0, 6, 0, 0, 0]; // deflate, level 6 + let mut header = simple[..header_end - 4].to_vec(); + header[7..9].copy_from_slice(&(pipeline.len() as u16).to_le_bytes()); + header.extend_from_slice(&100u64.to_le_bytes()); // root block's stored size + header.extend_from_slice(&0u32.to_le_bytes()); // its filter mask + header.extend_from_slice(&pipeline); + let sum = crate::checksum::jenkins_lookup3(&header); + header.extend_from_slice(&sum.to_le_bytes()); + let mut file = header.clone(); + file.resize(256, 0); + let hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap(); + assert!(hdr.filter_pipeline.is_some()); + for cut in 0..=file.len() { + let f = &file[..cut]; + let storage = CountingStorage::new(f.to_vec()); + assert_eq!( + format!("{:?}", FractalHeapHeader::parse_in(&storage, 0, 8, 8)), + format!("{:?}", FractalHeapHeader::parse(f, 0, 8, 8)), + "cut {cut}" + ); + } + } } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index fb19f32..4608b2d 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -206,11 +206,52 @@ pub fn read_exact_at( if bytes.len() < len { // The storage shrank or the backend served a short read inside the // file: never parse a partial structure. - return Err(eof()); + return Err(short_read()); } Ok(bytes) } +fn short_read() -> FormatError { + FormatError::Storage( + "short read inside the file (the storage shrank or the backend failed)".into(), + ) +} + +/// A window of the file: up to `max` bytes read at `base`, fewer only at +/// the end of the file. Its [`Window::ensure`] reports a bounds failure +/// exactly as the whole-file check `ensure_len(file_data, base + rel, n)` +/// did — with the absolute position and the file's length — as long as +/// every position checked lies within the `max` bytes the window was asked +/// for: then a position past the window is past the end of the file. +pub(crate) struct Window<'a> { + /// The bytes, from `base` on. + pub bytes: Cow<'a, [u8]>, + base: usize, + file_len: usize, +} + +impl<'a> Window<'a> { + /// Read up to `max` bytes at `base`. + pub fn read(file: &'a dyn Storage, base: u64, max: usize) -> Result { + Ok(Window { + bytes: read_upto(file, base, max)?, + base: usize::try_from(base).unwrap_or(usize::MAX), + file_len: len_usize(file), + }) + } + + /// Check that `[rel, rel + needed)` (relative to `base`) is in the file. + pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> { + match rel.checked_add(needed) { + Some(end) if end <= self.bytes.len() => Ok(()), + _ => Err(FormatError::UnexpectedEof { + expected: self.base.saturating_add(rel).saturating_add(needed), + available: self.file_len, + }), + } + } +} + /// Up to `max` bytes from `offset` on: fewer only at the end of the /// storage. For structures whose size is only known once their prefix has /// been parsed and whose parsers bound-check what they are given. @@ -224,9 +265,7 @@ pub fn read_upto( let len = usize::try_from(avail).map_or(max, |a| a.min(max)); let bytes = file.read_at(offset, len)?; if bytes.len() < len { - return Err(FormatError::Storage( - "short read inside the file (the storage shrank or the backend failed)".into(), - )); + return Err(short_read()); } Ok(bytes) } -- 2.54.0 From 23a4784e72a30914bf49b74887ed5a7c568cf8f2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:56:07 -0500 Subject: [PATCH 12/48] format: ZFP decoder (filter 32013), pure Rust, read-only A port of the zfp 1.0.1 decoder and of H5Z-ZFP 1.1.1's decompression path behind a new `zfp` feature (in `plugin-filters`): every mode (fixed rate, precision, accuracy, reversible, expert), int32, int64, float and double, 1-4 dimensional fields with partial blocks, and headers written big-endian (values byte-swapped as H5Z-ZFP does). H5Z-ZFP keeps the zfp header in cd_values (version word, then the magic/metadata/mode bit stream); each chunk is the bare stream. The decoder reproduces libzfp bit for bit: integer arithmetic wraps as libzfp's, block exponents scale by exact powers of two, and the mode goes through zfp_stream_mode as H5Z-ZFP hands it to zfp. A stream that ends early is an error (libzfp reads past its buffer), as is a field whose size is not the chunk's; the output is allocated only once the stream holds a bit per block. Tests: header/mode unit tests, and a counting-allocator fuzz of random headers (expert parameters at their edges) and streams. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 5 +- crates/clawhdf5-format/src/filter_pipeline.rs | 3 +- crates/clawhdf5-format/src/filter_registry.rs | 8 +- crates/clawhdf5-format/src/filters.rs | 7 + crates/clawhdf5-format/src/filters_zfp.rs | 1083 +++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 2 + .../clawhdf5-format/tests/zfp_alloc_bounds.rs | 274 +++++ 7 files changed, 1377 insertions(+), 5 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_zfp.rs create mode 100644 crates/clawhdf5-format/tests/zfp_alloc_bounds.rs diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index c1ec6df..21bc626 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -78,8 +78,11 @@ bzip2 = ["dep:bzip2", "std"] blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] # Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above. blosc2 = ["blosc"] +# ZFP (32013, H5Z-ZFP), read-only: every mode, for int32, int64, float and +# double fields of 1 to 4 dimensions. +zfp = [] # Every plugin filter above. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2", "zfp"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/filter_pipeline.rs b/crates/clawhdf5-format/src/filter_pipeline.rs index 13ed69f..72f61bc 100644 --- a/crates/clawhdf5-format/src/filter_pipeline.rs +++ b/crates/clawhdf5-format/src/filter_pipeline.rs @@ -27,7 +27,8 @@ pub const FILTER_LZF: u16 = 32000; pub const FILTER_BLOSC: u16 = 32001; /// Bitshuffle, optionally with LZ4 or Zstandard (hdf5plugin's `Bitshuffle`). pub const FILTER_BITSHUFFLE: u16 = 32008; -/// ZFP lossy floating-point compression (hdf5plugin's `Zfp`). Not supported. +/// ZFP lossy (and lossless) compression of numeric arrays (H5Z-ZFP; +/// hdf5plugin's `Zfp`). Read-only, with the `zfp` feature. pub const FILTER_ZFP: u16 = 32013; /// Blosc 2 (hdf5plugin's `Blosc2`). pub const FILTER_BLOSC2: u16 = 32026; diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 442e27c..7670a8b 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -6,7 +6,7 @@ //! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip, //! N-Bit, scale-offset) and the plugin filters whose cargo features are //! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc, -//! blosc2). +//! blosc2, zfp). //! [`builtin_filters`] lists them. //! * **Registered filters** (`std` only) — codecs the application supplies //! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a @@ -162,7 +162,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> { 32001 => ("Blosc", Some("blosc")), 32004 => ("LZ4", Some("lz4")), 32008 => ("bitshuffle", Some("bitshuffle")), - 32013 => ("ZFP", None), + 32013 => ("ZFP", Some("zfp")), 32015 => ("Zstandard", Some("zstd")), 32019 => ("JPEG", None), 32022 => ("BitGroom", None), @@ -455,8 +455,10 @@ pub(crate) mod tests { let msg = FormatError::UnsupportedFilter(32026).to_string(); assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}"); let msg = FormatError::UnsupportedFilter(32013).to_string(); + assert!(msg.contains("ZFP") && msg.contains("`zfp`"), "{msg}"); + let msg = FormatError::UnsupportedFilter(32019).to_string(); assert!( - msg.contains("ZFP") && msg.contains("not implemented"), + msg.contains("JPEG") && msg.contains("not implemented"), "{msg}" ); let msg = FormatError::UnsupportedFilter(32000).to_string(); diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 40d6cd9..b261284 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -432,6 +432,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ decode: crate::filters_bitshuffle::bitshuffle_decode, encode: Some(crate::filters_bitshuffle::bitshuffle_encode), }, + #[cfg(feature = "zfp")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_ZFP, + name: "zfp", + decode: crate::filters_zfp::zfp_decode, + encode: None, + }, #[cfg(feature = "zstd")] BuiltinFilter { id: FILTER_ZSTD, diff --git a/crates/clawhdf5-format/src/filters_zfp.rs b/crates/clawhdf5-format/src/filters_zfp.rs new file mode 100644 index 0000000..7ac775c --- /dev/null +++ b/crates/clawhdf5-format/src/filters_zfp.rs @@ -0,0 +1,1083 @@ +//! ZFP (HDF5 filter 32013, `H5Z-ZFP`, hdf5plugin's `Zfp`) in pure Rust, +//! read-only: a port of the zfp 1.0.1 decoder (`src/zfp.c`, +//! `src/template/decode*.c`, `revdecode*.c`, `include/zfp/bitstream.inl`) +//! and of the decompression half of H5Z-ZFP 1.1.1's `H5Zzfp.c`. +//! +//! **The filter.** H5Z-ZFP keeps the zfp header in the filter's +//! `cd_values`, not in the chunks: `[0]` packs the zfp library version +//! (bits 16-31), the zfp codec version (bits 12-15; 0 before H5Z-ZFP 1.1.0, +//! when it is inferred from the library version) and the filter version +//! (bits 0-11); `[1..]` (at most 6 words) are a zfp bit stream holding the +//! full header written by `zfp_write_header`. Each chunk is the bare +//! compressed stream. The words are read as little-endian bytes; if the +//! magic is not there they are read byte-swapped (a big-endian writer), and +//! then the decoded values are byte-swapped too, as H5Z-ZFP does, because +//! the dataset's datatype is big-endian. +//! +//! **The header** (LSB-first bit stream; H5Z-ZFP requires zfp built with +//! 8-bit stream words, so the stream is plain bytes): 32 bits of magic +//! (`'z' 'f' 'p'` and codec version 5); 52 bits of field metadata (scalar +//! type: int32, int64, float, double; dimensionality 1-4; the sizes, 48 +//! bits shared between the dimensions, the fastest-varying first); and a +//! 12-bit mode, or 64 bits when the 12 are all ones: fixed rate (maxbits +//! per block), fixed precision (bit planes), fixed accuracy (the smallest +//! bit plane kept), reversible (lossless), or the four expert parameters +//! (minbits, maxbits, maxprec, minexp). +//! +//! **The stream.** The field is cut into blocks of 4^d values (partial at +//! the array's edges; the decoder decodes whole blocks and keeps the part +//! inside the array), decoded one after another. A floating-point block is +//! a "nonzero" bit, the block's common exponent (8 or 11 bits) and a block +//! of integers; an integer block of the array is that block of integers +//! directly. Integers are coded as negabinary bit planes, most significant +//! first, each plane as the bits of the values already significant plus a +//! unary run-length ("group test") code for the rest, within the budget of +//! maxbits and maxprec bits (the precision of a float block also depends on +//! its exponent and minexp). The coefficients are stored in order of +//! sequency, then run through the inverse decorrelating transform (a +//! lifting scheme along each dimension) and, for floats, scaled by the +//! block exponent. Blocks that used fewer than minbits bits are padded. +//! Reversible mode stores the precision in the block, uses an integer +//! Lorenzo transform, and stores float blocks either as integers scaled by +//! a common exponent or as their bit patterns. +//! +//! The decoder is deterministic and reproduces libzfp's output bit for bit +//! (libzfp built with its default `ZFP_ROUND_NEVER` rounding, as hdf5plugin +//! builds it): integer arithmetic wraps as libzfp's does, and floats are +//! scaled by an exact power of two. A stream that ends before the decoder +//! is done is an error (libzfp reads past the buffer). + +#[cfg(not(feature = "std"))] +use alloc::{format, vec, vec::Vec}; + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("zfp: {msg}")) +} + +const ZFP_MIN_BITS: u32 = 1; +const ZFP_MAX_BITS: u32 = 16658; +const ZFP_MAX_PREC: u32 = 64; +const ZFP_MIN_EXP: i32 = -1074; +const ZFP_META_BITS: u32 = 52; +const ZFP_MODE_SHORT_BITS: u32 = 12; +const ZFP_MODE_LONG_BITS: u32 = 64; +const ZFP_MODE_SHORT_MAX: u64 = (1 << ZFP_MODE_SHORT_BITS) - 2; +/// The zfp codec this decoder implements (zfp 0.5.0 to 1.0.1). +const ZFP_CODEC: u64 = 5; +/// `H5Z_ZFP_CD_NELMTS_MAX`: header words after the version word. +const H5Z_ZFP_CD_NELMTS_MAX: usize = 6; + +/// An LSB-first bit stream over bytes (zfp's `bitstream` with 8-bit words). +struct Bits<'a> { + data: &'a [u8], + pos: u64, +} + +fn truncated() -> FormatError { + err("compressed stream ends early") +} + +impl<'a> Bits<'a> { + fn new(data: &'a [u8]) -> Self { + Bits { data, pos: 0 } + } + + #[inline] + fn bit(&mut self) -> Result { + let byte = *self + .data + .get((self.pos >> 3) as usize) + .ok_or_else(truncated)?; + let v = (byte >> (self.pos & 7)) & 1; + self.pos += 1; + Ok(v as u32) + } + + /// The next `n` (0..=64) bits, the first in the least significant place. + fn bits(&mut self, n: u32) -> Result { + if n == 0 { + return Ok(0); + } + if self.pos + n as u64 > self.data.len() as u64 * 8 { + return Err(truncated()); + } + let mut v = 0u64; + let mut got = 0u32; + while got < n { + let byte = self.data[(self.pos >> 3) as usize] as u64; + let off = (self.pos & 7) as u32; + let take = (8 - off).min(n - got); + v |= ((byte >> off) & ((1u64 << take) - 1)) << got; + got += take; + self.pos += take as u64; + } + Ok(v) + } + + /// Skip `n` bits. Nothing is read, so skipping past the end is not an + /// error until a bit there is read. + fn skip(&mut self, n: u32) { + self.pos += n as u64; + } + + fn tell(&self) -> u64 { + self.pos + } +} + +/// A zfp stream's four parameters (`zfp_stream` minus the bit stream). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Params { + minbits: u32, + maxbits: u32, + maxprec: u32, + minexp: i32, +} + +impl Params { + fn reversible(&self) -> bool { + self.minexp < ZFP_MIN_EXP + } + + /// `zfp_stream_set_params`. + fn new(minbits: u32, maxbits: u32, maxprec: u32, minexp: i32) -> Option { + if minbits > maxbits || !(0 < maxprec && maxprec <= 64) { + return None; + } + Some(Params { + minbits, + maxbits, + maxprec, + minexp, + }) + } + + /// `zfp_stream_set_mode`. + fn from_mode(mut mode: u64) -> Option { + let (minbits, maxbits, maxprec, minexp); + if mode <= ZFP_MODE_SHORT_MAX { + if mode < 2048 { + minbits = mode as u32 + 1; + maxbits = minbits; + maxprec = ZFP_MAX_PREC; + minexp = ZFP_MIN_EXP; + } else if mode < 2048 + 128 { + minbits = ZFP_MIN_BITS; + maxbits = ZFP_MAX_BITS; + maxprec = mode as u32 + 1 - 2048; + minexp = ZFP_MIN_EXP; + } else if mode == 2048 + 128 { + minbits = ZFP_MIN_BITS; + maxbits = ZFP_MAX_BITS; + maxprec = ZFP_MAX_PREC; + minexp = ZFP_MIN_EXP - 1; + } else { + minbits = ZFP_MIN_BITS; + maxbits = ZFP_MAX_BITS; + maxprec = ZFP_MAX_PREC; + minexp = mode as i32 + ZFP_MIN_EXP - (2048 + 128 + 1); + } + } else { + mode >>= 12; + minbits = (mode & 0x7fff) as u32 + 1; + mode >>= 15; + maxbits = (mode & 0x7fff) as u32 + 1; + mode >>= 15; + maxprec = (mode & 0x7f) as u32 + 1; + mode >>= 7; + minexp = (mode & 0x7fff) as i32 - 16495; + } + Params::new(minbits, maxbits, maxprec, minexp) + } + + /// `zfp_stream_compression_mode`: 1 expert, 2 fixed rate, 3 fixed + /// precision, 4 fixed accuracy, 5 reversible. + fn compression_mode(&self) -> u32 { + let p = self; + if p.minbits == ZFP_MIN_BITS + && p.maxbits == ZFP_MAX_BITS + && p.maxprec == ZFP_MAX_PREC + && p.minexp == ZFP_MIN_EXP + { + return 1; + } + if p.minbits == p.maxbits + && 1 <= p.maxbits + && p.maxbits <= ZFP_MAX_BITS + && p.maxprec >= ZFP_MAX_PREC + && p.minexp == ZFP_MIN_EXP + { + return 2; + } + if p.minbits <= ZFP_MIN_BITS + && p.maxbits >= ZFP_MAX_BITS + && p.maxprec >= 1 + && p.minexp == ZFP_MIN_EXP + { + return 3; + } + if p.minbits <= ZFP_MIN_BITS + && p.maxbits >= ZFP_MAX_BITS + && p.maxprec >= ZFP_MAX_PREC + && p.minexp >= ZFP_MIN_EXP + { + return 4; + } + if p.minbits <= ZFP_MIN_BITS + && p.maxbits >= ZFP_MAX_BITS + && p.maxprec >= ZFP_MAX_PREC + && p.minexp < ZFP_MIN_EXP + { + return 5; + } + 1 + } + + /// `zfp_stream_mode`: the mode word these parameters are written as. + fn mode(&self) -> u64 { + let p = self; + match p.compression_mode() { + 2 if p.maxbits <= 2048 => return (p.maxbits - 1) as u64, + 3 if p.maxprec <= 128 => return (p.maxprec - 1 + 2048) as u64, + 4 if p.minexp <= 843 => return (p.minexp - ZFP_MIN_EXP) as u64 + (2048 + 128 + 1), + 5 => return 2048 + 128, + _ => {} + } + let minbits = p.minbits.clamp(1, 0x8000) - 1; + let maxbits = p.maxbits.clamp(1, 0x8000) - 1; + let maxprec = p.maxprec.clamp(1, 0x80) - 1; + let minexp = (p.minexp.saturating_add(16495)).clamp(0, 0x7fff) as u64; + let mut mode = minexp; + mode = (mode << 7) + maxprec as u64; + mode = (mode << 15) + maxbits as u64; + mode = (mode << 15) + minbits as u64; + (mode << 12) + 0xfff + } +} + +/// Scalar types (`zfp_type`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ZType { + Int32, + Int64, + Float, + Double, +} + +impl ZType { + fn size(self) -> usize { + match self { + ZType::Int32 | ZType::Float => 4, + ZType::Int64 | ZType::Double => 8, + } + } +} + +/// A field's type and sizes (`zfp_field` without the data), from the +/// header's 52-bit metadata (`zfp_field_set_metadata`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Field { + ztype: ZType, + dims: usize, + /// nx (fastest), ny, nz, nw; 1 for the unused ones. + n: [usize; 4], +} + +impl Field { + fn from_meta(mut meta: u64) -> Field { + let ztype = match meta & 3 { + 0 => ZType::Int32, + 1 => ZType::Int64, + 2 => ZType::Float, + _ => ZType::Double, + }; + meta >>= 2; + let dims = (meta & 3) as usize + 1; + meta >>= 2; + let mut n = [1usize; 4]; + match dims { + // zfp limits 1-D sizes to 32 bits (of the 48 in the header). + 1 => n[0] = (meta & 0xffff_ffff) as usize + 1, + 2 => { + for v in n.iter_mut().take(2) { + *v = (meta & 0xff_ffff) as usize + 1; + meta >>= 24; + } + } + 3 => { + for v in n.iter_mut().take(3) { + *v = (meta & 0xffff) as usize + 1; + meta >>= 16; + } + } + _ => { + for v in n.iter_mut() { + *v = (meta & 0xfff) as usize + 1; + meta >>= 12; + } + } + } + Field { ztype, dims, n } + } + + fn values(&self) -> Option { + self.n.iter().try_fold(1usize, |a, &v| a.checked_mul(v)) + } + + fn blocks(&self) -> Option { + self.n + .iter() + .try_fold(1usize, |a, &v| a.checked_mul(v.div_ceil(4))) + } +} + +/// What the filter's `cd_values` say: the stream parameters, the field, +/// and whether the header was written big-endian. +#[derive(Debug)] +struct Header { + params: Params, + field: Field, + swap: bool, +} + +/// `zfp_read_header` with `ZFP_HEADER_MAGIC`. +fn read_magic(b: &mut Bits<'_>) -> Result { + Ok(b.bits(8)? == b'z' as u64 + && b.bits(8)? == b'f' as u64 + && b.bits(8)? == b'p' as u64 + && b.bits(8)? == ZFP_CODEC) +} + +/// `get_zfp_info_from_cd_values` and `zfp_codec_version_mismatch`. +fn parse_cd_values(cd: &[u32]) -> Result { + let (&version, words) = cd + .split_first() + .ok_or_else(|| err("no filter parameters"))?; + if words.len() > H5Z_ZFP_CD_NELMTS_MAX { + return Err(err("too many filter parameters")); + } + let bytes = |swap: bool| -> Vec { + words + .iter() + .flat_map(|w| { + if swap { + w.to_be_bytes() + } else { + w.to_le_bytes() + } + }) + .collect() + }; + let mut swap = false; + let mut hdr = bytes(false); + if !read_magic(&mut Bits::new(&hdr)).unwrap_or(false) { + swap = true; + hdr = bytes(true); + if !read_magic(&mut Bits::new(&hdr)).unwrap_or(false) { + return Err(err( + "no zfp header in the filter parameters, or a zfp codec other than 5", + )); + } + } + let mut b = Bits::new(&hdr); + let bad = |_| err("truncated zfp header in the filter parameters"); + b.skip(32); + let meta = b.bits(ZFP_META_BITS).map_err(bad)?; + let field = Field::from_meta(meta); + let mut mode = b.bits(ZFP_MODE_SHORT_BITS).map_err(bad)?; + if mode > ZFP_MODE_SHORT_MAX { + let rest = b + .bits(ZFP_MODE_LONG_BITS - ZFP_MODE_SHORT_BITS) + .map_err(bad)?; + mode += rest << ZFP_MODE_SHORT_BITS; + } + let params = Params::from_mode(mode).ok_or_else(|| err("invalid compression mode"))?; + // H5Z-ZFP hands the decoder `zfp_stream_mode()` of the header's + // parameters, which maps some expert settings to a standard mode's. + let params = Params::from_mode(params.mode()).ok_or_else(|| err("invalid compression mode"))?; + + // Data written by a newer codec than this decoder's. + let h5z_version = version & 0xfff; + let codec = (version >> 12) & 0xf; + let writer_codec = if h5z_version < 0x110 { + let v = (version >> 16) << 4; + if v < 0x0500 { + 4 + } else if v < 0x1000 { + (v & 0x0f00) >> 8 + } else { + 5 + } + } else { + codec + }; + if writer_codec as u64 > ZFP_CODEC { + return Err(err("data written by a newer zfp codec")); + } + Ok(Header { + params, + field, + swap, + }) +} + +/// Coefficient order by sequency (zfp's `perm_1` .. `perm_4`). +const PERM_1: [u8; 4] = [0, 1, 2, 3]; +const PERM_2: [u8; 16] = [0, 1, 4, 5, 2, 8, 6, 9, 3, 12, 10, 7, 13, 11, 14, 15]; +const PERM_3: [u8; 64] = [ + 0, 1, 4, 16, 20, 17, 5, 2, 8, 32, 21, 6, 18, 24, 9, 33, 36, 3, 12, 48, 22, 25, 37, 40, 34, 10, + 7, 19, 28, 13, 49, 52, 41, 38, 26, 23, 29, 53, 11, 35, 44, 14, 50, 56, 42, 27, 39, 45, 30, 54, + 57, 60, 51, 15, 43, 46, 58, 61, 55, 31, 62, 59, 47, 63, +]; +const PERM_4: [u8; 256] = [ + 0, 1, 4, 16, 64, 5, 80, 17, 68, 65, 20, 2, 8, 32, 128, 84, 81, 69, 21, 6, 18, 66, 24, 72, 9, + 96, 33, 36, 129, 132, 144, 3, 12, 48, 192, 85, 82, 70, 22, 73, 25, 88, 37, 100, 97, 148, 145, + 133, 10, 160, 34, 136, 130, 40, 7, 19, 67, 28, 76, 13, 112, 49, 52, 193, 196, 208, 86, 89, 101, + 149, 161, 137, 41, 134, 38, 164, 26, 152, 146, 104, 98, 74, 83, 71, 23, 77, 29, 92, 53, 116, + 113, 212, 209, 197, 11, 35, 131, 44, 140, 14, 176, 50, 56, 194, 200, 224, 90, 165, 102, 153, + 150, 105, 168, 162, 138, 42, 87, 93, 117, 213, 27, 75, 99, 39, 135, 147, 108, 45, 141, 156, 30, + 78, 177, 180, 54, 114, 120, 57, 198, 210, 216, 201, 225, 228, 15, 240, 51, 204, 195, 60, 169, + 166, 154, 106, 91, 103, 151, 109, 157, 94, 181, 118, 121, 214, 217, 229, 163, 139, 43, 142, 46, + 172, 58, 184, 178, 232, 226, 202, 241, 205, 61, 199, 55, 244, 31, 220, 211, 124, 115, 79, 170, + 167, 155, 107, 158, 110, 173, 122, 185, 182, 233, 230, 218, 95, 245, 119, 221, 215, 125, 242, + 206, 62, 203, 59, 248, 47, 236, 227, 188, 179, 143, 171, 174, 186, 234, 246, 222, 126, 219, + 123, 249, 111, 237, 231, 189, 183, 159, 252, 243, 207, 63, 175, 250, 187, 238, 235, 190, 253, + 247, 223, 127, 254, 251, 239, 191, 255, +]; + +fn perm(dims: usize) -> &'static [u8] { + match dims { + 1 => &PERM_1, + 2 => &PERM_2, + 3 => &PERM_3, + _ => &PERM_4, + } +} + +/// `with_maxbits`: whether the block's bit budget may run out before its +/// precision does. +fn with_maxbits(maxbits: u32, maxprec: u32, size: u32) -> bool { + ((maxprec + 1) * size).wrapping_sub(1) > maxbits +} + +/// Apply `lift` to every line of a block of `dims` dimensions, one axis at +/// a time from the slowest to the fastest, as zfp's `inv_xform` does. +fn xform(p: &mut [T], dims: usize, lift: impl Fn(&mut [T], usize, usize)) { + let size = 1usize << (2 * dims); + for axis in (0..dims).rev() { + let s = 1usize << (2 * axis); + for i in 0..size { + if (i / s).is_multiple_of(4) { + lift(p, i, s); + } + } + } +} + +/// The integer codec (`decode.c`, `revdecode.c`) for 32- or 64-bit +/// integers. +macro_rules! int_codec { + ($m:ident, $int:ty, $uint:ty, $nbmask:expr, $pbits:expr) => { + mod $m { + use super::{Bits, FormatError, perm, with_maxbits, xform}; + + pub(super) type Int = $int; + type UInt = $uint; + const INTPREC: u32 = <$uint>::BITS; + const NBMASK: UInt = $nbmask; + const PBITS: u32 = $pbits; + + fn uint2int(x: UInt) -> Int { + (x ^ NBMASK).wrapping_sub(NBMASK) as Int + } + + fn inv_lift(p: &mut [Int], at: usize, s: usize) { + let (mut x, mut y, mut z, mut w) = (p[at], p[at + s], p[at + 2 * s], p[at + 3 * s]); + y = y.wrapping_add(w >> 1); + w = w.wrapping_sub(y >> 1); + y = y.wrapping_add(w); + w = w.wrapping_shl(1); + w = w.wrapping_sub(y); + z = z.wrapping_add(x); + x = x.wrapping_shl(1); + x = x.wrapping_sub(z); + y = y.wrapping_add(z); + z = z.wrapping_shl(1); + z = z.wrapping_sub(y); + w = w.wrapping_add(x); + x = x.wrapping_shl(1); + x = x.wrapping_sub(w); + p[at] = x; + p[at + s] = y; + p[at + 2 * s] = z; + p[at + 3 * s] = w; + } + + fn rev_inv_lift(p: &mut [Int], at: usize, s: usize) { + let (x, mut y, mut z, mut w) = (p[at], p[at + s], p[at + 2 * s], p[at + 3 * s]); + w = w.wrapping_add(z); + z = z.wrapping_add(y); + w = w.wrapping_add(z); + y = y.wrapping_add(x); + z = z.wrapping_add(y); + w = w.wrapping_add(z); + p[at + s] = y; + p[at + 2 * s] = z; + p[at + 3 * s] = w; + } + + fn kmin(maxprec: u32) -> u32 { + INTPREC.saturating_sub(maxprec) + } + + /// `decode_few_ints` (size <= 64) and `decode_many_ints`: bit + /// planes within a budget of `maxbits` bits. + fn decode_ints_rate( + s: &mut Bits<'_>, + maxbits: u32, + maxprec: u32, + data: &mut [UInt], + ) -> Result { + let size = data.len(); + let kmin = kmin(maxprec); + let mut bits = maxbits; + data.fill(0); + let mut k = INTPREC; + let mut n = 0usize; + while bits != 0 && k > kmin { + k -= 1; + // step 1: the first n bits of bit plane k + let m = n.min(bits as usize); + bits -= m as u32; + if size <= 64 { + let mut x = s.bits(m as u32)?; + // step 2: unary run-length decode the rest + while bits != 0 && n < size { + bits -= 1; + if s.bit()? != 0 { + while bits != 0 && n < size - 1 { + bits -= 1; + if s.bit()? != 0 { + break; + } + n += 1; + } + x += 1u64 << n; + } else { + break; + } + n += 1; + } + // step 3: deposit the bit plane + let mut i = 0; + while x != 0 { + data[i] += ((x & 1) as UInt) << k; + x >>= 1; + i += 1; + } + } else { + for v in data.iter_mut().take(m) { + if s.bit()? != 0 { + *v += (1 as UInt) << k; + } + } + while bits != 0 && n < size { + bits -= 1; + if s.bit()? != 0 { + while bits != 0 && n < size - 1 { + bits -= 1; + if s.bit()? != 0 { + break; + } + n += 1; + } + data[n] += (1 as UInt) << k; + } else { + break; + } + n += 1; + } + } + } + Ok(maxbits - bits) + } + + /// `decode_few_ints_prec` and `decode_many_ints_prec`: whole bit + /// planes, no bit budget. + fn decode_ints_prec( + s: &mut Bits<'_>, + maxprec: u32, + data: &mut [UInt], + ) -> Result { + let size = data.len(); + let start = s.tell(); + let kmin = kmin(maxprec); + data.fill(0); + let mut k = INTPREC; + let mut n = 0usize; + while k > kmin { + k -= 1; + if size <= 64 { + let mut x = s.bits(n as u32)?; + while n < size && s.bit()? != 0 { + while n < size - 1 && s.bit()? == 0 { + n += 1; + } + x += 1u64 << n; + n += 1; + } + let mut i = 0; + while x != 0 { + data[i] += ((x & 1) as UInt) << k; + x >>= 1; + i += 1; + } + } else { + for v in data.iter_mut().take(n) { + if s.bit()? != 0 { + *v += (1 as UInt) << k; + } + } + while n < size && s.bit()? != 0 { + while n < size - 1 && s.bit()? == 0 { + n += 1; + } + data[n] += (1 as UInt) << k; + n += 1; + } + } + } + Ok((s.tell() - start) as u32) + } + + /// `decode_ints`. + fn decode_ints( + s: &mut Bits<'_>, + maxbits: u32, + maxprec: u32, + data: &mut [UInt], + ) -> Result { + if with_maxbits(maxbits, maxprec, data.len() as u32) { + decode_ints_rate(s, maxbits, maxprec, data) + } else { + decode_ints_prec(s, maxprec, data) + } + } + + /// Undo the coefficient order and the negabinary mapping. + fn inv_order(ublock: &[UInt], iblock: &mut [Int], dims: usize) { + for (&u, &p) in ublock.iter().zip(perm(dims)) { + iblock[p as usize] = uint2int(u); + } + } + + /// `decode_block` for integers: a block of `4^dims` values in + /// `iblock`. + pub(super) fn decode_block( + s: &mut Bits<'_>, + dims: usize, + minbits: u32, + maxbits: u32, + maxprec: u32, + iblock: &mut [Int], + ) -> Result { + let mut ublock = [0 as UInt; 256]; + let ublock = &mut ublock[..iblock.len()]; + let mut bits = decode_ints(s, maxbits, maxprec, ublock)?; + if bits < minbits { + s.skip(minbits - bits); + bits = minbits; + } + inv_order(ublock, iblock, dims); + xform(iblock, dims, inv_lift); + Ok(bits) + } + + /// `rev_decode_block` for integers. + pub(super) fn rev_decode_block( + s: &mut Bits<'_>, + dims: usize, + minbits: u32, + maxbits: u32, + iblock: &mut [Int], + ) -> Result { + let mut bits = PBITS; + let prec = s.bits(PBITS)? as u32 + 1; + let mut ublock = [0 as UInt; 256]; + let ublock = &mut ublock[..iblock.len()]; + bits += decode_ints(s, maxbits.wrapping_sub(bits), prec, ublock)?; + if bits < minbits { + s.skip(minbits - bits); + bits = minbits; + } + inv_order(ublock, iblock, dims); + xform(iblock, dims, rev_inv_lift); + Ok(bits) + } + } + }; +} + +int_codec!(int32, i32, u32, 0xaaaa_aaaa, 5); +int_codec!(int64, i64, u64, 0xaaaa_aaaa_aaaa_aaaa, 6); + +/// `ldexp(1, e)` for `f32`, exactly: 0 below the smallest subnormal (the +/// exact value, at most half of it, rounds to 0). +fn pow2_f32(e: i32) -> f32 { + if e >= -126 { + f32::from_bits(((e.min(128) + 127) as u32) << 23) + } else if e >= -149 { + f32::from_bits(1 << (e + 149)) + } else { + 0.0 + } +} + +/// `ldexp(1, e)` for `f64`, exactly, as [`pow2_f32`]. +fn pow2_f64(e: i32) -> f64 { + if e >= -1022 { + f64::from_bits(((e.min(1024) + 1023) as u64) << 52) + } else if e >= -1074 { + f64::from_bits(1 << (e + 1074)) + } else { + 0.0 + } +} + +/// The floating-point codec (`decodef.c`, `revdecodef.c`). +macro_rules! float_codec { + ($m:ident, $f:ty, $ints:ident, $ebits:expr, $tcmask:expr, $pow2:ident) => { + mod $m { + use super::{Bits, FormatError, Params, $ints, $pow2}; + use $ints::Int; + + const EBITS: u32 = $ebits; + const EBIAS: i32 = (1 << (EBITS - 1)) - 1; + const INTBITS: i32 = (core::mem::size_of::<$f>() * 8) as i32; + + /// `precision`: the bit planes to decode for a block whose + /// largest exponent is `maxexp`. + fn precision(maxexp: i32, maxprec: u32, minexp: i32, dims: usize) -> u32 { + let p = maxexp as i64 - minexp as i64 + 2 * dims as i64 + 2; + maxprec.min(p.max(0).min(u32::MAX as i64) as u32) + } + + /// `inv_cast`: scale the block's integers by its exponent. + fn inv_cast(iblock: &[Int], fblock: &mut [$f], emax: i32) { + let s = $pow2(emax - (INTBITS - 2)); + for (f, &i) in fblock.iter_mut().zip(iblock) { + *f = s * i as $f; + } + } + + fn decode_lossy( + s: &mut Bits<'_>, + p: &Params, + dims: usize, + fblock: &mut [$f], + ) -> Result<(), FormatError> { + let mut bits = 1u32; + if s.bit()? != 0 { + let mut iblock = [0 as Int; 256]; + let iblock = &mut iblock[..fblock.len()]; + bits += EBITS; + let emax = s.bits(EBITS)? as i32 - EBIAS; + let maxprec = precision(emax, p.maxprec, p.minexp, dims); + $ints::decode_block( + s, + dims, + p.minbits - bits.min(p.minbits), + p.maxbits.wrapping_sub(bits), + maxprec, + iblock, + )?; + inv_cast(iblock, fblock, emax); + } else { + fblock.fill(0.0); + if p.minbits > bits { + s.skip(p.minbits - bits); + } + } + Ok(()) + } + + fn decode_reversible( + s: &mut Bits<'_>, + p: &Params, + dims: usize, + fblock: &mut [$f], + ) -> Result<(), FormatError> { + let mut bits = 1u32; + if s.bit()? != 0 { + let mut iblock = [0 as Int; 256]; + let iblock = &mut iblock[..fblock.len()]; + bits += 1; + if s.bit()? != 0 { + // the values' bit patterns, as sign-magnitude + // integers stored in two's complement + $ints::rev_decode_block( + s, + dims, + p.minbits - bits.min(p.minbits), + p.maxbits.wrapping_sub(bits), + iblock, + )?; + for (f, &i) in fblock.iter_mut().zip(iblock.iter()) { + let i = if i < 0 { i ^ $tcmask } else { i }; + *f = <$f>::from_bits(i as _); + } + } else { + bits += EBITS; + let emax = s.bits(EBITS)? as i32 - EBIAS; + $ints::rev_decode_block( + s, + dims, + p.minbits - bits.min(p.minbits), + p.maxbits.wrapping_sub(bits), + iblock, + )?; + if emax != -EBIAS { + inv_cast(iblock, fblock, emax); + } else { + fblock.fill(0.0); + } + } + } else { + fblock.fill(0.0); + if p.minbits > bits { + s.skip(p.minbits - bits); + } + } + Ok(()) + } + + /// `zfp_decode_block` for floats: `4^dims` values. + pub(super) fn decode_block( + s: &mut Bits<'_>, + p: &Params, + dims: usize, + fblock: &mut [$f], + ) -> Result<(), FormatError> { + if p.reversible() { + decode_reversible(s, p, dims, fblock) + } else { + decode_lossy(s, p, dims, fblock) + } + } + } + }; +} + +float_codec!(float32, f32, int32, 8, 0x7fff_ffff, pow2_f32); +float_codec!(float64, f64, int64, 11, 0x7fff_ffff_ffff_ffff, pow2_f64); + +/// A decoded scalar, stored into the output in little- or big-endian order. +trait Scalar: Copy + Default { + const SIZE: usize; + fn store(self, out: &mut [u8], big_endian: bool); +} + +macro_rules! scalar { + ($t:ty) => { + impl Scalar for $t { + const SIZE: usize = core::mem::size_of::<$t>(); + fn store(self, out: &mut [u8], big_endian: bool) { + out.copy_from_slice(&if big_endian { + self.to_be_bytes() + } else { + self.to_le_bytes() + }); + } + } + }; +} +scalar!(i32); +scalar!(i64); +scalar!(f32); +scalar!(f64); + +/// `zfp_decompress`: decode the field's blocks in order (x fastest) and +/// keep the part of each inside the array. +fn decompress( + s: &mut Bits<'_>, + field: &Field, + out: &mut [u8], + big_endian: bool, + mut decode: impl FnMut(&mut Bits<'_>, &mut [T]) -> Result<(), FormatError>, +) -> Result<(), FormatError> { + let [nx, ny, nz, nw] = field.n; + let bsize = 1usize << (2 * field.dims); + let mut block = [T::default(); 256]; + let block = &mut block[..bsize]; + let ext = |n: usize, at: usize, d: usize| { + if d < field.dims { (n - at).min(4) } else { 1 } + }; + for w in (0..nw).step_by(4) { + for z in (0..nz).step_by(4) { + for y in (0..ny).step_by(4) { + for x in (0..nx).step_by(4) { + decode(s, block)?; + let (bx, by, bz, bw) = + (ext(nx, x, 0), ext(ny, y, 1), ext(nz, z, 2), ext(nw, w, 3)); + for j in 0..bw { + for k in 0..bz { + for i in 0..by { + let row = (((w + j) * nz + z + k) * ny + y + i) * nx + x; + let q = 64 * j + 16 * k + 4 * i; + for (h, &v) in block[q..q + bx].iter().enumerate() { + let at = (row + h) * T::SIZE; + v.store(&mut out[at..at + T::SIZE], big_endian); + } + } + } + } + } + } + } + } + Ok(()) +} + +/// Decode one chunk: the H5Z-ZFP filter's decompression direction. +pub(crate) fn zfp_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + zfp_decompress(input, ctx.client_data(), ctx.max_output) +} + +/// Decode `input`, one chunk compressed by H5Z-ZFP with the filter +/// parameters `cd`. `max_output` is the chunk's size in bytes (0 when +/// unknown, in which case the ceiling is 256 MiB); a field of another size +/// is an error. +pub fn zfp_decompress(input: &[u8], cd: &[u32], max_output: usize) -> Result, FormatError> { + let h = parse_cd_values(cd)?; + let field = &h.field; + let size = field + .values() + .and_then(|n| n.checked_mul(field.ztype.size())) + .ok_or_else(|| err("field too large"))?; + if max_output != 0 && size != max_output { + return Err(err(&format!( + "the filter parameters describe {size} bytes, the chunk holds {max_output}" + ))); + } + if size > crate::filters::MAX_DECOMPRESS_SIZE.max(max_output) { + return Err(err("field too large")); + } + // Every block takes at least one bit of the stream. + let blocks = field.blocks().ok_or_else(|| err("field too large"))?; + if blocks as u64 > input.len() as u64 * 8 { + return Err(truncated()); + } + let mut out = vec![0u8; size]; + let mut s = Bits::new(input); + let p = h.params; + let dims = field.dims; + let be = h.swap; + match field.ztype { + ZType::Float => decompress::(&mut s, field, &mut out, be, |s, b| { + float32::decode_block(s, &p, dims, b) + })?, + ZType::Double => decompress::(&mut s, field, &mut out, be, |s, b| { + float64::decode_block(s, &p, dims, b) + })?, + ZType::Int32 => decompress::(&mut s, field, &mut out, be, |s, b| { + if p.reversible() { + int32::rev_decode_block(s, dims, p.minbits, p.maxbits, b) + } else { + int32::decode_block(s, dims, p.minbits, p.maxbits, p.maxprec, b) + } + .map(|_| ()) + })?, + ZType::Int64 => decompress::(&mut s, field, &mut out, be, |s, b| { + if p.reversible() { + int64::rev_decode_block(s, dims, p.minbits, p.maxbits, b) + } else { + int64::decode_block(s, dims, p.minbits, p.maxbits, p.maxprec, b) + } + .map(|_| ()) + })?, + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn modes_round_trip() { + // Fixed rate 16 bits/value in 2-D: 256 bits per block. + let p = Params::from_mode(255).unwrap(); + assert_eq!((p.minbits, p.maxbits), (256, 256)); + assert_eq!(p.mode(), 255); + // Reversible. + let p = Params::from_mode(2176).unwrap(); + assert!(p.reversible()); + assert_eq!(p.mode(), 2176); + // Fixed accuracy 2^-10. + let p = Params::from_mode((-10 - ZFP_MIN_EXP) as u64 + 2177).unwrap(); + assert_eq!(p.minexp, -10); + // An expert maxbits above ZFP_MAX_BITS is fixed precision to + // H5Z-ZFP, which hands zfp the standard mode's parameters. + let long = Params::new(1, 20000, 20, ZFP_MIN_EXP).unwrap().mode(); + assert_eq!(long, 2048 + 19); + let p = Params { + minbits: 3, + maxbits: 20000, + maxprec: 20, + minexp: -3, + }; + assert_eq!(Params::from_mode(p.mode()), Some(p)); + } + + #[test] + fn pow2_is_exact() { + // 2^e by repeated doubling or halving, each step exact down to the + // smallest subnormal (the next halving rounds to 0, as ldexp does). + let pow2 = |e: i32| { + let mut v = 1f64; + for _ in 0..e.unsigned_abs() { + v = if e < 0 { v / 2.0 } else { v * 2.0 }; + } + v + }; + for e in -160..128 { + let want = if e < -149 { 0.0 } else { pow2(e) }; + assert_eq!(pow2_f32(e) as f64, want, "{e}"); + } + for e in -1080..1000 { + assert_eq!(pow2_f64(e), pow2(e), "{e}"); + } + } + + /// The filter parameters of the conformance corpus's `h5ex_d_zfp.h5` + /// (H5Z-ZFP 1.0.1, zfp 0.5.5; no codec version in `cd_values[0]`): a + /// 2-D float field of 4x8 chunks. + #[test] + fn corpus_header_parses() { + let cd = [ + 5570817u32, 91252346, 805306486, 4293918720, 3767009280, 493487, + ]; + let h = parse_cd_values(&cd).unwrap(); + assert!(!h.swap); + assert_eq!(h.field.ztype, ZType::Float); + assert_eq!(h.field.dims, 2); + assert_eq!(h.field.n[..2], [8, 4]); + } + + #[test] + fn hostile_headers_are_errors() { + assert!(parse_cd_values(&[]).is_err()); + assert!(parse_cd_values(&[0]).is_err()); + assert!(parse_cd_values(&[0, 0x0570_667a]).is_err()); + assert!(parse_cd_values(&[0; 8]).is_err()); + // A newer codec. + let mut cd = [ + 5570817u32, 91252346, 805306486, 4293918720, 3767009280, 493487, + ]; + cd[0] = (0x1100 << 16) | (6 << 12) | 0x111; + assert!(parse_cd_values(&cd).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index a197c9f..7042a13 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -94,6 +94,8 @@ mod filters_bzip2; #[cfg(feature = "lzf")] pub mod filters_lzf; mod filters_szip; +#[cfg(feature = "zfp")] +pub mod filters_zfp; pub mod fixed_array; pub mod float16; pub mod fractal_heap; diff --git a/crates/clawhdf5-format/tests/zfp_alloc_bounds.rs b/crates/clawhdf5-format/tests/zfp_alloc_bounds.rs new file mode 100644 index 0000000..90119a6 --- /dev/null +++ b/crates/clawhdf5-format/tests/zfp_alloc_bounds.rs @@ -0,0 +1,274 @@ +//! Crafted ZFP filter parameters and streams cannot make the decoder panic +//! or allocate out of proportion to the chunk it decodes. +//! +//! The field size comes from the filter's `cd_values` (up to 2^48 values), +//! not from the chunk: the decoder allocates the output only when it matches +//! the chunk's size (or, when that is unknown, is within the 256 MiB +//! ceiling), and only when the stream is long enough to hold a bit per +//! block. Peak heap use is measured with a counting global allocator; the +//! tests share it, so each holds `SERIAL` for its whole run. +#![cfg(feature = "zfp")] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use clawhdf5_format::filters_zfp::zfp_decompress; + +struct Counting; + +static CURRENT: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +static SERIAL: Mutex<()> = Mutex::new(()); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + CURRENT.fetch_sub(layout.size(), Ordering::Relaxed); + } +} + +#[global_allocator] +static ALLOC: Counting = Counting; + +/// Bytes allocated at the peak of `f`, above what was live when it started. +fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { + let base = CURRENT.load(Ordering::Relaxed); + PEAK.store(base, Ordering::Relaxed); + let out = f(); + (out, PEAK.load(Ordering::Relaxed).saturating_sub(base)) +} + +fn lock() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// What decoding may hold: the output (at most the limit, and at most a +/// 4-D block of doubles, 2 KiB, per bit of input), and a little more. +fn bound(max_output: usize, input: &[u8]) -> usize { + let limit = if max_output == 0 { + 256 << 20 + } else { + max_output + }; + limit.min(input.len() * 8 * 2048) + 4096 +} + +/// An LSB-first bit writer. +#[derive(Default)] +struct Bits { + v: Vec, + n: usize, +} + +impl Bits { + fn put(&mut self, x: u64, bits: usize) { + for i in 0..bits { + if self.n.is_multiple_of(8) { + self.v.push(0); + } + if (x >> i) & 1 == 1 { + *self.v.last_mut().unwrap() |= 1 << (self.n % 8); + } + self.n += 1; + } + } +} + +/// H5Z-ZFP `cd_values`: a version word and a zfp header for a field of +/// `ztype` (0 int32, 1 int64, 2 float, 3 double) and sizes `n` (fastest +/// first), with `mode` (12 bits, or 64 when `long`). +fn cd_values(ztype: u64, n: &[u64], mode: u64, long: bool) -> Vec { + let mut b = Bits::default(); + for c in b"zfp" { + b.put(*c as u64, 8); + } + b.put(5, 8); + let dims = n.len(); + let mut meta = 0u64; + let bits = [48, 24, 16, 12][dims - 1]; + for &v in n.iter().rev() { + meta = (meta << bits) + v - 1; + } + meta = (meta << 2) + dims as u64 - 1; + meta = (meta << 2) + ztype; + b.put(meta, 52); + b.put(mode, if long { 64 } else { 12 }); + b.v.resize(b.v.len().div_ceil(4) * 4, 0); + let mut cd = vec![0x1001_1111u32]; + cd.extend( + b.v.chunks(4) + .map(|w| u32::from_le_bytes(w.try_into().unwrap())), + ); + cd +} + +fn elem(ztype: u64) -> usize { + if ztype & 1 == 0 { 4 } else { 8 } +} + +/// A 1-D field of 2^32 doubles (32 GiB) in a 1-byte chunk: refused for +/// its size, with or without the chunk size known, before anything is +/// allocated. +#[test] +fn huge_fields_are_refused_without_allocating() { + let _g = lock(); + for (n, ztype) in [ + (vec![1u64 << 32], 3), + (vec![1 << 24, 1 << 24], 3), + (vec![4096; 4], 1), + ] { + let cd = cd_values(ztype, &n, 2176, false); + for max_output in [0usize, 1 << 20] { + let (r, peak) = peak_during(|| zfp_decompress(&[0xff], &cd, max_output)); + assert!(r.is_err(), "{n:?}: decoded {:?} bytes", r.map(|v| v.len())); + assert!(peak < 4096, "{n:?}: peak {peak} bytes"); + } + } +} + +/// A field of the chunk's size whose stream is too short for its blocks +/// is refused before the output is allocated. +#[test] +fn short_streams_are_refused_before_allocating() { + let _g = lock(); + let n = [1u64 << 18]; + let cd = cd_values(2, &n, 2176, false); + let size = (1 << 18) * 4; + let (r, peak) = peak_during(|| zfp_decompress(&[0u8; 100], &cd, size)); + assert!(r.is_err()); + assert!(peak < 4096, "peak {peak} bytes"); + // A stream with a bit per block: all-zero blocks, which decode. + let input = vec![0u8; (1 << 16) / 8]; + let out = zfp_decompress(&input, &cd, size).unwrap(); + assert_eq!(out, vec![0u8; size]); +} + +/// xorshift64*: deterministic, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n.max(1) + } +} + +/// A mode word: one of the four short forms, or the 64-bit expert form +/// with parameters at and past their edges (maxbits below a float block's +/// exponent, minbits past maxbits, precision 0, minexp at the reversible +/// boundary). +fn mode(rng: &mut Rng) -> (u64, bool) { + match rng.below(6) { + 0 => (rng.below(2048), false), + 1 => (2048 + rng.below(128), false), + 2 => (2176, false), + 3 => (2177 + rng.below(4094 - 2177 + 1), false), + _ => { + fn pick(rng: &mut Rng, v: [u64; 6]) -> u64 { + v[rng.below(6) as usize] + } + let r = [rng.below(2000), rng.below(0x8000), rng.below(40)]; + let minbits = pick(rng, [0, 1, 11, r[0], r[1], 1]); + let r = [rng.below(0x8000), rng.below(40)]; + let maxbits = pick(rng, [minbits, minbits + r[1], 7, 11, r[0], 0x7fff]); + let maxprec = rng.below(0x80); + let r = [rng.below(0x8000), rng.below(400)]; + let minexp = pick( + rng, + [ + r[0], + 16495 - 1074, + 16495 - 1075, + 16495 + r[1] - 200, + 16495 - 1074, + r[0], + ], + ); + let m = ((((minexp << 7) + maxprec) << 15) + maxbits) << 15; + ((m + minbits) << 12 | 0xfff, true) + } + } +} + +/// Random fields, modes and streams (random bytes, runs of ones, mostly +/// zeros; of every length), decoded with the chunk size known and not, +/// and with header words that are random too. +#[test] +fn fuzzed_headers_and_streams_stay_within_the_allocation_bound() { + let _g = lock(); + let mut rng = Rng(0x2f9); + let mut decoded = 0; + for i in 0..20_000 { + let ztype = rng.below(4); + let dims = 1 + rng.below(4) as usize; + let max = [300, 40, 14, 8][dims - 1]; + let n: Vec = (0..dims).map(|_| 1 + rng.below(max)).collect(); + let (m, long) = mode(&mut rng); + let mut cd = cd_values(ztype, &n, m, long); + if rng.below(10) == 0 { + let at = rng.below(cd.len() as u64) as usize; + cd[at] ^= 1 << rng.below(32); + } + if rng.below(20) == 0 { + cd.truncate(rng.below(cd.len() as u64 + 1) as usize); + } + let len = match rng.below(4) { + 0 => rng.below(8), + 1 => rng.below(300), + _ => rng.below(20_000), + } as usize; + let input: Vec = match rng.below(3) { + 0 => (0..len).map(|_| rng.next() as u8).collect(), + 1 => (0..len) + .map(|_| [0, 0xff, rng.next() as u8][rng.below(3) as usize]) + .collect(), + _ => (0..len) + .map(|_| [0, 0, 0, 1, 0x80, rng.next() as u8][rng.below(6) as usize]) + .collect(), + }; + let size = n.iter().product::() as usize * elem(ztype); + let max_output = if rng.below(4) == 0 { 0 } else { size }; + let (r, peak) = peak_during(|| zfp_decompress(&input, &cd, max_output)); + if let Ok(out) = &r { + decoded += 1; + if max_output != 0 { + assert_eq!(out.len(), max_output, "iteration {i}"); + } + } + assert!( + peak <= bound(max_output, &input), + "iteration {i}: peak {peak} bytes for {n:?} from {} bytes ({:?})", + input.len(), + r.map(|v| v.len()) + ); + } + // Most inputs are streams zfp decodes without running out. + assert!(decoded > 5_000, "only {decoded} decoded"); +} -- 2.54.0 From d97b3d703af46d2bfdddf5f2dbccc5a11697d657 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:57:02 -0500 Subject: [PATCH 13/48] facade: `zfp` feature; ZFP read bit-exact against libhdf5 + libzfp The facade forwards `zfp` and adds it to `plugin-filters`. tests/zfp_interop.rs: h5py + hdf5plugin (H5Z-ZFP 1.1.1, zfp 1.0.1) write 2205 datasets over 16 modes (rate, precision, accuracy, reversible, expert settings at their edges) x int32/int64/float/double x 1-4-D shapes with partial edge chunks, partial blocks and unit chunk dimensions x smooth/noisy/wide-range/zero/inf-NaN data; clawhdf5 must read each byte for byte as h5py does (read back after closing the file: h5py returns a chunk still in libhdf5's cache without decoding it). A second test swaps a file's header words to what a big-endian writer stores and checks the byte-swapped values match h5py's. The left-out-filter test now expects ZFP only in builds without it; CI lints `zfp` alone and runs the new test with the plugin-filter step. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/Cargo.toml | 4 +- .../clawhdf5/tests/plugin_filters_interop.rs | 16 +- crates/clawhdf5/tests/zfp_interop.rs | 277 ++++++++++++++++++ scripts/ci-test.sh | 9 +- 4 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 crates/clawhdf5/tests/zfp_interop.rs diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 1eca373..9acc7e9 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -48,8 +48,10 @@ bitshuffle = ["clawhdf5-format/bitshuffle"] bzip2 = ["clawhdf5-format/bzip2"] blosc = ["clawhdf5-format/blosc"] blosc2 = ["clawhdf5-format/blosc2"] +# ZFP (32013), read-only. +zfp = ["clawhdf5-format/zfp"] # Every plugin filter. -plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2", "zfp"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 00911c6..2b30de4 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -495,16 +495,15 @@ fn lzf_written_by_clawhdf5_reads_in_h5py() { }); } -/// ZFP is not implemented, and Blosc2 is not in a build without the -/// `blosc2` feature: reading them must be a clear error naming the filter, -/// never data. +/// Blosc2 and ZFP are not in a build without their features: reading them +/// must be a clear error naming the filter, never data. #[test] -fn unimplemented_filters_are_a_clear_error() { +fn filters_left_out_of_the_build_are_a_clear_error() { if !have_python("h5py, hdf5plugin") { return; } let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("unimplemented.h5"); + let path = dir.path().join("left_out.h5"); run_python( r#" import sys @@ -517,7 +516,10 @@ with h5py.File(sys.argv[1], 'w') as f: &[path.to_str().unwrap()], ); let file = File::open(&path).unwrap(); - let mut missing = vec![("zfp", 32013u16, "ZFP")]; + let mut missing = Vec::new(); + if !cfg!(feature = "zfp") { + missing.push(("zfp", 32013u16, "ZFP")); + } if !cfg!(feature = "blosc2") { missing.push(("blosc2", 32026, "Blosc2")); } @@ -526,7 +528,7 @@ with h5py.File(sys.argv[1], 'w') as f: .dataset(name) .unwrap() .read_selection(&Selection::All) - .expect_err("an unimplemented filter must not read"); + .expect_err("a filter left out of the build must not read"); let msg = err.to_string(); assert!( msg.contains(&id.to_string()) && msg.contains(label), diff --git a/crates/clawhdf5/tests/zfp_interop.rs b/crates/clawhdf5/tests/zfp_interop.rs new file mode 100644 index 0000000..e6e0bdb --- /dev/null +++ b/crates/clawhdf5/tests/zfp_interop.rs @@ -0,0 +1,277 @@ +//! ZFP (H5Z-ZFP, filter 32013) against libhdf5 + libzfp. +//! +//! h5py with hdf5plugin (H5Z-ZFP 1.1.1, zfp 1.0.1) writes datasets in every +//! ZFP mode (fixed rate, precision and accuracy, reversible, expert) for +//! each type ZFP supports (int32, int64, float, double), in 1 to 4 +//! dimensions, with chunks that are partial at the dataset's edges, blocks +//! that are partial at the chunks' edges, and chunks with unit dimensions +//! (a lower-dimensional ZFP field). Next to each it stores what h5py reads +//! back, unfiltered, and clawhdf5 must read the ZFP dataset bit for bit +//! equal to that: the decoder is deterministic, so lossy modes have one +//! right answer. +//! +//! Skipped when python3 with h5py and hdf5plugin is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. +#![cfg(feature = "zfp")] + +use std::collections::BTreeMap; +use std::process::Command; + +use clawhdf5::File; +use clawhdf5_format::selection::Selection; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn have_python() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, hdf5plugin"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if ok { + return true; + } + assert!( + !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py and hdf5plugin is not available" + ); + eprintln!("SKIP: python3 with h5py and hdf5plugin not available"); + false +} + +fn run_python(script: &str, args: &[&str]) -> String { + let output = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Writes `f{i}` (ZFP) and `r{i}` (h5py's reading of `f{i}`, unfiltered) +/// for every mode x dtype x shape x data kind H5Z-ZFP accepts; each `f{i}` +/// has a `case` attribute. Prints the number of pairs. +const GENERATE: &str = r#" +import sys +import numpy as np, h5py, hdf5plugin +path = sys.argv[1] +MODES = [ + ('rate 2.5', dict(rate=2.5)), + ('rate 8', dict(rate=8)), + ('rate 16', dict(rate=16)), + ('rate 31', dict(rate=31)), + ('rate 64', dict(rate=64)), + ('precision 6', dict(precision=6)), + ('precision 20', dict(precision=20)), + ('precision 64', dict(precision=64)), + ('accuracy 0.5', dict(accuracy=0.5)), + ('accuracy 1e-3', dict(accuracy=1e-3)), + ('accuracy 1e-12', dict(accuracy=1e-12)), + ('reversible', dict(reversible=True)), + ('expert', dict(minbits=24, maxbits=600, maxprec=30, minexp=-20)), + # maxbits past ZFP_MAX_BITS: H5Z-ZFP stores it as fixed precision. + ('expert maxbits 20000', dict(minbits=1, maxbits=20000, maxprec=40, minexp=-1074)), + # A budget of 10 bits a block: one bit left after a float's exponent. + # (Doubles are left out: 12 bits of exponent and flag overrun it, and + # libzfp's encoder then writes past its buffer.) + ('expert maxbits 10', dict(minbits=5, maxbits=10, maxprec=64, minexp=-100)), + ('expert minbits 900', dict(minbits=900, maxbits=4000, maxprec=64, minexp=-60)), +] +DTYPES = [' dtypes read + let mut seen: BTreeMap> = BTreeMap::new(); + let mut failures = Vec::new(); + for i in 0..n { + let ds = file.dataset(&format!("f{i}")).unwrap(); + let case = match ds.attrs().unwrap().get("case") { + Some(clawhdf5::AttrValue::String(s)) => s.clone(), + other => panic!("f{i}: case attribute {other:?}"), + }; + let want = file + .dataset(&format!("r{i}")) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + match ds.read_selection(&Selection::All) { + Ok(got) if got == want => { + let mut parts = case.split('|'); + let mode = parts.next().unwrap().to_string(); + let dt = parts.next().unwrap().to_string(); + let e = seen.entry(mode).or_default(); + if !e.contains(&dt) { + e.push(dt); + } + } + Ok(got) => { + let first = got.iter().zip(&want).position(|(a, b)| a != b); + failures.push(format!("f{i} {case}: differs from byte {first:?}")); + } + Err(e) => failures.push(format!("f{i} {case}: {e}")), + } + } + assert!( + failures.is_empty(), + "{} of {n} datasets:\n{}", + failures.len(), + failures.join("\n") + ); + // Every mode was exercised on every type H5Z-ZFP accepts it for. + for (mode, dts) in &seen { + assert!(dts.len() >= 2, "{mode}: only {dts:?}"); + } + assert_eq!(seen.len(), 16, "{:?}", seen.keys()); + eprintln!("{n} ZFP datasets bit-exact; modes and types: {seen:?}"); +} + +/// A header written on a big-endian machine: H5Z-ZFP finds the magic only +/// after byte-swapping the `cd_values`, and then byte-swaps the decoded +/// values, since the dataset's datatype is big-endian there. The file is +/// made by swapping the header words of a little-endian one in place, so +/// the datatype stays little-endian and libhdf5 reads the values swapped; +/// clawhdf5 must read the same bytes. +#[test] +fn big_endian_header_swaps_the_values() { + if !have_python() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("zfp_be.h5"); + let n: usize = run_python( + r#" +import sys, struct +import numpy as np, h5py, hdf5plugin +path = sys.argv[1] +cases = [('{len(cd) - 1}I', *cd[1:]) + at = raw.find(le) + assert at > 0 and raw.find(le, at + 1) < 0, 'cd_values not found once' + raw[at:at + len(le)] = swapped +open(path, 'wb').write(raw) +with h5py.File(path, 'a') as f: + for k in range(len(cases)): + f.create_dataset(f'r{k}', data=f[f'f{k}'][()]) + f.create_dataset(f'o{k}', data=f[f'f{k}'][()].byteswap()) +print(len(cases)) +"#, + &[path.to_str().unwrap()], + ) + .parse() + .unwrap(); + let file = File::open(&path).unwrap(); + for k in 0..n { + let read = |name: String| { + file.dataset(&name) + .unwrap() + .read_selection(&Selection::All) + .unwrap_or_else(|e| panic!("{name}: {e}")) + }; + let got = read(format!("f{k}")); + assert!(got == read(format!("r{k}")), "f{k}: differs from h5py"); + // Byte-swapped back, the values are the right ones. + assert!(got != read(format!("o{k}")), "f{k}: not swapped"); + } +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 3012cba..a3fa3b8 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -67,7 +67,7 @@ run_step "cargo clippy --all-targets" cargo clippy \ # 3. Clippy over clawhdf5-format's optional features, which the default # workspace build never compiles (szip is left out: it needs libaec). -# plugin-filters = bitshuffle, bzip2, blosc, blosc2 (and the default-on lzf). +# plugin-filters = bitshuffle, bzip2, blosc, blosc2, zfp (and the default-on lzf). run_step "cargo clippy (format feature matrix)" cargo clippy \ -p clawhdf5-format \ --all-targets \ @@ -78,7 +78,7 @@ run_step "cargo clippy (format feature matrix)" cargo clippy \ # dependencies (bitshuffle and blosc share code). plugin_filters_alone() { local f - for f in bitshuffle bzip2 blosc blosc2; do + for f in bitshuffle bzip2 blosc blosc2 zfp; do echo "--- $f" cargo clippy -p clawhdf5-format --all-targets --features "$f" -- -D warnings || return 1 done @@ -208,9 +208,10 @@ if "$PYTHON" -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:- # libhdf5's registered plugins) compile and run too. run_step "h5py interop (format, ignored tests)" cargo test \ -p clawhdf5-format --features lz4,zstd --test writer_h5py_tests -- --include-ignored - # LZF, bitshuffle, bzip2 and Blosc both ways against h5py + hdf5plugin. + # LZF, bitshuffle, bzip2 and Blosc both ways against h5py + hdf5plugin; + # Blosc2 and ZFP (read-only) against what h5py reads. run_step "h5py interop (plugin filters)" cargo test \ - -p clawhdf5 --features plugin-filters --test plugin_filters_interop + -p clawhdf5 --features plugin-filters --test plugin_filters_interop --test zfp_interop else echo "" echo "==> [h5py interop] SKIPPED: no h5py in $PYTHON" -- 2.54.0 From cf2b408a63bd7ea630875183ea82506e56add14b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 12:57:08 -0500 Subject: [PATCH 14/48] format: read fixed array chunk indexes over Storage FixedArrayHeader::parse_in reads the header in one read, and read_fixed_array_chunks_in reads the data block's prefix and then the whole block, paged or not, as one window; checksums and elements are checked in it, with bounds errors reported as the whole-file checks did (also in builds without the checksum feature, where the per-element checks are the only ones). The open-ended &file_data[offset..] slices are gone. The &[u8] functions are wrappers. New test: non-paged and paged, filtered and unfiltered arrays, cut through the data block and with damaged bytes, read identically through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/fixed_array.rs | 280 ++++++++++++++++------ 1 file changed, 208 insertions(+), 72 deletions(-) diff --git a/crates/clawhdf5-format/src/fixed_array.rs b/crates/clawhdf5-format/src/fixed_array.rs index 8546224..de936a7 100644 --- a/crates/clawhdf5-format/src/fixed_array.rs +++ b/crates/clawhdf5-format/src/fixed_array.rs @@ -9,16 +9,19 @@ use alloc::{format, vec, vec::Vec}; use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; +use crate::storage::{Storage, Window, len_usize, read_exact_at}; /// Verify the Jenkins lookup3 checksum stored immediately after -/// `data[start..end]`, as every Fixed Array structure carries one. +/// `data[start..end]`, as every Fixed Array structure carries one. `w` is +/// a window of the file and `start`/`end` are relative to it. /// /// A corrupt chunk index silently yields addresses pointing at the wrong /// bytes, so a mismatch has to be an error rather than a shrug: without this /// the damage surfaces as plausible-looking data from the wrong chunk. #[cfg(feature = "checksum")] -fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> { - ensure_len(data, end, 4)?; +fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> { + w.ensure(end, 4)?; + let data = &w.bytes; let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]); let computed = crate::checksum::jenkins_lookup3(&data[start..end]); if computed != stored { @@ -31,7 +34,7 @@ fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatEr } #[cfg(not(feature = "checksum"))] -fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> { +fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> { Ok(()) } @@ -73,19 +76,6 @@ fn read_length(data: &[u8], pos: usize, size: u8) -> Result { read_offset(data, pos, size) } -fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { - if offset - .checked_add(needed) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: offset.saturating_add(needed), - available: data.len(), - }); - } - Ok(()) -} - fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool { let s = size as usize; if pos + s > data.len() { @@ -101,13 +91,24 @@ impl FixedArrayHeader { offset: usize, offset_size: u8, length_size: u8, + ) -> Result { + Self::parse_in(&file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the header. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + length_size: u8, ) -> Result { // FAHD signature(4) + version(1) + client_id(1) + element_size(1) + // max_nelmts_bits(1) + num_elements(length_size) + data_block_addr(offset_size) + checksum(4) let min_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + offset_size as usize + 4; - ensure_len(file_data, offset, min_size)?; + let w = Window::read(file, offset, min_size)?; + w.ensure(0, min_size)?; - let d = &file_data[offset..]; + let d: &[u8] = &w.bytes; if &d[0..4] != b"FAHD" { return Err(FormatError::ChunkedReadError( "invalid Fixed Array header signature".into(), @@ -130,7 +131,7 @@ impl FixedArrayHeader { pos += length_size as usize; let data_block_address = read_offset(d, pos, offset_size)?; pos += offset_size as usize; - verify_checksum(file_data, offset, offset + pos)?; + verify_checksum(&w, 0, pos)?; Ok(FixedArrayHeader { client_id, @@ -156,15 +157,39 @@ pub fn read_fixed_array_chunks( chunk_dimensions: &[u32], element_size: u32, offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + read_fixed_array_chunks_in( + &file_data, + header, + dataset_dims, + max_dims, + chunk_dimensions, + element_size, + offset_size, + length_size, + ) +} + +/// [`read_fixed_array_chunks`] over any [`Storage`]: one read of the data +/// block's prefix, one of the whole data block (pages included). +#[allow(clippy::too_many_arguments)] +pub fn read_fixed_array_chunks_in( + file: &dyn Storage, + header: &FixedArrayHeader, + dataset_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dimensions: &[u32], + element_size: u32, + offset_size: u8, _length_size: u8, ) -> Result, FormatError> { + let file_len = len_usize(file); let db_offset = header.data_block_address as usize; // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size) let db_header_size = 4 + 1 + 1 + offset_size as usize; - ensure_len(file_data, db_offset, db_header_size)?; - - let d = &file_data[db_offset..]; + let d = read_exact_at(file, db_offset as u64, db_header_size)?; if &d[0..4] != b"FADB" { return Err(FormatError::ChunkedReadError( "invalid Fixed Array data block signature".into(), @@ -178,7 +203,7 @@ pub fn read_fixed_array_chunks( // A chunk index cannot describe more elements than the file has bytes (each // element occupies at least `offset_size` bytes). Reject a corrupt count // before it can drive a huge loop or overflow an offset computation. - if num_elements > file_data.len() { + if num_elements > file_len { return Err(FormatError::ChunkedReadError( "Fixed Array element count exceeds file size".into(), )); @@ -208,30 +233,34 @@ pub fn read_fixed_array_chunks( chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; let mut chunks = Vec::new(); - let push_element = - |i: usize, abs: usize, chunks: &mut Vec| -> Result<(), FormatError> { - if let Some((address, chunk_size, filter_mask)) = parse_fa_element( - file_data, - abs, - header.client_id, - offset_size, - header.element_size, - chunk_byte_size, - )? { - // A slot beyond the current extent is ignored, as the - // library does. - let Some(offsets) = grid.offsets(i as u64) else { - return Ok(()); - }; - chunks.push(ChunkInfo { - chunk_size, - filter_mask, - offsets, - address, - }); - } - Ok(()) - }; + // `rel` is relative to the data block, whose bytes are in `w`. + let push_element = |w: &Window<'_>, + i: usize, + rel: usize, + chunks: &mut Vec| + -> Result<(), FormatError> { + if let Some((address, chunk_size, filter_mask)) = parse_fa_element( + w, + rel, + header.client_id, + offset_size, + header.element_size, + chunk_byte_size, + )? { + // A slot beyond the current extent is ignored, as the + // library does. + let Some(offsets) = grid.offsets(i as u64) else { + return Ok(()); + }; + chunks.push(ChunkInfo { + chunk_size, + filter_mask, + offsets, + address, + }); + } + Ok(()) + }; // A data block is paged when it holds more elements than fit in one page. // `max_nelmts_bits` is an untrusted u8; a shift >= the pointer width would @@ -246,10 +275,13 @@ pub fn read_fixed_array_chunks( if !is_paged { // Non-paged: prefix, then `num_elements` elements packed directly, - // then a checksum over both. - verify_checksum(file_data, db_offset, elem_at(elements_start, num_elements)?)?; + // then a checksum over both. One window holds all of it (or ends at + // the end of the file), so its bounds checks are the whole-file ones. + let end = elem_at(elements_start, num_elements)?; + let w = Window::read(file, db_offset as u64, end.saturating_add(4) - db_offset)?; + verify_checksum(&w, 0, end - db_offset)?; for i in 0..num_elements { - push_element(i, elem_at(elements_start, i)?, &mut chunks)?; + push_element(&w, i, elem_at(elements_start, i)? - db_offset, &mut chunks)?; } return Ok(chunks); } @@ -272,22 +304,27 @@ pub fn read_fixed_array_chunks( .and_then(|x| x.checked_add(4)) .ok_or_else(stride_overflow)?; - if bitmap_start + bitmap_size > file_data.len() { + if bitmap_start + bitmap_size > file_len { return Err(FormatError::UnexpectedEof { expected: bitmap_start + bitmap_size, - available: file_data.len(), + available: file_len, }); } + // The whole data block in one window: every page slot is at most + // `page_stride` bytes, so every position checked below lies inside it + // (or past the end of the file). + let block_len = (pages_start - db_offset).saturating_add(npages.saturating_mul(page_stride)); + let w = Window::read(file, db_offset as u64, block_len)?; // The prefix and page bitmap are covered by their own checksum, and each // initialised page by one of its own. - verify_checksum(file_data, db_offset, bitmap_start + bitmap_size)?; + verify_checksum(&w, 0, bitmap_start + bitmap_size - db_offset)?; for p in 0..npages { let page_first = p * page_nelmts; // < num_elements, cannot overflow let page_count = core::cmp::min(page_nelmts, num_elements - page_first); // Check the page-init bit (MSB-first within each byte). - let bit_byte = file_data[bitmap_start + p / 8]; + let bit_byte = w.bytes[bitmap_start + p / 8 - db_offset]; let bit_mask = 1u8 << (7 - (p % 8)); if bit_byte & bit_mask == 0 { continue; // entire page unallocated @@ -297,21 +334,30 @@ pub fn read_fixed_array_chunks( .checked_mul(page_stride) .and_then(|o| pages_start.checked_add(o)) .ok_or_else(stride_overflow)?; - verify_checksum(file_data, page_off, elem_at(page_off, page_count)?)?; + verify_checksum( + &w, + page_off - db_offset, + elem_at(page_off, page_count)? - db_offset, + )?; for e in 0..page_count { - push_element(page_first + e, elem_at(page_off, e)?, &mut chunks)?; + push_element( + &w, + page_first + e, + elem_at(page_off, e)? - db_offset, + &mut chunks, + )?; } } Ok(chunks) } -/// Parse a single Fixed Array element at absolute file offset `abs`. +/// Parse a single Fixed Array element at offset `abs` of the window `w`. /// /// Returns `Some((address, chunk_size, filter_mask))` for an allocated chunk, or /// `None` if the element is undefined (an unallocated chunk, address all-`0xFF`). fn parse_fa_element( - file_data: &[u8], + w: &Window<'_>, abs: usize, client_id: u8, offset_size: u8, @@ -321,12 +367,8 @@ fn parse_fa_element( let os = offset_size as usize; if client_id == 0 { // Non-filtered: element is just the chunk address. - if abs + os > file_data.len() { - return Err(FormatError::UnexpectedEof { - expected: abs + os, - available: file_data.len(), - }); - } + w.ensure(abs, os)?; + let file_data: &[u8] = &w.bytes; if is_undefined(file_data, abs, offset_size) { return Ok(None); } @@ -341,17 +383,14 @@ fn parse_fa_element( )); } let chunk_size_bytes = es - os - 4; - if abs + es > file_data.len() { - return Err(FormatError::UnexpectedEof { - expected: abs + es, - available: file_data.len(), - }); - } + w.ensure(abs, es)?; + let file_data: &[u8] = &w.bytes; if is_undefined(file_data, abs, offset_size) { return Ok(None); } let address = read_offset(file_data, abs, offset_size)?; - let chunk_size = read_variable_length(&file_data[abs + os..], chunk_size_bytes)?; + let chunk_size = + read_variable_length(&file_data[abs + os..abs + es - 4], chunk_size_bytes)?; let fm_off = abs + os + chunk_size_bytes; let filter_mask = u32::from_le_bytes([ file_data[fm_off], @@ -813,4 +852,101 @@ mod tests { .collect(); assert_eq!(got, expect); } + + /// A fixed array (header at 0x100, data block at 0x200) of `n` chunks, + /// filtered or not, paged when `n` exceeds `1 << page_bits`; every + /// page initialised except page 1. + fn build_fixed_array(n: usize, filtered: bool, page_bits: u8) -> Vec { + let os = 8usize; + let es = if filtered { os + 4 + 4 } else { os }; + let (fahd, db) = (0x100usize, 0x200usize); + let mut f = vec![0u8; 0x2000]; + f[fahd..fahd + 4].copy_from_slice(b"FAHD"); + f[fahd + 5] = u8::from(filtered); + f[fahd + 6] = es as u8; + f[fahd + 7] = page_bits; + f[fahd + 8..fahd + 16].copy_from_slice(&(n as u64).to_le_bytes()); + f[fahd + 16..fahd + 24].copy_from_slice(&(db as u64).to_le_bytes()); + stamp_checksum(&mut f, fahd, fahd + 24); + f[db..db + 4].copy_from_slice(b"FADB"); + f[db + 5] = u8::from(filtered); + f[db + 6..db + 14].copy_from_slice(&(fahd as u64).to_le_bytes()); + let elems = db + 6 + os; + let write = |f: &mut Vec, at: usize, i: usize| { + let addr = if i == 2 { + u64::MAX + } else { + 0x1000 + i as u64 * 0x100 + }; + f[at..at + os].copy_from_slice(&addr.to_le_bytes()); + if filtered { + f[at + os..at + os + 4].copy_from_slice(&(100 + i as u32).to_le_bytes()); + f[at + os + 4..at + os + 8].copy_from_slice(&(i as u32 & 1).to_le_bytes()); + } + }; + let page = 1usize << page_bits; + if n <= page { + for i in 0..n { + write(&mut f, elems + i * es, i); + } + stamp_checksum(&mut f, db, elems + n * es); + } else { + let npages = n.div_ceil(page); + let bitmap = npages.div_ceil(8); + for p in 0..npages { + if p != 1 { + f[elems + p / 8] |= 0x80 >> (p % 8); + } + } + stamp_checksum(&mut f, db, elems + bitmap); + let pages_start = elems + bitmap + 4; + for p in (0..npages).filter(|&p| p != 1) { + let at = pages_start + p * (page * es + 4); + let count = page.min(n - p * page); + for e in 0..count { + write(&mut f, at + e * es, p * page + e); + } + stamp_checksum(&mut f, at, at + count * es); + } + } + f + } + + /// Non-paged and paged, filtered and unfiltered arrays, cut at every + /// length through the data block and with a damaged byte, read + /// identically through a `read_at`-only storage. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + for (n, filtered, bits) in [(3, false, 10), (3, true, 10), (11, false, 2), (11, true, 2)] { + let full = build_fixed_array(n, filtered, bits); + let es = if filtered { 16 } else { 8 }; + let dims = [n as u64 * 20]; + let h = FixedArrayHeader::parse(&full, 0x100, 8, 8).unwrap(); + let chunks = read_fixed_array_chunks(&full, &h, &dims, None, &[20], 8, 8, 8).unwrap(); + // Chunk 2 is unallocated, and so is page 1 of a paged array. + let expect = if n > 4 { n - 1 - 4 } else { n - 1 }; + assert_eq!(chunks.len(), expect); + let mut files = Vec::new(); + for cut in (0x100..0x200 + 40 + n * (es + 4) + 16).step_by(3) { + files.push(full[..cut].to_vec()); + } + for at in [0x104, 0x210, 0x21a, 0x230] { + let mut damaged = full.clone(); + damaged[at] ^= 1; + files.push(damaged); + } + files.push(full); + for f in files { + let storage = CountingStorage::new(f.clone()); + let want = FixedArrayHeader::parse(&f, 0x100, 8, 8); + let got = FixedArrayHeader::parse_in(&storage, 0x100, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + let Ok(h) = want else { continue }; + let want = read_fixed_array_chunks(&f, &h, &dims, None, &[20], 8, 8, 8); + let got = read_fixed_array_chunks_in(&storage, &h, &dims, None, &[20], 8, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len()); + } + } + } } -- 2.54.0 From ff6d644391485ae5af05cd030ccad4318be779b5 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:01:11 -0500 Subject: [PATCH 15/48] format: read extensible array chunk indexes over Storage ExtensibleArrayHeader::parse_in reads the header in one read, and read_extensible_array_chunks_in reads each index block, super block and data block as a prefix read and then one window of the whole structure (paged data blocks included); checksums and elements are checked in the window with bounds errors reported as the whole-file checks did. The &[u8] functions are wrappers. New test: an array with inline elements and a data block, cut at every length and damaged in each structure, reads identically through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../clawhdf5-format/src/extensible_array.rs | 278 ++++++++++++------ crates/clawhdf5-format/src/storage.rs | 10 + 2 files changed, 193 insertions(+), 95 deletions(-) diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index ba1c822..529955d 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -12,16 +12,19 @@ use alloc::{format, vec, vec::Vec}; use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; +use crate::storage::{Storage, Window, read_exact_at}; /// Verify the Jenkins lookup3 checksum stored immediately after -/// `data[start..end]`, as every Extensible Array structure carries one. +/// `data[start..end]`, as every Extensible Array structure carries one. `w` +/// is a window of the file and `start`/`end` are relative to it. /// /// A corrupt chunk index yields addresses pointing at the wrong bytes, so a /// mismatch is an error: otherwise the damage surfaces as plausible data read /// from the wrong chunk. #[cfg(feature = "checksum")] -fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> { - ensure_len(data, end, 4)?; +fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> { + w.ensure(end, 4)?; + let data: &[u8] = &w.bytes; let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]); let computed = crate::checksum::jenkins_lookup3(&data[start..end]); if computed != stored { @@ -34,7 +37,7 @@ fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatEr } #[cfg(not(feature = "checksum"))] -fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> { +fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> { Ok(()) } @@ -80,19 +83,6 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { }) } -fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { - if offset - .checked_add(needed) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: offset.saturating_add(needed), - available: data.len(), - }); - } - Ok(()) -} - fn is_undefined_addr(addr: u64, offset_size: u8) -> bool { match offset_size { 2 => addr == 0xFFFF, @@ -130,6 +120,16 @@ impl ExtensibleArrayHeader { offset: usize, offset_size: u8, length_size: u8, + ) -> Result { + Self::parse_in(&file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the header. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + length_size: u8, ) -> Result { // EAHD: signature(4) + version(1) + client_id(1) + element_size(1) + // max_nelmts_bits(1) + idx_blk_elmts(1) + min_dblk_nelmts(1) + @@ -137,9 +137,10 @@ impl ExtensibleArrayHeader { // 6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4) let min_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4; - ensure_len(file_data, offset, min_size)?; + let w = Window::read(file, offset, min_size)?; + w.ensure(0, min_size)?; - let d = &file_data[offset..]; + let d: &[u8] = &w.bytes; if &d[0..4] != b"EAHD" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array header signature".into(), @@ -172,7 +173,7 @@ impl ExtensibleArrayHeader { pos += ls; // skip max_idx_set (6th stats field) let index_block_address = read_offset(d, pos, offset_size)?; pos += offset_size as usize; - verify_checksum(file_data, offset, offset + pos)?; + verify_checksum(&w, 0, pos)?; Ok(ExtensibleArrayHeader { client_id, @@ -193,11 +194,11 @@ impl ExtensibleArrayHeader { } } -/// Read a single element from the extensible array element data. +/// Read a single element at offset `pos` of the window `w`. /// Returns (chunk_info, bytes_consumed) or None if unallocated. #[allow(clippy::too_many_arguments)] fn read_element( - data: &[u8], + w: &Window<'_>, pos: usize, client_id: u8, element_size: u8, @@ -207,15 +208,11 @@ fn read_element( grid: &ChunkGrid, ) -> Result<(Option, usize), FormatError> { let os = offset_size as usize; + let data: &[u8] = &w.bytes; if client_id == 0 { // Non-filtered: just address - if pos + os > data.len() { - return Err(FormatError::UnexpectedEof { - expected: pos + os, - available: data.len(), - }); - } + w.ensure(pos, os)?; if is_undefined(data, pos, offset_size) { return Ok((None, os)); } @@ -243,15 +240,7 @@ fn read_element( } let chunk_size_bytes = es - os - 4; let elem_total = os + chunk_size_bytes + 4; - if pos - .checked_add(elem_total) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: pos.saturating_add(elem_total), - available: data.len(), - }); - } + w.ensure(pos, elem_total)?; if is_undefined(data, pos, offset_size) { return Ok((None, elem_total)); } @@ -316,8 +305,8 @@ fn page_nelmts(header: &ExtensibleArrayHeader) -> Option { /// stores only its prefix, then one slot per page. #[allow(clippy::too_many_arguments)] fn read_data_block_elements( - file_data: &[u8], - db_offset: usize, + file: &dyn Storage, + db_offset: u64, nelmts: usize, header: &ExtensibleArrayHeader, offset_size: u8, @@ -330,21 +319,28 @@ fn read_data_block_elements( // EADB: signature(4) + version(1) + client_id(1) + header_address(offset_size) // + block offset(arr_off_size) let db_header_size = 4 + 1 + 1 + offset_size as usize + arr_off_size(header); - ensure_len(file_data, db_offset, db_header_size)?; + let prefix = read_exact_at(file, db_offset, db_header_size)?; - if &file_data[db_offset..db_offset + 4] != b"EADB" { + if &prefix[0..4] != b"EADB" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array data block signature".into(), )); } - let mut pos = db_offset + db_header_size; + // Positions below are relative to the data block. + let mut pos = db_header_size; let page = page_nelmts(header).ok_or_else(|| { FormatError::Overflow("Extensible Array page element count overflows usize".into()) })?; + let elem_bytes = if header.client_id == 0 { + offset_size as usize + } else { + header.element_size as usize + }; let mut chunks = Vec::new(); - let read_run = |from: usize, + let read_run = |w: &Window<'_>, + from: usize, count: usize, first_index: usize, chunks: &mut Vec| @@ -352,7 +348,7 @@ fn read_data_block_elements( let mut p = from; for i in 0..count { let (info, consumed) = read_element( - file_data, + w, p, header.client_id, header.element_size, @@ -370,18 +366,16 @@ fn read_data_block_elements( }; if nelmts <= page { - // Prefix and elements are covered by one checksum. - let elem_bytes = if header.client_id == 0 { - offset_size as usize - } else { - header.element_size as usize - }; + // Prefix and elements are covered by one checksum. One window holds + // all of it (or ends at the end of the file), so its bounds checks + // are the whole-file ones. let end = nelmts .checked_mul(elem_bytes) .and_then(|b| pos.checked_add(b)) .ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?; - verify_checksum(file_data, db_offset, end)?; - read_run(pos, nelmts, start_index, &mut chunks)?; + let w = Window::read(file, db_offset, end.saturating_add(4))?; + verify_checksum(&w, 0, end)?; + read_run(&w, pos, nelmts, start_index, &mut chunks)?; return Ok(chunks); } @@ -389,18 +383,19 @@ fn read_data_block_elements( // each holding `page` elements followed by a checksum. Pages whose bit is // clear were never written; their slot still occupies the file, so stride // over it rather than reading zeros as addresses. - verify_checksum(file_data, db_offset, pos)?; + let npages = nelmts.div_ceil(page); + // The whole data block in one window: every position checked below lies + // inside it (or past the end of the file). + let block_len = pos + .saturating_add(4) + .saturating_add(npages.saturating_mul(page.saturating_mul(elem_bytes).saturating_add(4))); + let w = Window::read(file, db_offset, block_len)?; + verify_checksum(&w, 0, pos)?; pos += 4; - let elem_bytes = if header.client_id == 0 { - offset_size as usize - } else { - header.element_size as usize - }; let page_stride = page .checked_mul(elem_bytes) .and_then(|b| b.checked_add(4)) .ok_or_else(|| FormatError::Overflow("Extensible Array page stride".into()))?; - let npages = nelmts.div_ceil(page); for p in 0..npages { // One bit per page across the whole super block, packed contiguously // and MSB-first within each byte, as H5VM_bit_get reads it. @@ -412,8 +407,8 @@ fn read_data_block_elements( let count = core::cmp::min(page, nelmts - p * page); // Each page carries its own checksum, over a full page's worth of // slots even when the last one holds fewer live elements. - verify_checksum(file_data, pos, pos + page * elem_bytes)?; - read_run(pos, count, start_index + p * page, &mut chunks)?; + verify_checksum(&w, pos, pos + page * elem_bytes)?; + read_run(&w, pos, count, start_index + p * page, &mut chunks)?; } pos = pos .checked_add(page_stride) @@ -435,6 +430,32 @@ pub fn read_extensible_array_chunks( chunk_dimensions: &[u32], element_size: u32, offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + read_extensible_array_chunks_in( + &file_data, + header, + dataset_dims, + max_dims, + chunk_dimensions, + element_size, + offset_size, + length_size, + ) +} + +/// [`read_extensible_array_chunks`] over any [`Storage`]: one read of the +/// index block's prefix, one of the whole index block, and the same for +/// every super block and data block it references. +#[allow(clippy::too_many_arguments)] +pub fn read_extensible_array_chunks_in( + file: &dyn Storage, + header: &ExtensibleArrayHeader, + dataset_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dimensions: &[u32], + element_size: u32, + offset_size: u8, _length_size: u8, ) -> Result, FormatError> { let os = offset_size as usize; @@ -451,16 +472,17 @@ pub fn read_extensible_array_chunks( // Parse index block (EAIB): signature(4) + version(1) + client_id(1) // + header address(offset_size), then the inline elements, then the // direct data block addresses, then the super block addresses. - let ib_offset = header.index_block_address as usize; + // Positions below are relative to the index block. + let ib_offset = header.index_block_address; let ib_header_size = 4 + 1 + 1 + os; - ensure_len(file_data, ib_offset, ib_header_size)?; + let prefix = read_exact_at(file, ib_offset, ib_header_size)?; - if &file_data[ib_offset..ib_offset + 4] != b"EAIB" { + if &prefix[0..4] != b"EAIB" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array index block signature".into(), )); } - let mut pos = ib_offset + ib_header_size; + let mut pos = ib_header_size; let mut chunks = Vec::new(); let total_elements = header.num_elements as usize; @@ -520,13 +542,16 @@ pub fn read_extensible_array_chunks( .and_then(|n| n.checked_mul(os).and_then(|b| p.checked_add(b))) }) .ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?; - verify_checksum(file_data, ib_offset, ib_end)?; + // The whole index block in one window: every position read below is + // before `ib_end`. + let w = Window::read(file, ib_offset, ib_end.saturating_add(4))?; + verify_checksum(&w, 0, ib_end)?; // 1. Elements stored inline in the index block. let n_inline = (header.idx_blk_elmts as usize).min(total_elements); for i in 0..n_inline { let (info, consumed) = read_element( - file_data, + &w, pos, header.client_id, header.element_size, @@ -550,8 +575,8 @@ pub fn read_extensible_array_chunks( if global_index >= total_elements { return Ok(chunks); } - ensure_len(file_data, pos, os)?; - let addr = read_offset(file_data, pos, offset_size)?; + w.ensure(pos, os)?; + let addr = read_offset(&w.bytes, pos, offset_size)?; pos += os; if !is_undefined_addr(addr, offset_size) { if dblk_nelmts > page_nelmts(header).unwrap_or(usize::MAX) { @@ -562,8 +587,8 @@ pub fn read_extensible_array_chunks( )); } chunks.extend(read_data_block_elements( - file_data, - addr as usize, + file, + addr, dblk_nelmts, header, offset_size, @@ -583,16 +608,16 @@ pub fn read_extensible_array_chunks( if global_index >= total_elements { break; } - ensure_len(file_data, pos, os)?; - let sb_addr = read_offset(file_data, pos, offset_size)?; + w.ensure(pos, os)?; + let sb_addr = read_offset(&w.bytes, pos, offset_size)?; pos += os; let (ndblks, dblk_nelmts) = sblk_info(u, dmin).ok_or_else(|| { FormatError::Overflow("Extensible Array super block layout overflows usize".into()) })?; if !is_undefined_addr(sb_addr, offset_size) { chunks.extend(read_super_block( - file_data, - sb_addr as usize, + file, + sb_addr, ndblks, dblk_nelmts, header, @@ -618,8 +643,8 @@ pub fn read_extensible_array_chunks( /// + one address per data block + checksum. #[allow(clippy::too_many_arguments)] fn read_super_block( - file_data: &[u8], - sb_offset: usize, + file: &dyn Storage, + sb_offset: u64, ndblks: usize, dblk_nelmts: usize, header: &ExtensibleArrayHeader, @@ -630,9 +655,9 @@ fn read_super_block( ) -> Result, FormatError> { let os = offset_size as usize; let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header); - ensure_len(file_data, sb_offset, sb_header_size)?; + let prefix = read_exact_at(file, sb_offset, sb_header_size)?; - if &file_data[sb_offset..sb_offset + 4] != b"EASB" { + if &prefix[0..4] != b"EASB" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array super block signature".into(), )); @@ -654,9 +679,19 @@ fn read_super_block( let bitmap_bytes = per_dblk_bitmap .checked_mul(ndblks) .ok_or_else(|| FormatError::Overflow("Extensible Array page bitmap size".into()))?; - let bitmap_start = sb_offset + sb_header_size; - ensure_len(file_data, bitmap_start, bitmap_bytes)?; - let bitmap = &file_data[bitmap_start..bitmap_start + bitmap_bytes]; + // Positions below are relative to the super block, whose bytes (up to + // its checksum) are all in one window. + let bitmap_start = sb_header_size; + let w = Window::read( + file, + sb_offset, + bitmap_start + .saturating_add(bitmap_bytes) + .saturating_add(ndblks.saturating_mul(os)) + .saturating_add(4), + )?; + w.ensure(bitmap_start, bitmap_bytes)?; + let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes]; let mut pos = bitmap_start + bitmap_bytes; let mut chunks = Vec::new(); @@ -667,16 +702,16 @@ fn read_super_block( .checked_mul(os) .and_then(|b| pos.checked_add(b)) .ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?; - verify_checksum(file_data, sb_offset, sb_end)?; + verify_checksum(&w, 0, sb_end)?; for i in 0..ndblks { - ensure_len(file_data, pos, os)?; - let addr = read_offset(file_data, pos, offset_size)?; + w.ensure(pos, os)?; + let addr = read_offset(&w.bytes, pos, offset_size)?; pos += os; if !is_undefined_addr(addr, offset_size) { chunks.extend(read_data_block_elements( - file_data, - addr as usize, + file, + addr, dblk_nelmts, header, offset_size, @@ -887,11 +922,11 @@ mod tests { assert_eq!(chunks[1].offsets, vec![20]); } - /// Build a synthetic EA with inline elements + one direct data block. - #[test] - fn read_inline_plus_data_blocks() { + /// A synthetic EA with inline elements + one direct data block: the + /// file, with the header at 0x100 (8-byte offsets and lengths, 4 chunks + /// of 10 elements from 0x1000 on). + fn build_inline_plus_data_blocks() -> Vec { let os: u8 = 8; - let ls: u8 = 8; let osv = os as usize; let chunk_byte_size = 10u64 * 8; // 10 elements × 8 bytes let idx_blk_elmts = 2u8; @@ -981,8 +1016,17 @@ mod tests { dbpos += osv; } stamp_checksum(&mut file_data, aedb_offset, dbpos); + file_data + } - let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap(); + /// Build a synthetic EA with inline elements + one direct data block. + #[test] + fn read_inline_plus_data_blocks() { + let (os, ls) = (8u8, 8u8); + let chunk_byte_size = 10u64 * 8; + let base_addr = 0x1000u64; + let file_data = build_inline_plus_data_blocks(); + let header = ExtensibleArrayHeader::parse(&file_data, 0x100, os, ls).unwrap(); let ds_dims = vec![40u64]; let chunk_dims = vec![10u32]; let chunks = read_extensible_array_chunks( @@ -1018,7 +1062,8 @@ mod tests { fn read_element_unallocated() { let data = vec![0xFFu8; 16]; let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); - let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap(); + let (info, consumed) = + read_element(&Window::whole(&data), 0, 0, 8, 8, 80, 0, &grid).unwrap(); assert!(info.is_none()); assert_eq!(consumed, 8); } @@ -1038,8 +1083,17 @@ mod tests { data[12..16].copy_from_slice(&0u32.to_le_bytes()); let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); - let (info, consumed) = - read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap(); + let (info, consumed) = read_element( + &Window::whole(&data), + 0, + 1, + elem_size as u8, + os, + 80, + 2, + &grid, + ) + .unwrap(); let ci = info.unwrap(); assert_eq!(ci.address, 0x2000); assert_eq!(ci.chunk_size, 120); @@ -1047,4 +1101,38 @@ mod tests { assert_eq!(ci.offsets, vec![20]); assert_eq!(consumed, elem_size); } + + /// The Storage path reads exactly what the slice path reads: the array + /// whole, cut at every length through its structures, and with a byte + /// damaged in each of them, through a read_at-only CountingStorage. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + let full = build_inline_plus_data_blocks(); + let mut files = Vec::new(); + for cut in 0x100..0x340 { + files.push(full[..cut].to_vec()); + } + for at in [0x104, 0x150, 0x204, 0x216, 0x230, 0x304, 0x318] { + let mut damaged = full.clone(); + damaged[at] ^= 1; + files.push(damaged); + } + files.push(full); + let mut compared = 0; + for f in files { + let storage = CountingStorage::new(f.clone()); + let want = ExtensibleArrayHeader::parse(&f, 0x100, 8, 8); + let got = ExtensibleArrayHeader::parse_in(&storage, 0x100, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + let Ok(h) = want else { continue }; + for dims in [&[40u64][..], &[25]] { + let want = read_extensible_array_chunks(&f, &h, dims, None, &[10], 8, 8, 8); + let got = read_extensible_array_chunks_in(&storage, &h, dims, None, &[10], 8, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len()); + compared += 1; + } + } + assert!(compared > 100); + } } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 4608b2d..c0a1c37 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -240,6 +240,16 @@ impl<'a> Window<'a> { }) } + /// A whole in-memory file as one window (base 0). + #[cfg(test)] + pub fn whole(bytes: &'a [u8]) -> Self { + Window { + bytes: Cow::Borrowed(bytes), + base: 0, + file_len: bytes.len(), + } + } + /// Check that `[rel, rel + needed)` (relative to `base`) is in the file. pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> { match rel.checked_add(needed) { -- 2.54.0 From ba3f476be69a98daea0a52dd5ce60edf4d6118fa Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:02:58 -0500 Subject: [PATCH 16/48] format: resolve shared messages over Storage Every shared-message entry point has an *_in(&dyn Storage, ..) core: message_data(_with_sohm), resolve_shared_message(_with_sohm), resolve_sohm_message, load_sohm_table, and the SMTB/SMLI parsers, which read the signature and then all entries in one bounded read (the list's open-ended &file_data[pos..] slice is gone). Object headers and the SOHM fractal heap are read through their Storage parsers; a SOHM B-tree index still needs the v2 B-tree over a slice, so over a backend without one it is a clean ContiguousStorageRequired error. New signature::find_signature_in probes the candidate offsets with 8-byte reads. The &[u8] functions are wrappers. New tests: SOHM tables and lists with 4- and 8-byte offsets, at two offsets, cut at every length and with a bad signature, parse identically through a read_at-only CountingStorage in at most two reads; the signature search matches the slice search. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/shared_message.rs | 256 ++++++++++++++++--- crates/clawhdf5-format/src/signature.rs | 37 +++ 2 files changed, 264 insertions(+), 29 deletions(-) diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 9667926..5c8b1cb 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -28,6 +28,7 @@ use crate::error::FormatError; use crate::fractal_heap::FractalHeapHeader; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; +use crate::storage::{Storage, Window, read_exact_at, require_contiguous}; /// Fractal heap ID length for SOHM entries (fixed at 8 bytes). const FHEAP_ID_LEN: usize = 8; @@ -253,17 +254,31 @@ pub fn parse_sohm_table( nindexes: u8, offset_size: u8, ) -> Result { - ensure_len(file_data, table_addr, 4)?; - if &file_data[table_addr..table_addr + 4] != b"SMTB" { + parse_sohm_table_in(&file_data, table_addr as u64, nindexes, offset_size) +} + +/// [`parse_sohm_table`] over any [`Storage`]: one read of the signature, +/// one of every index entry. +pub fn parse_sohm_table_in( + file: &dyn Storage, + table_addr: u64, + nindexes: u8, + offset_size: u8, +) -> Result { + let sig = read_exact_at(file, table_addr, 4)?; + if *sig != *b"SMTB" { return Err(FormatError::InvalidSohmTableSignature); } - let mut pos = table_addr + 4; let os = offset_size as usize; let entry_size = 1 + 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 14 + 2*offset_size + // Positions below are relative to the table. + let w = Window::read(file, table_addr, 4 + nindexes as usize * entry_size)?; + let file_data: &[u8] = &w.bytes; + let mut pos = 4; let mut indexes = Vec::with_capacity(nindexes as usize); for _ in 0..nindexes { - ensure_len(file_data, pos, entry_size)?; + w.ensure(pos, entry_size)?; let version = file_data[pos]; if version != 0 { return Err(FormatError::InvalidSohmTableVersion(version)); @@ -369,16 +384,29 @@ pub fn parse_sohm_list( num_messages: u16, offset_size: u8, ) -> Result, FormatError> { - ensure_len(file_data, list_addr, 4)?; - if &file_data[list_addr..list_addr + 4] != b"SMLI" { + parse_sohm_list_in(&file_data, list_addr as u64, num_messages, offset_size) +} + +/// [`parse_sohm_list`] over any [`Storage`]: one read of the signature, one +/// of every entry. +pub fn parse_sohm_list_in( + file: &dyn Storage, + list_addr: u64, + num_messages: u16, + offset_size: u8, +) -> Result, FormatError> { + let sig = read_exact_at(file, list_addr, 4)?; + if *sig != *b"SMLI" { return Err(FormatError::InvalidSohmListSignature); } let entry_sz = sohm_entry_size(offset_size); - let mut pos = list_addr + 4; + // Positions below are relative to the list. + let w = Window::read(file, list_addr, 4 + num_messages as usize * entry_sz)?; + let mut pos = 4; let mut entries = Vec::with_capacity(num_messages as usize); for _ in 0..num_messages { - ensure_len(file_data, pos, entry_sz)?; - let entry = parse_sohm_entry(&file_data[pos..], offset_size)?; + w.ensure(pos, entry_sz)?; + let entry = parse_sohm_entry(&w.bytes[pos..], offset_size)?; entries.push(entry); pos += entry_sz; } @@ -392,6 +420,20 @@ pub fn parse_sohm_btree_entries( offset_size: u8, length_size: u8, ) -> Result, FormatError> { + parse_sohm_btree_entries_in(&file_data, btree_addr as u64, offset_size, length_size) +} + +/// [`parse_sohm_btree_entries`] over any [`Storage`]. The v2 B-tree is not +/// read over [`Storage`] yet, so this needs the whole file in memory +/// ([`FormatError::ContiguousStorageRequired`] otherwise). +pub fn parse_sohm_btree_entries_in( + file: &dyn Storage, + btree_addr: u64, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let file_data = require_contiguous(file, "a shared-message B-tree index")?; + let btree_addr = usize::try_from(btree_addr).unwrap_or(usize::MAX); let header = BTreeV2Header::parse(file_data, btree_addr, offset_size, length_size)?; let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?; let mut entries = Vec::with_capacity(records.len()); @@ -413,15 +455,24 @@ pub fn load_sohm_table( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - let sig = crate::signature::find_signature(file_data)?; - let sb = crate::superblock::Superblock::parse(file_data, sig)?; + load_sohm_table_in(&file_data, offset_size, length_size) +} + +/// [`load_sohm_table`] over any [`Storage`]. +pub fn load_sohm_table_in( + file_data: &dyn Storage, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let sig = crate::signature::find_signature_in(file_data)?; + let sb = crate::superblock::Superblock::parse_in(file_data, sig)?; let Some(ext_addr) = sb .superblock_extension_address .filter(|&a| !is_undefined(a, offset_size)) else { return Ok(None); }; - let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?; + let ext = ObjectHeader::parse_in(file_data, ext_addr, offset_size, length_size)?; let Some(msg) = ext .messages .iter() @@ -430,9 +481,9 @@ pub fn load_sohm_table( return Ok(None); }; let table_msg = parse_sohm_table_message(&msg.data, offset_size)?; - parse_sohm_table( + parse_sohm_table_in( file_data, - table_msg.table_address as usize, + table_msg.table_address, table_msg.nindexes, offset_size, ) @@ -446,17 +497,27 @@ pub fn message_data_with_sohm<'a>( msg: &'a crate::object_header::HeaderMessage, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + message_data_with_sohm_in(&file_data, msg, offset_size, length_size) +} + +/// [`message_data_with_sohm`] over any [`Storage`]. +pub fn message_data_with_sohm_in<'a>( + file_data: &dyn Storage, + msg: &'a crate::object_header::HeaderMessage, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { if !is_shared(msg.flags) { return Ok(Cow::Borrowed(&msg.data)); } let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?; let table = if shared_ref.heap_id.is_some() { - load_sohm_table(file_data, offset_size, length_size)? + load_sohm_table_in(file_data, offset_size, length_size)? } else { None }; - resolve_shared_message_with_sohm( + resolve_shared_message_with_sohm_in( file_data, &shared_ref, msg.msg_type, @@ -495,6 +556,25 @@ pub fn resolve_sohm_message( target_msg_type: MessageType, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + resolve_sohm_message_in( + &file_data, + heap_id, + sohm_table, + target_msg_type, + offset_size, + length_size, + ) +} + +/// [`resolve_sohm_message`] over any [`Storage`]. +pub fn resolve_sohm_message_in( + file_data: &dyn Storage, + heap_id: &[u8; FHEAP_ID_LEN], + sohm_table: &SohmTable, + target_msg_type: MessageType, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let index = find_index_for_msg_type(sohm_table, target_msg_type) .ok_or(FormatError::InvalidSharedMessageVersion(2))?; @@ -503,13 +583,9 @@ pub fn resolve_sohm_message( return Err(FormatError::InvalidSharedMessageVersion(2)); } - let fh_header = FractalHeapHeader::parse( - file_data, - index.heap_addr as usize, - offset_size, - length_size, - )?; - fh_header.read_managed_object(file_data, heap_id, offset_size) + let fh_header = + FractalHeapHeader::parse_in(file_data, index.heap_addr, offset_size, length_size)?; + fh_header.read_managed_object_in(file_data, heap_id, offset_size) } /// The payload of an object-header message, following the indirection if the @@ -526,12 +602,22 @@ pub fn message_data<'a>( msg: &'a crate::object_header::HeaderMessage, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + message_data_in(&file_data, msg, offset_size, length_size) +} + +/// [`message_data`] over any [`Storage`]. +pub fn message_data_in<'a>( + file_data: &dyn Storage, + msg: &'a crate::object_header::HeaderMessage, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { if !is_shared(msg.flags) { return Ok(Cow::Borrowed(&msg.data)); } let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?; - resolve_shared_message( + resolve_shared_message_in( file_data, &shared_ref, msg.msg_type, @@ -553,13 +639,30 @@ pub fn resolve_shared_message( target_msg_type: MessageType, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + resolve_shared_message_in( + &file_data, + shared_ref, + target_msg_type, + offset_size, + length_size, + ) +} + +/// [`resolve_shared_message`] over any [`Storage`]. +pub fn resolve_shared_message_in( + file_data: &dyn Storage, + shared_ref: &SharedMessageRef, + target_msg_type: MessageType, + offset_size: u8, + length_size: u8, ) -> Result, FormatError> { let table = if shared_ref.heap_id.is_some() { - load_sohm_table(file_data, offset_size, length_size)? + load_sohm_table_in(file_data, offset_size, length_size)? } else { None }; - resolve_shared_message_with_sohm( + resolve_shared_message_with_sohm_in( file_data, shared_ref, target_msg_type, @@ -577,6 +680,25 @@ pub fn resolve_shared_message_with_sohm( offset_size: u8, length_size: u8, sohm_table: Option<&SohmTable>, +) -> Result, FormatError> { + resolve_shared_message_with_sohm_in( + &file_data, + shared_ref, + target_msg_type, + offset_size, + length_size, + sohm_table, + ) +} + +/// [`resolve_shared_message_with_sohm`] over any [`Storage`]. +pub fn resolve_shared_message_with_sohm_in( + file_data: &dyn Storage, + shared_ref: &SharedMessageRef, + target_msg_type: MessageType, + offset_size: u8, + length_size: u8, + sohm_table: Option<&SohmTable>, ) -> Result, FormatError> { // Dispatch on what the reference carries rather than on `ref_type`: v1/v2 // references are always an object-header address whatever their type @@ -586,8 +708,7 @@ pub fn resolve_shared_message_with_sohm( shared_ref.heap_id.as_ref(), ) { (Some(addr), _) => { - let target_header = - ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?; + let target_header = ObjectHeader::parse_in(file_data, addr, offset_size, length_size)?; for msg in &target_header.messages { if msg.msg_type == target_msg_type && !is_shared(msg.flags) { return Ok(msg.data.clone()); @@ -614,7 +735,7 @@ pub fn resolve_shared_message_with_sohm( } (None, Some(heap_id)) => { let table = sohm_table.ok_or(FormatError::InvalidSharedMessageVersion(2))?; - resolve_sohm_message( + resolve_sohm_message_in( file_data, heap_id, table, @@ -1052,4 +1173,81 @@ mod tests { // With 2-byte offsets: OH=2+2=4, heap=12, entry=1+4+12=17 assert_eq!(sohm_entry_size(2), 17); } + + /// SOHM tables and lists parse identically through a read_at-only + /// CountingStorage: at two offsets, with 4- and 8-byte offsets, cut at + /// every length and with a bad signature. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + let idx = |t: u8, n: u16| SohmIndex { + index_type: t, + mesg_types: 0x0008, + min_mesg_size: 50, + list_max: 50, + btree_min: 40, + num_messages: n, + index_addr: 0x3000, + heap_addr: 0x4000, + }; + let heap_entry = |h: u32| SohmEntry { + location: 0, + hash: h, + heap_id: Some([1, 2, 3, 4, 5, 6, 7, h as u8]), + ref_count: Some(h), + mesg_index: None, + oh_addr: None, + }; + let oh_entry = SohmEntry { + location: 1, + hash: 9, + heap_id: None, + ref_count: None, + mesg_index: Some(3), + oh_addr: Some(0x7000), + }; + let mut compared = 0; + for os in [4u8, 8] { + let smtb = build_smtb(&[idx(0, 2), idx(1, 7)], os); + let smli = build_smli(&[heap_entry(1), oh_entry.clone(), heap_entry(2)], os); + for (body, n) in [(smtb, 2u16), (smli, 3)] { + let is_table = &body[..4] == b"SMTB"; + for at in [0usize, 0x40] { + let mut full = vec![0u8; at]; + full.extend_from_slice(&body); + let mut files = Vec::new(); + for cut in at..=full.len() { + files.push(full[..cut].to_vec()); + } + let mut bad = full.clone(); + bad[at] = b'X'; + files.push(bad); + for f in files { + let st = CountingStorage::new(f.clone()); + let (want, got) = if is_table { + ( + format!("{:?}", parse_sohm_table(&f, at, n as u8, os)), + format!("{:?}", parse_sohm_table_in(&st, at as u64, n as u8, os)), + ) + } else { + ( + format!("{:?}", parse_sohm_list(&f, at, n, os)), + format!("{:?}", parse_sohm_list_in(&st, at as u64, n, os)), + ) + }; + assert_eq!(got, want, "{} bytes", f.len()); + assert!(st.reads() <= 2); + compared += 1; + } + } + } + } + assert!(compared > 200); + // The B-tree index is not read over Storage yet: a clean error. + let st = CountingStorage::new(vec![0u8; 64]); + assert_eq!( + parse_sohm_btree_entries_in(&st, 0, 8, 8).unwrap_err(), + FormatError::ContiguousStorageRequired("a shared-message B-tree index") + ); + } } diff --git a/crates/clawhdf5-format/src/signature.rs b/crates/clawhdf5-format/src/signature.rs index 600b650..4b152ee 100644 --- a/crates/clawhdf5-format/src/signature.rs +++ b/crates/clawhdf5-format/src/signature.rs @@ -1,6 +1,7 @@ //! HDF5 file signature (magic bytes) detection. use crate::error::FormatError; +use crate::storage::{Storage, read_exact_at}; /// The 8-byte HDF5 magic signature. pub const HDF5_SIGNATURE: [u8; 8] = [0x89, b'H', b'D', b'F', b'\r', b'\n', 0x1A, b'\n']; @@ -39,6 +40,20 @@ pub fn find_signature(data: &[u8]) -> Result { Err(FormatError::SignatureNotFound) } +/// [`find_signature`] over any [`Storage`]: one 8-byte read per candidate +/// offset. +pub fn find_signature_in(file: &dyn Storage) -> Result { + let len = file.len(); + let mut offset = 0u64; + while offset.checked_add(8).is_some_and(|end| end <= len) { + if *read_exact_at(file, offset, 8)? == HDF5_SIGNATURE { + return Ok(offset); + } + offset = if offset == 0 { 512 } else { offset * 2 }; + } + Err(FormatError::SignatureNotFound) +} + /// Split a file into its user block and its HDF5 bytes. /// /// Returns `(user_block, hdf5)`: `user_block` is everything before the @@ -132,4 +147,26 @@ mod tests { data[512..520].copy_from_slice(&HDF5_SIGNATURE); assert_eq!(find_signature(&data), Ok(0)); } + + #[test] + fn find_signature_in_matches_slice_search() { + use crate::storage::CountingStorage; + for (len, at) in [ + (0, None), + (7, None), + (8, Some(0)), + (600, Some(512)), + (5000, Some(4096)), + (3000, Some(2048)), + (3000, None), + ] { + let mut data = vec![0u8; len]; + if let Some(at) = at { + data[at..at + 8].copy_from_slice(&HDF5_SIGNATURE); + } + let want = find_signature(&data).map(|o| o as u64); + let got = find_signature_in(&CountingStorage::new(data)); + assert_eq!(got, want, "{len} {at:?}"); + } + } } -- 2.54.0 From 2c292404d2305528e8ab7448ccec9278e346fb73 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:05:48 -0500 Subject: [PATCH 17/48] format: read attributes over Storage AttributeMessage::parse_in_storage, extract_attributes_full_in and extract_attributes_tolerant_in take the file as &dyn Storage: shared datatypes, dataspaces and attributes are resolved through the Storage shared-message path, and dense attributes' fractal heap through FractalHeapHeader::parse_in / read_managed_object_in. The dense-storage name index is a v2 B-tree, which is not read over Storage yet: over a backend without the whole file in memory it is a clean ContiguousStorageRequired error, never a partial list. The &[u8] functions are wrappers. New test: every object in five h5py-written fixtures (compact, shared and dense attributes) reads identically through a slice as Storage, and through a read_at-only CountingStorage except the dense ones, which give the clean error. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/attribute.rs | 144 +++++++++++++++++++++--- 1 file changed, 127 insertions(+), 17 deletions(-) diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 6c4e9ae..815719e 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -15,6 +15,7 @@ use crate::fractal_heap::FractalHeapHeader; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; use crate::shared_message; +use crate::storage::{Storage, require_contiguous}; use crate::vl_data; /// A parsed HDF5 attribute message. @@ -65,13 +66,24 @@ impl AttributeMessage { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_impl(data, length_size, Some((file_data, offset_size))) + Self::parse_in_storage(data, &file_data, offset_size, length_size) + } + + /// [`AttributeMessage::parse_in_file`] with the file behind any + /// [`Storage`]. + pub fn parse_in_storage( + data: &[u8], + file: &dyn Storage, + offset_size: u8, + length_size: u8, + ) -> Result { + Self::parse_impl(data, length_size, Some((file, offset_size))) } fn parse_impl( data: &[u8], length_size: u8, - file: Option<(&[u8], u8)>, + file: Option<(&dyn Storage, u8)>, ) -> Result { ensure_len(data, 0, 2)?; let version = data[0]; @@ -91,14 +103,14 @@ impl AttributeMessage { shared: bool, msg_type: MessageType, length_size: u8, - file: Option<(&[u8], u8)>, + file: Option<(&dyn Storage, u8)>, ) -> Result, FormatError> { if !shared { return Ok(Cow::Borrowed(bytes)); } let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?; let shared_ref = shared_message::parse_shared_ref_sized(bytes, offset_size, length_size)?; - shared_message::resolve_shared_message( + shared_message::resolve_shared_message_in( file_data, &shared_ref, msg_type, @@ -146,7 +158,7 @@ impl AttributeMessage { fn parse_v2( data: &[u8], length_size: u8, - file: Option<(&[u8], u8)>, + file: Option<(&dyn Storage, u8)>, ) -> Result { // Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared. let flags = data.get(1).copied().unwrap_or(0); @@ -200,7 +212,7 @@ impl AttributeMessage { fn parse_v3( data: &[u8], length_size: u8, - file: Option<(&[u8], u8)>, + file: Option<(&dyn Storage, u8)>, ) -> Result { // Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared. let flags = data.get(1).copied().unwrap_or(0); @@ -415,7 +427,20 @@ pub fn extract_attributes_full( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - extract_attributes_with(file_data, header, offset_size, length_size, &mut Err) + extract_attributes_full_in(&file_data, header, offset_size, length_size) +} + +/// [`extract_attributes_full`] over any [`Storage`]. Dense attribute +/// storage is indexed by a v2 B-tree, which is not read over [`Storage`] +/// yet: on a backend without the whole file in memory an object with dense +/// attributes is [`FormatError::ContiguousStorageRequired`]. +pub fn extract_attributes_full_in( + file: &dyn Storage, + header: &ObjectHeader, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + extract_attributes_with(file, header, offset_size, length_size, &mut Err) } /// Like [`extract_attributes_full`], but an attribute that cannot be read @@ -431,6 +456,17 @@ pub fn extract_attributes_tolerant( header: &ObjectHeader, offset_size: u8, length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + extract_attributes_tolerant_in(&file_data, header, offset_size, length_size) +} + +/// [`extract_attributes_tolerant`] over any [`Storage`] (see +/// [`extract_attributes_full_in`] for dense storage). +pub fn extract_attributes_tolerant_in( + file_data: &dyn Storage, + header: &ObjectHeader, + offset_size: u8, + length_size: u8, ) -> Result<(Vec, Vec), FormatError> { let mut errors = Vec::new(); let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| { @@ -443,7 +479,7 @@ pub fn extract_attributes_tolerant( /// Read every attribute; each one that fails goes to `on_error`, which /// either stops the read (returns the error) or skips that attribute. fn extract_attributes_with( - file_data: &[u8], + file_data: &dyn Storage, header: &ObjectHeader, offset_size: u8, length_size: u8, @@ -460,7 +496,7 @@ fn extract_attributes_with( // Shared attribute: resolve the reference to get actual attribute data shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size) .and_then(|shared_ref| { - shared_message::resolve_shared_message( + shared_message::resolve_shared_message_in( file_data, &shared_ref, MessageType::Attribute, @@ -469,7 +505,7 @@ fn extract_attributes_with( ) }) .and_then(|resolved| { - AttributeMessage::parse_in_file( + AttributeMessage::parse_in_storage( &resolved, file_data, offset_size, @@ -477,7 +513,7 @@ fn extract_attributes_with( ) }) } else { - AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size) + AttributeMessage::parse_in_storage(&msg.data, file_data, offset_size, length_size) }; let attr = attr.and_then(|a| check_in_header(a, header)); match attr { @@ -537,7 +573,7 @@ fn find_attribute_info( /// each one's creation order into `orders`. #[allow(clippy::too_many_arguments)] fn extract_dense_attributes( - file_data: &[u8], + file_data: &dyn Storage, attr_info: &AttributeInfoMessage, fh_addr: u64, offset_size: u8, @@ -547,7 +583,7 @@ fn extract_dense_attributes( on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result<(), FormatError> { // Parse fractal heap - let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?; + let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?; // Parse B-tree v2 for name index (type 8) let btree_addr = attr_info @@ -556,8 +592,10 @@ fn extract_dense_attributes( expected: 1, available: 0, })?; - let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?; - let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; + let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?; + let btree_hdr = + BTreeV2Header::parse(contiguous, btree_addr as usize, offset_size, length_size)?; + let records = collect_btree_v2_records(contiguous, &btree_hdr, offset_size, length_size)?; for record in &records { // Per HDF5 spec, both type 8 and type 9 records start with heap_id: @@ -574,9 +612,9 @@ fn extract_dense_attributes( // The data in the heap is a complete attribute message let attr = fh - .read_managed_object(file_data, id_bytes, offset_size) + .read_managed_object_in(file_data, id_bytes, offset_size) .and_then(|attr_data| { - AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size) + AttributeMessage::parse_in_storage(&attr_data, file_data, offset_size, length_size) }); match attr { Ok(attr) => { @@ -986,4 +1024,76 @@ mod tests { let strs = attr.read_as_strings().unwrap(); assert_eq!(strs, vec!["abcd", "EFGH"]); } + + /// Every object's attributes in h5py-written files read identically + /// through a read_at-only CountingStorage — compact ones, shared ones + /// and those behind an Attribute Info message — except dense storage, + /// whose v2 B-tree index is not read over Storage yet: that is the clean + /// ContiguousStorageRequired error, never a partial list. Through a + /// slice as Storage every object matches. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + let files: [(&str, &[u8]); 5] = [ + ("attrs", include_bytes!("../tests/fixtures/attrs.h5")), + ( + "mixed_attrs", + include_bytes!("../tests/fixtures/mixed_attrs.h5"), + ), + ( + "dense_attrs", + include_bytes!("../tests/fixtures/dense_attrs.h5"), + ), + ( + "dense_attrs_root", + include_bytes!("../tests/fixtures/dense_attrs_root.h5"), + ), + ( + "shared_fill_value", + include_bytes!("../tests/fixtures/shared_fill_value.h5"), + ), + ]; + let (mut same, mut dense, mut attrs) = (0, 0, 0); + for (name, file) in files { + let sb = crate::superblock::Superblock::parse(file, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + let mut addrs = vec![sb.root_group_address]; + addrs.extend( + crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address) + .unwrap() + .iter() + .map(|e| e.object_header_address), + ); + let storage = CountingStorage::new(file.to_vec()); + for addr in addrs { + let header = ObjectHeader::parse(file, addr as usize, os, ls).unwrap(); + let want = extract_attributes_full(file, &header, os, ls); + let slice_storage = extract_attributes_full_in(&file, &header, os, ls); + assert_eq!(format!("{slice_storage:?}"), format!("{want:?}")); + let got = extract_attributes_full_in(&storage, &header, os, ls); + let got_t = extract_attributes_tolerant_in(&storage, &header, os, ls); + let is_dense = find_attribute_info(&header, os) + .unwrap() + .is_some_and(|i| i.fractal_heap_address.is_some()); + if is_dense { + let e = FormatError::ContiguousStorageRequired( + "dense attribute storage (a v2 B-tree)", + ); + assert_eq!(got.unwrap_err(), e, "{name}"); + assert_eq!(got_t.unwrap_err(), e, "{name}"); + dense += 1; + } else { + attrs += want.as_ref().map_or(0, Vec::len); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}"); + let want_t = extract_attributes_tolerant(file, &header, os, ls); + assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}"); + same += 1; + } + } + } + assert!( + same >= 5 && dense >= 2 && attrs >= 5, + "{same} {dense} {attrs}" + ); + } } -- 2.54.0 From a65a2b7f18856190c76d320ed9a414a8dbf38c57 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:06:12 -0500 Subject: [PATCH 18/48] format: resolve fill values over Storage dataset_fill_value_in now takes the file as any Storage (generic, so every &[u8] caller compiles unchanged) and follows a shared fill value message through message_data_with_sohm_in. The raw-data helpers in the module (read_full_with_fill, apply_to_unallocated_chunks) walk chunk indexes and stay on &[u8] until milestone M2. New test: the four datasets of shared_fill_value.h5, two with their fill value in the SOHM heap, resolve identically through a read_at-only CountingStorage. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/fill_value.rs | 44 ++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5-format/src/fill_value.rs b/crates/clawhdf5-format/src/fill_value.rs index 3b18790..88cbc3f 100644 --- a/crates/clawhdf5-format/src/fill_value.rs +++ b/crates/clawhdf5-format/src/fill_value.rs @@ -111,14 +111,19 @@ pub fn dataset_fill_value(messages: &[HeaderMessage]) -> Result>, /// fill value message to where it lives: another object header, or the /// file's shared-message (SOHM) heap, as libhdf5 writes it when the file has /// a SOHM index for fill values. -pub fn dataset_fill_value_in( - file_data: &[u8], +/// +/// `file_data` is any [`Storage`](crate::storage::Storage): the file as a +/// `&[u8]`, or a backend that serves it by range. (The trait is not +/// imported here: its `len` would shadow the slice method in this module.) +pub fn dataset_fill_value_in( + file_data: &S, messages: &[HeaderMessage], offset_size: u8, length_size: u8, ) -> Result>, FormatError> { + let file: &dyn crate::storage::Storage = &file_data; fill_value_from(messages, |msg| { - crate::shared_message::message_data_with_sohm(file_data, msg, offset_size, length_size) + crate::shared_message::message_data_with_sohm_in(file, msg, offset_size, length_size) .map(|data| data.into_owned()) }) } @@ -439,4 +444,37 @@ mod tests { .collect(); assert_eq!(filled, [2, 3, 7, 8]); } + + /// Fill values, shared ones in the SOHM heap included, resolve + /// identically through a read_at-only CountingStorage. + #[test] + fn storage_reads_match_slice_reads() { + use crate::object_header::ObjectHeader; + use crate::storage::CountingStorage; + let file: &[u8] = include_bytes!("../tests/fixtures/shared_fill_value.h5"); + let sb = crate::superblock::Superblock::parse(file, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + let storage = CountingStorage::new(file.to_vec()); + let mut shared = 0; + let children = + crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address).unwrap(); + assert!(children.len() >= 3); + for child in children { + let h = + ObjectHeader::parse(file, child.object_header_address as usize, os, ls).unwrap(); + shared += h + .messages + .iter() + .filter(|m| { + m.msg_type == MessageType::FillValue + && crate::shared_message::is_shared(m.flags) + }) + .count(); + let want = dataset_fill_value_in(file, &h.messages, os, ls); + assert_eq!(want, Ok(Some((-7i32).to_le_bytes().to_vec()))); + let got = dataset_fill_value_in(&storage, &h.messages, os, ls); + assert_eq!(got, want, "{}", child.name); + } + assert!(shared >= 2); + } } -- 2.54.0 From 6a535e26511aff376e786556737d888641a60ba4 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:06:12 -0500 Subject: [PATCH 19/48] format: resolve virtual dataset mappings over Storage DataLayout::resolve_vds_mappings_in reads the global heap collection holding a virtual dataset's mappings through GlobalHeapCollection:: parse_in; resolve_vds_mappings is a wrapper. The rest of the data layout module parses message payloads and does not read the file. New test: the virtual dataset of vds_same_file.h5 resolves to the same mappings through a read_at-only CountingStorage, in two reads. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_layout.rs | 58 +++++++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 59a9065..480f459 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -7,6 +7,7 @@ use alloc::{format, string::String, vec::Vec}; use std::string::String; use crate::error::FormatError; +use crate::storage::Storage; /// A single VDS (Virtual Dataset) source mapping. /// @@ -309,6 +310,16 @@ impl DataLayout { &mut self, file_data: &[u8], length_size: u8, + ) -> Result<(), FormatError> { + self.resolve_vds_mappings_in(&file_data, length_size) + } + + /// [`Self::resolve_vds_mappings`] over any [`Storage`]: one read of the + /// global heap collection holding the mappings. + pub fn resolve_vds_mappings_in( + &mut self, + file_data: &dyn Storage, + length_size: u8, ) -> Result<(), FormatError> { if let DataLayout::Virtual { global_heap_address, @@ -318,11 +329,8 @@ impl DataLayout { } = self && let Some(addr) = *global_heap_address { - let coll = crate::global_heap::GlobalHeapCollection::parse( - file_data, - addr as usize, - length_size, - )?; + let coll = + crate::global_heap::GlobalHeapCollection::parse_in(file_data, addr, length_size)?; let obj = coll.get_object(*global_heap_index as u16).ok_or( FormatError::GlobalHeapObjectNotFound { collection_address: addr, @@ -1305,4 +1313,44 @@ mod tests { let blob = [0x01u8, 0, 0, 0, 0, 0, 0, 0, 0]; assert!(parse_vds_mappings(&blob, 8).unwrap().is_empty()); } + + /// A virtual dataset's mappings resolve identically through a + /// read_at-only CountingStorage, in two reads of the global heap. + #[test] + fn vds_mappings_through_storage_match_slice() { + use crate::message_type::MessageType; + use crate::object_header::ObjectHeader; + use crate::storage::CountingStorage; + let file: &[u8] = include_bytes!("../tests/fixtures/vds_same_file.h5"); + let sb = crate::superblock::Superblock::parse(file, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + let storage = CountingStorage::new(file.to_vec()); + let mut virtuals = 0; + for child in + crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address).unwrap() + { + let h = + ObjectHeader::parse(file, child.object_header_address as usize, os, ls).unwrap(); + let Some(msg) = h + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + else { + continue; + }; + let mut want = DataLayout::parse(&msg.data, os, ls).unwrap(); + if !matches!(want, DataLayout::Virtual { .. }) { + continue; + } + let mut got = want.clone(); + want.resolve_vds_mappings(file, ls).unwrap(); + storage.reset(); + got.resolve_vds_mappings_in(&storage, ls).unwrap(); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + assert!(matches!(&got, DataLayout::Virtual { mappings, .. } if !mappings.is_empty())); + assert_eq!(storage.reads(), 2); + virtuals += 1; + } + assert!(virtuals >= 1); + } } -- 2.54.0 From 2b7065998adf1a552541a8ec1df6c391b481a1bd Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:08:47 -0500 Subject: [PATCH 20/48] docs: ZFP reads (changelog, README feature table, known issues) ZFP (32013) was the one plugin filter still listed as UnsupportedFilter. Conformance on tank, `conformance/run.sh --no-fetch` (2026-09-26): 600 of 697 files ok (baseline 599); h5ex_d_zfp.h5 is newly ok. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 24 ++++++++++++++++++++++++ CLAUDE.md | 2 +- README.md | 10 +++++----- docs/known-issues.md | 12 ++++++++---- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f4316..badf440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ ## Unreleased +### ZFP (2026-09-26) +- **ZFP (filter 32013, H5Z-ZFP) reads, in pure Rust.** It was the last + filter in the conformance corpus that failed with `UnsupportedFilter`. + New feature `zfp` (`clawhdf5-format` and `clawhdf5`, included in + `plugin-filters`, no dependencies) ports the zfp 1.0.1 decoder and the + decompression half of H5Z-ZFP 1.1.1: every mode (fixed rate, fixed + precision, fixed accuracy, reversible, expert), int32, int64, float and + double, 1-4-D fields with partial blocks, and headers written by + big-endian machines (values byte-swapped as H5Z-ZFP does). Read only: + there is no ZFP encoder. The decoder is deterministic, so lossy modes have + one right answer, and clawhdf5 returns exactly libzfp's values. +- Tests: `crates/clawhdf5/tests/zfp_interop.rs` has h5py + hdf5plugin 7.1 + (H5Z-ZFP 1.1.1, zfp 1.0.1) write 2205 datasets over 16 mode settings + (including expert parameters at their edges) x the four types x 1-4-D + shapes with partial edge chunks, partial blocks and unit chunk + dimensions x smooth, noisy, wide-range, zero and inf/NaN data; each must + read byte for byte as h5py reads it. `tests/zfp_alloc_bounds.rs` fuzzes + headers and streams under a counting allocator: no panics, and the output + is allocated only when it matches the chunk's size and the stream holds at + least a bit per block. A stream that ends before the decoder is done is an + error (libzfp reads past its buffer). +- Conformance on tank (2026-09-26, `conformance/run.sh --no-fetch`): 600 of + 697 files ok (599 before); `h5ex_d_zfp.h5` now reads. + ### Chunked full reads (2026-09-26) - **Chunks are decoded straight into the output, into reused buffers.** A full read of a chunked dataset faulted in about three times its size in diff --git a/CLAUDE.md b/CLAUDE.md index b029c5d..cf9f9ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F |-------|------| | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants | | `clawhdf5-io` | Read/write implementation | -| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1, and Blosc2 read-only) live in `clawhdf5-format`. No ZFP. | +| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1, and Blosc2 and ZFP read-only) live in `clawhdf5-format`. | | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs | | `clawhdf5` | Main facade crate | | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer | diff --git a/README.md b/README.md index 25f43a7..f74fc71 100644 --- a/README.md +++ b/README.md @@ -780,14 +780,14 @@ stores keep their setting. Opt out with `float16 = false` or | `bzip2` | no | bzip2 filter (id 307): read and write. Pure Rust (the `bzip2` crate's libbz2-rs-sys backend compiles no C) | | `blosc` | no | Blosc 1 filter (id 32001): reads BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard frames with byte or bit shuffle; writes LZ4, Snappy, Zlib or Zstandard (not BloscLZ). Pure Rust | | `blosc2` | no | Blosc2 filter (id 32026), read only: hdf5plugin's frames and B2ND (n-D) chunks, BloscLZ, LZ4/LZ4HC, Zlib and Zstandard, with shuffle, bit shuffle, delta or truncated precision. Pure Rust | -| `plugin-filters` | no | All five above | +| `zfp` | no | ZFP filter (id 32013, H5Z-ZFP), read only: every mode (rate, precision, accuracy, reversible, expert) for int32, int64, float and double, 1-4-D, returning exactly libzfp's values. Pure Rust, no dependencies | +| `plugin-filters` | no | All six above | -ZFP (32013) is not implemented: reading it fails with `UnsupportedFilter`, -whose message names the filter. clawhdf5 cannot write Blosc2. Any other +clawhdf5 cannot write Blosc2 or ZFP. Any other filter can be supplied at run time with `filter_registry::register_filter` (a decoder closure, or a `FilterCodec` that also encodes). The facade -(`clawhdf5`) forwards `lzf`, `bitshuffle`, `bzip2`, `blosc`, `blosc2` and -`plugin-filters`. Write +(`clawhdf5`) forwards `lzf`, `bitshuffle`, `bzip2`, `blosc`, `blosc2`, `zfp` +and `plugin-filters`. Write with `DatasetBuilder::with_lzf()`, `with_bitshuffle(..)`, `with_bzip2(..)` and `with_blosc(..)`; h5py + hdf5plugin read the result (tested both ways in `crates/clawhdf5/tests/plugin_filters_interop.rs`). The pure-Rust Zstandard diff --git a/docs/known-issues.md b/docs/known-issues.md index 287e501..4cbd25b 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -293,10 +293,14 @@ fill-value item that did is fixed). 697 ok, tank, `conformance/run.sh --no-fetch`). Blosc2 frames using dictionaries, lazy chunks, variable-length blocks, user-defined codecs or registered filters (e.g. bytedelta) are refused with an error. - **Still open:** ZFP (32013) fails with an `UnsupportedFilter` error that - names the filter, and can be plugged in with - `filter_registry::register_filter` (32023, Granular BitRound, too, since - 2026-09-26 even with the `pcodec` feature). clawhdf5 cannot write Blosc2. + **Fixed 2026-09-26** for ZFP (32013, `zfp` feature, also in + `plugin-filters`), read only: every H5Z-ZFP mode and type, bit-exact + against h5py + hdf5plugin 7.1 (`crates/clawhdf5/tests/zfp_interop.rs`); + h5ex_d_zfp now reads (conformance 600 of 697 ok, tank, + `conformance/run.sh --no-fetch`). **Still open:** clawhdf5 cannot write + Blosc2 or ZFP. Other filters (32023, Granular BitRound, too, since + 2026-09-26 even with the `pcodec` feature) can be plugged in with + `filter_registry::register_filter`. - **Wrong data: a chunk whose filters decode to fewer bytes than the chunk read with zeros for the missing bytes** (any filter; found reviewing the plugin filters). **Fixed 2026-09-26:** it is an error naming the chunk. A corrupt -- 2.54.0 From 5705866d40d64c9e64081cae008b9eff5e854544 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:09:32 -0500 Subject: [PATCH 21/48] format: equivalence harness for the Storage migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/storage_equivalence.rs walks real files and runs every metadata parser converted to Storage twice per object — over the file as a slice and over a read_at-only CountingStorage — and requires identical results, values and errors alike; the only allowed difference is the clean ContiguousStorageRequired from the structures still indexed by a v2 B-tree (dense attributes, a SOHM B-tree index), which is counted. It covers superblock and extension, cache image, SOHM table/list/B-tree, object headers, attributes, fill values, shared messages, symbol-table groups (local heap, B-tree, nodes, names), fractal heaps and their objects, VDS mappings, and fixed/extensible array chunk indexes. Inputs: every fixture; files h5py writes for what the fixtures lack (extensible arrays with super blocks and paged data blocks, paged fixed arrays, a 400-group v1 file with a user block, a 300-link dense group, dense, shared and committed-type attributes, SOHM list and B-tree indexes; honours CLAWHDF5_PYTHON / CLAWHDF5_REQUIRE_INTEROP); and, with CLAWHDF5_STORAGE_CORPUS set, a corpus such as conformance/.cache/corpus. Milestones M2/M3 add their parsers to check_object. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/storage_equivalence.rs | 610 ++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 crates/clawhdf5-format/tests/storage_equivalence.rs diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs new file mode 100644 index 0000000..a046ffb --- /dev/null +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -0,0 +1,610 @@ +//! Equivalence harness for the range-read migration +//! (`docs/design/range-reads.md`, milestone M1). +//! +//! Every metadata parser converted to [`Storage`] must give exactly what its +//! `&[u8]` form gives. This walks real files — the fixtures, files h5py +//! writes to exercise the less common structures, and optionally the +//! conformance corpus — and, for every object, runs each converted parser +//! twice: over the file as a slice, and over a [`CountingStorage`] that +//! serves the same bytes through `read_at` only (`as_contiguous()` is +//! `None`, so no parser can fall back to the whole slice). The results must +//! be identical, value for value and error for error. +//! +//! The one allowed difference is [`FormatError::ContiguousStorageRequired`] +//! from the storage path: the structures still indexed by a v2 B-tree (dense +//! attributes, a SOHM B-tree index, huge fractal-heap objects), which fail +//! cleanly instead of reading the whole file. Those are counted. +//! +//! Milestones M2/M3 extend `check_object` with the raw-data and group +//! parsers as they are converted. +//! +//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/ +//! `.nc`/`.h5ad` file under those directories (the conformance corpus is +//! `conformance/.cache/corpus`); `CLAWHDF5_STORAGE_REPORT=1` prints the +//! per-file read counts. +//! - The h5py-written files honour `CLAWHDF5_PYTHON` and +//! `CLAWHDF5_REQUIRE_INTEROP` like the facade's interop tests. + +use std::collections::{HashSet, VecDeque}; +use std::fmt::Debug; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use clawhdf5_format::attribute::{ + extract_attributes_full, extract_attributes_full_in, extract_attributes_tolerant, + extract_attributes_tolerant_in, +}; +use clawhdf5_format::attribute_info::AttributeInfoMessage; +use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in}; +use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::dataspace::Dataspace; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::extensible_array::{ + ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in, +}; +use clawhdf5_format::fill_value::dataset_fill_value_in; +use clawhdf5_format::fixed_array::{ + FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in, +}; +use clawhdf5_format::fractal_heap::FractalHeapHeader; +use clawhdf5_format::link_info::LinkInfoMessage; +use clawhdf5_format::local_heap::LocalHeap; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::shared_message::{ + self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in, + parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in, +}; +use clawhdf5_format::signature::split_user_block; +use clawhdf5_format::storage::{CountingStorage, Storage}; +use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext::{ + cache_image_state, cache_image_state_in, read_superblock_extension, + read_superblock_extension_in, +}; +use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode}; + +/// Objects visited per file, heap objects read per heap: enough to cover +/// every structure kind while keeping a 35 000-group file fast. +const MAX_OBJECTS: usize = 1500; +const MAX_HEAP_IDS: usize = 200; + +#[derive(Default, Debug)] +struct Tally { + files: usize, + objects: usize, + checks: usize, + contiguous_required: usize, + reads: u64, + bytes: u64, +} + +struct Walk<'a> { + slice: &'a [u8], + storage: &'a CountingStorage, + name: String, + tally: &'a mut Tally, +} + +impl Walk<'_> { + /// The storage result must equal the slice result, or be the clean + /// "needs the whole file" error. + fn same( + &mut self, + what: &str, + want: &Result, + got: &Result, + ) { + self.tally.checks += 1; + if let Err(FormatError::ContiguousStorageRequired(_)) = got { + self.tally.contiguous_required += 1; + return; + } + let (w, g) = (format!("{want:?}"), format!("{got:?}")); + assert!( + w == g, + "{}: {what} differs\n slice: {}\n storage: {}", + self.name, + &w[..w.len().min(600)], + &g[..g.len().min(600)] + ); + } + + fn st(&self) -> &dyn Storage { + self.storage + } + + fn run(&mut self) { + let slice = self.slice; + let sb = Superblock::parse(slice, 0); + self.same("superblock", &sb, &Superblock::parse_in(self.st(), 0)); + let Ok(sb) = sb else { return }; + let (os, ls) = (sb.offset_size, sb.length_size); + + let want = read_superblock_extension(slice, &sb); + self.same( + "superblock extension", + &want, + &read_superblock_extension_in(self.st(), &sb), + ); + let want = cache_image_state(slice, &sb); + self.same("cache image", &want, &cache_image_state_in(self.st(), &sb)); + let table = load_sohm_table(slice, os, ls); + self.same("SOHM table", &table, &load_sohm_table_in(self.st(), os, ls)); + if let Ok(Some(table)) = &table { + for idx in &table.indexes { + if idx.index_type == 0 { + let want = + parse_sohm_list(slice, idx.index_addr as usize, idx.num_messages, os); + let got = parse_sohm_list_in(self.st(), idx.index_addr, idx.num_messages, os); + self.same("SOHM list", &want, &got); + } else { + let want = parse_sohm_btree_entries(slice, idx.index_addr as usize, os, ls); + let got = parse_sohm_btree_entries_in(self.st(), idx.index_addr, os, ls); + self.same("SOHM B-tree", &want, &got); + } + } + } + + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([sb.root_group_address]); + while let Some(addr) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + self.tally.objects += 1; + self.check_object(&sb, addr); + // Traversal only (group lookups are milestone M0/M3 work). + if let Ok(children) = + clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr) + { + queue.extend(children.iter().map(|c| c.object_header_address)); + } + } + } + + fn check_object(&mut self, sb: &Superblock, addr: u64) { + let slice = self.slice; + let (os, ls) = (sb.offset_size, sb.length_size); + let header = ObjectHeader::parse(slice, addr as usize, os, ls); + self.same( + "object header", + &header, + &ObjectHeader::parse_in(self.st(), addr, os, ls), + ); + let Ok(header) = header else { return }; + + let want = extract_attributes_full(slice, &header, os, ls); + self.same( + "attributes", + &want, + &extract_attributes_full_in(self.st(), &header, os, ls), + ); + let want = extract_attributes_tolerant(slice, &header, os, ls); + let got = extract_attributes_tolerant_in(self.st(), &header, os, ls); + self.same("attributes (tolerant)", &want, &got); + let want = dataset_fill_value_in(slice, &header.messages, os, ls); + self.same( + "fill value", + &want, + &dataset_fill_value_in(self.storage, &header.messages, os, ls), + ); + + for msg in &header.messages { + if shared_message::is_shared(msg.flags) { + let want = message_data_with_sohm(slice, msg, os, ls); + let got = message_data_with_sohm_in(self.st(), msg, os, ls); + self.same("shared message", &want, &got); + } + match msg.msg_type { + MessageType::SymbolTable => { + if let Ok(stm) = SymbolTableMessage::parse(&msg.data, os) { + self.check_v1_group(&stm, os, ls); + } + } + MessageType::LinkInfo => { + if let Ok(li) = LinkInfoMessage::parse(&msg.data, os) { + self.check_heap( + li.fractal_heap_address, + li.btree_name_index_address, + 4, + os, + ls, + ); + } + } + MessageType::AttributeInfo => { + if let Ok(ai) = AttributeInfoMessage::parse(&msg.data, os) { + self.check_heap( + ai.fractal_heap_address, + ai.btree_name_index_address, + 0, + os, + ls, + ); + } + } + _ => {} + } + } + self.check_layout(&header, os, ls); + } + + /// A symbol-table group: its local heap, B-tree, nodes and names. + fn check_v1_group(&mut self, stm: &SymbolTableMessage, os: u8, ls: u8) { + let slice = self.slice; + let heap = LocalHeap::parse(slice, stm.local_heap_address as usize, os, ls); + self.same( + "local heap", + &heap, + &LocalHeap::parse_in(self.st(), stm.local_heap_address, os, ls), + ); + let nodes = collect_symbol_table_nodes(slice, stm.btree_address, os, ls); + let got = collect_symbol_table_nodes_in(self.st(), stm.btree_address, os, ls); + self.same("group B-tree", &nodes, &got); + let Ok(heap) = heap else { return }; + let want = heap.validate_free_list(slice, ls); + self.same( + "local heap free list", + &want, + &heap.validate_free_list_in(self.st(), ls), + ); + let Ok(nodes) = nodes else { return }; + for &node in nodes.iter().take(MAX_HEAP_IDS) { + let snod = SymbolTableNode::parse(slice, node as usize, os); + self.same( + "symbol table node", + &snod, + &SymbolTableNode::parse_in(self.st(), node, os), + ); + let Ok(snod) = snod else { continue }; + for e in &snod.entries { + let want = heap.read_string(slice, e.link_name_offset); + self.same( + "link name", + &want, + &heap.read_string_in(self.st(), e.link_name_offset), + ); + } + } + } + + /// A dense group's or dense attributes' fractal heap: the header, and + /// the objects its name index points at. `id_at` is where the heap ID + /// starts in a name-index record (after the hash for links). + fn check_heap(&mut self, heap: Option, index: Option, id_at: usize, os: u8, ls: u8) { + let slice = self.slice; + let Some(heap_addr) = heap else { return }; + let fh = FractalHeapHeader::parse(slice, heap_addr as usize, os, ls); + self.same( + "fractal heap", + &fh, + &FractalHeapHeader::parse_in(self.st(), heap_addr, os, ls), + ); + let (Ok(fh), Some(index)) = (fh, index) else { + return; + }; + let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else { + return; + }; + let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else { + return; + }; + let id_len = fh.heap_id_length as usize; + for rec in records.iter().take(MAX_HEAP_IDS) { + let Some(id) = rec.data.get(id_at..id_at + id_len) else { + continue; + }; + let want = fh.read_managed_object(slice, id, os); + self.same( + "heap object", + &want, + &fh.read_managed_object_in(self.st(), id, os), + ); + } + } + + /// A dataset's layout: VDS mappings, and fixed/extensible array chunk + /// indexes. + fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) { + let slice = self.slice; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok()) + }; + let Some(layout) = find(MessageType::DataLayout) else { + return; + }; + let Ok(layout) = DataLayout::parse(&layout, os, ls) else { + return; + }; + match &layout { + DataLayout::Virtual { .. } => { + let (mut want, mut got) = (layout.clone(), layout.clone()); + let w = want.resolve_vds_mappings(slice, ls).map(|()| want); + let g = got.resolve_vds_mappings_in(self.st(), ls).map(|()| got); + self.same("VDS mappings", &w, &g); + } + DataLayout::Chunked { + chunk_dimensions, + btree_address: Some(addr), + version: 4, + chunk_index_type: Some(kind @ (3 | 4)), + .. + } => { + let (Some(ds), Some(dt)) = + (find(MessageType::Dataspace), find(MessageType::Datatype)) + else { + return; + }; + let (Ok(ds), Ok((dt, _))) = (Dataspace::parse(&ds, ls), Datatype::parse(&dt)) + else { + return; + }; + let rank = ds.dimensions.len(); + if chunk_dimensions.len() < rank { + return; + } + let dims = &chunk_dimensions[..rank]; + let max = ds.max_dimensions.as_deref(); + let es = dt.type_size(); + if *kind == 3 { + let h = FixedArrayHeader::parse(slice, *addr as usize, os, ls); + self.same( + "fixed array header", + &h, + &FixedArrayHeader::parse_in(self.st(), *addr, os, ls), + ); + let Ok(h) = h else { return }; + let want = + read_fixed_array_chunks(slice, &h, &ds.dimensions, max, dims, es, os, ls); + let got = read_fixed_array_chunks_in( + self.st(), + &h, + &ds.dimensions, + max, + dims, + es, + os, + ls, + ); + self.same("fixed array chunks", &want, &got); + } else { + let h = ExtensibleArrayHeader::parse(slice, *addr as usize, os, ls); + let got = ExtensibleArrayHeader::parse_in(self.st(), *addr, os, ls); + self.same("extensible array header", &h, &got); + let Ok(h) = h else { return }; + let want = read_extensible_array_chunks( + slice, + &h, + &ds.dimensions, + max, + dims, + es, + os, + ls, + ); + let got = read_extensible_array_chunks_in( + self.st(), + &h, + &ds.dimensions, + max, + dims, + es, + os, + ls, + ); + self.same("extensible array chunks", &want, &got); + } + } + _ => {} + } + } +} + +fn check_file(path: &Path, tally: &mut Tally) { + let Ok(bytes) = std::fs::read(path) else { + return; + }; + let Ok((_, hdf5)) = split_user_block(&bytes) else { + return; + }; + let storage = CountingStorage::new(hdf5.to_vec()); + let before = (tally.checks, tally.objects); + let mut walk = Walk { + slice: hdf5, + storage: &storage, + name: path.display().to_string(), + tally, + }; + walk.run(); + tally.files += 1; + tally.reads += storage.reads(); + tally.bytes += storage.bytes_read(); + if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") { + eprintln!( + "{:>6} objects {:>7} checks {:>8} reads {:>12} bytes {}", + tally.objects - before.1, + tally.checks - before.0, + storage.reads(), + storage.bytes_read(), + path.display() + ); + } +} + +fn hdf5_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + hdf5_files(&p, out); + } else if p + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf")) + { + out.push(p); + } + } +} + +#[test] +fn fixtures_parse_identically_through_storage() { + let mut files = Vec::new(); + hdf5_files( + &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"), + &mut files, + ); + files.sort(); + assert!(files.len() >= 40, "{} fixtures", files.len()); + let mut tally = Tally::default(); + for f in &files { + check_file(f, &mut tally); + } + eprintln!("fixtures: {tally:?}"); + assert!(tally.objects >= 150 && tally.checks >= 1000, "{tally:?}"); + // Reads really went through read_at. + assert!(tally.reads > tally.objects as u64); +} + +#[test] +fn corpus_parses_identically_through_storage() { + let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else { + eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus"); + return; + }; + let mut files = Vec::new(); + for d in std::env::split_paths(&dirs) { + hdf5_files(&d, &mut files); + } + files.sort(); + let mut tally = Tally::default(); + for f in &files { + check_file(f, &mut tally); + } + eprintln!("corpus: {tally:?}"); + assert!(tally.files > 0); +} + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +/// Files h5py writes to cover what the fixtures do not: extensible arrays +/// deep enough for super blocks and paged data blocks, paged fixed arrays, +/// big symbol-table and dense groups, dense and shared attributes, a SOHM +/// list and a SOHM B-tree, committed datatypes, and a user block. +const GENERATE: &str = r#" +import ctypes, glob, os, sys, h5py, numpy as np +out = sys.argv[1] +def p(n): return os.path.join(out, n) + +with h5py.File(p('ea.h5'), 'w', libver='latest') as f: + # One unlimited dimension: extensible array. 3000 chunks reach super + # blocks; 2-element chunks of int8 keep data small. + d = f.create_dataset('ea', shape=(6000,), maxshape=(None,), chunks=(2,), dtype='i1') + d[:] = np.arange(6000) % 100 + d2 = f.create_dataset('ea_deflate', shape=(4000, 3), maxshape=(None, 3), chunks=(2, 3), + dtype='f4', compression='gzip') + d2[:] = np.random.default_rng(1).random((4000, 3)) + d3 = f.create_dataset('ea_sparse', shape=(100000,), maxshape=(None,), chunks=(4,), dtype='i2') + d3[0:8] = 1; d3[50000:50004] = 2; d3[99996:] = 3 + fa = f.create_dataset('fa_paged', shape=(5000,), chunks=(1,), dtype='u1') + fa[::3] = 7 + fa2 = f.create_dataset('fa', shape=(40, 40), chunks=(8, 8), dtype='f8', compression='gzip') + fa2[:] = 1.5 + for i in range(30): + f.attrs[f'a{i}'] = np.arange(i + 1) + g = f.create_group('dense') + for i in range(300): + g.create_group(f'child{i:04d}').attrs['i'] = i + f['committed'] = np.dtype([('x', 'i4'), ('y', 'f8')]) + f.create_dataset('uses_committed', shape=(3,), dtype=f['committed']) + f['uses_committed'].attrs.create('ta', data=np.zeros(2, dtype=f['committed'].dtype), dtype=f['committed']) + f.attrs['vl'] = ['alpha', 'beta', 'gamma'] + +with h5py.File(p('v1_groups.h5'), 'w', libver='earliest', userblock_size=512) as f: + for i in range(400): + g = f.create_group(f'g{i:04d}') + g.attrs['n'] = i + f.create_dataset('x', data=np.arange(10)) + +libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*')) +if libs: + lib = ctypes.CDLL(libs[0]) + lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint] + lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint] + lib.H5Pset_shared_mesg_phase_change.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint] + for name, list_max in [('sohm_list.h5', 50), ('sohm_btree.h5', 0)]: + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE) + assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0 + # datatype, dataspace, fill value, filter pipeline, attribute + assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, 0x02 | 0x04 | 0x08 | 0x10 | 0x20, 1) >= 0 + assert lib.H5Pset_shared_mesg_phase_change(fcpl.id, list_max, 0) >= 0 + fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS) + fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST) + fid = h5py.h5f.create(p(name).encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl) + with h5py.File(fid) as f: + for i in range(20): + d = f.create_dataset(f'd{i}', shape=(10, i + 1), dtype='f4', fillvalue=-1.0, + chunks=(5, 1), compression='gzip') + d.attrs['units'] = 'metres per second, a long enough string to share' + d.attrs['scale'] = np.arange(20, dtype='f8') +print('ok') +"#; + +#[test] +fn h5py_files_parse_identically_through_storage() { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("storage_equivalence"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let out = Command::new(python()) + .args(["-c", GENERATE, dir.to_str().unwrap()]) + .output(); + match out { + Ok(o) if o.status.success() => {} + Ok(o) if interop_required() => panic!( + "h5py generation failed:\n{}\n{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ), + Err(e) if interop_required() => panic!("python not available: {e}"), + _ => { + eprintln!("python3 with h5py unavailable; skipping"); + return; + } + } + let mut files = Vec::new(); + hdf5_files(&dir, &mut files); + files.sort(); + let names: Vec<_> = files + .iter() + .map(|f| f.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + for want in ["ea.h5", "v1_groups.h5"] { + assert!(names.iter().any(|n| n == want), "{names:?}"); + } + if interop_required() { + assert!(names.iter().any(|n| n == "sohm_btree.h5"), "{names:?}"); + } + let mut tally = Tally::default(); + for f in &files { + check_file(f, &mut tally); + } + eprintln!("h5py files: {tally:?}"); + assert!(tally.objects >= 700, "{tally:?}"); + // Dense attributes and the SOHM B-tree are the known clean errors. + assert!(tally.contiguous_required > 0, "{tally:?}"); +} -- 2.54.0 From 24f0c71939b085adce56dece64e3e73181356213 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:11:46 -0500 Subject: [PATCH 22/48] format: keep dataset_fill_value_in's &[u8] signature Making dataset_fill_value_in generic over Storage broke callers that pass an array (`include_bytes!`): a generic parameter does not unsize-coerce `&[u8; N]`. It takes &[u8] again, as before this branch, and wraps the new dataset_fill_value_from_storage(&dyn Storage, ..). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/fill_value.rs | 23 ++++++++++++------- .../tests/storage_equivalence.rs | 4 ++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/crates/clawhdf5-format/src/fill_value.rs b/crates/clawhdf5-format/src/fill_value.rs index 88cbc3f..6ab3460 100644 --- a/crates/clawhdf5-format/src/fill_value.rs +++ b/crates/clawhdf5-format/src/fill_value.rs @@ -111,17 +111,24 @@ pub fn dataset_fill_value(messages: &[HeaderMessage]) -> Result>, /// fill value message to where it lives: another object header, or the /// file's shared-message (SOHM) heap, as libhdf5 writes it when the file has /// a SOHM index for fill values. -/// -/// `file_data` is any [`Storage`](crate::storage::Storage): the file as a -/// `&[u8]`, or a backend that serves it by range. (The trait is not -/// imported here: its `len` would shadow the slice method in this module.) -pub fn dataset_fill_value_in( - file_data: &S, +pub fn dataset_fill_value_in( + file_data: &[u8], + messages: &[HeaderMessage], + offset_size: u8, + length_size: u8, +) -> Result>, FormatError> { + dataset_fill_value_from_storage(&file_data, messages, offset_size, length_size) +} + +/// [`dataset_fill_value_in`] with the file behind any +/// [`Storage`](crate::storage::Storage). (The trait is not imported here: +/// its `len` would shadow the slice method in this module.) +pub fn dataset_fill_value_from_storage( + file: &dyn crate::storage::Storage, messages: &[HeaderMessage], offset_size: u8, length_size: u8, ) -> Result>, FormatError> { - let file: &dyn crate::storage::Storage = &file_data; fill_value_from(messages, |msg| { crate::shared_message::message_data_with_sohm_in(file, msg, offset_size, length_size) .map(|data| data.into_owned()) @@ -472,7 +479,7 @@ mod tests { .count(); let want = dataset_fill_value_in(file, &h.messages, os, ls); assert_eq!(want, Ok(Some((-7i32).to_le_bytes().to_vec()))); - let got = dataset_fill_value_in(&storage, &h.messages, os, ls); + let got = dataset_fill_value_from_storage(&storage, &h.messages, os, ls); assert_eq!(got, want, "{}", child.name); } assert!(shared >= 2); diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index a046ffb..ada6274 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -44,7 +44,7 @@ use clawhdf5_format::error::FormatError; use clawhdf5_format::extensible_array::{ ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in, }; -use clawhdf5_format::fill_value::dataset_fill_value_in; +use clawhdf5_format::fill_value::{dataset_fill_value_from_storage, dataset_fill_value_in}; use clawhdf5_format::fixed_array::{ FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in, }; @@ -189,7 +189,7 @@ impl Walk<'_> { self.same( "fill value", &want, - &dataset_fill_value_in(self.storage, &header.messages, os, ls), + &dataset_fill_value_from_storage(self.st(), &header.messages, os, ls), ); for msg in &header.messages { -- 2.54.0 From 476960f4b88c75eb85e19a7e76b3000912afcac6 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:11:46 -0500 Subject: [PATCH 23/48] format: fix a clippy lint in the global heap test fixture With Storage in scope, `data.len()` on a `&&[u8]` resolves to Storage::len (already a u64), so `as u64` was a no-op cast; name the slice method explicitly. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/global_heap.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index c6de363..e635d4f 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -296,8 +296,10 @@ mod tests { buf.extend_from_slice(&ref_count.to_le_bytes()); buf.extend_from_slice(&[0u8; 4]); // reserved match length_size { - 4 => buf.extend_from_slice(&(data.len() as u32).to_le_bytes()), - 8 => buf.extend_from_slice(&(data.len() as u64).to_le_bytes()), + // `<[u8]>::len`: with `Storage` in scope `data.len()` on a + // `&&[u8]` resolves to `Storage::len` (a `u64`). + 4 => buf.extend_from_slice(&(<[u8]>::len(data) as u32).to_le_bytes()), + 8 => buf.extend_from_slice(&(<[u8]>::len(data) as u64).to_le_bytes()), _ => panic!("unsupported"), } buf.resize(buf.len() + (pad8(8 + ls) - (8 + ls)), 0); -- 2.54.0 From 5d17712adb3a8f2a656223f30e4aa24bb350f788 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:29:11 -0500 Subject: [PATCH 24/48] docs: changelog and design status for range-read milestone M1 (part 1) Lists the parsers now reading through Storage, what still needs the whole file (v2 B-tree-indexed structures: a clean error; raw data: M2), the equivalence harness, and the evidence that nothing changed: existing tests, a byte-identical conformance results.json and per-file probe output against f2ff2c4, and identical slice-API transcripts over 748 files between the two builds. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 43 ++++++++++++++++++++++++++++++++++++++ docs/design/range-reads.md | 6 ++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c6d6e6..f33cba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,49 @@ fast path; implemented for `[u8]`, `Vec`, and references, `Box`es and (with `std`) `Arc`s of a `Storage`. Slices and `Vec`s serve borrowed bytes, so parsing an in-memory file costs no copy. +- **The metadata parsers read through `Storage`.** Each converted parser has + an `*_in(&dyn Storage, ..)` core, and its `&[u8]` function is now a thin + wrapper over it, so no caller changes: the superblock + (`Superblock::parse_in`), its extension and cache image + (`read_superblock_extension_in`, `cache_image_state_in`), object headers + with their continuation chunks (`ObjectHeader::parse_in`), local and + global heaps, symbol-table nodes and the group B-tree (v1), fractal heaps, + fixed and extensible array chunk indexes, shared messages and the SOHM + table (`message_data_in`, `message_data_with_sohm_in`, + `load_sohm_table_in`, …), attributes (`extract_attributes_full_in`, + `extract_attributes_tolerant_in`, `AttributeMessage::parse_in_storage`), + fill values (`dataset_fill_value_from_storage`) and virtual-dataset + mappings (`DataLayout::resolve_vds_mappings_in`); also + `signature::find_signature_in`. Each structure is read with bounded reads + (a prefix, then the structure) instead of slicing the whole file; the + open-ended `&file_data[addr..]` slices in these modules are gone. Bounds + errors keep their values (absolute position, file length). +- Structures still indexed by a v2 B-tree — dense attribute storage, a SOHM + B-tree index and huge fractal-heap objects found through their B-tree — + are not converted yet (the + v2 B-tree and dense groups come with milestone M3); over a backend without + the whole file in memory they are the clean `ContiguousStorageRequired` + error, never a partial result. Raw data, chunk B-tree (v1) indexes and VL + data are milestone M2. +- **No behaviour change**, checked three ways (2026-09-26, tank): every + existing test passes unchanged; the conformance sweep + (`conformance/run.sh --no-fetch`) gives a byte-identical `results.json` + and identical per-file probe output for all 697 files at `f2ff2c4` and on + this branch; and a transcript of every converted `&[u8]` function's + result over the fixtures, the conformance corpus and the h5py-written + files below (748 files, 7 603 object headers) is byte-identical between + the two builds. +- New equivalence harness `clawhdf5-format/tests/storage_equivalence.rs`: + every converted parser runs over the file as a slice and over + `storage::CountingStorage` — a `Storage` that serves an in-memory buffer + through `read_at` only (`as_contiguous()` is `None`), copying what it + serves and counting reads — and must give identical results. It walks + the fixtures, files h5py writes for it (extensible arrays with super + blocks and paged data blocks, paged fixed arrays, large v1 and dense + groups, a user block, SOHM list and B-tree indexes, dense, shared and + committed-type attributes), and with `CLAWHDF5_STORAGE_CORPUS=` a + corpus (all 653 HDF5 files of the conformance corpus pass). Milestones M2 + and M3 extend it. ### Chunked full reads (2026-09-26) - **Chunks are decoded straight into the output, into reused buffers.** A diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 2e82a6c..488db01 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -1,7 +1,9 @@ # Design: range reads (reading HDF5 without holding the whole file) -Status: proposal, 2026-09-26. No library code has changed; this document is -the plan for Phase 3's largest architectural change. Every count below was +Status: proposal, 2026-09-26; the plan for Phase 3's largest architectural +change. Progress: M1, first part (the `Storage` trait and the metadata +parsers listed in `CHANGELOG.md` under "Range reads, milestone M1") is done; +group B-tree v2 lookups, dense groups and the facade are not converted yet. Every count below was taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given next to it. No timing numbers appear here on purpose: the machine was shared with other build jobs when this was written. -- 2.54.0 From 02e89c1d2d6cf65f3eae207cfb23c25a1bd9ae87 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:33:19 -0500 Subject: [PATCH 25/48] read: look names up through the dense name indexes Finding one link or attribute by name read every entry: Group::dataset and Group::group (File, MmapFile, LazyFile) listed the whole group per call, and path resolution scanned each group's links. Opening every child of a 35 001-link group by name decoded ~1.2e9 links. Now a dense group's v2 B-tree name index (type 5, lookup3 hash of the name) is descended to the records with the name's hash (btree_v2::find_btree_v2_records reads only the nodes whose key interval overlaps), and only those links are read and compared; all hash-equal records are compared, so libhdf5's tie order does not matter. Dense attributes the same through their type 8 index (attribute::find_attribute_in_file, facade attr(name)); huge heap objects through their ID-ordered index. group_v2::resolve_child returns what the listing has under a name (soft links followed, dangling/external ones not found). Group::entries and File::group_at hand out a listing's addresses. The lookup-stats feature counts heap objects read. Tests: one lookup in an h5py-written 35 001-link group with colliding hashes reads at most two links (before: 35 001, failing), attribute lookups likewise (before: 3 000, failing), every child opens through all three readers and matches h5py, every link kind resolves as h5py resolves it in dense and compact groups, 300 huge attributes are found, and a range search matches a full scan at every tree depth. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 3 + crates/clawhdf5-format/src/attribute.rs | 181 ++++++-- crates/clawhdf5-format/src/btree_v2.rs | 216 +++++++-- crates/clawhdf5-format/src/btree_v2_write.rs | 53 +++ crates/clawhdf5-format/src/fractal_heap.rs | 34 +- crates/clawhdf5-format/src/group_v2.rs | 217 +++++++-- crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/lookup_stats.rs | 31 ++ crates/clawhdf5/Cargo.toml | 3 +- crates/clawhdf5/src/lazy.rs | 57 ++- crates/clawhdf5/src/mmap_file.rs | 57 ++- crates/clawhdf5/src/reader.rs | 88 +++- crates/clawhdf5/src/types.rs | 29 ++ .../clawhdf5/tests/dense_storage_interop.rs | 24 + .../clawhdf5/tests/indexed_lookup_interop.rs | 410 ++++++++++++++++++ 15 files changed, 1245 insertions(+), 159 deletions(-) create mode 100644 crates/clawhdf5-format/src/lookup_stats.rs create mode 100644 crates/clawhdf5/tests/indexed_lookup_interop.rs diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index c1ec6df..e6a1258 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -80,6 +80,9 @@ blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] blosc2 = ["blosc"] # Every plugin filter above. plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"] +# Test instrumentation: per-thread counts of heap objects read (see +# `lookup_stats`), so tests can bound the cost of a name lookup. +lookup-stats = ["std"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 6c4e9ae..d22a552 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -5,8 +5,10 @@ use alloc::{borrow::Cow, string::String, vec::Vec}; #[cfg(feature = "std")] use std::borrow::Cow; +use crate::addr::to_usize; use crate::attribute_info::AttributeInfoMessage; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::checksum::jenkins_lookup3; use crate::data_read; use crate::dataspace::Dataspace; use crate::datatype::Datatype; @@ -341,7 +343,8 @@ fn compute_raw_data( dataspace: &Dataspace, datatype: &Datatype, ) -> Vec { - let num_elements = dataspace.num_elements() as usize; + // Saturating, like the product: the size is capped at what is there. + let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX); let elem_size = datatype.type_size() as usize; let expected_size = num_elements.saturating_mul(elem_size); let available = data.len().saturating_sub(pos); @@ -453,7 +456,145 @@ fn extract_attributes_with( // Each attribute's creation order, where the file records one. let mut orders: Vec = Vec::new(); - // Collect compact attributes (inline in OH) + extract_compact_attributes( + file_data, + header, + offset_size, + length_size, + &mut attrs, + &mut orders, + on_error, + )?; + + // Check for dense attributes via AttributeInfo message + let attr_info = find_attribute_info(header, offset_size)?; + if let Some(info) = &attr_info + && let Some(fh_addr) = info.fractal_heap_address + { + extract_dense_attributes( + file_data, + info, + fh_addr, + offset_size, + length_size, + &mut attrs, + &mut orders, + on_error, + )?; + } + + // An object that tracks attribute creation order lists its attributes + // in that order (h5py's `track_order=True`), as libhdf5 does; otherwise + // they come in storage order. + if attr_info.is_some_and(|i| i.max_creation_index.is_some()) { + let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect(); + paired.sort_by_key(|(o, _)| *o); + attrs = paired.into_iter().map(|(_, a)| a).collect(); + } + + Ok(attrs) +} + +/// B-tree v2 record type of dense attribute storage's name index. +const ATTRIBUTE_NAME_INDEX: u8 = 8; + +/// The attribute called `name` on the object with header `header`: the +/// first one [`extract_attributes_tolerant`] returns under that name, or +/// `None` if it returns none (an attribute that cannot be read is not +/// returned there either). +/// +/// Compact attributes are in the header and are scanned. Dense attributes +/// are found through the name index (a v2 B-tree of lookup3 name hashes, +/// record type 8): only the attributes whose names hash like `name` are read +/// from the heap, O(log n) instead of all of them. Errors in the structures +/// that index the attributes fail the call, as they fail a listing. +pub fn find_attribute_in_file( + file_data: &[u8], + header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let attr_info = find_attribute_info(header, offset_size)?; + let dense = attr_info + .as_ref() + .and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?))); + let Some((fh_addr, btree_addr)) = dense else { + // Compact only (or dense storage without a name index, which a + // listing reports): as a listing finds it. + return Ok( + extract_attributes_tolerant(file_data, header, offset_size, length_size)? + .0 + .into_iter() + .find(|a| a.name == name), + ); + }; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 { + return Ok( + extract_attributes_tolerant(file_data, header, offset_size, length_size)? + .0 + .into_iter() + .find(|a| a.name == name), + ); + } + + // A listing has the compact attributes first. + let mut compact = Vec::new(); + extract_compact_attributes( + file_data, + header, + offset_size, + length_size, + &mut compact, + &mut Vec::new(), + &mut |_| Ok(()), + )?; + if let Some(a) = compact.into_iter().find(|a| a.name == name) { + return Ok(Some(a)); + } + + // Record: heap ID + message flags(1) + creation order(4) + hash(4); the + // hash is the last field. + let hash = jenkins_lookup3(name.as_bytes()); + let hash_at = usize::from(btree_hdr.record_size) - 4; + let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| match r + .get(hash_at..hash_at + 4) + { + Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), + None => core::cmp::Ordering::Less, + })?; + let id_len = usize::from(fh.heap_id_length); + for record in &records { + let Some(id_bytes) = record.data.get(..id_len) else { + continue; + }; + let attr = fh + .read_managed_object(file_data, id_bytes, offset_size) + .and_then(|d| AttributeMessage::parse_in_file(&d, file_data, offset_size, length_size)); + // One that cannot be read is left out, as from a listing. + if let Ok(attr) = attr + && attr.name == name + { + return Ok(Some(attr)); + } + } + Ok(None) +} + +/// The attributes stored in the object header itself (compact storage), and +/// each one's creation order into `orders`. +fn extract_compact_attributes( + file_data: &[u8], + header: &ObjectHeader, + offset_size: u8, + length_size: u8, + attrs: &mut Vec, + orders: &mut Vec, + on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, +) -> Result<(), FormatError> { for msg in &header.messages { if msg.msg_type == MessageType::Attribute { let attr = if shared_message::is_shared(msg.flags) { @@ -489,34 +630,7 @@ fn extract_attributes_with( } } } - - // Check for dense attributes via AttributeInfo message - let attr_info = find_attribute_info(header, offset_size)?; - if let Some(info) = &attr_info - && let Some(fh_addr) = info.fractal_heap_address - { - extract_dense_attributes( - file_data, - info, - fh_addr, - offset_size, - length_size, - &mut attrs, - &mut orders, - on_error, - )?; - } - - // An object that tracks attribute creation order lists its attributes - // in that order (h5py's `track_order=True`), as libhdf5 does; otherwise - // they come in storage order. - if attr_info.is_some_and(|i| i.max_creation_index.is_some()) { - let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect(); - paired.sort_by_key(|(o, _)| *o); - attrs = paired.into_iter().map(|(_, a)| a).collect(); - } - - Ok(attrs) + Ok(()) } /// Find and parse the Attribute Info message from an object header. @@ -547,7 +661,7 @@ fn extract_dense_attributes( on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result<(), FormatError> { // Parse fractal heap - let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?; + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; // Parse B-tree v2 for name index (type 8) let btree_addr = attr_info @@ -556,7 +670,8 @@ fn extract_dense_attributes( expected: 1, available: 0, })?; - let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index 9159253..7582c05 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -2,10 +2,12 @@ #[cfg(not(feature = "std"))] use alloc::vec::Vec; +use core::cmp::Ordering; #[cfg(feature = "checksum")] use byteorder::{ByteOrder, LittleEndian}; +use crate::addr::to_usize; use crate::error::FormatError; /// Parsed B-tree v2 header (signature "BTHD"). @@ -216,7 +218,7 @@ pub fn collect_btree_v2_records( // Root is a leaf parse_leaf_records( file_data, - header.root_node_address as usize, + to_usize(header.root_node_address)?, header.num_records_in_root, header.record_size, ) @@ -225,7 +227,7 @@ pub fn collect_btree_v2_records( let mut records = Vec::new(); collect_internal_records( file_data, - header.root_node_address as usize, + to_usize(header.root_node_address)?, header.num_records_in_root, header.depth, header.record_size, @@ -289,9 +291,10 @@ fn parse_leaf_records( Ok(records) } -/// Recursively collect records from an internal node. -#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)] -fn collect_internal_records( +/// An internal node's layout: where its records start, and its children as +/// `(address, record count)`. +#[allow(clippy::too_many_arguments)] +fn read_internal_node( file_data: &[u8], offset: usize, num_records: u16, @@ -299,11 +302,8 @@ fn collect_internal_records( record_size: u16, node_size: u32, offset_size: u8, - length_size: u8, max_leaf_nrec: u64, - budget: &mut usize, - out: &mut Vec, -) -> Result<(), FormatError> { +) -> Result<(usize, Vec<(u64, u16)>), FormatError> { // signature(4) + version(1) + type(1) = 6 ensure_len(file_data, offset, 6)?; if &file_data[offset..offset + 4] != b"BTIN" { @@ -314,7 +314,7 @@ fn collect_internal_records( let rs = record_size as usize; let mut pos = offset + 6; - // Read all records first + // Records first let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), @@ -346,7 +346,6 @@ fn collect_internal_records( let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width; ensure_len(file_data, pos, num_children * child_ptr_size)?; - // Read child pointers let mut children = Vec::with_capacity(num_children); for _ in 0..num_children { let addr = read_offset(file_data, pos, offset_size)?; @@ -356,6 +355,61 @@ fn collect_internal_records( pos += total_nrec_width; // skip total records in subtree children.push((addr, child_nrec)); } + Ok((records_start, children)) +} + +/// Record `i` of an internal node whose records start at `records_start`. +fn internal_record( + file_data: &[u8], + records_start: usize, + i: usize, + rs: usize, +) -> Result<&[u8], FormatError> { + let overflow = || FormatError::UnexpectedEof { + expected: usize::MAX, + available: file_data.len(), + }; + let rec_start = i + .checked_mul(rs) + .and_then(|o| records_start.checked_add(o)) + .ok_or_else(overflow)?; + let rec_end = rec_start.checked_add(rs).ok_or_else(overflow)?; + file_data + .get(rec_start..rec_end) + .ok_or(FormatError::UnexpectedEof { + expected: rec_end, + available: file_data.len(), + }) +} + +/// Recursively collect records from an internal node. +#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)] +fn collect_internal_records( + file_data: &[u8], + offset: usize, + num_records: u16, + depth: u16, + record_size: u16, + node_size: u32, + offset_size: u8, + length_size: u8, + max_leaf_nrec: u64, + budget: &mut usize, + out: &mut Vec, +) -> Result<(), FormatError> { + let nr = num_records as usize; + let rs = record_size as usize; + let (records_start, children) = read_internal_node( + file_data, + offset, + num_records, + depth, + record_size, + node_size, + offset_size, + max_leaf_nrec, + )?; + let child_depth = depth - 1; // Interleave: child[0], record[0], child[1], record[1], ..., child[nr] // We collect child[0] records, then record[0], then child[1], etc. @@ -364,12 +418,12 @@ fn collect_internal_records( // Before parsing, so a refused tree is not also a large allocation. spend(budget, usize::from(child_nrec))?; let leaf_recs = - parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?; + parse_leaf_records(file_data, to_usize(child_addr)?, child_nrec, record_size)?; out.extend(leaf_recs); } else { collect_internal_records( file_data, - child_addr as usize, + to_usize(child_addr)?, child_nrec, child_depth, record_size, @@ -384,32 +438,10 @@ fn collect_internal_records( // Add record[i] (except after the last child) if i < nr { - let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - })?; - let rec_start = - records_start - .checked_add(rec_offset) - .ok_or(FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - })?; - let rec_end = rec_start - .checked_add(rs) - .ok_or(FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - })?; - if rec_end > file_data.len() { - return Err(FormatError::UnexpectedEof { - expected: rec_end, - available: file_data.len(), - }); - } + let data = internal_record(file_data, records_start, i, rs)?; spend(budget, 1)?; out.push(BTreeV2Record { - data: file_data[rec_start..rec_end].to_vec(), + data: data.to_vec(), }); } } @@ -417,6 +449,116 @@ fn collect_internal_records( Ok(()) } +/// The records of a B-tree v2 that fall in one key range, found by +/// descending the tree instead of reading all of it. +/// +/// `cmp` places a record relative to the range: `Less` if the record sorts +/// before it, `Greater` if after, `Equal` if the record is in it. The tree +/// must be ordered consistently with `cmp`, as libhdf5 orders it (a link or +/// attribute name index by name hash, so all records with one hash form a +/// range whatever order their names are in). Only the nodes whose key +/// interval overlaps the range are read: O(depth) nodes plus those holding +/// the matches. Matches come in tree order. +pub fn find_btree_v2_records( + file_data: &[u8], + header: &BTreeV2Header, + offset_size: u8, + cmp: &mut dyn FnMut(&[u8]) -> Ordering, +) -> Result, FormatError> { + if header.total_records == 0 || header.num_records_in_root == 0 { + return Ok(Vec::new()); + } + if header.depth > MAX_DEPTH { + return Err(FormatError::NestingDepthExceeded); + } + // As in `collect_btree_v2_records`: a valid tree cannot hold more + // records than the file has room for, however its children are shared. + let mut budget = file_data.len() / usize::from(header.record_size.max(1)); + let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size); + let mut out = Vec::new(); + find_in_node( + file_data, + header, + to_usize(header.root_node_address)?, + header.num_records_in_root, + header.depth, + offset_size, + max_leaf_nrec, + cmp, + &mut budget, + &mut out, + )?; + Ok(out) +} + +#[allow(clippy::too_many_arguments)] +fn find_in_node( + file_data: &[u8], + header: &BTreeV2Header, + offset: usize, + num_records: u16, + depth: u16, + offset_size: u8, + max_leaf_nrec: u64, + cmp: &mut dyn FnMut(&[u8]) -> Ordering, + budget: &mut usize, + out: &mut Vec, +) -> Result<(), FormatError> { + spend(budget, usize::from(num_records))?; + if depth == 0 { + let records = parse_leaf_records(file_data, offset, num_records, header.record_size)?; + out.extend( + records + .into_iter() + .filter(|r| cmp(&r.data) == Ordering::Equal), + ); + return Ok(()); + } + let rs = usize::from(header.record_size); + let (records_start, children) = read_internal_node( + file_data, + offset, + num_records, + depth, + header.record_size, + header.node_size, + offset_size, + max_leaf_nrec, + )?; + let nr = usize::from(num_records); + let mut order = Vec::with_capacity(nr); + for i in 0..nr { + order.push(cmp(internal_record(file_data, records_start, i, rs)?)); + } + // Child `i` holds the keys between record `i - 1` and record `i`: it can + // hold a match unless the record before it is already past the range or + // the record after it is still before it. + for (i, &(child_addr, child_nrec)) in children.iter().enumerate() { + let after_left = i == 0 || order[i - 1] != Ordering::Greater; + let before_right = i == nr || order[i] != Ordering::Less; + if after_left && before_right { + find_in_node( + file_data, + header, + to_usize(child_addr)?, + child_nrec, + depth - 1, + offset_size, + max_leaf_nrec, + cmp, + budget, + out, + )?; + } + if i < nr && order[i] == Ordering::Equal { + out.push(BTreeV2Record { + data: internal_record(file_data, records_start, i, rs)?.to_vec(), + }); + } + } + Ok(()) +} + /// Most records a subtree whose root is at `depth` can hold (libhdf5's /// `cum_max_nrec`). See [`node_info`]. fn cum_max_records( diff --git a/crates/clawhdf5-format/src/btree_v2_write.rs b/crates/clawhdf5-format/src/btree_v2_write.rs index 1a18984..795fb80 100644 --- a/crates/clawhdf5-format/src/btree_v2_write.rs +++ b/crates/clawhdf5-format/src/btree_v2_write.rs @@ -385,6 +385,59 @@ mod tests { assert!(nodes > 0); } + /// Descending to a key range finds exactly the records a full read + /// holds in it — runs of equal keys that straddle node boundaries + /// included — at every depth, and nothing for keys not in the tree. + #[test] + fn a_key_range_search_matches_a_full_scan() { + use crate::btree_v2::find_btree_v2_records; + use core::cmp::Ordering; + let rs = 11usize; + // Keys 0, 0, 0, 2, 2, 2, 4, ...: runs of three, odd keys missing. + for n in [1usize, 45, 46, 1150, 30_000] { + let mut recs = Vec::with_capacity(n * rs); + for i in 0..n { + let mut r = vec![0u8; rs]; + r[..8].copy_from_slice(&((i / 3 * 2) as u64).to_be_bytes()); + r[8..].copy_from_slice(&[(i % 3) as u8, 0, 0]); + recs.extend_from_slice(&r); + } + let base = 4096u64; + let tree = build_btree_v2(params(512, 11), &recs, base, 8, 8).unwrap(); + let mut file = vec![0u8; base as usize]; + file.extend_from_slice(&tree); + let hdr = BTreeV2Header::parse(&file, base as usize, 8, 8).unwrap(); + let all = collect_btree_v2_records(&file, &hdr, 8, 8).unwrap(); + let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap()); + let last = key(&all[n - 1].data); + let probes = (0..=last + 1).step_by(if n > 1000 { 37 } else { 1 }); + for k in probes.chain([last, last + 1, u64::MAX]) { + let found = + find_btree_v2_records(&file, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)).unwrap(); + let want: Vec<&[u8]> = all + .iter() + .map(|r| r.data.as_slice()) + .filter(|r| key(r) == k) + .collect(); + let got: Vec<&[u8]> = found.iter().map(|r| r.data.as_slice()).collect(); + assert_eq!(got, want, "n {n} key {k}"); + assert_eq!( + got.len(), + if k % 2 == 0 && k <= last { + want.len() + } else { + 0 + } + ); + } + // Every record, or none, when the whole tree is in or out of range. + let every = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Equal).unwrap(); + assert_eq!(every.len(), n); + let none = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Less).unwrap(); + assert!(none.is_empty()); + } + } + #[test] fn a_node_too_small_or_too_big_is_an_error() { assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err()); diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index 9bc4ba4..d896156 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -6,7 +6,8 @@ use alloc::{format, vec::Vec}; #[cfg(feature = "checksum")] use byteorder::{ByteOrder, LittleEndian}; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::addr::to_usize; +use crate::btree_v2::{BTreeV2Header, find_btree_v2_records}; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; @@ -354,6 +355,7 @@ impl FractalHeapHeader { id_bytes: &[u8], offset_size: u8, ) -> Result, FormatError> { + crate::lookup_stats::heap_object_read(); let Some(&first) = id_bytes.first() else { return Err(FormatError::UnexpectedEof { expected: 1, @@ -448,7 +450,7 @@ impl FractalHeapHeader { } let hdr = BTreeV2Header::parse( file_data, - self.huge_btree_address as usize, + to_usize(self.huge_btree_address)?, self.offset_size, self.length_size, )?; @@ -463,8 +465,12 @@ impl FractalHeapHeader { if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len { return Err(heap_error("unexpected huge-object B-tree record type")); } - let records = - collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?; + // Records are ordered by ID (the last field): descend to the ones + // equal to `key` instead of reading the whole index. + let id_at = rec_len - ls; + let records = find_btree_v2_records(file_data, &hdr, self.offset_size, &mut |r| { + le_uint(&r[id_at..id_at + ls]).cmp(&key) + })?; for rec in &records { let d = &rec.data; if d.len() < rec_len { @@ -533,24 +539,24 @@ impl FractalHeapHeader { self.read_from_direct_block( file_data, DirectBlock { - addr: self.root_block_address as usize, + addr: to_usize(self.root_block_address)?, size: self.starting_block_size, heap_offset: 0, filtered_size: self.root_direct_block_filtered_size, filter_mask: self.root_direct_block_filter_mask, }, heap_offset, - obj_len as usize, + to_usize(obj_len)?, ) } else { // Root is an indirect block — limit recursion to 64 levels self.read_from_indirect_block( file_data, - self.root_block_address as usize, + to_usize(self.root_block_address)?, self.current_rows_in_root_indirect_block, 0, // block offset heap_offset, - obj_len as usize, + to_usize(obj_len)?, offset_size, 64, // max recursion depth ) @@ -572,11 +578,11 @@ impl FractalHeapHeader { ) -> Result, FormatError> { if target_offset < block.heap_offset { return Err(FormatError::UnexpectedEof { - expected: block.heap_offset as usize, - available: target_offset as usize, + expected: to_usize(block.heap_offset)?, + available: to_usize(target_offset)?, }); } - let local_offset = (target_offset - block.heap_offset) as usize; + let local_offset = to_usize(target_offset - block.heap_offset)?; if let Some(pipeline) = &self.filter_pipeline { let stored_len = usize::try_from(block.filtered_size) .map_err(|_| heap_error("direct block size"))?; @@ -673,7 +679,7 @@ impl FractalHeapHeader { return self.read_from_direct_block( file_data, DirectBlock { - addr: child_addr as usize, + addr: to_usize(child_addr)?, size: block_size, heap_offset: current_heap_offset, filtered_size, @@ -705,7 +711,7 @@ impl FractalHeapHeader { { return self.read_from_indirect_block( file_data, - child_addr as usize, + to_usize(child_addr)?, child_nrows, current_heap_offset, target_offset, @@ -719,7 +725,7 @@ impl FractalHeapHeader { } Err(FormatError::UnexpectedEof { - expected: target_offset as usize + length, + expected: to_usize(target_offset)?.saturating_add(length), available: file_data.len(), }) } diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index b645bd6..e1bd9f5 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -6,7 +6,9 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; -use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use crate::addr::to_usize; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; +use crate::checksum::jenkins_lookup3; use crate::error::FormatError; use crate::fractal_heap::FractalHeapHeader; use crate::group_v1::{self, GroupEntry}; @@ -93,13 +95,14 @@ fn for_each_dense_link( mut visit: impl FnMut(LinkMessage), ) -> Result<(), FormatError> { // Parse fractal heap - let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?; + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; // Parse B-tree v2 for name index let btree_addr = link_info .btree_name_index_address .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; - let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; for record in &records { @@ -181,10 +184,55 @@ fn find_symbolic_link( if !is_v2_group(object_header) { return Ok(None); } - let is_symbolic = |t: &LinkTarget| !matches!(t, LinkTarget::Hard { .. }); + // As when all links were scanned: the last symbolic link of that name. + Ok( + links_named(file_data, object_header, name, offset_size, length_size)? + .into_iter() + .rev() + .map(|link| link.link_target) + .find(|t| !matches!(t, LinkTarget::Hard { .. })), + ) +} + +/// B-tree v2 record type of a dense group's link name index. +const LINK_NAME_INDEX: u8 = 5; + +/// The links called `name` in a v2 group (a valid group has at most one). +/// +/// In dense storage the link name index (a v2 B-tree of lookup3 name +/// hashes, record type 5) is descended to the records with the name's hash, +/// and only their links are read from the heap — O(log n) instead of every +/// link. libhdf5 orders records with equal hashes by name; all of them are +/// read and compared here, so that order does not matter. An index of +/// another type is scanned in full. +fn links_named( + file_data: &[u8], + object_header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let mut found = Vec::new(); let link_info = find_link_info(object_header, offset_size)?; - let mut found = None; - if let Some(fh_addr) = link_info.fractal_heap_address { + let Some(fh_addr) = link_info.fractal_heap_address else { + for msg in &object_header.messages { + if msg.msg_type == MessageType::Link + && let Some(link) = parse_link(&msg.data, offset_size)? + && link.name == name + { + found.push(link); + } + } + return Ok(found); + }; + + let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?; + let btree_addr = link_info + .btree_name_index_address + .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?; + let btree_hdr = + BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?; + if btree_hdr.tree_type != LINK_NAME_INDEX { for_each_dense_link( file_data, &link_info, @@ -192,26 +240,134 @@ fn find_symbolic_link( offset_size, length_size, |link| { - if link.name == name && is_symbolic(&link.link_target) { - found = Some(link.link_target); + if link.name == name { + found.push(link); } }, )?; - } else { - for msg in &object_header.messages { - if msg.msg_type == MessageType::Link { - let Some(link) = parse_link(&msg.data, offset_size)? else { - continue; - }; - if link.name == name && is_symbolic(&link.link_target) { - found = Some(link.link_target); - } - } + return Ok(found); + } + + // Record: hash(4) + heap ID. + let hash = jenkins_lookup3(name.as_bytes()); + let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| { + match r.get(..4) { + Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash), + // Too short to hold a hash (a corrupt record size): never a match. + None => core::cmp::Ordering::Less, + } + })?; + let id_len = usize::from(fh.heap_id_length); + for record in &records { + let Some(id_bytes) = record.data.get(4..4 + id_len) else { + continue; + }; + let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; + if let Some(link) = parse_link(&link_data, offset_size)? + && link.name == name + { + found.push(link); } } Ok(found) } +/// The link [`resolve_path_any`] follows for one path component `name` of +/// the group with header `object_header`: a hard link (as `Hard`), else a +/// soft or external link of that name, else `None`. Fails with +/// `PathNotFound` if the object is not a group. +fn lookup_link( + file_data: &[u8], + object_header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + if is_v1_group(object_header) { + let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?; + if let Some(e) = entries + .iter() + .find(|e| e.name == name && e.object_header_address != u64::MAX) + { + return Ok(Some(LinkTarget::Hard { + object_header_address: e.object_header_address, + })); + } + return find_symbolic_link(file_data, object_header, name, offset_size, length_size); + } + if !is_v2_group(object_header) { + return Err(FormatError::PathNotFound(String::from( + "object header is not a group", + ))); + } + let links = links_named(file_data, object_header, name, offset_size, length_size)?; + if let Some(addr) = links.iter().find_map(|l| match l.link_target { + LinkTarget::Hard { + object_header_address, + } if object_header_address != u64::MAX => Some(object_header_address), + _ => None, + }) { + return Ok(Some(LinkTarget::Hard { + object_header_address: addr, + })); + } + Ok(links + .into_iter() + .rev() + .map(|link| link.link_target) + .find(|t| !matches!(t, LinkTarget::Hard { .. }))) +} + +/// The object header address of the child called `name` of the group at +/// `group_address`: the address [`resolve_group_children`] lists under that +/// name, or `PathNotFound` if it lists none. +/// +/// A dense group's child is found through its link name index (see +/// [`links_named`]) and only the named link is read and, if it is a soft +/// link, followed — not every link in the group. A v1 group is listed. +pub fn resolve_child( + file_data: &[u8], + superblock: &Superblock, + group_address: u64, + name: &str, +) -> Result { + let os = superblock.offset_size; + let ls = superblock.length_size; + let not_found = || FormatError::PathNotFound(String::from(name)); + let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; + if !is_v2_group(&header) || is_v1_group(&header) { + return resolve_group_children(file_data, superblock, group_address)? + .into_iter() + .find(|e| e.name == name) + .map(|e| e.object_header_address) + .ok_or_else(not_found); + } + let links = links_named(file_data, &header, name, os, ls)?; + // The listing puts hard links before resolved soft links. + if let Some(addr) = links.iter().find_map(|l| match l.link_target { + LinkTarget::Hard { + object_header_address, + } => Some(object_header_address), + _ => None, + }) { + return Ok(addr); + } + for link in links { + if let LinkTarget::Soft { target_path } = link.link_target { + return match resolve_path_from(file_data, superblock, group_address, &target_path) { + // Left out of the listing: dangling, cyclic, or in another file. + Err( + FormatError::PathNotFound(_) + | FormatError::NestingDepthExceeded + | FormatError::ExternalLinkUnsupported { .. }, + ) => Err(not_found()), + other => other, + }; + } + } + Err(not_found()) +} + /// Find and parse the Link Info message from an object header. fn find_link_info( object_header: &ObjectHeader, @@ -298,7 +454,7 @@ pub fn resolve_group_children( ) -> Result, FormatError> { let os = superblock.offset_size; let ls = superblock.length_size; - let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?; + let header = ObjectHeader::parse(file_data, to_usize(group_address)?, os, ls)?; let mut entries = Vec::new(); let mut soft = Vec::new(); @@ -383,24 +539,21 @@ fn resolve_path_following_links( let ls = superblock.length_size; let mut current_addr = start; - let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?; + let mut current_header = ObjectHeader::parse(file_data, to_usize(start)?, os, ls)?; for (i, component) in components.iter().enumerate() { - let entries = resolve_group_entries(file_data, ¤t_header, os, ls)?; - - let found = entries - .iter() - .find(|e| e.name == *component && e.object_header_address != u64::MAX); - match found { - Some(entry) => { + match lookup_link(file_data, ¤t_header, component, os, ls)? { + Some(LinkTarget::Hard { + object_header_address, + }) => { if i == components.len() - 1 { - return Ok(entry.object_header_address); + return Ok(object_header_address); } - current_addr = entry.object_header_address; - current_header = ObjectHeader::parse(file_data, current_addr as usize, os, ls)?; + current_addr = object_header_address; + current_header = ObjectHeader::parse(file_data, to_usize(current_addr)?, os, ls)?; } - None => { - return match find_symbolic_link(file_data, ¤t_header, component, os, ls)? { + found => { + return match found { Some(LinkTarget::Soft { target_path }) => { if depth >= MAX_SOFT_LINK_DEPTH { return Err(FormatError::NestingDepthExceeded); diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 2737c9b..345c923 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -108,6 +108,7 @@ pub mod lane_partition; pub mod link_info; pub mod link_message; pub mod local_heap; +pub mod lookup_stats; pub mod message_type; pub mod metadata_cache; pub mod metadata_index; diff --git a/crates/clawhdf5-format/src/lookup_stats.rs b/crates/clawhdf5-format/src/lookup_stats.rs new file mode 100644 index 0000000..0206729 --- /dev/null +++ b/crates/clawhdf5-format/src/lookup_stats.rs @@ -0,0 +1,31 @@ +//! Work counters for tests of lookup cost (feature `lookup-stats`). +//! +//! Counts fractal-heap objects read — each is one link or attribute message +//! decoded out of a dense group or dense attribute storage — so a test can +//! check that finding one name reads a handful of them, not the whole group. +//! Per thread, so tests running in parallel do not see each other's reads. +//! Without the feature the counting compiles to nothing. + +#[cfg(feature = "lookup-stats")] +std::thread_local! { + static HEAP_OBJECTS: core::cell::Cell = const { core::cell::Cell::new(0) }; +} + +/// Record one heap object read. +#[inline(always)] +pub(crate) fn heap_object_read() { + #[cfg(feature = "lookup-stats")] + HEAP_OBJECTS.with(|c| c.set(c.get() + 1)); +} + +/// Heap objects read on this thread since the last [`reset`]. +#[cfg(feature = "lookup-stats")] +pub fn heap_objects_read() -> u64 { + HEAP_OBJECTS.with(core::cell::Cell::get) +} + +/// Zero this thread's counters. +#[cfg(feature = "lookup-stats")] +pub fn reset() { + HEAP_OBJECTS.with(|c| c.set(0)); +} diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 1eca373..4f68e58 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -19,7 +19,8 @@ rayon = { version = "1", optional = true } tempfile = { workspace = true } criterion = { workspace = true } clawhdf5-io = { path = "../clawhdf5-io", version = "2.7.0", features = ["mmap"] } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum"] } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0", features = ["parallel", "fast-checksum", "lookup-stats"] } +serde_json = "1" clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.7.0" } [[bench]] diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index df41688..7ad7e78 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -28,7 +28,7 @@ use clawhdf5_format::superblock::Superblock; use clawhdf5_io::HDF5Read; use crate::error::Error; -use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; +use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; /// A lazy HDF5 file handle that parses metadata on demand. /// @@ -304,12 +304,7 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { /// Get a dataset within this group by name. pub fn dataset(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; - let hdr = self.file.get_or_parse_header(entry.object_header_address)?; + let hdr = self.file.get_or_parse_header(self.child_address(name)?)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } @@ -322,17 +317,38 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { /// Get a subgroup within this group by name. pub fn group(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; Ok(LazyGroup { file: self.file, - address: entry.object_header_address, + address: self.child_address(name)?, }) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let hdr = self.file.get_or_parse_header(self.address)?; + let data = self.file.hdf5_bytes(); + read_attr( + data, + &hdr, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// The object header address of the child called `name`: the entry of + /// the group's listing with that name, looked up through the group's + /// name index rather than by listing the group (see + /// [`group_v2::resolve_child`]). + fn child_address(&self, name: &str) -> Result { + let data = self.file.hdf5_bytes(); + group_v2::resolve_child(data, &self.file.superblock, self.address, name) + .map_err(Error::Format) + } + /// This group's links that can be opened: hard links, and soft links /// resolved to their targets (see /// [`group_v2::resolve_group_children`]); dangling, external and @@ -555,6 +571,21 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { ) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let data = self.file.hdf5_bytes(); + read_attr( + data, + &self.header, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + /// A header message's payload, resolved through the shared-message /// indirection when needed (e.g. a committed datatype). See /// [`clawhdf5_format::shared_message::message_data`]. diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index b6df1d8..edb6dd4 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock; use clawhdf5_io::MmapReader; use crate::error::Error; -use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; +use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; /// An HDF5 file opened via memory mapping. /// @@ -242,12 +242,7 @@ impl<'f> MmapGroup<'f> { /// Get a dataset within this group by name. pub fn dataset(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; - let hdr = self.file.parse_header(entry.object_header_address)?; + let hdr = self.file.parse_header(self.child_address(name)?)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } @@ -260,17 +255,38 @@ impl<'f> MmapGroup<'f> { /// Get a subgroup within this group by name. pub fn group(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; Ok(MmapGroup { file: self.file, - address: entry.object_header_address, + address: self.child_address(name)?, }) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let hdr = self.file.parse_header(self.address)?; + let data = self.file.hdf5_bytes(); + read_attr( + data, + &hdr, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// The object header address of the child called `name`: the entry of + /// the group's listing with that name, looked up through the group's + /// name index rather than by listing the group (see + /// [`group_v2::resolve_child`]). + fn child_address(&self, name: &str) -> Result { + let data = self.file.meta()?; + group_v2::resolve_child(data, &self.file.superblock, self.address, name) + .map_err(Error::Format) + } + /// This group's links that can be opened: hard links, and soft links /// resolved to their targets (see /// [`group_v2::resolve_group_children`]); dangling, external and @@ -506,6 +522,21 @@ impl<'f> MmapDataset<'f> { ) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let data = self.file.hdf5_bytes(); + read_attr( + data, + &self.header, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + /// A header message's payload, resolved through the shared-message /// indirection when needed (e.g. a committed datatype). See /// [`clawhdf5_format::shared_message::message_data`]. diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 1b20dd6..041a318 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -23,7 +23,7 @@ use clawhdf5_format::superblock::Superblock; use crate::cache_image::{self, ImageView}; use crate::error::Error; -use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; +use crate::types::{AttrValue, DType, classify_datatype, read_attr, read_attrs}; // --------------------------------------------------------------------------- // FileData — internal storage for either owned bytes or an mmap @@ -230,9 +230,10 @@ impl File { /// A `Dataset` handle for the object header at `address` (an address /// from a group listing, or one kept from an earlier lookup), without - /// resolving a path. Resolving a path walks every group on it, which in - /// a large group costs a scan of its links; keep the address instead to - /// open the same dataset repeatedly. + /// resolving a path. Resolving a path looks each component up in its + /// group (through the name index of a dense group; a v1 group's entries + /// are scanned); keep the address instead to open the same dataset + /// repeatedly. pub fn dataset_at(&self, address: u64) -> Result, Error> { let hdr = self.parse_header(address)?; if !has_message(&hdr, MessageType::DataLayout) { @@ -245,6 +246,17 @@ impl File { .check_open() } + /// A `Group` handle for the object header at `address` (from + /// [`Group::entries`], or kept from an earlier lookup), without + /// resolving a path. Like [`group`](Self::group), the object is not + /// checked to be a group; a non-group has no children. + pub fn group_at(&self, address: u64) -> Group<'_> { + Group { + file: self, + address, + } + } + /// Resolve a path and return a `Group` handle. /// /// The path uses `/` separators (e.g., `"sensors"`). @@ -483,12 +495,7 @@ impl<'f> Group<'f> { /// Get a dataset within this group by name. pub fn dataset(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; - let hdr = self.file.parse_header(entry.object_header_address)?; + let hdr = self.file.parse_header(self.child_address(name)?)?; if !has_message(&hdr, MessageType::DataLayout) { return Err(Error::NotADataset(name.to_string())); } @@ -501,17 +508,51 @@ impl<'f> Group<'f> { /// Get a subgroup within this group by name. pub fn group(&self, name: &str) -> Result, Error> { - let entries = self.children()?; - let entry = entries - .iter() - .find(|e| e.name == name) - .ok_or_else(|| Error::Format(FormatError::PathNotFound(name.to_string())))?; Ok(Group { file: self.file, - address: entry.object_header_address, + address: self.child_address(name)?, }) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let hdr = self.file.parse_header(self.address)?; + let data = self.file.data.as_bytes(); + read_attr( + data, + &hdr, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// The object header address of the child called `name`: the entry of + /// the group's listing with that name, looked up through the group's + /// name index rather than by listing the group (see + /// [`group_v2::resolve_child`]). + fn child_address(&self, name: &str) -> Result { + let data = self.file.data.meta()?; + group_v2::resolve_child(data, &self.file.superblock, self.address, name) + .map_err(Error::Format) + } + + /// This group's children that can be opened, as `(name, object header + /// address)` in listing order — the entries [`datasets`](Self::datasets) + /// and [`groups`](Self::groups) are drawn from. Open one with + /// [`File::dataset_at`] or [`File::group_at`] to skip looking its name + /// up again, or keep the addresses to revisit the objects. + pub fn entries(&self) -> Result, Error> { + Ok(self + .children()? + .into_iter() + .map(|e| (e.name, e.object_header_address)) + .collect()) + } + /// This group's links that can be opened: hard links, and soft links /// resolved to their targets (see /// [`group_v2::resolve_group_children`]); dangling, external and @@ -1051,6 +1092,21 @@ impl<'f> Dataset<'f> { ) } + /// The attribute called `name`, or `None` if it has none by that name + /// (or it cannot be read) — the value [`attrs`](Self::attrs) has under + /// that name, found without reading the other attributes when they are + /// stored densely. + pub fn attr(&self, name: &str) -> Result, Error> { + let data = self.file.data.as_bytes(); + read_attr( + data, + &self.header, + name, + self.file.offset_size(), + self.file.length_size(), + ) + } + /// Verify this dataset's content against its stored provenance hash /// (`_provenance_sha256`, written automatically on save when a /// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see diff --git a/crates/clawhdf5/src/types.rs b/crates/clawhdf5/src/types.rs index 7023b14..acae280 100644 --- a/crates/clawhdf5/src/types.rs +++ b/crates/clawhdf5/src/types.rs @@ -182,6 +182,35 @@ pub(crate) fn read_attrs( )) } +/// The attribute called `name` on the object with header `header`, decoded +/// as [`read_attrs`] decodes it, or `None` (see +/// [`find_attribute_in_file`](clawhdf5_format::attribute::find_attribute_in_file)). +pub(crate) fn read_attr( + file_data: &[u8], + header: &clawhdf5_format::object_header::ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, crate::Error> { + let Some(msg) = clawhdf5_format::attribute::find_attribute_in_file( + file_data, + header, + name, + offset_size, + length_size, + )? + else { + return Ok(None); + }; + Ok(attrs_to_map( + std::slice::from_ref(&msg), + file_data, + offset_size, + length_size, + ) + .remove(name)) +} + pub(crate) fn attrs_to_map( attrs: &[clawhdf5_format::attribute::AttributeMessage], file_data: &[u8], diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index b5aa8d1..3f741f6 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -211,6 +211,30 @@ fn dense_attribute_stored_as_a_huge_heap_object() { assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger)); } +/// Enough huge attributes that the huge-object B-tree has internal nodes: +/// each one is found by descending it by heap ID (libhdf5 orders the +/// records of indirectly addressed huge objects by ID), and every value +/// matches what was written. +#[test] +fn many_huge_attributes_are_found_through_their_index() { + skip_if_no_python!(); + let (_dir, path) = h5py_file( + "d = f.create_dataset('d', data=[1.0])\n\ + for i in range(300):\n\ + \x20 d.attrs['h%03d' % i] = np.arange(600, dtype='i8') + i\n", + ); + let f = File::open(&path).unwrap(); + let d = f.dataset("d").unwrap(); + let attrs = d.attrs().unwrap(); + assert_eq!(attrs.len(), 300); + for i in 0..300i64 { + let name = format!("h{i:03}"); + let want: Vec = (0..600).map(|v| v + i).collect(); + assert!(matches!(&attrs[&name], AttrValue::I64Array(v) if *v == want), "{name}"); + assert!(matches!(d.attr(&name).unwrap(), Some(AttrValue::I64Array(v)) if v == want), "{name}"); + } +} + /// A group whose link heap has a deflate I/O filter (set on the group /// creation property list), with 3 000 links and one link whose message is /// larger than the heap's managed-object limit, so it is a huge object. diff --git a/crates/clawhdf5/tests/indexed_lookup_interop.rs b/crates/clawhdf5/tests/indexed_lookup_interop.rs new file mode 100644 index 0000000..518af1e --- /dev/null +++ b/crates/clawhdf5/tests/indexed_lookup_interop.rs @@ -0,0 +1,410 @@ +//! Looking one name up in a dense group (links in a fractal heap, indexed by +//! a v2 B-tree of name hashes) or in dense attribute storage reads the name +//! index, not every link: O(log n) index nodes and only the links whose +//! lookup3 hash equals the name's. Before, every lookup decoded all n links, +//! so opening each child of a 35 001-link group by name decoded ~1.2e9. +//! +//! The file is written by h5py (libhdf5 orders the index), with names whose +//! hashes collide, and every result is compared with what h5py reads. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::collections::{BTreeMap, HashMap}; +use std::process::Command; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use clawhdf5::{AttrValue, File, LazyFile, MmapFile}; +use clawhdf5_format::checksum::jenkins_lookup3; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::lookup_stats; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Links in the big group, as libhdf5's `h5stat_newgrat.h5` has. +const LINKS: usize = 35_001; +/// Attributes on the dense-attribute dataset. +const ATTRS: usize = 3_000; + +/// Pairs of distinct names with equal lookup3 hashes, found by search (the +/// hash is fixed, so the pairs are too). +fn colliding_pairs(count: usize) -> Vec<(String, String)> { + let mut seen: HashMap = HashMap::new(); + let mut pairs = Vec::new(); + for i in 0.. { + let name = format!("c{i}"); + let h = jenkins_lookup3(name.as_bytes()); + if let Some(first) = seen.insert(h, name.clone()) { + pairs.push((first, name)); + if pairs.len() == count { + break; + } + } + } + pairs +} + +/// What h5py reads: link values, attribute values, and which of the +/// missing names it finds as links and as attributes (none). +type H5pyView = ( + BTreeMap, + BTreeMap, + Vec, + Vec, +); + +struct Fixture { + _dir: tempfile::TempDir, + path: String, + /// Names in the big group, with the value of the scalar dataset each + /// links to, as h5py reads them. + links: BTreeMap, + /// Names that are not links but hash like one that is. + missing_links: Vec, + /// Attributes of `/x`, as h5py reads them. + attrs: BTreeMap, + missing_attrs: Vec, +} + +fn fixture() -> &'static Fixture { + static FIXTURE: OnceLock = OnceLock::new(); + FIXTURE.get_or_init(|| { + let pairs = colliding_pairs(6); + for (a, b) in &pairs { + assert_ne!(a, b); + assert_eq!(jenkins_lookup3(a.as_bytes()), jenkins_lookup3(b.as_bytes())); + } + // Pairs 0-2 both present (either can be the one libhdf5 orders + // first), pairs 3-5 only the first: its partner must not be found. + // "k69209"/"k155448" is the pair the writer once misordered. + let mut present: Vec = vec!["k69209".into(), "k155448".into()]; + let mut missing: Vec = Vec::new(); + for (i, (a, b)) in pairs.into_iter().enumerate() { + present.push(a); + if i < 3 { + present.push(b); + } else { + missing.push(b); + } + } + missing.extend(["", "nope", "n35001x", "N1"].map(String::from)); + let mut links = present.clone(); + let mut i = 0; + while links.len() < LINKS { + links.push(format!("n{i}")); + i += 1; + } + let mut attrs = present.clone(); + attrs.extend((0..ATTRS - present.len()).map(|i| format!("a{i}"))); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("big.h5").display().to_string(); + // The names go through a file: 35 001 of them overflow an argument. + let names = dir.path().join("names.json"); + std::fs::write( + &names, + serde_json::to_string(&(&links, &attrs, &missing)).unwrap(), + ) + .unwrap(); + let names = names.display(); + let out = run_python(&format!( + "import h5py, json, numpy as np\n\ + links, attrs, missing = json.load(open(r'{names}'))\n\ + with h5py.File(r'{path}', 'w', libver='latest') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i, n in enumerate(links):\n\ + \x20 g.create_dataset(n, data=np.int64(i))\n\ + \x20 x = f.create_dataset('x', data=np.int64(0))\n\ + \x20 for i, n in enumerate(attrs):\n\ + \x20 x.attrs[n] = np.int64(1000 + i)\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 g, a = f['g'], f['x'].attrs\n\ + \x20 print(json.dumps([{{n: int(g[n][()]) for n in g}}, {{n: int(a[n]) for n in a}},\n\ + \x20 [n for n in missing if n and n in g], [n for n in missing if n and n in a]]))", + )); + let (links, attrs, found_links, found_attrs): H5pyView = + serde_json::from_str(&out).unwrap(); + assert_eq!(links.len(), LINKS); + assert_eq!(attrs.len(), ATTRS); + assert!(found_links.is_empty() && found_attrs.is_empty()); + Fixture { + _dir: dir, + path, + links, + missing_links: missing.clone(), + attrs, + missing_attrs: missing, + } + }) +} + +fn is_not_found(e: &clawhdf5::Error) -> bool { + matches!(e, clawhdf5::Error::Format(FormatError::PathNotFound(_))) +} + +#[test] +fn one_link_lookup_reads_the_index_not_every_link() { + skip_if_no_python!(); + let fx = fixture(); + let f = File::open(&fx.path).unwrap(); + let g = f.group("g").unwrap(); + for (name, value) in &fx.links { + lookup_stats::reset(); + let ds = g.dataset(name).unwrap(); + // One link decoded per lookup, two where hashes collide — not 35 001. + let read = lookup_stats::heap_objects_read(); + assert!(read <= 2, "looking up {name} read {read} heap objects"); + assert_eq!(ds.read_i64().unwrap(), vec![*value], "{name}"); + } + + for name in &fx.missing_links { + lookup_stats::reset(); + let err = g.dataset(name).unwrap_err(); + assert!(is_not_found(&err), "{name:?}: {err:?}"); + assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}"); + } + + // A path resolves each component the same way. + for name in ["k155448", "n0", "n34000"] { + lookup_stats::reset(); + let ds = f.dataset(&format!("/g/{name}")).unwrap(); + assert!(lookup_stats::heap_objects_read() <= 2); + assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]); + } +} + +#[test] +fn one_attribute_lookup_reads_the_index_not_every_attribute() { + skip_if_no_python!(); + let fx = fixture(); + let f = File::open(&fx.path).unwrap(); + let x = f.dataset("x").unwrap(); + let all = x.attrs().unwrap(); + assert_eq!(all.len(), ATTRS); + for (name, value) in &fx.attrs { + lookup_stats::reset(); + let got = x.attr(name).unwrap(); + assert!(lookup_stats::heap_objects_read() <= 2, "{name}"); + assert!( + matches!(got, Some(AttrValue::I64(v)) if v == *value), + "{name}: {got:?}" + ); + assert!(matches!(all.get(name), Some(AttrValue::I64(v)) if v == value)); + } + for name in &fx.missing_attrs { + lookup_stats::reset(); + assert!(x.attr(name).unwrap().is_none(), "{name:?}"); + assert!(lookup_stats::heap_objects_read() <= 2, "{name:?}"); + } + // Compact attributes (on the root group: none) and a group's attributes. + assert!(f.root().attr("k69209").unwrap().is_none()); +} + +/// Every child of the big group opened by name through each file type, +/// within `limit`: with a scan per lookup this is ~1.2e9 link decodes. +#[test] +fn opening_every_child_of_a_35001_link_group_by_name_is_quick() { + skip_if_no_python!(); + let fx = fixture(); + let limit = Duration::from_secs(120); + let started = Instant::now(); + let check_time = |n: usize| { + assert!( + started.elapsed() < limit, + "{n} lookups took {:?}", + started.elapsed() + ); + }; + + let f = File::open(&fx.path).unwrap(); + let g = f.group("g").unwrap(); + for (n, (name, value)) in fx.links.iter().enumerate() { + assert_eq!(g.dataset(name).unwrap().read_i64().unwrap(), vec![*value]); + check_time(n); + } + // The listing hands out entries: open each by address. + let entries = g.entries().unwrap(); + assert_eq!(entries.len(), LINKS); + for (name, address) in &entries { + let ds = f.dataset_at(*address).unwrap(); + assert_eq!(ds.read_i64().unwrap(), vec![fx.links[name]]); + } + assert!(f.group_at(g_address(&f)).dataset("n0").is_ok()); + + let m = MmapFile::open(&fx.path).unwrap(); + let mg = m.group("g").unwrap(); + for (n, (name, value)) in fx.links.iter().enumerate() { + assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]); + check_time(n); + } + assert!(mg.group("nope").is_err_and(|e| is_not_found(&e))); + + let l = LazyFile::open_mmap(&fx.path).unwrap(); + let lg = l.group("g").unwrap(); + for (n, (name, value)) in fx.links.iter().enumerate() { + assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![*value]); + check_time(n); + } + let lx = l.dataset("x").unwrap(); + assert!( + matches!(lx.attr("k155448").unwrap(), Some(AttrValue::I64(v)) if v == fx.attrs["k155448"]) + ); + assert!( + lg.dataset(&fx.missing_links[0]) + .is_err_and(|e| is_not_found(&e)) + ); +} + +fn g_address(f: &File) -> u64 { + f.root() + .entries() + .unwrap() + .into_iter() + .find(|(n, _)| n == "g") + .unwrap() + .1 +} + +/// Every kind of link, looked up by name in a dense group (through the name +/// index) and in a compact one, opens what h5py opens and nothing it cannot: +/// hard links, soft links (absolute, relative, to a group), and not a +/// dangling soft link, an external link or a missing name. +#[test] +fn links_of_every_kind_resolve_by_name_as_in_h5py() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("links.h5").display().to_string(); + // For each group and name: "dataset ", "group", or "none" as + // h5py sees it. + let out = run_python(&format!( + "import h5py, json, numpy as np\n\ + with h5py.File(r'{path}', 'w', libver='latest') as f:\n\ + \x20 for gname, n in (('dense', 20), ('compact', 2)):\n\ + \x20 g = f.create_group(gname)\n\ + \x20 for i in range(n):\n\ + \x20 g.create_dataset(f'd{{i}}', data=np.int64(100 + i))\n\ + \x20 s = g.create_group('sub')\n\ + \x20 s.create_dataset('x', data=np.int64(7))\n\ + \x20 g['abs'] = h5py.SoftLink(f'/{{gname}}/d1')\n\ + \x20 g['rel'] = h5py.SoftLink('sub/x')\n\ + \x20 g['tosub'] = h5py.SoftLink('sub')\n\ + \x20 g['dangling'] = h5py.SoftLink('/nowhere')\n\ + \x20 g['ext'] = h5py.ExternalLink('other.h5', '/y')\n\ + names = ['d0', 'd1', 'sub', 'abs', 'rel', 'tosub', 'dangling', 'ext', 'nope', '']\n\ + seen = {{}}\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 for gname in ('dense', 'compact'):\n\ + \x20 g = f[gname]\n\ + \x20 for n in names:\n\ + \x20 try:\n\ + \x20 o = g[n] if n else None\n\ + \x20 except (KeyError, OSError):\n\ + \x20 o = None\n\ + \x20 if isinstance(o, h5py.Dataset):\n\ + \x20 seen[f'{{gname}}/{{n}}'] = f'dataset {{int(o[()])}}'\n\ + \x20 elif isinstance(o, h5py.Group):\n\ + \x20 seen[f'{{gname}}/{{n}}'] = 'group'\n\ + \x20 else:\n\ + \x20 seen[f'{{gname}}/{{n}}'] = 'none'\n\ + print(json.dumps(seen))", + )); + let seen: BTreeMap = serde_json::from_str(&out).unwrap(); + assert_eq!(seen.len(), 20); + + let f = File::open(&path).unwrap(); + // The dense group's links are in a heap, the compact group's in its + // header. + for (gname, dense) in [("dense", true), ("compact", false)] { + let g = f.group(gname).unwrap(); + lookup_stats::reset(); + g.dataset("d0").unwrap(); + assert_eq!(lookup_stats::heap_objects_read() > 0, dense, "{gname}"); + } + let m = MmapFile::open(&path).unwrap(); + let l = LazyFile::open_mmap(&path).unwrap(); + for (key, want) in &seen { + let (gname, name) = key.split_once('/').unwrap(); + let got = { + let g = f.group(gname).unwrap(); + match (g.dataset(name), g.group(name)) { + (Ok(ds), _) => format!("dataset {}", ds.read_i64().unwrap()[0]), + (Err(clawhdf5::Error::NotADataset(_)), Ok(sub)) => { + // A group: it has the child `x` (checks the address). + assert!(sub.dataset("x").is_ok() || name == "sub" || name == "tosub"); + "group".to_string() + } + (Err(e), Err(e2)) => { + assert!(is_not_found(&e) && is_not_found(&e2), "{key}: {e:?} / {e2:?}"); + "none".to_string() + } + (Err(e), Ok(_)) => panic!("{key}: dataset {e:?} but group ok"), + } + }; + assert_eq!(&got, want, "{key}"); + // The other readers agree, and a path through the group resolves the + // same way. + let mg = m.group(gname).unwrap(); + let lg = l.group(gname).unwrap(); + match want.strip_prefix("dataset ") { + Some(v) => { + let v: i64 = v.parse().unwrap(); + assert_eq!(mg.dataset(name).unwrap().read_i64().unwrap(), vec![v]); + assert_eq!(lg.dataset(name).unwrap().read_i64().unwrap(), vec![v]); + let ds = f.dataset(&format!("/{gname}/{name}")).unwrap(); + assert_eq!(ds.read_i64().unwrap(), vec![v], "{key}"); + } + None if want == "group" => { + assert!(mg.group(name).unwrap().dataset("x").is_ok(), "{key}"); + assert!(lg.group(name).unwrap().dataset("x").is_ok(), "{key}"); + let ds = f.dataset(&format!("/{gname}/{name}/x")).unwrap(); + assert_eq!(ds.read_i64().unwrap(), vec![7]); + } + None => { + assert!(mg.dataset(name).is_err_and(|e| is_not_found(&e)), "{key}"); + assert!(lg.group(name).is_err_and(|e| is_not_found(&e)), "{key}"); + } + } + } +} -- 2.54.0 From b41583113a883db41a14e5442c38bc29c1644d23 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:33:24 -0500 Subject: [PATCH 26/48] format: no truncating u64 -> usize casts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every `u64 as usize` cast in clawhdf5-format (115 on wasm32) now goes through addr::to_usize for values read from the file — addresses, lengths, counts, dimensions: FormatError::Overflow where the value does not fit instead of wrapping onto another part of the file on a 32-bit target — or addr::saturating_usize for counts bounded by something in memory (codec progress counters, writer sizes), which fail a bounds check or allocation rather than wrap. A chunk whose offset does not fit lies outside the dataset and is skipped; partial reads treat such an offset as out of the buffers. On 64-bit targets nothing changes. scripts/check-32bit-casts.sh (run by ci-test.sh) lints the wasm32 build with clippy's cast_possible_truncation and fails on any u64 -> usize finding; before this commit it listed 115. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/addr.rs | 22 +++++++++ crates/clawhdf5-format/src/btree_v1.rs | 8 +++- crates/clawhdf5-format/src/btree_v2_write.rs | 7 ++- crates/clawhdf5-format/src/chunk_index.rs | 11 ++++- crates/clawhdf5-format/src/chunked_read.rs | 46 +++++++++++++------ crates/clawhdf5-format/src/chunked_write.rs | 23 +++++----- crates/clawhdf5-format/src/data_layout.rs | 5 +- crates/clawhdf5-format/src/data_read.rs | 17 +++---- crates/clawhdf5-format/src/ea_writer.rs | 3 +- .../clawhdf5-format/src/extensible_array.rs | 11 +++-- crates/clawhdf5-format/src/file_writer.rs | 20 ++++---- crates/clawhdf5-format/src/fill_value.rs | 9 +++- crates/clawhdf5-format/src/filters.rs | 16 +++++-- crates/clawhdf5-format/src/filters_blosc2.rs | 3 +- crates/clawhdf5-format/src/filters_bzip2.rs | 7 +-- crates/clawhdf5-format/src/fixed_array.rs | 5 +- crates/clawhdf5-format/src/group_v1.rs | 11 +++-- crates/clawhdf5-format/src/link_message.rs | 3 +- crates/clawhdf5-format/src/local_heap.rs | 7 +-- crates/clawhdf5-format/src/object_header.rs | 11 +++-- crates/clawhdf5-format/src/partial_read.rs | 15 ++++-- crates/clawhdf5-format/src/selection.rs | 9 ++-- crates/clawhdf5-format/src/shared_message.rs | 9 ++-- crates/clawhdf5-format/src/vds.rs | 13 +++--- crates/clawhdf5-format/src/vl_data.rs | 7 +-- scripts/check-32bit-casts.sh | 35 ++++++++++++++ scripts/ci-test.sh | 2 + 27 files changed, 236 insertions(+), 99 deletions(-) create mode 100755 scripts/check-32bit-casts.sh diff --git a/crates/clawhdf5-format/src/addr.rs b/crates/clawhdf5-format/src/addr.rs index ec25e3d..09c2cef 100644 --- a/crates/clawhdf5-format/src/addr.rs +++ b/crates/clawhdf5-format/src/addr.rs @@ -23,6 +23,19 @@ pub fn to_usize(value: u64) -> Result { usize::try_from(value).map_err(|_| too_large(value)) } +/// A count or offset into an in-memory buffer (a codec's progress counter, +/// a size the writer computed from data it holds) as a `usize`, saturating +/// at `usize::MAX` instead of truncating. +/// +/// For values that are bounded by the length of something in memory, so +/// always fit; if one ever did not, a saturated index fails its bounds check +/// or allocation instead of silently addressing the wrong bytes. A value +/// read from the file uses [`to_usize`]. +#[inline] +pub fn saturating_usize(value: u64) -> usize { + usize::try_from(value).unwrap_or(usize::MAX) +} + #[cold] #[inline(never)] fn too_large(value: u64) -> FormatError { @@ -42,6 +55,15 @@ mod tests { assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX)); } + #[test] + fn saturating_conversion_never_wraps() { + assert_eq!(saturating_usize(0), 0); + assert_eq!(saturating_usize(0x1234), 0x1234); + assert_eq!(saturating_usize(usize::MAX as u64), usize::MAX); + // Past usize::MAX (32-bit targets) or at u64::MAX: saturates. + assert_eq!(saturating_usize(u64::MAX), usize::MAX); + } + #[test] fn values_past_usize_max_are_an_error_not_truncated() { // Only reachable where usize is narrower than u64; on a 64-bit host diff --git a/crates/clawhdf5-format/src/btree_v1.rs b/crates/clawhdf5-format/src/btree_v1.rs index b652dcc..0e9b7fe 100644 --- a/crates/clawhdf5-format/src/btree_v1.rs +++ b/crates/clawhdf5-format/src/btree_v1.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::vec::Vec; +use crate::addr::to_usize; use crate::error::FormatError; /// A parsed B-tree v1 node. @@ -164,7 +165,12 @@ fn collect_symbol_table_nodes_inner( return Err(FormatError::NestingDepthExceeded); } - let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?; + let node = BTreeV1Node::parse( + file_data, + to_usize(btree_address)?, + offset_size, + length_size, + )?; if node.node_type != 0 { return Err(FormatError::InvalidBTreeNodeType(node.node_type)); diff --git a/crates/clawhdf5-format/src/btree_v2_write.rs b/crates/clawhdf5-format/src/btree_v2_write.rs index 795fb80..38fa7c1 100644 --- a/crates/clawhdf5-format/src/btree_v2_write.rs +++ b/crates/clawhdf5-format/src/btree_v2_write.rs @@ -6,6 +6,7 @@ //! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so //! the pointer widths the writer encodes are the ones every reader expects. +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; @@ -105,7 +106,9 @@ pub(crate) fn build_btree_v2( first_node: addr + hdr_len as u64, nodes: Vec::new(), }; - let root = (n > 0).then(|| w.node(depth, 0, n as usize)).transpose()?; + let root = (n > 0) + .then(|| w.node(depth, 0, saturating_usize(n))) + .transpose()?; let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize); out.extend_from_slice(b"BTHD"); @@ -201,7 +204,7 @@ impl TreeWriter<'_> { "cannot spread {n} B-tree v2 records over {k} children at depth {depth}" ))); } - let k = k as usize; + let k = saturating_usize(k); let in_children = n - (k - 1); let (base, extra) = (in_children / k, in_children % k); diff --git a/crates/clawhdf5-format/src/chunk_index.rs b/crates/clawhdf5-format/src/chunk_index.rs index c307572..706bad0 100644 --- a/crates/clawhdf5-format/src/chunk_index.rs +++ b/crates/clawhdf5-format/src/chunk_index.rs @@ -18,6 +18,7 @@ use alloc::collections::BTreeMap; #[cfg(feature = "std")] use std::collections::HashMap; +use crate::addr::to_usize; use crate::chunk_cache::ChunkCoord; use crate::chunked_read::ChunkInfo; @@ -167,7 +168,15 @@ impl ChunkLayout { for (_coord, ci) in index.iter() { let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect(); - let chunk_offsets: Vec = coord.iter().map(|&o| o as usize).collect(); + // `ds_dims` are `usize`: a chunk at an offset past `usize::MAX` + // (only on a 32-bit target) lies outside the dataset. + let Ok(chunk_offsets) = coord + .iter() + .map(|&o| to_usize(o)) + .collect::, _>>() + else { + continue; + }; let copies = if rank == 0 { // Scalar dataset — single copy diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 6dda3cb..e249168 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -6,6 +6,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::addr::to_usize; #[cfg(feature = "std")] use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache}; use crate::data_layout::DataLayout; @@ -265,7 +266,7 @@ fn fill_from_chunks( ))); } let offsets = &c.offsets[..rank]; - let c_addr = c.address as usize; + let c_addr = to_usize(c.address)?; let size = c.chunk_size as usize; ensure_len(file_data, c_addr, size)?; let raw = &file_data[c_addr..c_addr + size]; @@ -825,7 +826,7 @@ fn parse_chunk_node( return Err(FormatError::NestingDepthExceeded); } - let offset = btree_address as usize; + let offset = to_usize(btree_address)?; let os = offset_size as usize; // Parse B-tree v1 header @@ -945,7 +946,8 @@ pub fn generate_implicit_chunks( } let total_chunks: u64 = num_chunks_per_dim.iter().product(); - let mut chunks = Vec::with_capacity(total_chunks as usize); + // A capacity hint only (a count past `usize::MAX` could not be pushed). + let mut chunks = Vec::with_capacity(usize::try_from(total_chunks).unwrap_or(0)); for linear_idx in 0..total_chunks { let mut offsets = vec![0u64; rank]; let mut remaining = linear_idx; @@ -993,7 +995,7 @@ fn read_btree_v2_chunks( use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}")); - let header = BTreeV2Header::parse(file_data, addr as usize, offset_size, length_size)?; + let header = BTreeV2Header::parse(file_data, to_usize(addr)?, offset_size, length_size)?; let rank = chunk_dims.len(); let os = offset_size as usize; let record_size = header.record_size as usize; @@ -1122,7 +1124,11 @@ pub fn list_chunks( // Both v3 and v4 include element size as last dim (rank+1) let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; // Collect chunks based on version and index type let mut chunks = match (version, chunk_index_type) { @@ -1158,7 +1164,7 @@ pub fn list_chunks( // Fixed Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; let header = - FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; + FixedArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; read_fixed_array_chunks( file_data, &header, @@ -1174,7 +1180,7 @@ pub fn list_chunks( // Extensible Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; let header = - ExtensibleArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; + ExtensibleArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; read_extensible_array_chunks( file_data, &header, @@ -1349,7 +1355,11 @@ pub(crate) fn read_chunked_full( // dimension the total is 0 even if other dimensions are huge. return Ok(output); } - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; let placer = ChunkPlacer::new(&chunk_dims, &ds_dims, elem_size); let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?; // Chunks are cached only when the whole dataset fits: pushing a larger @@ -1603,7 +1613,11 @@ pub fn read_chunked_data_sweep( check_chunk_element_size(layout, datatype, offset_size)?; let elem_size = datatype.type_size() as usize; let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; // The per-file cache is shared across datasets (and threads); every // lookup is keyed by this dataset's chunk-index address, so another @@ -1659,7 +1673,7 @@ pub fn read_chunked_data_sweep( cached } else { // Decompress from file - let c_addr = chunk_info.address as usize; + let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; @@ -1682,8 +1696,8 @@ pub fn read_chunked_data_sweep( .offsets .iter() .take(rank) - .map(|&o| o as usize) - .collect(); + .map(|&o| to_usize(o)) + .collect::>()?; if rank == 0 { let copy_len = decompressed.len().min(output.len()); @@ -1743,7 +1757,11 @@ pub fn read_chunked_data_indexed( check_chunk_element_size(layout, datatype, offset_size)?; let elem_size = datatype.type_size() as usize; let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; // Chunk index and assembly plan for this dataset, built on first access // and kept per dataset (keyed by chunk-index address) in the shared cache. @@ -1776,7 +1794,7 @@ pub fn read_chunked_data_indexed( if let Some(cached) = cache.get_decompressed_in(addr, coord) { chunk_buffers.push(cached); } else { - let c_addr = *file_offset as usize; + let c_addr = to_usize(*file_offset)?; let size = *file_size as usize; ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 2a8bd09..cf5c958 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; @@ -414,18 +415,18 @@ pub fn split_into_chunks( // Dataset strides (row-major) let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * shape[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * saturating_usize(shape[i + 1]); } // Chunk strides let mut chunk_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1] as usize; + chunk_strides[i] = chunk_strides[i + 1] * saturating_usize(chunk_dims[i + 1]); } - let chunk_total_elements: usize = chunk_dims.iter().map(|&d| d as usize).product(); + let chunk_total_elements: usize = chunk_dims.iter().map(|&d| saturating_usize(d)).product(); - let mut result = Vec::with_capacity(total_chunks as usize); + let mut result = Vec::with_capacity(saturating_usize(total_chunks)); for linear_idx in 0..total_chunks { // Convert linear index to chunk grid coordinates @@ -453,8 +454,8 @@ pub fn split_into_chunks( let coord_in_chunk = remaining_idx / chunk_strides[d]; remaining_idx %= chunk_strides[d]; - let global_coord = offsets[d] as usize + coord_in_chunk; - if global_coord >= shape[d] as usize { + let global_coord = saturating_usize(offsets[d]) + coord_in_chunk; + if global_coord >= saturating_usize(shape[d]) { out_of_bounds = true; break; } @@ -1036,7 +1037,7 @@ impl ChunkIndexPlan { Ok(Self::SingleChunk) } else { let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?; - Ok(Self::FixedArray(grid, nslots as usize)) + Ok(Self::FixedArray(grid, saturating_usize(nslots))) } } 1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array( @@ -1251,7 +1252,7 @@ pub fn write_selection_to_buffer( let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]); } let mut src_offset = 0usize; @@ -1301,7 +1302,7 @@ pub fn write_selection_to_buffer( buffer, new_data, src_offset, - current_ds_offset + coord as usize * ds_strides[d], + current_ds_offset + saturating_usize(coord) * ds_strides[d], ); } } @@ -1328,14 +1329,14 @@ pub fn write_selection_to_buffer( let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]); } for (pi, pt) in pts.iter().enumerate() { let flat: usize = pt .iter() .zip(ds_strides.iter()) - .map(|(&p, &s)| p as usize * s) + .map(|(&p, &s)| saturating_usize(p) * s) .sum(); let dst = flat * elem_size; let src = pi * elem_size; diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 59a9065..c9b93c7 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -6,6 +6,7 @@ use alloc::{format, string::String, vec::Vec}; #[cfg(feature = "std")] use std::string::String; +use crate::addr::to_usize; use crate::error::FormatError; /// A single VDS (Virtual Dataset) source mapping. @@ -207,7 +208,7 @@ pub fn parse_vds_mappings( "VDS mapping shares a name with a later entry".into(), )); } - Ok(idx as usize) + to_usize(idx) }; let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 { @@ -320,7 +321,7 @@ impl DataLayout { { let coll = crate::global_heap::GlobalHeapCollection::parse( file_data, - addr as usize, + to_usize(addr)?, length_size, )?; let obj = coll.get_object(*global_heap_index as u16).ok_or( diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index ef73a04..bcf1a18 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -6,6 +6,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] use std::collections::BTreeMap; +use crate::addr::to_usize; #[cfg(feature = "std")] use crate::chunk_cache::ChunkCache; use crate::chunked_read::read_chunked_data; @@ -117,7 +118,7 @@ pub fn read_raw_data_zerocopy<'a>( dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { - let num_elements = dataspace.num_elements() as usize; + let num_elements = to_usize(dataspace.num_elements())?; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( @@ -128,7 +129,7 @@ pub fn read_raw_data_zerocopy<'a>( match layout { DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; - let addr = addr as usize; + let addr = to_usize(addr)?; let sz = contiguous_read_len(*size, expected_size)?; ensure_len(file_data, addr, sz)?; Ok(Some(&file_data[addr..addr + sz])) @@ -219,7 +220,7 @@ fn read_raw_data_full_impl( length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { - let num_elements = dataspace.num_elements() as usize; + let num_elements = to_usize(dataspace.num_elements())?; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( @@ -239,7 +240,7 @@ fn read_raw_data_full_impl( } DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; - let addr = addr as usize; + let addr = to_usize(addr)?; let sz = contiguous_read_len(*size, expected_size)?; ensure_len(file_data, addr, sz)?; let mut out = crate::bulk_alloc::vec_for_bulk(sz); @@ -582,7 +583,7 @@ pub fn extract_selection_from_buffer( let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * to_usize(dims[i + 1])?; } let mut output = Vec::with_capacity(pts.len() * elem_size); @@ -590,8 +591,8 @@ pub fn extract_selection_from_buffer( let flat: usize = pt .iter() .zip(ds_strides.iter()) - .map(|(&p, &s)| p as usize * s) - .sum(); + .map(|(&p, &s)| Ok(to_usize(p)? * s)) + .sum::>()?; let src = flat * elem_size; if src + elem_size <= full_data.len() { output.extend_from_slice(&full_data[src..src + elem_size]); @@ -1341,7 +1342,7 @@ pub fn read_compound_fields( let mut fields = Vec::with_capacity(members.len()); for m in members { let field_size = m.datatype.type_size() as usize; - let offset = m.byte_offset as usize; + let offset = to_usize(m.byte_offset)?; if offset .checked_add(field_size) .is_none_or(|end| end > elem_size) diff --git a/crates/clawhdf5-format/src/ea_writer.rs b/crates/clawhdf5-format/src/ea_writer.rs index f669534..de7859c 100644 --- a/crates/clawhdf5-format/src/ea_writer.rs +++ b/crates/clawhdf5-format/src/ea_writer.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{vec, vec::Vec}; @@ -247,7 +248,7 @@ pub fn build_extensible_array_at( // Header (EAHD). The six statistics are, in order: super blocks, their // bytes, data blocks, their bytes, max index set, elements realised. - let mut out = Vec::with_capacity((cursor - ea_base_address) as usize); + let mut out = Vec::with_capacity(saturating_usize(cursor - ea_base_address)); out.extend_from_slice(b"EAHD"); out.push(0); // version out.push(client_id); diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index ba1c822..9256326 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -9,6 +9,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::addr::to_usize; use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; @@ -451,7 +452,7 @@ pub fn read_extensible_array_chunks( // Parse index block (EAIB): signature(4) + version(1) + client_id(1) // + header address(offset_size), then the inline elements, then the // direct data block addresses, then the super block addresses. - let ib_offset = header.index_block_address as usize; + let ib_offset = to_usize(header.index_block_address)?; let ib_header_size = 4 + 1 + 1 + os; ensure_len(file_data, ib_offset, ib_header_size)?; @@ -463,7 +464,7 @@ pub fn read_extensible_array_chunks( let mut pos = ib_offset + ib_header_size; let mut chunks = Vec::new(); - let total_elements = header.num_elements as usize; + let total_elements = to_usize(header.num_elements)?; let dmin = header.min_dblk_nelmts as usize; if dmin == 0 || !dmin.is_power_of_two() { @@ -563,7 +564,7 @@ pub fn read_extensible_array_chunks( } chunks.extend(read_data_block_elements( file_data, - addr as usize, + to_usize(addr)?, dblk_nelmts, header, offset_size, @@ -592,7 +593,7 @@ pub fn read_extensible_array_chunks( if !is_undefined_addr(sb_addr, offset_size) { chunks.extend(read_super_block( file_data, - sb_addr as usize, + to_usize(sb_addr)?, ndblks, dblk_nelmts, header, @@ -676,7 +677,7 @@ fn read_super_block( if !is_undefined_addr(addr, offset_size) { chunks.extend(read_data_block_elements( file_data, - addr as usize, + to_usize(addr)?, dblk_nelmts, header, offset_size, diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 2568582..1a1af83 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -3,6 +3,7 @@ //! Produces valid HDF5 files with v3 superblock, v2 object headers, //! link messages, contiguous datasets, inline and dense attributes. +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; @@ -336,7 +337,7 @@ pub(crate) fn build_single_block_fractal_heap( // An object must fit one direct block: the writer has no huge-object // path, and libhdf5 cannot read an object that overruns its block. - let max_managed = max_direct_block_size as usize - dblock_header_size; + let max_managed = saturating_usize(max_direct_block_size) - dblock_header_size; if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) { return Err(FormatError::SerializationError(format!( "a {}-byte message cannot go in dense storage: a fractal heap \ @@ -392,7 +393,7 @@ pub(crate) fn build_single_block_fractal_heap( let dblock_addr = frhp_addr + frhp_size as u64; let btree_addr = dblock_addr + starting_block_size; - let data_space = starting_block_size as usize - dblock_header_size; + let data_space = saturating_usize(starting_block_size) - dblock_header_size; let free_space = data_space - total_data_size; // Build fractal heap header @@ -428,7 +429,7 @@ pub(crate) fn build_single_block_fractal_heap( debug_assert_eq!(frhp.len(), frhp_size); // Build direct block: header (with checksum) + data + padding - let mut dblock = Vec::with_capacity(starting_block_size as usize); + let mut dblock = Vec::with_capacity(saturating_usize(starting_block_size)); dblock.extend_from_slice(b"FHDB"); dblock.push(0); // version write_offset(&mut dblock, frhp_addr, OFFSET_SIZE); @@ -446,12 +447,12 @@ pub(crate) fn build_single_block_fractal_heap( } // Pad to full block size - dblock.resize(starting_block_size as usize, 0); + dblock.resize(saturating_usize(starting_block_size), 0); // Checksum: computed over entire block with checksum field zeroed let dblock_checksum = crate::checksum::jenkins_lookup3(&dblock); dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&dblock_checksum.to_le_bytes()); - debug_assert_eq!(dblock.len(), starting_block_size as usize); + debug_assert_eq!(dblock.len(), saturating_usize(starting_block_size)); // Build heap IDs let heap_ids: Vec> = obj_offsets @@ -706,7 +707,7 @@ impl HeapIndirectBlock { let cksum_pos = out.len(); out.extend_from_slice(&[0u8; 4]); // checksum placeholder out.extend_from_slice(&b.data); - out.resize(d + b.size as usize, 0); + out.resize(d + saturating_usize(b.size), 0); let cksum = crate::checksum::jenkins_lookup3(&out[d..]); out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes()); child += b.size; @@ -740,7 +741,7 @@ impl HeapPacker<'_> { nrows: Option, ) -> Result { let geom = self.geom; - let width = geom.width as usize; + let width = saturating_usize(geom.width); let mut slots = Vec::new(); let mut off = heap_offset; let mut row = 0usize; @@ -763,7 +764,8 @@ impl HeapPacker<'_> { // A child whose biggest direct block cannot hold the // next object is skipped whole, not walked. let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1); - if self.objects[self.next].len() > (biggest as usize - geom.dblock_header_size) + if self.objects[self.next].len() + > (saturating_usize(biggest) - geom.dblock_header_size) { slots.push(HeapSlot::Empty); off += size; @@ -794,7 +796,7 @@ impl HeapPacker<'_> { /// objects as fit; leave it unallocated if not even the next one does. fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot { let header = self.geom.dblock_header_size; - let capacity = size as usize - header; + let capacity = saturating_usize(size) - header; let mut data = Vec::new(); while let Some(obj) = self.objects.get(self.next) { if data.len() + obj.len() > capacity { diff --git a/crates/clawhdf5-format/src/fill_value.rs b/crates/clawhdf5-format/src/fill_value.rs index 3b18790..d8c34f3 100644 --- a/crates/clawhdf5-format/src/fill_value.rs +++ b/crates/clawhdf5-format/src/fill_value.rs @@ -12,6 +12,7 @@ #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::addr::to_usize; use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; @@ -256,7 +257,11 @@ pub fn apply_to_unallocated_chunks( length_size, )?; let rank = chunk_dims.len(); - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; if rank == 0 || ds_dims.len() != rank || chunk_dims.contains(&0) { return Ok(()); } @@ -288,7 +293,7 @@ pub fn apply_to_unallocated_chunks( let mut cell = 0usize; let mut in_range = true; for d in 0..rank { - let coord = chunk.offsets[d] as usize / chunk_dims[d]; + let coord = to_usize(chunk.offsets[d])? / chunk_dims[d]; if coord >= grid[d] { in_range = false; break; diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 40d6cd9..672649b 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -3,6 +3,8 @@ #[cfg(not(feature = "std"))] extern crate alloc; +#[cfg(feature = "deflate")] +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{boxed::Box, format, vec, vec::Vec}; @@ -1114,7 +1116,11 @@ fn inflate_bounded_into( loop { let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); let status = inflater - .decompress_vec(&data[in_before as usize..], out, FlushDecompress::Finish) + .decompress_vec( + &data[saturating_usize(in_before)..], + out, + FlushDecompress::Finish, + ) .map_err(|e| format!("deflate: {e}"))?; if out.len() > limit { return Err("deflate: output exceeds size limit".into()); @@ -1132,7 +1138,7 @@ fn inflate_bounded_into( } Status::Ok | Status::BufError => { // Room left, so the decoder stopped for want of input. - if inflater.total_in() as usize >= data.len() + if saturating_usize(inflater.total_in()) >= data.len() || (inflater.total_in(), inflater.total_out()) == (in_before, out_before) { return Err("deflate: truncated stream".into()); @@ -1232,7 +1238,11 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result, String loop { let (in_before, out_before) = (deflater.total_in(), deflater.total_out()); let status = deflater - .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish) + .compress_vec( + &data[saturating_usize(in_before)..], + &mut out, + FlushCompress::Finish, + ) .map_err(|e| format!("deflate: {e}"))?; match status { Status::StreamEnd => return Ok(out), diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index f4d1690..a609a26 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -46,6 +46,7 @@ //! variable-length blocks, dictionaries, lazy chunks, user-defined codecs //! and registered filters (e.g. bytedelta), sparse frames. +use crate::addr::saturating_usize; use crate::error::FormatError; use crate::filter_registry::FilterContext; use crate::filters_bitshuffle::bitunshuffle_block; @@ -546,7 +547,7 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { return Err(err("negative size in frame header")); } let header_len = header_len as usize; - let buf = &buf[..frame_len as usize]; + let buf = &buf[..saturating_usize(frame_len)]; let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; let data_end = header_len .checked_add(cbytes) diff --git a/crates/clawhdf5-format/src/filters_bzip2.rs b/crates/clawhdf5-format/src/filters_bzip2.rs index 93e33a3..6989c5b 100644 --- a/crates/clawhdf5-format/src/filters_bzip2.rs +++ b/crates/clawhdf5-format/src/filters_bzip2.rs @@ -4,6 +4,7 @@ //! the compression level). Decoded with the `bzip2` crate's default backend, //! `libbz2-rs-sys`, a pure-Rust port of libbzip2. +use crate::addr::saturating_usize; use crate::error::FormatError; use crate::filter_registry::FilterContext; @@ -28,7 +29,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result limit { return Err(err("output exceeds the chunk size")); @@ -43,7 +44,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result= input.len() + } else if saturating_usize(dec.total_in()) >= input.len() || (dec.total_in(), dec.total_out()) == (in_before, out_before) { return Err(err("truncated stream")); @@ -61,7 +62,7 @@ pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result Result, FormatError> { - let db_offset = header.data_block_address as usize; + let db_offset = to_usize(header.data_block_address)?; // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size) let db_header_size = 4 + 1 + 1 + offset_size as usize; @@ -174,7 +175,7 @@ pub fn read_fixed_array_chunks( // Elements start immediately after the data block prefix. let elements_start = db_offset + db_header_size; - let num_elements = header.num_elements as usize; + let num_elements = to_usize(header.num_elements)?; // A chunk index cannot describe more elements than the file has bytes (each // element occupies at least `offset_size` bytes). Reject a corrupt count // before it can drive a huge loop or overflow an offset computation. diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index a4f97c8..3dff9cd 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; +use crate::addr::to_usize; use crate::btree_v1::collect_symbol_table_nodes; use crate::error::FormatError; use crate::local_heap::LocalHeap; @@ -54,7 +55,7 @@ pub(crate) fn v1_group_entries( // Parse local heap let heap = LocalHeap::parse( file_data, - sym_table_msg.local_heap_address as usize, + to_usize(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; @@ -70,7 +71,7 @@ pub(crate) fn v1_group_entries( let mut entries = Vec::new(); let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; + let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; for entry in &snod.entries { // Like libhdf5, look at the heap's free list only once a name is // needed: an empty group with a damaged heap still lists. @@ -152,7 +153,7 @@ fn for_each_v1_soft_link( ) -> Result<(), FormatError> { let heap = LocalHeap::parse( file_data, - sym_table_msg.local_heap_address as usize, + to_usize(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; @@ -164,7 +165,7 @@ fn for_each_v1_soft_link( )?; let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; + let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; for entry in &snod.entries { if entry.cache_type != CACHE_TYPE_SOFT_LINK { continue; @@ -242,7 +243,7 @@ pub fn resolve_path( // Not last — must be a group, parse its object header to get symbol table let obj_header = ObjectHeader::parse( file_data, - entry.object_header_address as usize, + to_usize(entry.object_header_address)?, offset_size, length_size, )?; diff --git a/crates/clawhdf5-format/src/link_message.rs b/crates/clawhdf5-format/src/link_message.rs index 27044cc..55cb269 100644 --- a/crates/clawhdf5-format/src/link_message.rs +++ b/crates/clawhdf5-format/src/link_message.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; +use crate::addr::to_usize; use crate::datatype::CharacterSet; use crate::error::FormatError; @@ -247,7 +248,7 @@ impl LinkMessage { }; // Link name length - let name_len = read_offset(data, pos, name_size_field_width)? as usize; + let name_len = to_usize(read_offset(data, pos, name_size_field_width)?)?; pos += name_size_field_width as usize; // Link name diff --git a/crates/clawhdf5-format/src/local_heap.rs b/crates/clawhdf5-format/src/local_heap.rs index 39e9b26..b2ef923 100644 --- a/crates/clawhdf5-format/src/local_heap.rs +++ b/crates/clawhdf5-format/src/local_heap.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::string::String; +use crate::addr::to_usize; use crate::error::FormatError; /// Parsed HDF5 Local Heap header. @@ -140,15 +141,15 @@ impl LocalHeap { /// Read a null-terminated string from the heap's data segment at the given byte offset. pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result { - let seg_addr = self.data_segment_address as usize; + let seg_addr = to_usize(self.data_segment_address)?; let str_start = seg_addr - .checked_add(string_offset as usize) + .checked_add(to_usize(string_offset)?) .ok_or(FormatError::Overflow( "local heap seg_addr + string_offset overflow".into(), ))?; let seg_end = seg_addr - .checked_add(self.data_segment_size as usize) + .checked_add(to_usize(self.data_segment_size)?) .ok_or(FormatError::Overflow( "local heap seg_addr + data_segment_size overflow".into(), ))?; diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 1fc8696..3d7b7f7 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -5,6 +5,7 @@ use alloc::vec::Vec; use byteorder::{ByteOrder, LittleEndian}; +use crate::addr::to_usize; use crate::error::FormatError; use crate::message_type::MessageType; @@ -264,8 +265,8 @@ impl ObjectHeader { // Follow continuations (v1 continuation chunks are just raw // messages, no signature); check_message has checked the body. if msg_type == MessageType::ObjectHeaderContinuation { - let cont_offset = read_offset(body, 0, offset_size)? as usize; - let cont_length = read_offset(body, offset_size as usize, length_size)? as usize; + let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?; + let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?; Self::parse_v1_chunk( data, cont_offset, @@ -339,7 +340,7 @@ impl ObjectHeader { _ => unreachable!(), }; ensure_len(data, pos, chunk_size_width as usize)?; - let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize; + let chunk0_size = to_usize(read_offset(data, pos, chunk_size_width)?)?; pos += chunk_size_width as usize; // Bit 2: attribute creation order tracked → messages include creation order field let has_creation_order = flags & 0x04 != 0; @@ -472,8 +473,8 @@ impl ObjectHeader { let msg_type = MessageType::from_u16(msg_type_raw); if msg_type == MessageType::ObjectHeaderContinuation { // check_message has checked the body holds both fields. - let cont_off = read_offset(body, 0, offset_size)? as usize; - let cont_len = read_offset(body, offset_size as usize, length_size)? as usize; + let cont_off = to_usize(read_offset(body, 0, offset_size)?)?; + let cont_len = to_usize(read_offset(body, offset_size as usize, length_size)?)?; continuations.push((cont_off, cont_len)); } else if msg_type == MessageType::Nil { null_count += 1; diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index 4c5a01e..ad5dc8e 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -203,7 +203,12 @@ fn copy_overlap( }; let (src_strides, out_strides) = (strides(src_shape), strides(box_extent)); let last = rank - 1; - let run = ((hi[last] - lo[last]) as usize) * elem_size; + // Byte offsets into the in-memory buffers; one that does not fit `usize` + // (a 32-bit target) is out of both buffers, like one past their ends. + let bytes = |elements: u64| usize::try_from(elements).ok()?.checked_mul(elem_size); + let Some(run) = bytes(hi[last] - lo[last]) else { + return; + }; let mut idx = lo.clone(); loop { @@ -213,8 +218,12 @@ fn copy_overlap( let out_at: u64 = (0..rank) .map(|d| (idx[d] - box_start[d]) * out_strides[d]) .sum(); - let (s, o) = (src_at as usize * elem_size, out_at as usize * elem_size); - if let (Some(from), Some(to)) = (src.get(s..s + run), out.get_mut(o..o + run)) { + if let (Some(s), Some(o)) = (bytes(src_at), bytes(out_at)) + && let (Some(from), Some(to)) = ( + src.get(s..s.saturating_add(run)), + out.get_mut(o..o.saturating_add(run)), + ) + { to.copy_from_slice(from); } // Advance over every dimension but the last. diff --git a/crates/clawhdf5-format/src/selection.rs b/crates/clawhdf5-format/src/selection.rs index de31e46..3dbc6ee 100644 --- a/crates/clawhdf5-format/src/selection.rs +++ b/crates/clawhdf5-format/src/selection.rs @@ -19,6 +19,7 @@ use alloc::{vec, vec::Vec}; use core::ops::Range; +use crate::addr::to_usize; use crate::error::FormatError; /// A selection describing which elements of a dataset to access. @@ -562,7 +563,7 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result 32 { @@ -625,11 +626,11 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result { let target_header = - ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?; + ObjectHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; for msg in &target_header.messages { if msg.msg_type == target_msg_type && !is_shared(msg.flags) { return Ok(msg.data.clone()); diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs index d67b832..df1b068 100644 --- a/crates/clawhdf5-format/src/vds.rs +++ b/crates/clawhdf5-format/src/vds.rs @@ -15,6 +15,7 @@ #[cfg(not(feature = "std"))] use alloc::{format, string::String, vec, vec::Vec}; +use crate::addr::to_usize; use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings}; use crate::dataspace::Dataspace; use crate::datatype::Datatype; @@ -208,7 +209,7 @@ fn load_mappings( return Ok(Vec::new()); }; let coll = - crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?; + crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?; let index = u16::try_from(*global_heap_index) .map_err(|_| vds_err("VDS mapping heap index out of range"))?; let obj = coll @@ -611,12 +612,12 @@ fn scatter( return Err(vds_err("virtual/source selection element counts differ")); } for (&v, &s) in vidx.iter().zip(sidx) { - let (vo, so) = (v as usize * elem_size, s as usize * elem_size); + let (vo, so) = (to_usize(v)? * elem_size, to_usize(s)? * elem_size); if vo + elem_size > out.len() || so + elem_size > src.len() { return Err(vds_err("virtual dataset selection out of bounds")); } out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]); - mapped[v as usize] = true; + mapped[to_usize(v)?] = true; } Ok(()) } @@ -747,7 +748,7 @@ fn selection_indices( return Err(vds_err("VDS selection blocks overlap")); } } - let mut out = Vec::with_capacity(volume as usize); + let mut out = Vec::with_capacity(to_usize(volume)?); for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { let mut cur = s.to_vec(); 'block: loop { @@ -868,7 +869,7 @@ fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> { // read as before, up to its length. let end = sb .data_end(base as u64, whole.len() as u64) - .map_or(whole.len(), |e| base + e as usize); + .map_or(Ok(whole.len()), |e| to_usize(e).map(|e| base + e))?; crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb) } @@ -921,7 +922,7 @@ fn open_source(file_data: &[u8], path: &str) -> Result, Forma Err(FormatError::PathNotFound(_)) => return Ok(None), Err(e) => return Err(e), }; - let header = crate::object_header::ObjectHeader::parse(file_data, addr as usize, os, ls)?; + let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?; let mut src = OpenSource { offset_size: os, length_size: ls, diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index 3a54c1a..a998a62 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -9,6 +9,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] use std::collections::BTreeMap; +use crate::addr::to_usize; use crate::error::FormatError; use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex}; @@ -55,7 +56,7 @@ pub fn parse_vl_references( ) -> Result, FormatError> { let elem_size = 4 + offset_size as usize + 4; // length + address + index let total = - (num_elements as usize) + to_usize(num_elements)? .checked_mul(elem_size) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, @@ -68,7 +69,7 @@ pub fn parse_vl_references( }); } - let mut elements = Vec::with_capacity(num_elements as usize); + let mut elements = Vec::with_capacity(to_usize(num_elements)?); let mut pos = 0; for _ in 0..num_elements { @@ -406,7 +407,7 @@ impl<'a> VlResolver<'a> { let index = GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?; // parse_index checked that the collection lies in the file. - let end = offset + index.collection_size as usize; + let end = offset + to_usize(index.collection_size)?; self.check_overlap(offset, end)?; let coll = CachedCollection::new(index); if self.cached_bytes.saturating_add(coll.cost()) > self.budget { diff --git a/scripts/check-32bit-casts.sh b/scripts/check-32bit-casts.sh new file mode 100755 index 0000000..cfed6f5 --- /dev/null +++ b/scripts/check-32bit-casts.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# CI check: clawhdf5-format has no truncating `u64 as usize` cast on a 32-bit +# target. HDF5 addresses and lengths are 64-bit; on wasm32 (or any 32-bit +# target) such a cast silently wraps an address past 4 GiB onto another part +# of the file. File values go through `addr::to_usize` (a clean error) and +# in-memory counts through `addr::saturating_usize`. +# +# Lints wasm32-unknown-unknown with clippy's cast_possible_truncation and +# fails on any u64 -> usize finding (other truncations are not checked here). +# +# Usage: +# ./scripts/check-32bit-casts.sh +# +# Prerequisites: +# rustup target add wasm32-unknown-unknown + +set -euo pipefail + +TARGET="wasm32-unknown-unknown" +echo "==> Checking for truncating u64 -> usize casts in clawhdf5-format ($TARGET)" + +out=$(cargo clippy -p clawhdf5-format --target "$TARGET" \ + --features plugin-filters --message-format short \ + -- -A clippy::all -W clippy::cast_possible_truncation 2>&1) || { + echo "$out" + echo "==> clippy failed" >&2 + exit 1 +} +found=$(grep -F 'casting `u64` to `usize`' <<<"$out" || true) +if [ -n "$found" ]; then + echo "$found" + echo "==> use addr::to_usize (file values) or addr::saturating_usize (in-memory counts)" >&2 + exit 1 +fi +echo "==> no truncating u64 -> usize casts" diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 3012cba..c7c77dd 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -147,6 +147,8 @@ run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \ --target wasm32-unknown-unknown \ --all-targets \ -- -D warnings +# A 64-bit file address must not wrap on a 32-bit target. +run_step "check-32bit-casts.sh" "$SCRIPT_DIR/check-32bit-casts.sh" # The built wasm package, run under Node against h5py/netCDF4-written files, # and the viewer page in headless Chromium when one is found. -- 2.54.0 From 1b4a93f65a555b414ab82d56dbf7156c1aefed3a Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:33:24 -0500 Subject: [PATCH 27/48] docs: indexed name lookups and checked address conversion (M0) Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++++ docs/design/range-reads.md | 9 +++++++++ 2 files changed, 44 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f4316..f3b0b6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,41 @@ ## Unreleased +### Name lookups through the name index (2026-09-26) +- **Finding one link or attribute by name reads the name index, not every + entry.** In a dense group (links in a fractal heap) the v2 B-tree name + index (record type 5, lookup3 hash of the name) is descended to the + records with the name's hash and only their links are read — O(log n) + instead of all n. Path resolution (`File::dataset`, `resolve_path_any`, + soft-link targets) and `Group::dataset`/`Group::group` (on `File`, + `MmapFile` and `LazyFile`, which listed the whole group per call) use it; + names whose hashes collide are all compared, so the order libhdf5 gives + them does not matter. New `clawhdf5_format::group_v2::resolve_child`, + `btree_v2::find_btree_v2_records` (records in one key range), and a + `lookup-stats` feature counting heap objects read, for tests. Huge heap + objects are found through their index the same way. +- **`attr(name)`** on the facade's groups and datasets (all three file + types): one attribute, found in dense storage through its name index + (record type 8) instead of reading every attribute + (`clawhdf5_format::attribute::find_attribute_in_file`). +- **`Group::entries()` and `File::group_at(address)`**: a listing's + `(name, address)` pairs, to open children without looking names up again. +- Test: `crates/clawhdf5/tests/indexed_lookup_interop.rs` — every child of + an h5py-written 35 001-link group (with colliding hashes) opened by name + reads at most two links per lookup (before: 35 001), matches h5py, and + every link kind (soft, relative, dangling, external) resolves as h5py + resolves it in dense and compact groups. + +### Checked address conversion (2026-09-26) +- **No 64-bit file value is truncated on a 32-bit target.** Every + `u64 as usize` cast in `clawhdf5-format` (115) is gone: file addresses, + lengths and counts go through `addr::to_usize`, which fails with + `FormatError::Overflow` where the value does not fit (wasm32 and other + 32-bit targets; it used to wrap onto another part of the file), and + in-memory counts through `addr::saturating_usize`. On 64-bit targets + nothing changes. `scripts/check-32bit-casts.sh` (run by `ci-test.sh`) + lints the wasm32 build and fails on any new truncating cast. + ### Chunked full reads (2026-09-26) - **Chunks are decoded straight into the output, into reused buffers.** A full read of a chunked dataset faulted in about three times its size in diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 2e82a6c..1edee1a 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -379,6 +379,15 @@ fast path within benchmark noise. n children decodes its links O(n) times. Look names up through the index (above) and let a listing hand out its entries, so the cache has less to absorb. +- *Status 2026-09-26:* done on branch `perf/p3-indexed-lookups` — link and + attribute names through the name indexes (`group_v2::resolve_child`, + `attribute::find_attribute_in_file`; creation-order lookups by name do + not exist in the API, so the creation-order index is still only listed), + `addr::to_usize`/`saturating_usize` for all 115 `u64 as usize` casts in + `clawhdf5-format` (the 133 above counted any `*addr*/*offset* as usize`, + mostly widening `u8`/`u32` casts; `scripts/check-32bit-casts.sh` lints + wasm32 for the truncating ones), and `Group::entries`/`File::group_at`. + The facade, io and ann casts are not converted. **M1 — metadata over the trait, in-memory impl identical to today (2–3 weeks).** - Add `Storage` (above) to `clawhdf5-format`, `no_std`-compatible, with -- 2.54.0 From 3c89a31df06b613c6f082691912b3b9b34b5de60 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:35:06 -0500 Subject: [PATCH 28/48] clawhdf5: FileEditor modifies existing files in place New clawhdf5::FileEditor opens an HDF5 file (h5py-written at any libver, HDF5 2.0 format included, or clawhdf5-written) under an exclusive flock and changes only what an edit touches: - write_selection/write_all/write_values: compact, contiguous (also late-allocated) and chunked datasets, any selection. Chunks are decoded, updated and re-encoded; a filtered chunk that no longer fits moves to the end of the file unless it is the file's last structure, which grows in place. New chunks go into v1 B-tree, Extensible Array (paged data blocks included), Fixed Array and single-chunk indexes, created on first use. - resize: grow chunked datasets up to maxshape. - set_attr: add/replace compact attributes, in a NIL slot or a new continuation chunk. Each edit is planned in an in-memory image and refused whole (Error::Unsupported) when any part is unsupported (v2 B-tree / implicit new chunks, shrinking, vlen/reference data, dense or order-tracked attributes, cache images, paged/persistent free space). Commit writes and syncs new space before patching existing bytes. Layout v5 (HDF5 2.0) array indexes use 8-byte filtered chunk sizes, as libhdf5 does. Error gains Unsupported/InvalidArgument/Locked and is #[non_exhaustive]; the Python bindings map them. build_attr_message is public. Tests (h5py, h5dump, h5rs check --data after every round; h5py r+ afterwards): appends crossing EA super/data blocks and B-tree splits, the same B-tree node counts and EA statistics as libhdf5 for the same writes (in order, reversed and shuffled; paged blocks), every layout and chunk index overwritten under random selections, attributes to continuation chunks, random operations against a model, refused edits leave the file byte-identical, locking. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/type_builders.rs | 3 +- crates/clawhdf5-py/src/lib.rs | 11 +- crates/clawhdf5-tools/tests/edit_interop.rs | 1189 ++++++++++++++++++ crates/clawhdf5/src/edit/btree1.rs | 403 ++++++ crates/clawhdf5/src/edit/earray.rs | 512 ++++++++ crates/clawhdf5/src/edit/farray.rs | 185 +++ crates/clawhdf5/src/edit/image.rs | 317 +++++ crates/clawhdf5/src/edit/mod.rs | 1213 +++++++++++++++++++ crates/clawhdf5/src/edit/ohdr.rs | 504 ++++++++ crates/clawhdf5/src/edit/select.rs | 142 +++ crates/clawhdf5/src/error.rs | 16 + crates/clawhdf5/src/lib.rs | 18 + crates/clawhdf5/src/reader.rs | 13 +- crates/clawhdf5/tests/edit_tests.rs | 138 +++ 14 files changed, 4657 insertions(+), 7 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/edit_interop.rs create mode 100644 crates/clawhdf5/src/edit/btree1.rs create mode 100644 crates/clawhdf5/src/edit/earray.rs create mode 100644 crates/clawhdf5/src/edit/farray.rs create mode 100644 crates/clawhdf5/src/edit/image.rs create mode 100644 crates/clawhdf5/src/edit/mod.rs create mode 100644 crates/clawhdf5/src/edit/ohdr.rs create mode 100644 crates/clawhdf5/src/edit/select.rs create mode 100644 crates/clawhdf5/tests/edit_tests.rs diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index 3019daa..9d32e5c 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -296,7 +296,8 @@ impl EnumTypeBuilder { // ---- Attribute helper ---- -pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage { +/// The attribute message the writers store for `value` under `name`. +pub fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage { match value { AttrValue::F64(v) => AttributeMessage { name: name.to_string(), diff --git a/crates/clawhdf5-py/src/lib.rs b/crates/clawhdf5-py/src/lib.rs index e3b5619..4055b82 100644 --- a/crates/clawhdf5-py/src/lib.rs +++ b/crates/clawhdf5-py/src/lib.rs @@ -66,7 +66,9 @@ fn _panic_for_test() -> PyResult<()> { /// - I/O errors -> `PyIOError` /// - Format/parsing errors -> `PyValueError` /// - Missing dataset/path errors -> `PyKeyError` -/// - Other errors -> `PyOSError` +/// - Invalid arguments -> `PyValueError` +/// - Unsupported operations -> `PyNotImplementedError` +/// - Other errors (a locked file, ...) -> `PyOSError` pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { use clawhdf5_rs::Error; match &e { @@ -79,9 +81,14 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { | Error::ZeroCopyNotContiguous | Error::ZeroCopyNonNativeEndian | Error::ZeroCopyTypeMismatch { .. } - | Error::ZeroCopyUnaligned { .. } => { + | Error::ZeroCopyUnaligned { .. } + | Error::InvalidArgument(_) => { PyErr::new::(e.to_string()) } + Error::Unsupported(_) => { + PyErr::new::(e.to_string()) + } + _ => PyErr::new::(e.to_string()), } } diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs new file mode 100644 index 0000000..2df54d7 --- /dev/null +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -0,0 +1,1189 @@ +//! `clawhdf5::FileEditor` against libhdf5: files written by h5py (default +//! `libver`, `v114`, and HDF5 2.0's `latest`) and by clawhdf5 are modified +//! in place — values overwritten, datasets grown and appended to many +//! times (crossing Extensible Array super-block / data-block boundaries and +//! version-1 B-tree node splits), attributes added and replaced — and after +//! each round h5py must read exactly the expected values, h5dump must read +//! the file, `h5rs check --data` must find nothing, and our reader must +//! agree. At the end h5py opens the file `r+` and modifies it further. +//! +//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips +//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::{Command, Output}; + +use clawhdf5::{AttrValue, Error, File, FileBuilder, FileEditor, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn available(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the tools are here; false (skip) or a panic when interop is +/// required. +fn tools_ok() -> bool { + let ok = + available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]); + if !ok { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available" + ); + eprintln!("SKIP: h5py/numpy or h5dump not available"); + } + ok +} + +fn py(script: &str) -> String { + let o = Command::new(python()) + .args(["-c", script]) + .output() + .expect("run python"); + assert!( + o.status.success(), + "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + String::from_utf8_lossy(&o.stdout).trim().to_string() +} + +fn text(o: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ) +} + +/// The file is valid for libhdf5's tools and for `h5rs check --data`. +/// HDF5 2.0 `latest` files are beyond h5dump 1.14. +fn check_tools(path: &Path, h5dump: bool) { + let p = path.to_str().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + if h5dump { + let o = Command::new("h5dump") + .args(["-o", "/dev/null", p]) + .output() + .unwrap(); + assert!( + o.status.success() && o.stderr.is_empty(), + "h5dump {p}:\n{}", + text(&o) + ); + } +} + +/// A dataset's expected contents. +#[derive(Clone, Debug)] +struct Model { + shape: Vec, + /// Row-major values. + data: Vec, +} + +impl Model { + fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self { + let n: u64 = shape.iter().product(); + Self { + shape: shape.to_vec(), + data: (0..n).map(f).collect(), + } + } + + fn index(&self, c: &[u64]) -> usize { + c.iter() + .zip(&self.shape) + .fold(0u64, |a, (&x, &d)| a * d + x) as usize + } + + /// Grow to `shape`, new elements `fill`. + fn resize(&mut self, shape: &[u64], fill: i32) { + let old = self.clone(); + *self = Self::new(shape, |_| fill); + let n: u64 = old.shape.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; old.shape.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % old.shape[d]; + r /= old.shape[d]; + } + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } + } + + /// Apply a hyperslab write of `vals` (row-major over the block). + fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) { + let n: u64 = count.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; count.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = start[d] + r % count[d]; + r /= count[d]; + } + let i = self.index(&c); + self.data[i] = vals[flat as usize]; + } + } +} + +fn block(start: &[u64], count: &[u64]) -> Selection { + Selection::Hyperslab { + start: start.to_vec(), + stride: vec![1; start.len()], + count: count.to_vec(), + block: vec![1; start.len()], + } +} + +/// h5py and our reader both read `m` from dataset `name`. +fn verify(path: &Path, name: &str, m: &Model) { + let f = File::open(path).unwrap(); + let ds = f.dataset(name).unwrap(); + assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}"); + assert!(ds.read_i32().unwrap() == m.data, "our values of {name}"); + let exp = path.with_extension("expect"); + let bytes: Vec = m.data.iter().flat_map(|v| v.to_le_bytes()).collect(); + std::fs::write(&exp, bytes).unwrap(); + let shape: Vec = m.shape.iter().map(|d| d.to_string()).collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + a = f[{name:?}][()]\n\ + e = np.fromfile({e:?}, dtype=' u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// `h5py.File(..., libver=...)` argument for each flavour tested; the last +/// is HDF5 2.0's own format, which h5dump 1.14 cannot read. +const LIBVERS: &[(&str, bool)] = &[("'earliest'", true), ("'v114'", true), ("'latest'", false)]; + +fn tmpdir() -> tempfile::TempDir { + let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + tempfile::TempDir::new_in(base).unwrap() +} + +/// Append to a 1-D unlimited dataset hundreds of times, in runs of random +/// length: enough chunks to fill an Extensible Array's index block, direct +/// data blocks and several super blocks (paged data blocks included), and +/// to split version-1 B-tree nodes several levels deep. +fn append_many(libver: &str, h5dump: bool, compression: &str, tag: &str) { + let dir = tmpdir(); + let path = dir.path().join(format!("append_{tag}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 f.create_dataset('x', shape=(5,), maxshape=(None,), chunks=(3,), dtype=' = (0..add).map(|k| (n + k) as i32 * 7 - 3).collect(); + ed.write_values("x", &block(&[n], &[add]), &vals).unwrap(); + m.write_block(&[n], &[add], &vals); + // Occasionally rewrite something earlier (in-place, or a filtered + // chunk that has to move). + if round % 7 == 0 { + let s = rng.below(m.shape[0]); + let c = 1 + rng.below((m.shape[0] - s).min(9)); + let vals: Vec = (0..c).map(|_| rng.next() as i32).collect(); + ed.write_values("x", &block(&[s], &[c]), &vals).unwrap(); + m.write_block(&[s], &[c], &vals); + } + if round % 150 == 149 { + drop(ed); + verify(&path, "x", &m); + check_tools(&path, h5dump); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "x", &m); + check_tools(&path, h5dump); + // h5py can go on appending to what we wrote. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 n = d.shape[0]\n\ + \x20 d.resize((n + 50,))\n\ + \x20 d[n:] = np.arange(50, dtype=' = (0..50).map(|k| 1000 + k).collect(); + m.write_block(&[n], &[50], &vals); + m.data[0] = -1; + verify(&path, "x", &m); + check_tools(&path, h5dump); +} + +#[test] +fn append_many_unfiltered() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + append_many(lv, *dump, "", &format!("plain{i}")); + } +} + +#[test] +fn append_many_gzip() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + append_many( + lv, + *dump, + ", compression='gzip', shuffle=True", + &format!("gzip{i}"), + ); + } +} + +/// Random operations — grow, hyperslab writes, point writes, attributes — +/// on a 2-D dataset with one unlimited dimension, checked against a model +/// after every few operations. +fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { + let dir = tmpdir(); + let path = dir.path().join(format!("rand_{tag}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \ + dtype=' = Vec::new(); + let mut rng = Rng(seed); + let mut ed = FileEditor::open(&path).unwrap(); + for step in 0..120 { + match rng.below(10) { + 0..=1 => { + let rows = m.shape[0] + 1 + rng.below(5); + ed.resize("m", &[rows, 7]).unwrap(); + m.resize(&[rows, 7], -9); + } + 2..=6 => { + let r0 = rng.below(m.shape[0]); + let c0 = rng.below(7); + let cnt = [ + 1 + rng.below((m.shape[0] - r0).min(6)), + 1 + rng.below(7 - c0), + ]; + let n = cnt[0] * cnt[1]; + let vals: Vec = (0..n).map(|_| (rng.next() % 100_000) as i32).collect(); + ed.write_values("m", &block(&[r0, c0], &cnt), &vals) + .unwrap(); + m.write_block(&[r0, c0], &cnt, &vals); + } + 7 => { + let pts: Vec> = (0..1 + rng.below(4)) + .map(|_| vec![rng.below(m.shape[0]), rng.below(7)]) + .collect(); + let vals: Vec = pts.iter().map(|_| rng.next() as i32).collect(); + ed.write_values("m", &Selection::Points(pts.clone()), &vals) + .unwrap(); + for (p, v) in pts.iter().zip(&vals) { + let i = m.index(p); + m.data[i] = *v; + } + } + _ => { + let k = rng.below(6); + let name = format!("a{k}"); + let v = rng.next() as i64; + match ed.set_attr("m", &name, &AttrValue::I64(v)) { + Ok(()) => { + attrs.retain(|(n, _)| *n != name); + attrs.push((name, v)); + } + Err(e) => panic!("set_attr {name}: {e}"), + } + } + } + if step % 30 == 29 { + drop(ed); + verify(&path, "m", &m); + check_tools(&path, h5dump); + check_attrs(&path, "m", &attrs); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "m", &m); + check_attrs(&path, "m", &attrs); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['m']\n\ + \x20 n = d.shape[0]\n\ + \x20 d.resize((n + 3, 7))\n\ + \x20 d[n:, :] = 42\n\ + \x20 d.attrs['from_h5py'] = 1.5\n", + p = path.to_str().unwrap() + )); + let n = m.shape[0]; + m.resize(&[n + 3, 7], -9); + m.write_block(&[n, 0], &[3, 7], &[42; 21]); + verify(&path, "m", &m); + check_tools(&path, h5dump); +} + +fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) { + let f = File::open(path).unwrap(); + let got = f.dataset(obj).unwrap().attrs().unwrap(); + for (n, v) in attrs { + match got.get(n) { + Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"), + other => panic!("attribute {n}: {other:?}"), + } + } + let want: Vec = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect(); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + want = {{{w}}}\n\ + got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\ + assert got == want, (got, want)\n", + p = path.to_str().unwrap(), + w = want.join(", ") + )); +} + +#[test] +fn random_operations_match_a_model() { + if !tools_ok() { + return; + } + let mut seed = 1; + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + // h5dump has no LZF decoder (h5py's own filter). + for (j, (extra, lzf)) in [ + ("", false), + (", compression='gzip', shuffle=True, fletcher32=True", false), + (", compression='lzf'", true), + ] + .iter() + .enumerate() + { + seed += 1; + random_ops(lv, *dump && !lzf, extra, &format!("{i}_{j}"), seed); + } + } +} + +fn unsupported(r: Result) { + match r { + Err(Error::Unsupported(_)) => {} + other => panic!("expected Error::Unsupported, got {other:?}"), + } +} + +/// The statistics an Extensible Array header keeps (super blocks and their +/// bytes, data blocks and their bytes, one past the highest index set, +/// elements realised), for the file's only Extensible Array. +fn ea_stats(path: &Path) -> [u64; 6] { + let b = std::fs::read(path).unwrap(); + let at = b + .windows(4) + .position(|w| w == b"EAHD") + .expect("an EA header"); + let mut s = [0u64; 6]; + for (k, v) in s.iter_mut().enumerate() { + let o = at + 12 + 8 * k; + *v = u64::from_le_bytes(b[o..o + 8].try_into().unwrap()); + } + s +} + +/// Version-1 B-tree nodes in the file: (leaves, internal nodes). +fn btree1_nodes(path: &Path) -> (usize, usize) { + let b = std::fs::read(path).unwrap(); + let mut out = (0, 0); + for i in 0..b.len().saturating_sub(6) { + if &b[i..i + 4] == b"TREE" && b[i + 4] == 1 { + if b[i + 5] == 0 { + out.0 += 1; + } else { + out.1 += 1; + } + } + } + out +} + +/// Growth one chunk at a time and in large steps: the editor's version-1 +/// B-tree must split the way libhdf5's does (the same number of leaves and +/// internal nodes for the same insertions). +#[test] +fn btree1_splits_match_libhdf5() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let mut steps: Vec = (1..=300).collect(); + steps.extend([1000, 1001, 5000, 9000]); + let a = dir.path().join("bt_h5py.h5"); + let b = dir.path().join("bt_edit.h5"); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 d = f.create_dataset('x', shape=(0,), maxshape=(None,), chunks=(2,), dtype=' = (n..s).map(|v| v as i32).collect(); + ed.write_values("x", &block(&[n], &[s - n]), &vals).unwrap(); + n = s; + } + drop(ed); + verify(&b, "x", &Model::new(&[n], |i| i as i32)); + check_tools(&b, true); + assert_eq!( + btree1_nodes(&b), + btree1_nodes(&a), + "B-tree shape differs from libhdf5's" + ); +} + +/// Enough chunks that the Extensible Array reaches its paged data blocks +/// (the first one holds element 131060 with h5py's parameters): the same +/// growth done by libhdf5 and by the editor must create the same blocks. +#[test] +fn extensible_array_paged_blocks_match_libhdf5() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let steps = [5u64, 131_000, 131_070, 133_000, 140_000]; + let a = dir.path().join("paged_h5py.h5"); + let b = dir.path().join("paged_edit.h5"); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v114') as f:\n\ + \x20 d = f.create_dataset('x', shape=(0,), maxshape=(None,), chunks=(1,), dtype=' = (n..s).map(|v| v as i32).collect(); + ed.write_values("x", &block(&[n], &[s - n]), &vals).unwrap(); + n = s; + } + drop(ed); + let m = Model::new(&[n], |i| i as i32); + verify(&b, "x", &m); + check_tools(&b, true); + assert_eq!( + ea_stats(&b), + ea_stats(&a), + "EA statistics differ from libhdf5's" + ); +} + +/// Every element of `sel` over `shape`, in selection order. +/// Every index of the box `ext`, row-major. +fn odometer(ext: &[u64]) -> Vec> { + let n: u64 = ext.iter().product(); + (0..n) + .map(|flat| { + let mut c = vec![0u64; ext.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % ext[d]; + r /= ext[d]; + } + c + }) + .collect() +} + +fn sel_coords(sel: &Selection, shape: &[u64]) -> Vec> { + match sel { + Selection::All => odometer(shape), + Selection::None => Vec::new(), + Selection::Points(p) => p.clone(), + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let ext: Vec = count.iter().zip(block).map(|(c, b)| c * b).collect(); + odometer(&ext) + .into_iter() + .map(|j| { + (0..j.len()) + .map(|d| start[d] + (j[d] / block[d]) * stride[d] + j[d] % block[d]) + .collect() + }) + .collect() + } + } +} + +impl Model { + fn write_sel(&mut self, sel: &Selection, vals: &[i32]) { + let cs = sel_coords(sel, &self.shape); + assert_eq!(cs.len(), vals.len()); + for (c, v) in cs.iter().zip(vals) { + let i = self.index(c); + self.data[i] = *v; + } + } +} + +/// A random selection of `shape`: a block, a strided hyperslab, points, +/// or everything. +fn random_sel(rng: &mut Rng, shape: &[u64]) -> Selection { + let rank = shape.len(); + match rng.below(4) { + 0 => Selection::All, + 1 => { + let pts = (0..1 + rng.below(5)) + .map(|_| shape.iter().map(|&d| rng.below(d)).collect()) + .collect(); + Selection::Points(pts) + } + _ => { + let mut start = vec![0; rank]; + let mut stride = vec![1; rank]; + let mut count = vec![1; rank]; + let mut block = vec![1; rank]; + for d in 0..rank { + start[d] = rng.below(shape[d]); + let room = shape[d] - start[d]; + block[d] = 1 + rng.below(room.min(3)); + stride[d] = block[d] + rng.below(3); + // count * stride may overshoot as long as the last block fits. + count[d] = 1 + (room - block[d]) / stride[d]; + count[d] = 1 + rng.below(count[d]); + } + Selection::Hyperslab { + start, + stride, + count, + block, + } + } + } +} + +/// Datasets of every layout and chunk index h5py writes, overwritten under +/// random selections: contiguous (allocated, and never written — late +/// allocation), compact, chunked with a Fixed Array (paged, too), single +/// chunk (unfiltered and filtered), implicit (early allocation), version-1 +/// B-tree (every chunked layout of the default libver), and version-2 +/// B-tree (existing unfiltered chunks only). +#[test] +fn overwrite_every_layout() { + if !tools_ok() { + return; + } + let names: &[(&str, &[u64])] = &[ + ("contig", &[10, 4]), + ("contig_late", &[6]), + ("compact", &[3, 4]), + ("fixed", &[10, 7]), + ("fixed_gz", &[10, 7]), + ("fa_paged", &[3000]), + ("single", &[5, 6]), + ("single_gz", &[5, 6]), + ("implicit", &[8, 8]), + ("fa_empty", &[10, 7]), + ("fa_gz_empty", &[3000]), + ("ea_gz_empty", &[20]), + ("bt2", &[6, 6]), + ]; + for (li, (lv, dump)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("overwrite_{li}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + from h5py import h5p, h5d, h5s, h5t\n\ + def ar(*s): return np.arange(int(np.prod(s)), dtype=' = vec![ + Model::new(&[10, 4], |i| i as i32), + Model::new(&[6], |_| 7), + Model::new(&[3, 4], |i| i as i32), + { + let mut m = Model::new(&[10, 7], |_| -1); + m.write_block(&[0, 0], &[3, 4], &[1; 12]); + m + }, + { + let mut m = Model::new(&[10, 7], |_| 0); + m.write_block(&[4, 3], &[6, 4], &(0..24).collect::>()); + m + }, + Model::new(&[3000], |i| if i % 7 == 0 { 3 } else { 0 }), + Model::new(&[5, 6], |_| 0), + Model::new(&[5, 6], |i| i as i32), + Model::new(&[8, 8], |_| 0), + Model::new(&[10, 7], |_| 4), + Model::new(&[3000], |_| 0), + Model::new(&[20], |_| 0), + Model::new(&[6, 6], |i| i as i32), + ]; + let mut rng = Rng(77 + li as u64); + for round in 0..3 { + let mut ed = FileEditor::open(&path).unwrap(); + for ((name, shape), m) in names.iter().zip(models.iter_mut()) { + for _ in 0..6 { + let sel = random_sel(&mut rng, shape); + let n = sel_coords(&sel, shape).len(); + let vals: Vec = (0..n).map(|_| rng.next() as i32).collect(); + ed.write_values(name, &sel, &vals) + .unwrap_or_else(|e| panic!("{name} round {round} {sel:?}: {e}")); + m.write_sel(&sel, &vals); + } + } + drop(ed); + for ((name, _), m) in names.iter().zip(&models) { + verify(&path, name, m); + } + check_tools(&path, *dump); + } + // A version-2 B-tree index can take new chunks only from libhdf5 for + // now: growing works, writing the new chunks is refused and changes + // nothing. + if *lv != "'earliest'" { + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("bt2", &[8, 6]).unwrap(); + let before = std::fs::read(&path).unwrap(); + unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12])); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); + models[12].resize(&[8, 6], 0); + } + // libhdf5 goes on modifying what we wrote. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for n in {names:?}:\n\ + \x20 d = f[n]\n\ + \x20 d[(0,) * d.ndim] = 123\n", + p = path.to_str().unwrap(), + names = names.iter().map(|(n, _)| *n).collect::>() + )); + for ((name, _), m) in names.iter().zip(models.iter_mut()) { + m.data[0] = 123; + verify(&path, name, m); + } + check_tools(&path, *dump); + } +} + +/// Attributes added to and replaced on the root group, a group and a +/// dataset, until the headers need continuation chunks; then h5py adds, +/// changes and deletes attributes in the same headers. +#[test] +fn attributes_in_place() { + if !tools_ok() { + return; + } + for (li, (lv, dump)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("attrs_{li}.h5")); + let p = path.to_str().unwrap(); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 f.attrs['title'] = 'root'\n\ + \x20 g = f.create_group('g')\n\ + \x20 g.attrs['n'] = 3\n\ + \x20 d = f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + let mut ed = FileEditor::open(&path).unwrap(); + // v2 headers hold 8 compact attributes by default. + let many = if *lv == "'earliest'" { 20 } else { 6 }; + for obj in ["/", "g", "d"] { + for i in 0..many { + let name = format!("x{i}"); + let v = AttrValue::I64Array((0..=i as i64).collect()); + ed.set_attr(obj, &name, &v).unwrap(); + want.push((obj, name, v)); + } + // Replace one with something much larger (moves it). + let big = AttrValue::String("z".repeat(700)); + ed.set_attr(obj, "x1", &big).unwrap(); + want.retain(|(o, n, _)| !(*o == obj && n == "x1")); + want.push((obj, "x1".into(), big)); + // And one smaller, in place. + ed.set_attr(obj, "x0", &AttrValue::F64(2.5)).unwrap(); + want.retain(|(o, n, _)| !(*o == obj && n == "x0")); + want.push((obj, "x0".into(), AttrValue::F64(2.5))); + } + ed.set_attr("d", "units", &AttrValue::String("km".into())) + .unwrap(); + want.push(("d", "units".into(), AttrValue::String("km".into()))); + if *lv != "'earliest'" { + // Up to the compact limit (8) and no further; attributes in + // dense storage and tracked creation order are refused, and a + // refused edit writes nothing. + ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap(); + want.push(("g", "eighth".into(), AttrValue::I64(8))); + let before = std::fs::read(&path).unwrap(); + unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9))); + unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1))); + unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1))); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); + } + drop(ed); + check_tools(&path, *dump); + let f = File::open(&path).unwrap(); + let expect_py: Vec = want + .iter() + .map(|(o, n, v)| { + let got = if *o == "/" { + f.root().attrs().unwrap() + } else if *o == "d" { + f.dataset(o).unwrap().attrs().unwrap() + } else { + f.group(o).unwrap().attrs().unwrap() + }; + let g = got.get(n).unwrap_or_else(|| panic!("{o}/{n} missing")); + assert_eq!(format!("{g:?}"), format!("{v:?}"), "{o}/{n}"); + let pv = match v { + AttrValue::I64Array(a) => format!("{a:?}"), + AttrValue::String(s) => format!("{s:?}"), + AttrValue::F64(x) => format!("{x:?}"), + AttrValue::I64(x) => format!("{x}"), + other => panic!("{other:?}"), + }; + format!("({o:?}, {n:?}, {pv})") + }) + .collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + for o, n, v in [{w}]:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 assert a == v, (o, n, a, v)\n\ + assert f['d'][()].tolist() == [0, 1, 2, 3]\n\ + assert f['g'].attrs['n'] == 3 and f.attrs['title'] == 'root'\n", + w = expect_py.join(", ") + )); + // libhdf5 modifies the headers we changed. + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'd']:\n\ + \x20 f[o].attrs['from_h5py'] = 'yes'\n\ + \x20 f[o].attrs['x0'] = 9\n\ + \x20 del f[o].attrs['x1']\n" + )); + check_tools(&path, *dump); + let f = File::open(&path).unwrap(); + for attrs in [ + f.root().attrs().unwrap(), + f.dataset("d").unwrap().attrs().unwrap(), + ] { + assert!(matches!(attrs.get("from_h5py"), Some(AttrValue::String(s)) if s == "yes")); + assert!(matches!(attrs.get("x0"), Some(AttrValue::I64(9)))); + assert!(!attrs.contains_key("x1")); + assert!(attrs.contains_key("x2")); + } + } +} + +/// Files written by clawhdf5's own writer: appended to, overwritten and +/// given attributes, then read by h5py. +#[test] +fn edit_clawhdf5_written_files() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("ours.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("ext") + .with_i32_data(&(0..10).collect::>()) + .with_shape(&[10]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[4]) + .with_deflate(4); + b.create_dataset("plain").with_i32_data(&[1, 2, 3, 4, 5, 6]); + b.create_dataset("grid") + .with_i32_data(&(0..24).collect::>()) + .with_shape(&[4, 6]) + .with_maxshape(&[u64::MAX, 6]) + .with_chunks(&[2, 3]); + b.set_attr("version", AttrValue::I64(1)); + b.write(&path).unwrap(); + let mut ext = Model::new(&[10], |i| i as i32); + let mut plain = Model::new(&[6], |i| i as i32 + 1); + let mut grid = Model::new(&[4, 6], |i| i as i32); + let mut ed = FileEditor::open(&path).unwrap(); + let mut rng = Rng(9); + for _ in 0..200 { + let n = ext.shape[0]; + let add = 1 + rng.below(9); + ed.resize("ext", &[n + add]).unwrap(); + ext.resize(&[n + add], 0); + let vals: Vec = (0..add).map(|_| rng.next() as i32).collect(); + ed.write_values("ext", &block(&[n], &[add]), &vals).unwrap(); + ext.write_block(&[n], &[add], &vals); + } + for _ in 0..30 { + let rows = grid.shape[0] + rng.below(3); + ed.resize("grid", &[rows, 6]).unwrap(); + grid.resize(&[rows, 6], 0); + let sel = random_sel(&mut rng, &grid.shape.clone()); + let vals: Vec = (0..sel_coords(&sel, &grid.shape).len()) + .map(|_| rng.next() as i32) + .collect(); + ed.write_values("grid", &sel, &vals).unwrap(); + grid.write_sel(&sel, &vals); + } + ed.write_values( + "plain", + &Selection::Points(vec![vec![5], vec![0]]), + &[60, 10], + ) + .unwrap(); + plain.write_sel(&Selection::Points(vec![vec![5], vec![0]]), &[60, 10]); + ed.set_attr("/", "version", &AttrValue::I64(2)).unwrap(); + ed.set_attr("ext", "note", &AttrValue::String("appended".into())) + .unwrap(); + drop(ed); + verify(&path, "ext", &ext); + verify(&path, "plain", &plain); + verify(&path, "grid", &grid); + check_tools(&path, true); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + assert f.attrs['version'] == 2\n\ + assert f['ext'].attrs['note'] in ('appended', b'appended')\n", + p = path.to_str().unwrap() + )); +} + +/// One writer at a time: a second editor (or libhdf5 with file locking) +/// is refused until the first is dropped. +#[test] +fn editor_locks_the_file() { + let dir = tmpdir(); + let path = dir.path().join("lock.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("x").with_i32_data(&[1, 2, 3]); + b.write(&path).unwrap(); + let ed = FileEditor::open(&path).unwrap(); + assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); + if available(&python(), &["-c", "import h5py"]) { + let o = Command::new(python()) + .args([ + "-c", + &format!("import h5py; h5py.File({:?}, 'r+')", path.to_str().unwrap()), + ]) + .env_remove("HDF5_USE_FILE_LOCKING") + .output() + .unwrap(); + assert!(!o.status.success(), "h5py opened a locked file r+"); + } + drop(ed); + FileEditor::open(&path).unwrap(); +} + +/// Refused edits leave the file byte for byte as it was. +#[test] +fn refused_edits_change_nothing() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("refuse.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 f.create_dataset('s', data=['a', 'bb'], dtype=h5py.string_dtype())\n\ + \x20 f.create_dataset('x', data=np.arange(6, dtype=' = (900..len).rev().collect(); + let mut rest: Vec = (0..900).collect(); + let mut rng = Rng(3); + for i in (1..rest.len()).rev() { + rest.swap(i, rng.below(i as u64 + 1) as usize); + } + order.extend(rest.iter().take(500)); + for (lv, tag) in [("'earliest'", "bt"), ("'v114'", "ea")] { + let dir = tmpdir(); + let a = dir.path().join(format!("ooo_{tag}_h5py.h5")); + let b = dir.path().join(format!("ooo_{tag}_edit.h5")); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 d = f.create_dataset('x', shape=({len},), maxshape=(None,), chunks=(1,), dtype=' = (n..n + add) + .map(|i| ((i as f64 / 50.0).sin() * 1000.0).round() / 1000.0) + .collect(); + ed.write_values("x", &block(&[n], &[add]), &vals).unwrap(); + } + drop(ed); + let size = |p: &Path| std::fs::metadata(p).unwrap().len(); + let repacked = |p: &Path| { + let out = p.with_extension("repacked.h5"); + let _ = std::fs::remove_file(&out); + let o = Command::new("h5repack") + .args([p.to_str().unwrap(), out.to_str().unwrap()]) + .output() + .unwrap(); + assert!(o.status.success(), "{}", text(&o)); + size(&out) + }; + println!( + "chunks {chunk}{comp}, {rounds} appends of {add}: editor {} bytes \ + (repacked {}), libhdf5 {} bytes (repacked {})", + size(&b), + repacked(&b), + size(&a), + repacked(&a) + ); + } +} diff --git a/crates/clawhdf5/src/edit/btree1.rs b/crates/clawhdf5/src/edit/btree1.rs new file mode 100644 index 0000000..7289c3f --- /dev/null +++ b/crates/clawhdf5/src/edit/btree1.rs @@ -0,0 +1,403 @@ +//! Inserting into (and updating) a version-1 B-tree chunk index (node type +//! 1; layout versions 1-3), as libhdf5's `H5B_insert` does: +//! +//! - keys compare lexicographically over the chunk offsets *and* the +//! element-size coordinate (0 in a chunk's own key), so a node's final +//! ("right") key after an append is the last chunk's offsets with the +//! element-size coordinate set to the element size — the smallest key +//! greater than that chunk, which is what libhdf5 writes; +//! - a full node (2K children) splits before the insertion: the right-most +//! node of a level keeps 90% of its children, the left-most 10%, any other +//! half (libhdf5's default split ratios); siblings are relinked; +//! - a full root splits by moving its left half to a new node, so the root +//! keeps its address (the layout message never changes). +//! +//! Version-1 B-tree nodes carry no checksum. + +use std::cmp::Ordering; + +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) size: u32, + pub(crate) mask: u32, + /// Offsets in every dimension, the element-size one last. + pub(crate) offs: Vec, +} + +fn cmp(a: &[u64], b: &[u64]) -> Ordering { + a.cmp(b) +} + +#[derive(Debug, Clone)] +struct Node { + addr: u64, + level: u8, + left: u64, + right: u64, + /// `children.len() + 1` keys. + keys: Vec, + children: Vec, +} + +pub(crate) struct BTree1 { + root: u64, + /// Children per node at most (2K). + two_k: usize, + ndims: usize, + elem_size: u64, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("chunk B-tree: {why}"), + )) +} + +enum Ins { + Done, + /// The node split; the new right sibling and its first key. + Split(Key, u64), +} + +impl BTree1 { + /// `k` is the file's chunk B-tree K (children per node are 2K); + /// `ndims` counts the element-size dimension. + pub(crate) fn new(root: u64, k: u16, ndims: usize, elem_size: u64) -> Result { + if k == 0 || ndims < 2 { + return Err(bad("bad parameters")); + } + Ok(Self { + root, + two_k: 2 * k as usize, + ndims, + elem_size, + }) + } + + fn key_size(&self) -> usize { + 8 + 8 * self.ndims + } + + fn node_size(&self, os: u8) -> usize { + let os = os as usize; + 8 + 2 * os + (self.two_k + 1) * self.key_size() + self.two_k * os + } + + fn read(&self, img: &Image<'_>, addr: u64) -> Result { + let os = img.os; + let osz = os as usize; + let d = img.read(addr, 8 + 2 * osz)?; + if &d[0..4] != b"TREE" || d[4] != 1 { + return Err(bad("not a chunk B-tree node")); + } + let level = d[5]; + let n = u16::from_le_bytes([d[6], d[7]]) as usize; + if n > self.two_k { + return Err(bad("node holds more children than 2K")); + } + let left = get_uint(&d[8..], os); + let right = get_uint(&d[8 + osz..], os); + let ks = self.key_size(); + let body = img.read(addr + 8 + 2 * osz as u64, (n + 1) * ks + n * osz)?; + let mut keys = Vec::with_capacity(n + 1); + let mut children = Vec::with_capacity(n); + let mut p = 0; + for i in 0..=n { + let k = &body[p..p + ks]; + keys.push(Key { + size: u32::from_le_bytes([k[0], k[1], k[2], k[3]]), + mask: u32::from_le_bytes([k[4], k[5], k[6], k[7]]), + offs: (0..self.ndims) + .map(|d| { + u64::from_le_bytes(k[8 + 8 * d..16 + 8 * d].try_into().unwrap_or([0; 8])) + }) + .collect(), + }); + p += ks; + if i < n { + children.push(get_uint(&body[p..], os)); + p += osz; + } + } + Ok(Node { + addr, + level, + left, + right, + keys, + children, + }) + } + + fn write(&self, img: &mut Image<'_>, node: &Node) -> Result<(), Error> { + let os = img.os; + let osz = os as usize; + let mut d = vec![0u8; self.node_size(os)]; + d[0..4].copy_from_slice(b"TREE"); + d[4] = 1; + d[5] = node.level; + d[6..8].copy_from_slice(&(node.children.len() as u16).to_le_bytes()); + put_uint(&mut d[8..], node.left, os); + put_uint(&mut d[8 + osz..], node.right, os); + let ks = self.key_size(); + let mut p = 8 + 2 * osz; + for (i, k) in node.keys.iter().enumerate() { + d[p..p + 4].copy_from_slice(&k.size.to_le_bytes()); + d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes()); + for (j, o) in k.offs.iter().enumerate() { + d[p + 8 + 8 * j..p + 16 + 8 * j].copy_from_slice(&o.to_le_bytes()); + } + p += ks; + if i < node.children.len() { + put_uint(&mut d[p..], node.children[i], os); + p += osz; + } + } + // Unused key/child slots stay zero, as libhdf5 leaves them. + img.write(node.addr, &d) + } + + /// Create a tree holding one chunk; returns it (its root is a new leaf). + pub(crate) fn create( + img: &mut Image<'_>, + k: u16, + ndims: usize, + elem_size: u64, + key: Key, + addr: u64, + ) -> Result { + let mut t = Self::new(0, k, ndims, elem_size)?; + let root = img.alloc(t.node_size(img.os) as u64)?; + t.root = root; + let right = t.right_key_after(&key); + let node = Node { + addr: root, + level: 0, + left: undef(img.os), + right: undef(img.os), + keys: vec![key, right], + children: vec![addr], + }; + t.write(img, &node)?; + Ok(t) + } + + pub(crate) fn root(&self) -> u64 { + self.root + } + + /// The smallest key above chunk `key`: its offsets with the element-size + /// coordinate one element in (what libhdf5 writes as a right key). + fn right_key_after(&self, key: &Key) -> Key { + let mut offs = key.offs.clone(); + if let Some(last) = offs.last_mut() { + *last = self.elem_size; + } + Key { + size: 0, + mask: 0, + offs, + } + } + + /// Insert chunk `key` at address `addr`, or update it when the tree + /// already has a chunk at those offsets. + pub(crate) fn insert(&mut self, img: &mut Image<'_>, key: Key, addr: u64) -> Result<(), Error> { + if key.offs.len() != self.ndims || key.offs[self.ndims - 1] != 0 { + return Err(bad("bad chunk key")); + } + let root = self.read(img, self.root)?; + if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? { + // The root split: move its (left) half to a new node so the root + // keeps its address, then make the root the parent of both. + let old = self.read(img, self.root)?; + let right = self.read(img, right_addr)?; + let new_left = img.alloc(self.node_size(img.os) as u64)?; + let mut moved = old.clone(); + moved.addr = new_left; + self.write(img, &moved)?; + let mut right = right; + right.left = new_left; + self.write(img, &right)?; + let first = old.keys[0].clone(); + let last = right + .keys + .last() + .cloned() + .ok_or_else(|| bad("empty node"))?; + let new_root = Node { + addr: self.root, + level: old.level + 1, + left: undef(img.os), + right: undef(img.os), + keys: vec![first, mid, last], + children: vec![new_left, right_addr], + }; + self.write(img, &new_root)?; + } + Ok(()) + } + + fn insert_at( + &self, + img: &mut Image<'_>, + mut node: Node, + key: &Key, + addr: u64, + depth: u8, + ) -> Result { + if depth == 0 { + return Err(bad("tree too deep")); + } + let n = node.children.len(); + if n == 0 { + return Err(bad("empty node")); + } + // The child whose range holds the key: the last i with + // keys[i] <= key (the first child when the key is below them all). + let mut i = node + .keys + .iter() + .take(n) + .rposition(|k| cmp(&k.offs, &key.offs) != Ordering::Greater) + .unwrap_or(0); + if node.level == 0 { + if node.keys[i].offs == key.offs { + node.keys[i].size = key.size; + node.keys[i].mask = key.mask; + node.children[i] = addr; + self.write(img, &node)?; + return Ok(Ins::Done); + } + // Insert after child i unless the key is below every child. + let pos = if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less { + 0 + } else { + i + 1 + }; + return self.add_child(img, node, pos, key.clone(), addr); + } + let child = self.read(img, node.children[i])?; + if child.level + 1 != node.level { + return Err(bad("inconsistent node levels")); + } + let ins = self.insert_at(img, child, key, addr, depth - 1)?; + let mut changed = false; + if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less && i == 0 { + node.keys[0] = key.clone(); + changed = true; + } + if cmp(&key.offs, &node.keys[n].offs) != Ordering::Less { + node.keys[n] = self.right_key_after(key); + changed = true; + } + match ins { + Ins::Done => { + if changed { + self.write(img, &node)?; + } + Ok(Ins::Done) + } + Ins::Split(mid, right) => { + i += 1; + self.add_child(img, node, i, mid, right) + } + } + } + + /// Insert child `addr` with left key `key` at position `pos` of `node` + /// (splitting it first when full), and write what changed. + fn add_child( + &self, + img: &mut Image<'_>, + mut node: Node, + pos: usize, + key: Key, + addr: u64, + ) -> Result { + let n = node.children.len(); + if n < self.two_k { + Self::insert_child(self, &mut node, pos, key, addr); + self.write(img, &node)?; + return Ok(Ins::Done); + } + // Split first (H5B__split): how many children stay left. + let undefined = undef(img.os); + let mut nleft = if node.right == undefined { + (self.two_k as f64 * 0.9) as usize + } else if node.left == undefined { + (self.two_k as f64 * 0.1) as usize + } else { + self.two_k / 2 + }; + if pos < nleft && nleft == self.two_k { + nleft -= 1; + } else if pos >= nleft && nleft == 0 { + nleft += 1; + } + let right_addr = img.alloc(self.node_size(img.os) as u64)?; + let mut right = Node { + addr: right_addr, + level: node.level, + left: node.addr, + right: node.right, + keys: node.keys[nleft..].to_vec(), + children: node.children[nleft..].to_vec(), + }; + if node.right != undefined { + let mut sib = self.read(img, node.right)?; + sib.left = right_addr; + self.write(img, &sib)?; + } + node.keys.truncate(nleft + 1); + node.children.truncate(nleft); + node.right = right_addr; + if pos <= nleft && !(pos == nleft && nleft < n && self.goes_right(&key, &right)) { + self.insert_child(&mut node, pos, key, addr); + } else { + self.insert_child(&mut right, pos - nleft, key, addr); + } + self.write(img, &node)?; + self.write(img, &right)?; + let mid = right.keys[0].clone(); + Ok(Ins::Split(mid, right_addr)) + } + + /// For an insertion exactly at the split point: whether the key belongs + /// to the right half (it is not below the right half's first key). + fn goes_right(&self, key: &Key, right: &Node) -> bool { + cmp(&key.offs, &right.keys[0].offs) != Ordering::Less + } + + fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) { + let n = node.children.len(); + if node.level == 0 { + // A leaf: the new chunk's key goes at `pos`. At the end, the + // node's right key moves up to stay above the new chunk. + if pos == n { + let right = self.right_key_after(&key); + let last = node.keys.len() - 1; + if cmp(&node.keys[last].offs, &right.offs) == Ordering::Less { + node.keys[last] = right; + } + node.keys.insert(n, key); + } else { + node.keys.insert(pos, key); + } + } else { + // An internal node: `key` is the new child's left key, taking + // position `pos` (the child's range starts there). + if pos == n { + // A child split off the last child: its right key is the + // parent's right key already. + node.keys.insert(n, key); + } else { + node.keys.insert(pos, key); + } + } + node.children.insert(pos, addr); + } +} diff --git a/crates/clawhdf5/src/edit/earray.rs b/crates/clawhdf5/src/edit/earray.rs new file mode 100644 index 0000000..49d9621 --- /dev/null +++ b/crates/clawhdf5/src/edit/earray.rs @@ -0,0 +1,512 @@ +//! Setting elements of an Extensible Array chunk index (layout v4, index +//! type 4), creating the index block, super blocks, data blocks and data +//! block pages the element needs, exactly as `H5EA__lookup_elmt` creates +//! them — including the header statistics libhdf5 keeps (blocks created, +//! their bytes, elements realised, one past the highest index set) and the +//! "block offset" each data block records. + +use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef}; +use crate::error::Error; + +/// A chunk index element. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Elem { + pub(crate) addr: u64, + pub(crate) size: u64, + pub(crate) mask: u32, +} + +/// Encode an element: the address, and for a filtered array the stored +/// size (in `elem_size - os - 4` bytes) and filter mask. `None` is the +/// fill element (undefined address, zero size and mask). +pub(crate) fn encode_elem( + e: Option, + filtered: bool, + elem_size: usize, + os: u8, +) -> Result, Error> { + let osz = os as usize; + let mut b = vec![0u8; if filtered { elem_size } else { osz }]; + let addr = e.map_or(undef(os), |e| e.addr); + put_uint(&mut b, addr, os); + if filtered { + let width = elem_size - osz - 4; + if let Some(e) = e { + if width < 8 && e.size >> (8 * width) != 0 { + return Err(Error::Unsupported(format!( + "filtered chunk of {} bytes does not fit the index's {width}-byte size field", + e.size + ))); + } + b[osz..osz + width].copy_from_slice(&e.size.to_le_bytes()[..width]); + b[osz + width..].copy_from_slice(&e.mask.to_le_bytes()); + } + } + Ok(b) +} + +/// The width libhdf5 gives the stored-size field of a filtered chunk index +/// element for chunks of `chunk_bytes` bytes (`H5D__earray_idx_create`, +/// `H5D__farray_idx_create`): one byte more than the nominal size needs — +/// except under layout message version 5 (HDF5 2.0's own format), which +/// always uses 8 bytes. +pub(crate) fn chunk_size_len(chunk_bytes: u64, layout_version: u8) -> usize { + if layout_version >= 5 { + return 8; + } + let log2 = if chunk_bytes <= 1 { + 0 + } else { + 63 - chunk_bytes.leading_zeros() + }; + (1 + ((log2 + 8) / 8) as usize).min(8) +} + +/// Creation parameters, in the layout message's order. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EaParams { + pub(crate) max_nelmts_bits: u8, + pub(crate) idx_blk_elmts: u8, + pub(crate) sup_blk_min_data_ptrs: u8, + pub(crate) data_blk_min_elmts: u8, + pub(crate) max_dblk_page_nelmts_bits: u8, +} + +#[derive(Debug, Clone, Copy)] +struct Level { + ndblks: u64, + dblk_nelmts: u64, + /// First element of the level, counted after the index block's own. + start_idx: u64, + /// Number of data blocks in the levels before this one. + start_dblk: u64, +} + +/// An open Extensible Array. +pub(crate) struct Ea { + hdr: u64, + filtered: bool, + elem_size: usize, + p: EaParams, + /// nsuper_blks, super_blk_size, ndata_blks, data_blk_size, + /// max_idx_set, nelmts. + stats: [u64; 6], + iblock: u64, + levels: Vec, + /// Levels whose data blocks the index block addresses directly. + direct_levels: usize, + ndblk_addrs: usize, + nsblk_addrs: usize, + dirty_hdr: bool, + /// Checksummed ranges changed by `set` (start -> checksum position), + /// recomputed once by `finish`. + dirty: std::collections::BTreeMap, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("Extensible Array: {why}"), + )) +} + +impl Ea { + fn layout(p: EaParams) -> Result<(Vec, usize, usize, usize), Error> { + let dmin = u64::from(p.data_blk_min_elmts); + if dmin == 0 || !dmin.is_power_of_two() || p.max_nelmts_bits > 64 { + return Err(bad("bad creation parameters")); + } + let nsblks = + 1 + (p.max_nelmts_bits as usize).saturating_sub(dmin.trailing_zeros() as usize); + let mut levels = Vec::with_capacity(nsblks); + let (mut start_idx, mut start_dblk) = (0u64, 0u64); + for u in 0..nsblks { + let ndblks = 1u64.checked_shl((u / 2) as u32).unwrap_or(u64::MAX); + let dblk_nelmts = dmin.checked_shl(u.div_ceil(2) as u32).unwrap_or(u64::MAX); + levels.push(Level { + ndblks, + dblk_nelmts, + start_idx, + start_dblk, + }); + start_idx = start_idx.saturating_add(ndblks.saturating_mul(dblk_nelmts)); + start_dblk = start_dblk.saturating_add(ndblks); + } + let ndblk_addrs = 2 * (p.sup_blk_min_data_ptrs as usize).saturating_sub(1); + let mut direct_levels = 0; + let mut n = 0u64; + while n < ndblk_addrs as u64 { + if direct_levels >= levels.len() { + return Err(bad("index block holds more data blocks than the array")); + } + n += levels[direct_levels].ndblks; + direct_levels += 1; + } + if n != ndblk_addrs as u64 { + return Err(bad("index block ends mid super block")); + } + Ok((levels, direct_levels, ndblk_addrs, nsblks - direct_levels)) + } + + fn arr_off_size(&self) -> usize { + (self.p.max_nelmts_bits as usize).div_ceil(8) + } + + fn page_nelmts(&self) -> u64 { + 1u64.checked_shl(u32::from(self.p.max_dblk_page_nelmts_bits)) + .unwrap_or(u64::MAX) + } + + fn slot_size(&self, os: u8) -> usize { + if self.filtered { + self.elem_size + } else { + os as usize + } + } + + /// Open the array whose header is at `hdr`. + pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result { + let os = img.os; + let ls = img.ls as usize; + let size = 12 + 6 * ls + os as usize + 4; + let d = img.read(hdr, size)?; + if &d[0..4] != b"EAHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let filtered = match d[5] { + 0 => false, + 1 => true, + _ => return Err(bad("unknown client")), + }; + let elem_size = d[6] as usize; + if filtered && elem_size < os as usize + 5 { + return Err(bad("element too small")); + } + let p = EaParams { + max_nelmts_bits: d[7], + idx_blk_elmts: d[8], + data_blk_min_elmts: d[9], + sup_blk_min_data_ptrs: d[10], + max_dblk_page_nelmts_bits: d[11], + }; + let mut stats = [0u64; 6]; + for (k, s) in stats.iter_mut().enumerate() { + *s = get_uint(&d[12 + k * ls..], img.ls); + } + let iblock = get_uint(&d[12 + 6 * ls..], os); + let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?; + Ok(Self { + hdr, + filtered, + elem_size, + p, + stats, + iblock, + levels, + direct_levels, + ndblk_addrs, + nsblk_addrs, + dirty_hdr: false, + dirty: Default::default(), + }) + } + + /// Create an empty array (header only; the index block comes with the + /// first element) and return it. + pub(crate) fn create( + img: &mut Image<'_>, + p: EaParams, + filtered: bool, + chunk_bytes: u64, + layout_version: u8, + ) -> Result { + let os = img.os; + let elem_size = if filtered { + os as usize + chunk_size_len(chunk_bytes, layout_version) + 4 + } else { + os as usize + }; + let size = 12 + 6 * img.ls as usize + os as usize + 4; + let hdr = img.alloc(size as u64)?; + let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?; + let mut ea = Self { + hdr, + filtered, + elem_size, + p, + stats: [0; 6], + iblock: undef(os), + levels, + direct_levels, + ndblk_addrs, + nsblk_addrs, + dirty_hdr: true, + dirty: Default::default(), + }; + ea.write_header(img)?; + Ok(ea) + } + + pub(crate) fn header_address(&self) -> u64 { + self.hdr + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + let ls = img.ls as usize; + let size = 12 + 6 * ls + os as usize + 4; + let mut d = vec![0u8; size]; + d[0..4].copy_from_slice(b"EAHD"); + d[4] = 0; + d[5] = u8::from(self.filtered); + d[6] = self.elem_size as u8; + d[7] = self.p.max_nelmts_bits; + d[8] = self.p.idx_blk_elmts; + d[9] = self.p.data_blk_min_elmts; + d[10] = self.p.sup_blk_min_data_ptrs; + d[11] = self.p.max_dblk_page_nelmts_bits; + for (k, s) in self.stats.iter().enumerate() { + put_uint(&mut d[12 + k * ls..], *s, img.ls); + } + put_uint(&mut d[12 + 6 * ls..], self.iblock, os); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]); + d[size - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(self.hdr, &d)?; + self.dirty_hdr = false; + Ok(()) + } + + /// Recompute the checksums of the blocks `set` changed; store changed + /// header statistics. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for (start, end) in std::mem::take(&mut self.dirty) { + rechecksum(img, start, end)?; + } + if self.dirty_hdr { + self.write_header(img)?; + } + Ok(()) + } + + fn fill_elems(&self, n: u64, os: u8) -> Result, Error> { + let one = encode_elem(None, self.filtered, self.elem_size, os)?; + let n = usize::try_from(n).map_err(|_| bad("block too large"))?; + Ok(one.repeat(n)) + } + + fn iblock_prefix(&self, os: u8) -> u64 { + 6 + u64::from(os) + } + + fn iblock_len(&self, os: u8) -> u64 { + let osz = os as u64; + self.iblock_prefix(os) + + u64::from(self.p.idx_blk_elmts) * self.slot_size(os) as u64 + + (self.ndblk_addrs + self.nsblk_addrs) as u64 * osz + } + + fn create_iblock(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + let len = self.iblock_len(os); + let addr = img.alloc(len + 4)?; + let mut d = Vec::with_capacity(len as usize + 4); + d.extend_from_slice(b"EAIB"); + d.push(0); + d.push(u8::from(self.filtered)); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.hdr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&self.fill_elems(u64::from(self.p.idx_blk_elmts), os)?); + let u = undef(os).to_le_bytes(); + for _ in 0..self.ndblk_addrs + self.nsblk_addrs { + d.extend_from_slice(&u[..os as usize]); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(addr, &d)?; + self.iblock = addr; + self.stats[5] += u64::from(self.p.idx_blk_elmts); + self.dirty_hdr = true; + Ok(()) + } + + fn block_prefix(&self, sig: &[u8; 4], off: u64, os: u8) -> Vec { + let mut d = Vec::new(); + d.extend_from_slice(sig); + d.push(0); + d.push(u8::from(self.filtered)); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.hdr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&off.to_le_bytes()[..self.arr_off_size()]); + d + } + + fn dblk_prefix_len(&self, os: u8) -> u64 { + 6 + u64::from(os) + self.arr_off_size() as u64 + } + + /// Create a data block of `nelmts` elements whose recorded block offset + /// is `off`; returns its address. + fn create_dblock(&mut self, img: &mut Image<'_>, nelmts: u64, off: u64) -> Result { + let os = img.os; + let es = self.slot_size(os) as u64; + let page = self.page_nelmts(); + let prefix = self.block_prefix(b"EADB", off, os); + let (size, body) = if nelmts > page { + // Paged: only the prefix (and its checksum) is written now; each + // page is written when an element in it is first set. + let npages = nelmts / page; + let size = prefix.len() as u64 + 4 + npages * (page * es + 4); + let mut d = prefix; + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + (size, d) + } else { + let mut d = prefix; + d.extend_from_slice(&self.fill_elems(nelmts, os)?); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + (d.len() as u64, d) + }; + let addr = img.alloc(size)?; + img.write(addr, &body)?; + self.stats[2] += 1; + self.stats[3] += size; + self.stats[5] += nelmts; + self.dirty_hdr = true; + Ok(addr) + } + + /// Set element `idx` to `e`. + pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + let os = img.os; + let osz = u64::from(os); + let es = self.slot_size(os) as u64; + let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + if self.iblock == undef(os) { + self.create_iblock(img)?; + } + let ib = self.iblock; + let ib_len = self.iblock_len(os); + let idx_blk = u64::from(self.p.idx_blk_elmts); + if idx < idx_blk { + img.write(ib + self.iblock_prefix(os) + idx * es, &enc)?; + self.dirty.insert(ib, ib + ib_len); + } else { + let rel = idx - idx_blk; + let u = self + .levels + .iter() + .position(|l| { + rel < l + .start_idx + .saturating_add(l.ndblks.saturating_mul(l.dblk_nelmts)) + }) + .ok_or_else(|| bad("index beyond the array's maximum"))?; + let l = self.levels[u]; + let dblks_at = ib + self.iblock_prefix(os) + idx_blk * es; + if u < self.direct_levels { + if l.dblk_nelmts > self.page_nelmts() { + return Err(Error::Unsupported( + "Extensible Array index block addressing a paged data block".into(), + )); + } + let local = (rel - l.start_idx) / l.dblk_nelmts; + let dblk_idx = l.start_dblk + local; + let slot = dblks_at + dblk_idx * osz; + let mut addr = get_uint(&img.read(slot, os as usize)?, os); + if addr == undef(os) { + // libhdf5 records start_idx + (global data block index) + // * nelmts here (H5EA__lookup_elmt), not the block's + // own first element; kept for byte-for-byte parity. + let off = l.start_idx + dblk_idx * l.dblk_nelmts; + addr = self.create_dblock(img, l.dblk_nelmts, off)?; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, addr, os); + img.write(slot, &a)?; + self.dirty.insert(ib, ib + ib_len); + } + let within = (rel - l.start_idx) % l.dblk_nelmts; + let at = addr + self.dblk_prefix_len(os) + within * es; + img.write(at, &enc)?; + self.dirty + .insert(addr, addr + self.dblk_prefix_len(os) + l.dblk_nelmts * es); + } else { + let s = (u - self.direct_levels) as u64; + let sslot = dblks_at + self.ndblk_addrs as u64 * osz + s * osz; + let page = self.page_nelmts(); + let npages = if l.dblk_nelmts > page { + l.dblk_nelmts / page + } else { + 0 + }; + let bitmap_len = npages.div_ceil(8) * l.ndblks; + let sb_prefix = self.dblk_prefix_len(os); + let sb_len = sb_prefix + bitmap_len + l.ndblks * osz; + let mut sb = get_uint(&img.read(sslot, os as usize)?, os); + if sb == undef(os) { + let mut d = self.block_prefix(b"EASB", l.start_idx, os); + d.resize(d.len() + bitmap_len as usize, 0); + let u8s = undef(os).to_le_bytes(); + for _ in 0..l.ndblks { + d.extend_from_slice(&u8s[..os as usize]); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + sb = img.alloc(d.len() as u64)?; + img.write(sb, &d)?; + self.stats[0] += 1; + self.stats[1] += d.len() as u64; + self.dirty_hdr = true; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, sb, os); + img.write(sslot, &a)?; + self.dirty.insert(ib, ib + ib_len); + } + let local = (rel - l.start_idx) / l.dblk_nelmts; + let dslot = sb + sb_prefix + bitmap_len + local * osz; + let mut addr = get_uint(&img.read(dslot, os as usize)?, os); + if addr == undef(os) { + let off = l.start_idx + local * l.dblk_nelmts; + addr = self.create_dblock(img, l.dblk_nelmts, off)?; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, addr, os); + img.write(dslot, &a)?; + self.dirty.insert(sb, sb + sb_len); + } + let within = (rel - l.start_idx) % l.dblk_nelmts; + let dprefix = self.dblk_prefix_len(os); + if npages == 0 { + img.write(addr + dprefix + within * es, &enc)?; + self.dirty.insert(addr, addr + dprefix + l.dblk_nelmts * es); + } else { + let pg = within / page; + let page_at = addr + dprefix + 4 + pg * (page * es + 4); + let bit = local * npages + pg; + let bpos = sb + sb_prefix + bit / 8; + let mut byte = img.read(bpos, 1)?[0]; + let mask = 0x80u8 >> (bit % 8); + if byte & mask == 0 { + let fill = self.fill_elems(page, os)?; + img.write(page_at, &fill)?; + byte |= mask; + img.write(bpos, &[byte])?; + self.dirty.insert(sb, sb + sb_len); + } + img.write(page_at + (within % page) * es, &enc)?; + self.dirty.insert(page_at, page_at + page * es); + } + } + } + if idx + 1 > self.stats[4] { + self.stats[4] = idx + 1; + self.dirty_hdr = true; + } + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/farray.rs b/crates/clawhdf5/src/edit/farray.rs new file mode 100644 index 0000000..0a25fe7 --- /dev/null +++ b/crates/clawhdf5/src/edit/farray.rs @@ -0,0 +1,185 @@ +//! Setting elements of a Fixed Array chunk index (layout v4, index type 3), +//! creating the array (header and data block) when the dataset has none +//! yet, and a data block page when an element in it is first set. + +use crate::edit::earray::{Elem, chunk_size_len, encode_elem}; +use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef}; +use crate::error::Error; + +pub(crate) struct Fa { + filtered: bool, + elem_size: usize, + page_bits: u8, + nelmts: u64, + dblk: u64, + /// Checksummed ranges changed by `set`, recomputed by `finish`. + dirty: std::collections::BTreeMap, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("Fixed Array: {why}"), + )) +} + +impl Fa { + fn slot(&self, os: u8) -> u64 { + if self.filtered { + self.elem_size as u64 + } else { + u64::from(os) + } + } + + fn page(&self) -> u64 { + 1u64.checked_shl(u32::from(self.page_bits)) + .unwrap_or(u64::MAX) + } + + /// Open the array whose header is at `hdr`. + pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result { + let os = img.os; + let size = 8 + img.ls as usize + os as usize + 4; + let d = img.read(hdr, size)?; + if &d[0..4] != b"FAHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let filtered = match d[5] { + 0 => false, + 1 => true, + _ => return Err(bad("unknown client")), + }; + let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let fa = Self { + filtered, + elem_size: d[6] as usize, + page_bits: d[7], + nelmts: get_uint(&d[8..], img.ls), + dblk: get_uint(&d[8 + img.ls as usize..], os), + dirty: Default::default(), + }; + if fa.filtered && fa.elem_size < os as usize + 5 { + return Err(bad("element too small")); + } + if fa.page_bits >= 64 || fa.dblk == undef(os) { + return Err(bad("bad header fields")); + } + Ok(fa) + } + + /// Create an array of `nelmts` fill elements; returns it and its + /// header address. + pub(crate) fn create( + img: &mut Image<'_>, + nelmts: u64, + page_bits: u8, + filtered: bool, + chunk_bytes: u64, + layout_version: u8, + ) -> Result<(Self, u64), Error> { + let os = img.os; + let osz = os as usize; + let elem_size = if filtered { + osz + chunk_size_len(chunk_bytes, layout_version) + 4 + } else { + osz + }; + let mut fa = Self { + filtered, + elem_size, + page_bits, + nelmts, + dblk: 0, + dirty: Default::default(), + }; + let hsize = 8 + img.ls as usize + osz + 4; + let hdr = img.alloc(hsize as u64)?; + // Data block. + let fill = encode_elem(None, filtered, elem_size, os)?; + let mut d = Vec::new(); + d.extend_from_slice(b"FADB"); + d.push(0); + d.push(u8::from(filtered)); + let mut a = vec![0u8; osz]; + put_uint(&mut a, hdr, os); + d.extend_from_slice(&a); + let page = fa.page(); + let n = usize::try_from(nelmts).map_err(|_| bad("too many elements"))?; + let total = if nelmts > page { + let npages = nelmts.div_ceil(page); + d.resize(d.len() + npages.div_ceil(8) as usize, 0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + // Pages are written when first used; their space is reserved. + d.len() as u64 + nelmts * fa.slot(os) + npages * 4 + } else { + d.extend_from_slice(&fill.repeat(n)); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + d.len() as u64 + }; + let dblk = img.alloc(total)?; + img.write(dblk, &d)?; + fa.dblk = dblk; + let mut h = vec![0u8; hsize]; + h[0..4].copy_from_slice(b"FAHD"); + h[5] = u8::from(filtered); + h[6] = elem_size as u8; + h[7] = page_bits; + put_uint(&mut h[8..], nelmts, img.ls); + put_uint(&mut h[8 + img.ls as usize..], dblk, os); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&h[..hsize - 4]); + h[hsize - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(hdr, &h)?; + Ok((fa, hdr)) + } + + /// Set element `idx` to `e`. + pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + let os = img.os; + if idx >= self.nelmts { + return Err(bad("index beyond the array")); + } + let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let es = self.slot(os); + let prefix = 6 + u64::from(os); + let page = self.page(); + if self.nelmts <= page { + img.write(self.dblk + prefix + idx * es, &enc)?; + self.dirty + .insert(self.dblk, self.dblk + prefix + self.nelmts * es); + return Ok(()); + } + let npages = self.nelmts.div_ceil(page); + let bitmap_len = npages.div_ceil(8); + let pages_at = self.dblk + prefix + bitmap_len + 4; + let p = idx / page; + let count = page.min(self.nelmts - p * page); + let page_at = pages_at + p * (page * es + 4); + let bpos = self.dblk + prefix + p / 8; + let mut byte = img.read(bpos, 1)?[0]; + let mask = 0x80u8 >> (p % 8); + if byte & mask == 0 { + let fill = encode_elem(None, self.filtered, self.elem_size, os)?; + img.write(page_at, &fill.repeat(count as usize))?; + byte |= mask; + img.write(bpos, &[byte])?; + self.dirty + .insert(self.dblk, self.dblk + prefix + bitmap_len); + } + img.write(page_at + (idx % page) * es, &enc)?; + self.dirty.insert(page_at, page_at + count * es); + Ok(()) + } + + /// Recompute the checksums of the blocks and pages `set` changed. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for (start, end) in std::mem::take(&mut self.dirty) { + rechecksum(img, start, end)?; + } + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs new file mode 100644 index 0000000..a100b66 --- /dev/null +++ b/crates/clawhdf5/src/edit/image.rs @@ -0,0 +1,317 @@ +//! The file as one edit sees it: the bytes on disk plus the edit's pending +//! writes, and an allocator that hands out space at the end of the file. +//! +//! An edit never writes to the file while it is being planned. Every change +//! is recorded here first (reads see them), so an edit that fails half-way — +//! a filter that cannot encode, a chunk index this code does not handle — +//! leaves the file exactly as it was. [`Image::commit`] then writes the +//! changes in an order that keeps the old metadata valid for as long as +//! possible (see there). + +use std::collections::BTreeMap; +use std::io::{Seek, SeekFrom, Write}; + +use crate::error::Error; + +/// Pending writes over the file's bytes, addressed as HDF5 addresses +/// (relative to the superblock). +pub(crate) struct Image<'a> { + /// The file from the superblock to its recorded end of allocation. + base: &'a [u8], + /// Pending writes: start address -> bytes. Never overlapping. + patches: BTreeMap>, + /// End of allocated space (grows with [`Self::alloc`]). + eoa: u64, + /// The end of allocated space when the edit started. + old_eoa: u64, + /// Width of addresses and lengths in the file. + pub(crate) os: u8, + pub(crate) ls: u8, +} + +impl<'a> Image<'a> { + pub(crate) fn new(base: &'a [u8], os: u8, ls: u8) -> Self { + let eoa = base.len() as u64; + Self { + base, + patches: BTreeMap::new(), + eoa, + old_eoa: eoa, + os, + ls, + } + } + + pub(crate) fn eoa(&self) -> u64 { + self.eoa + } + + pub(crate) fn old_eoa(&self) -> u64 { + self.old_eoa + } + + /// Whether the edit changes anything. + pub(crate) fn is_dirty(&self) -> bool { + !self.patches.is_empty() || self.eoa != self.old_eoa + } + + /// Allocate `size` bytes at the end of the file. The space reads as + /// zeros until written. Nothing is ever freed: space an edit stops + /// using (a relocated chunk, say) is leaked, as there is no free-space + /// manager. + pub(crate) fn alloc(&mut self, size: u64) -> Result { + let addr = self.eoa; + let end = addr + .checked_add(size) + .filter(|&e| self.os >= 8 || e < (1u64 << (8 * u32::from(self.os))) - 1) + .ok_or_else(|| Error::Unsupported("file would exceed its address size".into()))?; + self.eoa = end; + Ok(addr) + } + + /// If `[addr, addr + old_len)` is the last allocated space, grow it to + /// `new_len` bytes (a structure at the end of the file can grow where + /// it is) and return true. + pub(crate) fn grow_tail( + &mut self, + addr: u64, + old_len: u64, + new_len: u64, + ) -> Result { + if addr.checked_add(old_len) != Some(self.eoa) || new_len < old_len { + return Ok(false); + } + let old_end = self.eoa; + self.eoa = addr; + if let Err(e) = self.alloc(new_len) { + self.eoa = old_end; + return Err(e); + } + Ok(true) + } + + /// `len` bytes at `addr`, with the pending writes applied. + pub(crate) fn read(&self, addr: u64, len: usize) -> Result, Error> { + let end = addr + .checked_add(len as u64) + .filter(|&e| e <= self.eoa) + .ok_or_else(|| { + Error::Format(clawhdf5_format::error::FormatError::UnexpectedEof { + expected: addr.saturating_add(len as u64) as usize, + available: self.eoa as usize, + }) + })?; + let mut out = vec![0u8; len]; + let base_len = self.base.len() as u64; + if addr < base_len { + let b_end = end.min(base_len); + out[..(b_end - addr) as usize] + .copy_from_slice(&self.base[addr as usize..b_end as usize]); + } + // Patches overlapping [addr, end): the last one starting before + // `end`, walking back while they still reach `addr`. + for (&p_start, bytes) in self.patches.range(..end).rev() { + let p_end = p_start + bytes.len() as u64; + if p_end <= addr { + break; + } + let lo = p_start.max(addr); + let hi = p_end.min(end); + out[(lo - addr) as usize..(hi - addr) as usize] + .copy_from_slice(&bytes[(lo - p_start) as usize..(hi - p_start) as usize]); + } + Ok(out) + } + + /// Record a write of `bytes` at `addr` (inside allocated space). + pub(crate) fn write(&mut self, addr: u64, bytes: &[u8]) -> Result<(), Error> { + if bytes.is_empty() { + return Ok(()); + } + let end = addr + .checked_add(bytes.len() as u64) + .filter(|&e| e <= self.eoa) + .ok_or_else(|| Error::Unsupported("write past the end of allocated space".into()))?; + // Fast path: inside, or extending, the one patch that starts at or + // before `addr` and reaches it (sequential writes into a block, and + // chunks allocated back to back, stay linear). + if let Some((&p_start, p)) = self.patches.range_mut(..=addr).next_back() + && p_start + p.len() as u64 >= addr + && self + .patches + .range(addr + 1..end.max(addr + 1)) + .next() + .is_none() + { + let p = self.patches.get_mut(&p_start).expect("found above"); + let off = (addr - p_start) as usize; + if off + bytes.len() > p.len() { + p.resize(off + bytes.len(), 0); + } + p[off..off + bytes.len()].copy_from_slice(bytes); + return Ok(()); + } + // Patches that overlap or touch [addr, end) merge into one. + let touching: Vec = self + .patches + .range(..=end) + .rev() + .take_while(|(s, b)| **s + b.len() as u64 >= addr) + .map(|(s, _)| *s) + .collect(); + if touching.is_empty() { + self.patches.insert(addr, bytes.to_vec()); + return Ok(()); + } + let lo = touching.iter().copied().min().map_or(addr, |s| s.min(addr)); + let hi = touching + .iter() + .map(|s| s + self.patches[s].len() as u64) + .max() + .map_or(end, |e| e.max(end)); + let mut merged = self.read(lo, (hi - lo) as usize)?; + merged[(addr - lo) as usize..(end - lo) as usize].copy_from_slice(bytes); + for s in touching { + self.patches.remove(&s); + } + self.patches.insert(lo, merged); + Ok(()) + } + + /// Write the edit to `file`, whose superblock is at `user_block`. + /// + /// Order: first everything in newly allocated space (new chunks, new + /// index blocks, relocated structures), which nothing on disk refers to + /// yet, then a sync; then the changes to existing bytes — raw data + /// overwritten in place and the metadata that links the new space in + /// (superblock end of file, chunk index entries, object header + /// messages) — then a sync. A crash during the first phase leaves the + /// file as it was (plus unreferenced bytes past its end of file); a + /// crash during the second can leave it inconsistent, as with libhdf5 + /// without SWMR: there is no journal. + pub(crate) fn commit(self, file: &mut std::fs::File, user_block: u64) -> Result<(), Error> { + let old_eoa = self.old_eoa; + let mut in_place: Vec<(u64, &[u8])> = Vec::new(); + for (&addr, bytes) in &self.patches { + // A patch may run from existing bytes into new space (writes + // merge); its new part goes with the new space. + let split = old_eoa.saturating_sub(addr).min(bytes.len() as u64) as usize; + let (old, new) = bytes.split_at(split); + if !new.is_empty() { + write_at(file, user_block + addr + split as u64, new)?; + } + if !old.is_empty() { + in_place.push((addr, old)); + } + } + if self.eoa > old_eoa { + let want = user_block + self.eoa; + if file.metadata()?.len() < want { + file.set_len(want)?; + } + } + file.sync_data()?; + for (addr, bytes) in in_place { + write_at(file, user_block + addr, bytes)?; + } + file.sync_all()?; + Ok(()) + } +} + +fn write_at(file: &mut std::fs::File, pos: u64, bytes: &[u8]) -> Result<(), Error> { + file.seek(SeekFrom::Start(pos))?; + file.write_all(bytes)?; + Ok(()) +} + +/// Little-endian encode of `v` in `width` bytes. +pub(crate) fn put_uint(buf: &mut [u8], v: u64, width: u8) { + let w = width as usize; + buf[..w].copy_from_slice(&v.to_le_bytes()[..w]); +} + +/// Little-endian decode of `width` bytes. +pub(crate) fn get_uint(buf: &[u8], width: u8) -> u64 { + let mut b = [0u8; 8]; + b[..width as usize].copy_from_slice(&buf[..width as usize]); + u64::from_le_bytes(b) +} + +/// The undefined address for `os`-byte addresses. +pub(crate) fn undef(os: u8) -> u64 { + if os >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(os))) - 1 + } +} + +/// Recompute the Jenkins checksum over `[start, end)` and store it at `end`. +pub(crate) fn rechecksum(img: &mut Image<'_>, start: u64, end: u64) -> Result<(), Error> { + let bytes = img.read(start, (end - start) as usize)?; + let sum = clawhdf5_format::checksum::jenkins_lookup3(&bytes); + img.write(end, &sum.to_le_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_see_writes_and_merges() { + let base = vec![1u8; 32]; + let mut img = Image::new(&base, 8, 8); + img.write(4, &[9, 9]).unwrap(); + img.write(8, &[7]).unwrap(); + img.write(5, &[3, 3, 3]).unwrap(); // extends the first up to the second + assert_eq!(img.read(3, 7).unwrap(), vec![1, 9, 3, 3, 3, 7, 1]); + img.write(2, &[4, 4, 4, 4, 4, 4, 4, 4]).unwrap(); // covers both: merged + assert_eq!(img.patches.len(), 1); + assert_eq!(img.read(1, 10).unwrap(), vec![1, 4, 4, 4, 4, 4, 4, 4, 4, 1]); + let a = img.alloc(10).unwrap(); + assert_eq!(a, 32); + assert_eq!(img.read(30, 4).unwrap(), vec![1, 1, 0, 0]); + img.write(40, &[5]).unwrap(); + assert_eq!(img.read(39, 3).unwrap(), vec![0, 5, 0]); + assert!(img.write(42, &[1]).is_err()); + } + + /// Random reads and writes against a flat copy of the bytes. + #[test] + fn matches_a_flat_model() { + let base: Vec = (0..200u32).map(|i| i as u8).collect(); + let mut img = Image::new(&base, 8, 8); + img.alloc(100).unwrap(); + let mut flat = base.clone(); + flat.resize(300, 0); + let mut x = 12345u64; + let mut next = |n: u64| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x % n + }; + for step in 0..5000 { + let at = next(300); + let len = 1 + next(20).min(299 - at); + if step % 3 == 0 { + assert_eq!( + img.read(at, len as usize).unwrap(), + flat[at as usize..(at + len) as usize] + ); + } else { + let bytes: Vec = (0..len).map(|_| next(256) as u8).collect(); + img.write(at, &bytes).unwrap(); + flat[at as usize..(at + len) as usize].copy_from_slice(&bytes); + } + } + assert_eq!(img.read(0, 300).unwrap(), flat); + // Patches never overlap. + let mut end = 0; + for (s, b) in &img.patches { + assert!(*s >= end); + end = s + b.len() as u64; + } + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs new file mode 100644 index 0000000..b135a3f --- /dev/null +++ b/crates/clawhdf5/src/edit/mod.rs @@ -0,0 +1,1213 @@ +//! In-place modification of an existing HDF5 file: [`FileEditor`]. +//! +//! [`FileBuilder`](crate::FileBuilder) builds a whole file in memory; the +//! editor instead opens a file libhdf5 (or clawhdf5) wrote and changes it +//! where it lies: dataset values are overwritten in place, chunks are added +//! or relocated at the end of the file, the chunk index, dataspace and +//! object headers are patched, and every checksum of a structure it touched +//! is recomputed. +//! +//! Each operation is planned in memory first ([`image::Image`]): if any part +//! of it is unsupported, nothing is written. The plan is then committed with +//! the new space (new chunks, new index blocks) written and synced before +//! the existing bytes that link it in, then synced again. + +mod btree1; +mod earray; +mod farray; +mod image; +mod ohdr; +mod select; + +use std::collections::{BTreeMap, HashMap}; +use std::fs::{OpenOptions, TryLockError}; +use std::path::{Path, PathBuf}; + +use clawhdf5_format::attribute::AttributeMessage; +use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read::NativeElement; +use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::filter_pipeline::FilterPipeline; +use clawhdf5_format::selection::Selection; + +use crate::error::Error; +use crate::reader::File; +use crate::types::AttrValue; +use btree1::{BTree1, Key}; +use earray::{Ea, EaParams, Elem}; +use farray::Fa; +use image::{Image, get_uint, put_uint, undef}; +use ohdr::{Header, MSG_ATTRIBUTE}; + +const MSG_DATASPACE: u16 = 0x01; +const MSG_LAYOUT: u16 = 0x08; +const MSG_EXTERNAL: u16 = 0x07; +const MSG_ATTR_INFO: u16 = 0x15; +/// Message flag: the message is shared (stored elsewhere). +const MSG_FLAG_SHARED: u8 = 0x02; + +/// An HDF5 file opened for in-place modification. +/// +/// Opening takes an exclusive advisory lock on the file (`flock`, the lock +/// libhdf5 itself takes when file locking is on), so a second editor, or +/// h5py opening the file for writing, fails until the editor is dropped. +/// Readers that do not lock ([`File`]) can still open it, but see a file +/// that may be mid-update. +/// +/// Every method is one self-contained edit: it re-reads the file's +/// metadata, applies the change, and syncs the file before returning. +/// +/// # What it can change +/// +/// - [`write_selection`](Self::write_selection) / +/// [`write_all`](Self::write_all): overwrite values of a compact, +/// contiguous or chunked dataset with values of the dataset's own +/// datatype, under any selection. Chunks are decoded, updated and +/// re-encoded; an unfiltered chunk is rewritten in place, a filtered one +/// in place when it still fits and otherwise at the end of the file. New +/// chunks are added to the chunk index: version-1 B-tree (layout v1-v3, +/// what h5py's default `libver` writes), Extensible Array, Fixed Array and +/// single-chunk indexes. A version-2 B-tree index (two or more unlimited +/// dimensions) can only have existing chunks overwritten in place, and an +/// implicit index only in place. +/// - [`resize`](Self::resize): grow a chunked dataset up to its maximum +/// dimensions (h5py's `Dataset.resize`); new chunks come with the writes. +/// - [`set_attr`](Self::set_attr): add or replace an attribute of any +/// object whose attributes are stored in its object header. +/// +/// Anything else is an [`Error::Unsupported`] and leaves the file untouched. +/// +/// # Crash safety and space +/// +/// There is no journal (libhdf5 has none either, outside SWMR). An edit +/// writes all of its new space — chunks and index blocks past the old end +/// of file — and syncs it before it changes any existing byte, so a crash +/// before that point leaves the file as it was; a crash while the existing +/// structures are being patched can leave the file inconsistent. +/// +/// Space is never reused: a filtered chunk that grows moves to the end of +/// the file and its old bytes are leaked, as are index blocks that are +/// replaced. `h5repack` reclaims such space. +#[derive(Debug)] +pub struct FileEditor { + path: PathBuf, + file: std::fs::File, +} + +/// Where a layout message keeps the fields an edit may change (offsets in +/// the message body). +#[derive(Debug, Default, Clone, Copy)] +struct LayoutPos { + version: u8, + /// Contiguous data address, or the chunk index address. + addr: Option, + /// Filtered single chunk: its stored size and filter mask. + single_size: Option, + single_mask: Option, + /// First chunk index creation parameter (Fixed/Extensible Array). + params: Option, + /// Compact raw data. + compact_data: Option, +} + +fn layout_pos(d: &[u8], os: u8, ls: u8) -> Result { + let short = || Error::Unsupported("layout message too short".into()); + let os = os as usize; + let ls = ls as usize; + let version = *d.first().ok_or_else(short)?; + let mut p = LayoutPos { + version, + ..LayoutPos::default() + }; + match version { + 1 | 2 => { + let nd = *d.get(1).ok_or_else(short)? as usize; + match *d.get(2).ok_or_else(short)? { + 0 => p.compact_data = Some(8 + nd * 4 + 4), + _ => p.addr = Some(8), + } + } + 3 => match *d.get(1).ok_or_else(short)? { + 0 => p.compact_data = Some(4), + 1 => p.addr = Some(2), + _ => p.addr = Some(3), + }, + 4 | 5 => match *d.get(1).ok_or_else(short)? { + 0 => p.compact_data = Some(4), + 1 => p.addr = Some(2), + 2 => { + let flags = *d.get(2).ok_or_else(short)?; + let nd = *d.get(3).ok_or_else(short)? as usize; + let enc = *d.get(4).ok_or_else(short)? as usize; + let mut q = 5 + nd * enc; + let itype = *d.get(q).ok_or_else(short)?; + q += 1; + match itype { + 1 if flags & 0x02 != 0 => { + p.single_size = Some(q); + p.single_mask = Some(q + ls); + p.addr = Some(q + ls + 4); + } + 1 | 2 => p.addr = Some(q), + 3 => { + p.params = Some(q); + p.addr = Some(q + 1); + } + 4 => { + p.params = Some(q); + p.addr = Some(q + 5); + } + 5 => { + p.params = Some(q); + p.addr = Some(q + 6); + } + _ => return Err(Error::Unsupported(format!("chunk index type {itype}"))), + } + } + c => return Err(Error::Unsupported(format!("layout class {c}"))), + }, + v => return Err(Error::Unsupported(format!("layout message version {v}"))), + } + let end = p + .addr + .map_or(0, |a| a + os) + .max(p.compact_data.unwrap_or(0)); + if end > d.len() { + return Err(short()); + } + Ok(p) +} + +/// A dataset as an edit sees it. +struct Target { + addr: u64, + dt: Datatype, + es: usize, + ds: Dataspace, + layout: DataLayout, + pipeline: Option, + /// One element of fill value. + fill: Vec, +} + +impl Target { + fn load(f: &File, path: &str) -> Result { + let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let d = f.dataset_at(addr)?; + let dt = d.datatype()?; + let es = dt.type_size() as usize; + if es == 0 { + return Err(Error::Unsupported("datatype of size 0".into())); + } + let ds = d.dataspace()?; + let layout = d.data_layout()?; + let pipeline = d.filter_pipeline()?.filter(|p| !p.filters.is_empty()); + let os = f.superblock().offset_size; + let ls = f.superblock().length_size; + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + f.as_bytes(), + &d.header().messages, + os, + ls, + )?; + let fill = match fill { + None => vec![0u8; es], + Some(v) if v.len() == es => v, + Some(_) => { + return Err(Error::Unsupported( + "fill value size differs from the element size".into(), + )); + } + }; + if d.header() + .messages + .iter() + .any(|m| m.msg_type.to_u16() == MSG_EXTERNAL) + { + return Err(Error::Unsupported( + "dataset stored in external files".into(), + )); + } + Ok(Self { + addr, + dt, + es, + ds, + layout, + pipeline, + fill, + }) + } + + fn dims(&self) -> &[u64] { + &self.ds.dimensions + } +} + +/// Datatypes whose stored bytes point elsewhere in the file (variable-length +/// data in a global heap, references) cannot be written as raw values. +fn check_plain(dt: &Datatype) -> Result<(), Error> { + match dt { + Datatype::VariableLength { .. } => { + Err(Error::Unsupported("writing variable-length data".into())) + } + Datatype::Reference { .. } => Err(Error::Unsupported("writing references".into())), + Datatype::Compound { members, .. } => { + members.iter().try_for_each(|m| check_plain(&m.datatype)) + } + Datatype::Array { base_type, .. } => check_plain(base_type), + Datatype::Enumeration { base_type, .. } => check_plain(base_type), + _ => Ok(()), + } +} + +fn row_major(coords: &[u64], dims: &[u64]) -> u64 { + coords + .iter() + .zip(dims) + .fold(0u64, |acc, (&c, &d)| acc * d + c) +} + +/// Linear index of a chunk in a Fixed Array (`swizzle` false) or Extensible +/// Array (`swizzle` true) index; see `clawhdf5_format`'s `chunk_grid`. +fn array_index( + scaled: &[u64], + dims: &[u64], + max: Option<&[u64]>, + cd: &[u64], + swizzle: bool, +) -> Result { + let rank = cd.len(); + let max_chunks: Vec = (0..rank) + .map(|d| { + let m = max.map_or(dims[d], |m| m[d]); + if m == u64::MAX { + u64::MAX + } else { + m.div_ceil(cd[d]).max(dims[d].div_ceil(cd[d])) + } + }) + .collect(); + let mut order: Vec = (0..rank).collect(); + if swizzle && let Some(u) = max_chunks.iter().position(|&m| m == u64::MAX) { + order.remove(u); + order.insert(0, u); + } + let mut idx = 0u64; + let mut down = 1u64; + for p in (0..rank).rev() { + let d = order[p]; + idx = scaled[d] + .checked_mul(down) + .and_then(|v| idx.checked_add(v)) + .ok_or_else(|| Error::Unsupported("chunk index overflows".into()))?; + if p > 0 { + if max_chunks[d] == u64::MAX { + return Err(Error::Unsupported( + "array chunk index with more than one unlimited dimension".into(), + )); + } + down = down + .checked_mul(max_chunks[d]) + .ok_or_else(|| Error::Unsupported("chunk index overflows".into()))?; + } + } + Ok(idx) +} + +/// The chunk index of one dataset, opened for changes. Index structures a +/// dataset does not have yet (no chunk was ever written) are created on the +/// first insertion, and the layout message is pointed at them. +enum IndexEdit { + BTree1(Option), + Single, + Implicit, + Fixed(Option), + Extensible(Option), + BTree2, +} + +struct ChunkedEdit<'t> { + t: &'t Target, + /// Spatial chunk dimensions. + cd: Vec, + chunk_bytes: usize, + index: IndexEdit, + /// The dataset's object header (for layout message changes). + hdr: Header, + layout_msg: usize, + lpos: LayoutPos, + btree_k: u16, +} + +impl<'t> ChunkedEdit<'t> { + fn new(f: &File, img: &Image<'_>, t: &'t Target) -> Result { + let DataLayout::Chunked { + chunk_dimensions, + btree_address, + version, + chunk_index_type, + dont_filter_partial_edge_chunks, + .. + } = &t.layout + else { + return Err(Error::Unsupported("not a chunked dataset".into())); + }; + if *dont_filter_partial_edge_chunks { + return Err(Error::Unsupported( + "datasets whose partial edge chunks are not filtered".into(), + )); + } + let rank = t.dims().len(); + if chunk_dimensions.len() < rank { + return Err(Error::Unsupported( + "chunk rank differs from the dataspace".into(), + )); + } + let cd: Vec = chunk_dimensions[..rank] + .iter() + .map(|&c| u64::from(c)) + .collect(); + let chunk_bytes = cd + .iter() + .try_fold(t.es as u64, |a, &c| a.checked_mul(c)) + .and_then(|b| usize::try_from(b).ok()) + .filter(|&b| b <= u32::MAX as usize) + .ok_or_else(|| Error::Unsupported("chunk larger than 4 GiB".into()))?; + let hdr = Header::load(img, t.addr)?; + let layout_msg = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, layout_msg)?, img.os, img.ls)?; + let index = match (*version, *chunk_index_type) { + (3, _) => IndexEdit::BTree1( + btree_address + .map(|a| BTree1::new(a, chunk_btree_k(f)?, rank + 1, t.es as u64)) + .transpose()?, + ), + (_, Some(1)) => IndexEdit::Single, + (_, Some(2)) => IndexEdit::Implicit, + (_, Some(3)) => IndexEdit::Fixed(btree_address.map(|a| Fa::open(img, a)).transpose()?), + (_, Some(4)) => { + IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?) + } + (_, Some(5)) => IndexEdit::BTree2, + (v, i) => { + return Err(Error::Unsupported(format!( + "chunked layout version {v}, index type {i:?}" + ))); + } + }; + Ok(Self { + t, + cd, + chunk_bytes, + index, + hdr, + layout_msg, + lpos, + btree_k: chunk_btree_k(f)?, + }) + } + + fn patch_layout_addr(&mut self, img: &mut Image<'_>, addr: u64) -> Result<(), Error> { + let at = self + .lpos + .addr + .ok_or_else(|| Error::Unsupported("layout message has no address".into()))?; + if self.lpos.version < 3 { + return Err(Error::Unsupported( + "creating a chunk index in a version 1/2 layout message".into(), + )); + } + let mut a = vec![0u8; img.os as usize]; + put_uint(&mut a, addr, img.os); + self.hdr.patch(img, self.layout_msg, at, &a) + } + + /// Record chunk `scaled` at `e` in the index (new, moved or resized). + fn set(&mut self, img: &mut Image<'_>, scaled: &[u64], e: Elem) -> Result<(), Error> { + let t = self.t; + let filtered = t.pipeline.is_some(); + match &mut self.index { + IndexEdit::BTree1(tree) => { + let size = u32::try_from(e.size) + .map_err(|_| Error::Unsupported("chunk larger than 4 GiB".into()))?; + let mut offs: Vec = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect(); + offs.push(0); + let key = Key { + size, + mask: e.mask, + offs, + }; + match tree { + Some(tree) => tree.insert(img, key, e.addr)?, + None => { + let new = BTree1::create( + img, + self.btree_k, + scaled.len() + 1, + t.es as u64, + key, + e.addr, + )?; + let root = new.root(); + *tree = Some(new); + self.patch_layout_addr(img, root)?; + } + } + } + IndexEdit::Single => { + if scaled.iter().any(|&s| s != 0) { + return Err(Error::Unsupported( + "single-chunk index with a second chunk".into(), + )); + } + match (self.lpos.single_size, self.lpos.single_mask) { + (Some(s), Some(m)) => { + let mut b = vec![0u8; img.ls as usize]; + put_uint(&mut b, e.size, img.ls); + self.hdr.patch(img, self.layout_msg, s, &b)?; + self.hdr + .patch(img, self.layout_msg, m, &e.mask.to_le_bytes())?; + } + _ if filtered => { + return Err(Error::Unsupported( + "filtered single chunk without a filtered-size field".into(), + )); + } + _ => {} + } + self.patch_layout_addr(img, e.addr)?; + } + IndexEdit::Implicit => { + return Err(Error::Unsupported( + "adding a chunk to an implicit chunk index".into(), + )); + } + IndexEdit::Fixed(fa) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, false)?; + if fa.is_none() { + let pbits_at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("Fixed Array parameters".into()))?; + let page_bits = self.hdr.data(img, self.layout_msg)?[pbits_at]; + let nelmts = (0..self.cd.len()) + .map(|d| { + let m = max.map_or(t.dims()[d], |m| m[d]); + m.div_ceil(self.cd[d]) + }) + .try_fold(1u64, |a, n| a.checked_mul(n)) + .ok_or_else(|| Error::Unsupported("Fixed Array too large".into()))?; + let (new, hdr_addr) = Fa::create( + img, + nelmts, + page_bits, + filtered, + self.chunk_bytes as u64, + self.lpos.version, + )?; + *fa = Some(new); + self.patch_layout_addr(img, hdr_addr)?; + } + if let IndexEdit::Fixed(Some(fa)) = &mut self.index { + fa.set(img, idx, e)?; + } + } + IndexEdit::Extensible(ea) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, true)?; + if ea.is_none() { + let at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("Extensible Array parameters".into()))?; + let d = self.hdr.data(img, self.layout_msg)?; + // Layout message order: max_nelmts_bits, idx_blk_elmts, + // sup_blk_min_data_ptrs, data_blk_min_elmts, + // max_dblk_page_nelmts_bits. + let p = EaParams { + max_nelmts_bits: d[at], + idx_blk_elmts: d[at + 1], + sup_blk_min_data_ptrs: d[at + 2], + data_blk_min_elmts: d[at + 3], + max_dblk_page_nelmts_bits: d[at + 4], + }; + let new = + Ea::create(img, p, filtered, self.chunk_bytes as u64, self.lpos.version)?; + let hdr_addr = new.header_address(); + *ea = Some(new); + self.patch_layout_addr(img, hdr_addr)?; + } + if let IndexEdit::Extensible(Some(ea)) = &mut self.index { + ea.set(img, idx, e)?; + } + } + IndexEdit::BTree2 => { + return Err(Error::Unsupported( + "adding, moving or resizing a chunk in a version-2 B-tree chunk index \ + (datasets with more than one unlimited dimension)" + .into(), + )); + } + } + Ok(()) + } + + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + match &mut self.index { + IndexEdit::Extensible(Some(ea)) => ea.finish(img)?, + IndexEdit::Fixed(Some(fa)) => fa.finish(img)?, + _ => {} + } + self.hdr.finish(img) + } +} + +/// The chunk B-tree's K: the superblock's (version 1), the superblock +/// extension's (versions 2 and 3), or libhdf5's default of 32. +fn chunk_btree_k(f: &File) -> Result { + let sb = f.superblock(); + if let Some(k) = sb.indexed_storage_internal_node_k { + return Ok(k); + } + let ext = clawhdf5_format::superblock_ext::read_superblock_extension(f.as_bytes(), sb)?; + Ok(ext + .and_then(|e| e.btree_k) + .map_or(32, |(chunk, _, _)| chunk)) +} + +impl FileEditor { + /// Open `path` for modification, locking it exclusively. + /// + /// Refused ([`Error::Unsupported`]) for files the editor cannot keep + /// consistent: a metadata cache image, paged or persistent free-space + /// management, a multi-file driver, a file another writer has marked + /// open (superblock version 3 consistency flags). + pub fn open>(path: P) -> Result { + let path = path.as_ref().to_path_buf(); + let file = OpenOptions::new().read(true).write(true).open(&path)?; + match file.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => { + return Err(Error::Locked(format!( + "{} is open for writing elsewhere", + path.display() + ))); + } + Err(TryLockError::Error(e)) => return Err(Error::Io(e)), + } + let ed = Self { path, file }; + let f = File::open(&ed.path)?; + check_editable(&f)?; + Ok(ed) + } + + /// The file's path. + pub fn path(&self) -> &Path { + &self.path + } + + fn edit( + &mut self, + op: impl FnOnce(&File, &mut Image<'_>) -> Result, + ) -> Result { + let f = File::open(&self.path)?; + check_editable(&f)?; + let sb = f.superblock().clone(); + let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); + let r = op(&f, &mut img)?; + if img.is_dirty() { + if img.eoa() != img.old_eoa() { + set_superblock_eof(&mut img, &sb)?; + } + img.commit(&mut self.file, f.user_block_size())?; + } + Ok(r) + } + + /// Overwrite the dataset's values under `selection` with `data`: the + /// selected elements' bytes in the dataset's own datatype and byte + /// order, in selection order (row-major over a hyperslab, the listed + /// order for points). + pub fn write_selection( + &mut self, + path: &str, + selection: &Selection, + data: &[u8], + ) -> Result<(), Error> { + self.edit(|f, img| write_selection(f, img, path, selection, data)) + } + + /// Overwrite every value of the dataset (see + /// [`write_selection`](Self::write_selection)). + pub fn write_all(&mut self, path: &str, data: &[u8]) -> Result<(), Error> { + self.write_selection(path, &Selection::All, data) + } + + /// [`write_selection`](Self::write_selection) with typed values; the + /// dataset's datatype must be `T`'s native representation. + pub fn write_values( + &mut self, + path: &str, + selection: &Selection, + values: &[T], + ) -> Result<(), Error> { + // SAFETY: `NativeElement` types have no padding and no invalid bit + // patterns, so their memory is plain bytes. + let bytes = unsafe { + std::slice::from_raw_parts(values.as_ptr().cast::(), std::mem::size_of_val(values)) + }; + self.edit(|f, img| { + let t = Target::load(f, path)?; + if !T::is_native(&t.dt) { + return Err(Error::InvalidArgument(format!( + "dataset {path} does not store {}", + std::any::type_name::() + ))); + } + write_selection(f, img, path, selection, bytes) + }) + } + + /// Change a chunked dataset's current dimensions to `shape`, which may + /// only grow each dimension, up to the dataset's maximum dimensions + /// (h5py's `Dataset.resize`). New elements read as the fill value until + /// written. Shrinking is [`Error::Unsupported`]. + pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> { + self.edit(|f, img| { + let t = Target::load(f, path)?; + let dims = t.dims().to_vec(); + if shape.len() != dims.len() { + return Err(Error::InvalidArgument(format!( + "rank {} for a dataset of rank {}", + shape.len(), + dims.len() + ))); + } + if shape == dims.as_slice() { + return Ok(()); + } + let max = t.ds.max_dimensions.clone().unwrap_or_else(|| dims.clone()); + for d in 0..dims.len() { + if shape[d] > max[d] { + return Err(Error::InvalidArgument(format!( + "dimension {d}: {} exceeds the maximum {}", + shape[d], max[d] + ))); + } + if shape[d] < dims[d] { + return Err(Error::Unsupported("shrinking a dataset".into())); + } + } + if !matches!(t.layout, DataLayout::Chunked { .. }) { + return Err(Error::Unsupported( + "resizing a dataset that is not chunked".into(), + )); + } + // Only the dataspace changes: every chunk index is keyed + // independently of the current extent (the array indexes by the + // maximum dimensions), and new chunks come with the writes. + let mut hdr = Header::load(img, t.addr)?; + let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::Dataspace, + ))?; + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared dataspace message".into())); + } + let body = hdr.data(img, i)?; + let first = match body.first() { + Some(1) => 8, + Some(2) => 4, + _ => return Err(Error::Unsupported("dataspace message version".into())), + }; + let ls = img.ls as usize; + let mut dims_bytes = vec![0u8; shape.len() * ls]; + for (d, &n) in shape.iter().enumerate() { + if ls < 8 && n >> (8 * ls) != 0 { + return Err(Error::InvalidArgument("dimension too large".into())); + } + put_uint(&mut dims_bytes[d * ls..], n, img.ls); + } + hdr.patch(img, i, first, &dims_bytes)?; + hdr.finish(img) + }) + } + + /// Set attribute `name` of the object at `path` (a group or dataset; + /// `"/"` is the root group) to `value`, replacing an attribute of that + /// name. The attribute goes into free space in the object header, or a + /// new header continuation chunk at the end of the file. + /// + /// [`Error::Unsupported`] for an object whose attributes are in dense + /// storage (or would have to move there: more than the object's + /// compact-attribute limit), one that tracks attribute creation order, + /// or one with shared attribute messages. + pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { + if name.is_empty() { + return Err(Error::InvalidArgument("empty attribute name".into())); + } + self.edit(|f, img| { + let addr = + clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let mut hdr = Header::load(img, addr)?; + if hdr.version == 2 && hdr.flags & 0x04 != 0 { + return Err(Error::Unsupported( + "object tracks attribute creation order".into(), + )); + } + if let Some(i) = hdr.find(MSG_ATTR_INFO) { + let d = hdr.data(img, i)?; + // version(1) flags(1) [max creation index(2)] fractal heap + // address, name index address, [order index address]. + let mut p = 2; + if d.get(1).is_some_and(|f| f & 0x01 != 0) { + p += 2; + } + let os = img.os as usize; + if d.len() < p + os { + return Err(Error::Unsupported("short attribute info message".into())); + } + if get_uint(&d[p..], img.os) != undef(img.os) { + return Err(Error::Unsupported( + "object with attributes in dense storage".into(), + )); + } + } + let mut existing = None; + let mut count = 0usize; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + count += 1; + if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { + existing = Some(i); + } + } + if existing.is_none() && hdr.version == 2 { + let max_compact = max_compact_attrs(img, &hdr)?; + if count + 1 > usize::from(max_compact) { + return Err(Error::Unsupported(format!( + "object already has {count} compact attributes (its limit is \ + {max_compact}); more need dense storage" + ))); + } + } + let msg = clawhdf5_format::type_builders::build_attr_message(name, value); + check_plain(&msg.datatype)?; + let body = if hdr.version == 1 { + encode_attr_v1(&msg, img.ls) + } else { + let mut b = msg.serialize_v3(img.ls); + if !name.is_ascii() { + b[8] = 1; // UTF-8 name + } + b + }; + if let Some(i) = existing { + hdr.delete(img, i)?; + } + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + hdr.finish(img) + }) + } +} + +/// libhdf5's checks before it opens a file for writing, and what this +/// editor cannot keep consistent. +fn check_editable(f: &File) -> Result<(), Error> { + let sb = f.superblock(); + if sb.version > 3 { + return Err(Error::Unsupported(format!( + "superblock version {}", + sb.version + ))); + } + if sb.version >= 3 && (sb.is_write_access() || sb.is_swmr_write()) { + return Err(Error::Unsupported( + "the file is marked as open for writing by another process".into(), + )); + } + if sb + .driver_info_address + .is_some_and(|a| a != undef(sb.offset_size)) + { + return Err(Error::Unsupported("files with a driver info block".into())); + } + if let Some(ext) = clawhdf5_format::superblock_ext::read_superblock_extension(f.as_bytes(), sb)? + { + if ext.cache_image.is_some() { + return Err(Error::Unsupported( + "files with a metadata cache image".into(), + )); + } + // Strategy 1 is H5F_FSPACE_STRATEGY_PAGE. + if let Some(fs) = ext.file_space_info + && (fs.persist || fs.strategy == 1) + { + return Err(Error::Unsupported( + "files with paged or persistent free-space management".into(), + )); + } + } + Ok(()) +} + +/// Store the new end of file in the superblock. +fn set_superblock_eof( + img: &mut Image<'_>, + sb: &clawhdf5_format::superblock::Superblock, +) -> Result<(), Error> { + let os = u64::from(sb.offset_size); + let (at, sum) = match sb.version { + 0 => (24 + 2 * os, None), + 1 => (28 + 2 * os, None), + 2 | 3 => (12 + 2 * os, Some(12 + 4 * os)), + v => return Err(Error::Unsupported(format!("superblock version {v}"))), + }; + let eof = sb + .base_address + .checked_add(img.eoa()) + .ok_or_else(|| Error::Unsupported("end of file overflows".into()))?; + let mut b = vec![0u8; os as usize]; + put_uint(&mut b, eof, sb.offset_size); + img.write(at, &b)?; + if let Some(end) = sum { + let bytes = img.read(0, end as usize)?; + let s = clawhdf5_format::checksum::jenkins_lookup3(&bytes); + img.write(end, &s.to_le_bytes())?; + } + Ok(()) +} + +/// The name bytes of an attribute message body (without the NUL). +fn attr_name(d: &[u8]) -> Result<&[u8], Error> { + let bad = || Error::Unsupported("malformed attribute message".into()); + let (len, at) = match d.first() { + Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), + Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), + _ => return Err(bad()), + }; + let name = d.get(at..at + len).ok_or_else(bad)?; + Ok(name.split(|&b| b == 0).next().unwrap_or(name)) +} + +/// A version-2 header's limit on compact attributes: stored when its flags +/// say so, else libhdf5's default of 8. +fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { + if hdr.flags & 0x10 == 0 { + return Ok(8); + } + let mut p = hdr.addr + 6; + if hdr.flags & 0x20 != 0 { + p += 16; + } + let b = img.read(p, 2)?; + Ok(u16::from_le_bytes([b[0], b[1]])) +} + +/// A version-1 attribute message (what libhdf5 writes in a version-1 object +/// header): name, datatype and dataspace each padded to 8 bytes, the +/// dataspace as a version-1 dataspace message. +fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { + let mut name = a.name.as_bytes().to_vec(); + name.push(0); + let dt = a.datatype.serialize(); + let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; + if a.dataspace.space_type == DataspaceType::Simple { + for &d in &a.dataspace.dimensions { + let mut b = vec![0u8; ls as usize]; + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + } else { + ds[1] = 0; + } + let mut out = vec![1u8, 0]; + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); + out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); + for part in [&name, &dt, &ds] { + out.extend_from_slice(part); + out.resize(out.len().next_multiple_of(8), 0); + } + out.extend_from_slice(&a.raw_data); + out +} + +fn write_selection( + f: &File, + img: &mut Image<'_>, + path: &str, + sel: &Selection, + data: &[u8], +) -> Result<(), Error> { + let t = Target::load(f, path)?; + check_plain(&t.dt)?; + if t.ds.space_type == DataspaceType::Null { + return Err(Error::InvalidArgument( + "dataset has a null dataspace".into(), + )); + } + let dims = t.dims().to_vec(); + clawhdf5_format::partial_read::validate(sel, &dims) + .map_err(|e| Error::InvalidArgument(e.to_string()))?; + if let Selection::Hyperslab { + stride, + count, + block, + .. + } = sel + && (0..dims.len()).any(|d| count[d] > 1 && stride[d] < block[d]) + { + return Err(Error::InvalidArgument( + "overlapping hyperslab blocks".into(), + )); + } + let n = select::for_each_run(sel, &dims, |_, _, _| Ok(()))?; + let want = n + .checked_mul(t.es as u64) + .ok_or_else(|| Error::InvalidArgument("selection too large".into()))?; + if data.len() as u64 != want { + return Err(Error::InvalidArgument(format!( + "{} bytes for {n} elements of {} bytes", + data.len(), + t.es + ))); + } + if n == 0 { + return Ok(()); + } + let es = t.es; + match &t.layout { + DataLayout::Compact { data: stored } => { + let mut hdr = Header::load(img, t.addr)?; + let li = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, li)?, img.os, img.ls)?; + let at = lpos + .compact_data + .ok_or_else(|| Error::Unsupported("compact layout".into()))?; + let total = dims.iter().product::() as usize * es; + if stored.len() < total { + return Err(Error::Unsupported( + "compact data shorter than the dataset".into(), + )); + } + select::for_each_run(sel, &dims, |c, len, src| { + let off = at + row_major(c, &dims) as usize * es; + let s = src as usize * es; + hdr.patch(img, li, off, &data[s..s + len as usize * es]) + })?; + hdr.finish(img) + } + DataLayout::Contiguous { address, size } => { + let total = dims + .iter() + .try_fold(es as u64, |a, &d| a.checked_mul(d)) + .ok_or_else(|| Error::Unsupported("dataset too large".into()))?; + let base = match address { + Some(a) => { + if *size < total { + return Err(Error::Unsupported("contiguous storage too small".into())); + } + *a + } + None => { + // Never written (late allocation): allocate it now, as + // libhdf5 does on the first write, with the fill value. + let mut hdr = Header::load(img, t.addr)?; + let li = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, li)?, img.os, img.ls)?; + if lpos.version < 3 { + return Err(Error::Unsupported( + "allocating storage for a version 1/2 layout message".into(), + )); + } + let at = lpos + .addr + .ok_or_else(|| Error::Unsupported("layout".into()))?; + let a = img.alloc(total)?; + if t.fill.iter().any(|&b| b != 0) { + let buf = t.fill.repeat(total as usize / es); + img.write(a, &buf)?; + } + let mut ab = vec![0u8; img.os as usize]; + put_uint(&mut ab, a, img.os); + hdr.patch(img, li, at, &ab)?; + // The size field follows the address. + let mut sb = vec![0u8; img.ls as usize]; + put_uint(&mut sb, total, img.ls); + hdr.patch(img, li, at + img.os as usize, &sb)?; + hdr.finish(img)?; + a + } + }; + select::for_each_run(sel, &dims, |c, len, src| { + let off = base + row_major(c, &dims) * es as u64; + let s = src as usize * es; + img.write(off, &data[s..s + len as usize * es]) + })?; + Ok(()) + } + DataLayout::Chunked { btree_address, .. } => { + let mut ce = ChunkedEdit::new(f, img, &t)?; + let existing: HashMap, ChunkInfo> = if btree_address.is_some() { + let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?; + chunks + .into_iter() + .map(|c| { + let s: Vec = + c.offsets.iter().zip(&ce.cd).map(|(o, cd)| o / cd).collect(); + (s, c) + }) + .collect() + } else { + HashMap::new() + }; + let cd = ce.cd.clone(); + let chunk_bytes = ce.chunk_bytes; + let rank = dims.len(); + let mut bufs: BTreeMap, Vec> = BTreeMap::new(); + let load = |scaled: &Vec| -> Result, Error> { + match existing.get(scaled) { + Some(info) => decode_chunk(img_read(f, info)?, &t, info, chunk_bytes), + None => Ok(t.fill.repeat(chunk_bytes / es)), + } + }; + select::for_each_run(sel, &dims, |coords, len, src| { + let mut c = coords.to_vec(); + let mut left = len; + let mut s = src as usize; + let l = rank.saturating_sub(1); + while left > 0 { + let scaled: Vec = c.iter().zip(&cd).map(|(x, d)| x / d).collect(); + let within: Vec = c.iter().zip(&cd).map(|(x, d)| x % d).collect(); + let n = if rank == 0 { + left + } else { + left.min(cd[l] - within[l]) + }; + if !bufs.contains_key(&scaled) { + let b = load(&scaled)?; + bufs.insert(scaled.clone(), b); + } + let buf = bufs.get_mut(&scaled).expect("inserted above"); + let at = row_major(&within, &cd) as usize * es; + let bytes = n as usize * es; + buf[at..at + bytes].copy_from_slice(&data[s * es..s * es + bytes]); + s += n as usize; + left -= n; + if rank > 0 { + c[l] += n; + } + } + Ok(()) + })?; + for (scaled, buf) in bufs { + let (bytes, mask) = match &t.pipeline { + Some(p) => ( + clawhdf5_format::filters::compress_chunk(&buf, p, es as u32)?, + 0u32, + ), + None => (buf, 0u32), + }; + let len = bytes.len() as u64; + let placed = match existing.get(&scaled) { + Some(info) if t.pipeline.is_none() => { + if u64::from(info.chunk_size) != len { + return Err(Error::Unsupported( + "unfiltered chunk stored at an unexpected size".into(), + )); + } + img.write(info.address, &bytes)?; + None + } + // Rewritten where it is when it still fits, or when it + // is the last thing in the file (the chunk an append + // keeps rewriting usually is) and can grow there. + Some(info) + if len <= u64::from(info.chunk_size) + || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => + { + img.write(info.address, &bytes)?; + (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some( + Elem { + addr: info.address, + size: len, + mask, + }, + ) + } + _ => { + let a = img.alloc(len)?; + img.write(a, &bytes)?; + Some(Elem { + addr: a, + size: len, + mask, + }) + } + }; + if let Some(e) = placed { + ce.set(img, &scaled, e)?; + } + } + ce.finish(img) + } + DataLayout::Virtual { .. } => Err(Error::Unsupported("writing a virtual dataset".into())), + } +} + +fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> { + let start = usize::try_from(info.address) + .map_err(|_| Error::Unsupported("chunk address out of range".into()))?; + f.as_bytes() + .get(start..start + info.chunk_size as usize) + .ok_or_else(|| { + Error::Format(clawhdf5_format::error::FormatError::UnexpectedEof { + expected: start + info.chunk_size as usize, + available: f.as_bytes().len(), + }) + }) +} + +fn decode_chunk( + raw: &[u8], + t: &Target, + info: &ChunkInfo, + chunk_bytes: usize, +) -> Result, Error> { + let out = match &t.pipeline { + Some(p) if !clawhdf5_format::filters::all_filters_skipped(p, info.filter_mask) => { + clawhdf5_format::filters::decompress_chunk_masked( + raw, + p, + chunk_bytes, + t.es as u32, + info.filter_mask, + )? + } + _ => raw.to_vec(), + }; + if out.len() != chunk_bytes { + return Err(Error::Format( + clawhdf5_format::error::FormatError::ChunkedReadError(format!( + "chunk decodes to {} bytes, expected {chunk_bytes}", + out.len() + )), + )); + } + Ok(out) +} diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs new file mode 100644 index 0000000..5446702 --- /dev/null +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -0,0 +1,504 @@ +//! An object header as an edit sees it: every chunk and every message +//! (NIL and continuation messages included) with its position in the file, +//! so single messages can be changed in place, deleted (turned into NIL +//! messages) and added (into a NIL message big enough, or into a new +//! continuation chunk at the end of the file). +//! +//! Version-2 chunks carry a checksum, recomputed by [`Header::finish`] for +//! every chunk the edit touched; a version-1 header's message count is kept +//! up to date there too. + +use std::collections::BTreeSet; + +use crate::edit::image::{Image, get_uint, put_uint, rechecksum}; +use crate::error::Error; +use clawhdf5_format::error::FormatError; + +pub(crate) const MSG_NIL: u16 = 0x00; +pub(crate) const MSG_CONTINUATION: u16 = 0x10; +pub(crate) const MSG_ATTRIBUTE: u16 = 0x0C; + +/// One message: where its header and body are, and what it is. +#[derive(Debug, Clone)] +pub(crate) struct Msg { + pub(crate) chunk: usize, + pub(crate) hdr_pos: u64, + pub(crate) data_pos: u64, + pub(crate) size: usize, + pub(crate) mtype: u16, + pub(crate) flags: u8, + pub(crate) corder: Option, +} + +/// One chunk of the header. +#[derive(Debug, Clone)] +struct Chunk { + /// Where the checksummed bytes start (the `OHDR`/`OCHK` signature). + start: u64, + /// Where the checksum is (version 2 only). + checksum_at: Option, + /// Where the chunk's messages end. + end: u64, + /// Bytes at the end too few for a message header (version 2 only). + /// libhdf5 refuses a chunk with both a gap and a NIL message, so a + /// NIL message made in such a chunk must absorb the gap. + gap: u64, +} + +#[derive(Debug)] +pub(crate) struct Header { + pub(crate) addr: u64, + pub(crate) version: u8, + /// Version-2 header flags (0 for version 1). + pub(crate) flags: u8, + chunks: Vec, + pub(crate) msgs: Vec, + dirty: BTreeSet, + /// Messages added (a split NIL message, a new chunk's messages), for a + /// version-1 header's message count. + added: usize, +} + +const MAX_CHUNKS: usize = 1024; + +fn corrupt(why: &'static str) -> Error { + Error::Format(FormatError::InvalidObjectHeader(why)) +} + +impl Header { + /// Locate every chunk and message of the header at `addr`. + pub(crate) fn load(img: &Image<'_>, addr: u64) -> Result { + let sig = img.read(addr, 4)?; + let mut h = Header { + addr, + version: 0, + flags: 0, + chunks: Vec::new(), + msgs: Vec::new(), + dirty: BTreeSet::new(), + added: 0, + }; + let mut pending: Vec<(u64, u64)> = Vec::new(); + if sig == b"OHDR" { + let pre = img.read(addr, 6)?; + if pre[4] != 2 { + return Err(corrupt("bad object header version")); + } + h.version = 2; + h.flags = pre[5]; + let mut pos = addr + 6; + if h.flags & 0x20 != 0 { + pos += 16; + } + if h.flags & 0x10 != 0 { + pos += 4; + } + let w = 1u8 << (h.flags & 0x03); + let size = get_uint(&img.read(pos, w as usize)?, w); + pos += u64::from(w); + h.chunks.push(Chunk { + start: addr, + checksum_at: Some(pos + size), + end: pos + size, + gap: 0, + }); + h.scan(img, 0, pos, pos + size, &mut pending)?; + } else { + let pre = img.read(addr, 16)?; + if pre[0] != 1 { + return Err(corrupt("bad object header version")); + } + h.version = 1; + let size = u64::from(u32::from_le_bytes([pre[8], pre[9], pre[10], pre[11]])); + h.chunks.push(Chunk { + start: addr, + checksum_at: None, + end: addr + 16 + size, + gap: 0, + }); + h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?; + } + while let Some((caddr, clen)) = pending.pop() { + if h.chunks.len() >= MAX_CHUNKS { + return Err(corrupt("too many object header chunks")); + } + let idx = h.chunks.len(); + if h.version == 2 { + if clen < 8 || img.read(caddr, 4)? != b"OCHK" { + return Err(corrupt("bad continuation chunk")); + } + h.chunks.push(Chunk { + start: caddr, + checksum_at: Some(caddr + clen - 4), + end: caddr + clen - 4, + gap: 0, + }); + h.scan(img, idx, caddr + 4, caddr + clen - 4, &mut pending)?; + } else { + h.chunks.push(Chunk { + start: caddr, + checksum_at: None, + end: caddr + clen, + gap: 0, + }); + h.scan(img, idx, caddr, caddr + clen, &mut pending)?; + } + } + Ok(h) + } + + /// Size of a message header in this object header. + pub(crate) fn hsize(&self) -> usize { + match (self.version, self.flags & 0x04 != 0) { + (1, _) => 8, + (_, true) => 6, + _ => 4, + } + } + + fn scan( + &mut self, + img: &Image<'_>, + chunk: usize, + start: u64, + end: u64, + pending: &mut Vec<(u64, u64)>, + ) -> Result<(), Error> { + let hs = self.hsize() as u64; + let bytes = img.read(start, (end - start) as usize)?; + let mut p = 0usize; + while (p as u64) + hs <= end - start { + let b = &bytes[p..]; + let (mtype, size, flags, corder) = if self.version == 1 { + ( + u16::from_le_bytes([b[0], b[1]]), + u16::from_le_bytes([b[2], b[3]]) as usize, + b[4], + None, + ) + } else { + ( + u16::from(b[0]), + u16::from_le_bytes([b[1], b[2]]) as usize, + b[3], + (hs == 6).then(|| u16::from_le_bytes([b[4], b[5]])), + ) + }; + let data_off = p + hs as usize; + if data_off + size > bytes.len() { + return Err(corrupt("message size exceeds buffer end")); + } + if mtype == MSG_CONTINUATION { + let d = &bytes[data_off..data_off + size]; + let os = img.os as usize; + let ls = img.ls as usize; + if d.len() < os + ls { + return Err(corrupt("short continuation message")); + } + pending.push((get_uint(d, img.os), get_uint(&d[os..], img.ls))); + } + self.msgs.push(Msg { + chunk, + hdr_pos: start + p as u64, + data_pos: start + data_off as u64, + size, + mtype, + flags, + corder, + }); + p = data_off + size; + } + self.chunks[chunk].gap = (end - start) - p as u64; + Ok(()) + } + + /// After message `i` became a NIL message: if its chunk ends in a gap, + /// grow the NIL message over it (it must be the chunk's last message). + fn absorb_gap(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> { + let m = self.msgs[i].clone(); + let c = &self.chunks[m.chunk]; + if c.gap == 0 { + return Ok(()); + } + if m.data_pos + m.size as u64 + c.gap != c.end { + return Err(Error::Unsupported( + "object header chunk ends in a gap that a free message cannot absorb".into(), + )); + } + let new_size = m.size + c.gap as usize; + if new_size > usize::from(u16::MAX) { + return Err(Error::Unsupported("object header message too large".into())); + } + self.write_msg_header(img, m.hdr_pos, MSG_NIL, new_size, 0, m.corder)?; + img.write(m.data_pos, &vec![0u8; new_size])?; + self.msgs[i].size = new_size; + self.chunks[m.chunk].gap = 0; + Ok(()) + } + + /// The first message of type `mtype`. + pub(crate) fn find(&self, mtype: u16) -> Option { + self.msgs.iter().position(|m| m.mtype == mtype) + } + + pub(crate) fn data(&self, img: &Image<'_>, i: usize) -> Result, Error> { + let m = &self.msgs[i]; + img.read(m.data_pos, m.size) + } + + /// Overwrite bytes of message `i`'s body, from `offset`. + pub(crate) fn patch( + &mut self, + img: &mut Image<'_>, + i: usize, + offset: usize, + bytes: &[u8], + ) -> Result<(), Error> { + let m = &self.msgs[i]; + if offset + bytes.len() > m.size { + return Err(Error::Unsupported( + "change does not fit the header message".into(), + )); + } + img.write(m.data_pos + offset as u64, bytes)?; + self.dirty.insert(m.chunk); + Ok(()) + } + + fn write_msg_header( + &mut self, + img: &mut Image<'_>, + hdr_pos: u64, + mtype: u16, + size: usize, + flags: u8, + corder: Option, + ) -> Result<(), Error> { + let mut h = vec![0u8; self.hsize()]; + if self.version == 1 { + h[0..2].copy_from_slice(&mtype.to_le_bytes()); + h[2..4].copy_from_slice(&(size as u16).to_le_bytes()); + h[4] = flags; + } else { + h[0] = mtype as u8; + h[1..3].copy_from_slice(&(size as u16).to_le_bytes()); + h[3] = flags; + if h.len() == 6 { + h[4..6].copy_from_slice(&corder.unwrap_or(0).to_le_bytes()); + } + } + img.write(hdr_pos, &h) + } + + /// Turn message `i` into a NIL message (its space becomes free). + pub(crate) fn delete(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> { + let m = self.msgs[i].clone(); + self.write_msg_header(img, m.hdr_pos, MSG_NIL, m.size, 0, m.corder)?; + img.write(m.data_pos, &vec![0u8; m.size])?; + self.msgs[i].mtype = MSG_NIL; + self.msgs[i].flags = 0; + self.dirty.insert(m.chunk); + self.absorb_gap(img, i) + } + + /// Body size a message of `len` bytes occupies (version 1 pads to 8). + fn padded(&self, len: usize) -> usize { + if self.version == 1 { + len.next_multiple_of(8) + } else { + len + } + } + + /// Whether a free slot of `slot` bytes can take a body of `need` bytes: + /// exactly, or with room left for a NIL message after it. + fn fits(&self, slot: usize, need: usize) -> bool { + slot == need || slot >= need + self.hsize() + } + + /// The smallest NIL message that can take `need` body bytes. + fn best_nil(&self, need: usize) -> Option { + self.msgs + .iter() + .enumerate() + .filter(|(_, m)| m.mtype == MSG_NIL && self.fits(m.size, need)) + .min_by_key(|(_, m)| m.size) + .map(|(i, _)| i) + } + + /// Put a message into slot `i` (a NIL message, or a message being + /// moved away), splitting off the rest as a NIL message. + fn place( + &mut self, + img: &mut Image<'_>, + i: usize, + mtype: u16, + flags: u8, + data: &[u8], + corder: Option, + ) -> Result<(), Error> { + let slot = self.msgs[i].clone(); + let need = self.padded(data.len()); + debug_assert!(self.fits(slot.size, need)); + let mut body = data.to_vec(); + body.resize(need, 0); + self.write_msg_header(img, slot.hdr_pos, mtype, need, flags, corder)?; + img.write(slot.data_pos, &body)?; + self.msgs[i] = Msg { + size: need, + mtype, + flags, + corder, + ..slot.clone() + }; + if slot.size > need { + let hs = self.hsize(); + let nil_hdr = slot.data_pos + need as u64; + let nil_size = slot.size - need - hs; + self.write_msg_header(img, nil_hdr, MSG_NIL, nil_size, 0, Some(0))?; + img.write(nil_hdr + hs as u64, &vec![0u8; nil_size])?; + self.msgs.push(Msg { + chunk: slot.chunk, + hdr_pos: nil_hdr, + data_pos: nil_hdr + hs as u64, + size: nil_size, + mtype: MSG_NIL, + flags: 0, + corder: (hs == 6).then_some(0), + }); + self.added += 1; + let nil = self.msgs.len() - 1; + self.absorb_gap(img, nil)?; + } + self.dirty.insert(slot.chunk); + Ok(()) + } + + /// Add a message: into free space in the header when there is some, + /// else into a new continuation chunk at the end of the file (whose + /// continuation message takes a NIL slot, or the slot of another + /// message — an attribute if possible — that moves into the new chunk + /// with it). + pub(crate) fn insert( + &mut self, + img: &mut Image<'_>, + mtype: u16, + flags: u8, + data: &[u8], + corder: Option, + ) -> Result<(), Error> { + if data.len() > usize::from(u16::MAX) { + return Err(Error::Unsupported( + "message larger than 64 KiB (would need dense storage)".into(), + )); + } + let need = self.padded(data.len()); + if let Some(i) = self.best_nil(need) { + return self.place(img, i, mtype, flags, data, corder); + } + let os = img.os as usize; + let ls = img.ls as usize; + let cont_need = self.padded(os + ls); + // Where the continuation message goes, and the message (if any) + // that moves out of that slot into the new chunk. + let (slot, moved) = match self.best_nil(cont_need) { + Some(i) => (i, None), + None => { + // Any message but a continuation can live in any chunk; + // prefer moving an attribute, then the smallest that fits. + let i = self + .msgs + .iter() + .enumerate() + .filter(|(_, m)| { + m.mtype != MSG_NIL + && m.mtype != MSG_CONTINUATION + && self.fits(m.size, cont_need) + }) + .min_by_key(|(_, m)| (m.mtype != MSG_ATTRIBUTE, m.size)) + .map(|(i, _)| i) + .ok_or_else(|| { + Error::Unsupported( + "no room in the object header for a continuation message".into(), + ) + })?; + let m = self.msgs[i].clone(); + let body = img.read(m.data_pos, m.size)?; + (i, Some((m, body))) + } + }; + + // The new chunk: [moved message] + new message + a NIL message + // holding spare room for later additions. + let hs = self.hsize(); + let spare = 64usize; + let mut payload = hs + need; + if let Some((m, _)) = &moved { + payload += hs + m.size; + } + let msgs_len = payload + hs + spare; + let (prefix, suffix) = if self.version == 2 { (4, 4) } else { (0, 0) }; + let chunk_len = prefix + msgs_len + suffix; + let caddr = img.alloc(chunk_len as u64)?; + if self.version == 2 { + img.write(caddr, b"OCHK")?; + } + let cidx = self.chunks.len(); + self.chunks.push(Chunk { + start: caddr, + checksum_at: (self.version == 2).then_some(caddr + (prefix + msgs_len) as u64), + end: caddr + (prefix + msgs_len) as u64, + gap: 0, + }); + let first = caddr + prefix as u64; + // Lay the chunk out as one NIL message, then place into it. + self.write_msg_header(img, first, MSG_NIL, msgs_len - hs, 0, Some(0))?; + self.msgs.push(Msg { + chunk: cidx, + hdr_pos: first, + data_pos: first + hs as u64, + size: msgs_len - hs, + mtype: MSG_NIL, + flags: 0, + corder: (hs == 6).then_some(0), + }); + self.added += 1; + if let Some((m, body)) = &moved { + let nil = self.msgs.len() - 1; + self.place(img, nil, m.mtype, m.flags, body, m.corder)?; + } + let nil = self.msgs.len() - 1; + self.place(img, nil, mtype, flags, data, corder)?; + + // Link it in. + let mut cont = vec![0u8; os + ls]; + put_uint(&mut cont, caddr, img.os); + put_uint(&mut cont[os..], chunk_len as u64, img.ls); + if moved.is_some() { + self.msgs[slot].mtype = MSG_NIL; // its content now lives in the new chunk + } + self.place(img, slot, MSG_CONTINUATION, 0, &cont, Some(0))?; + self.dirty.insert(cidx); + Ok(()) + } + + /// Recompute the checksum of every changed version-2 chunk; store a + /// version-1 header's new message count. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for &c in &self.dirty { + if let Some(at) = self.chunks[c].checksum_at { + rechecksum(img, self.chunks[c].start, at)?; + } + } + if self.version == 1 && self.added > 0 { + let old = u16::from_le_bytes(img.read(self.addr + 2, 2)?.try_into().unwrap_or([0; 2])); + let new = usize::from(old) + self.added; + let new = u16::try_from(new) + .map_err(|_| Error::Unsupported("too many object header messages".into()))?; + img.write(self.addr + 2, &new.to_le_bytes())?; + } + self.dirty.clear(); + self.added = 0; + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/select.rs b/crates/clawhdf5/src/edit/select.rs new file mode 100644 index 0000000..4c34336 --- /dev/null +++ b/crates/clawhdf5/src/edit/select.rs @@ -0,0 +1,142 @@ +//! A selection as runs of consecutive elements along the last dimension, in +//! the order the selection's elements are numbered (row-major over a +//! hyperslab, as h5py and libhdf5 number them; a point list in its order). + +use clawhdf5_format::selection::Selection; + +use crate::error::Error; + +/// Call `f(coords, len, src)` for each run: `len` elements starting at +/// `coords` (consecutive in the last dimension), which are elements +/// `src..src + len` of the selection. Returns the number of elements. +/// The selection must already be validated against `dims`. +pub(crate) fn for_each_run( + sel: &Selection, + dims: &[u64], + mut f: impl FnMut(&[u64], u64, u64) -> Result<(), Error>, +) -> Result { + let rank = dims.len(); + let mut src = 0u64; + match sel { + Selection::None => {} + Selection::Points(pts) => { + for p in pts { + f(p, 1, src)?; + src += 1; + } + } + Selection::All => { + if rank == 0 { + f(&[], 1, 0)?; + return Ok(1); + } + if dims.contains(&0) { + return Ok(0); + } + let last = dims[rank - 1]; + let mut coords = vec![0u64; rank]; + loop { + f(&coords, last, src)?; + src += last; + if !advance(&mut coords[..rank - 1], &dims[..rank - 1]) { + break; + } + } + } + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + if rank == 0 { + return Err(Error::InvalidArgument( + "hyperslab selection on a scalar dataset".into(), + )); + } + if (0..rank).any(|d| count[d] == 0 || block[d] == 0) { + return Ok(0); + } + // Per-dimension extent of the selection: j in 0..count*block. + let ext: Vec = (0..rank).map(|d| count[d] * block[d]).collect(); + let coord = |d: usize, j: u64| start[d] + (j / block[d]) * stride[d] + j % block[d]; + let l = rank - 1; + // Along the last dimension, blocks merge when they touch. + let merged = stride[l] == block[l] || count[l] == 1; + let mut js = vec![0u64; rank - 1]; + let mut coords = vec![0u64; rank]; + loop { + for (d, &j) in js.iter().enumerate() { + coords[d] = coord(d, j); + } + if merged { + coords[l] = start[l]; + f(&coords, ext[l], src)?; + src += ext[l]; + } else { + for c in 0..count[l] { + coords[l] = start[l] + c * stride[l]; + f(&coords, block[l], src)?; + src += block[l]; + } + } + if !advance(&mut js, &ext[..l]) { + break; + } + } + } + } + Ok(src) +} + +/// Odometer step over `0..lim[d]`; false when it wraps around. +fn advance(v: &mut [u64], lim: &[u64]) -> bool { + for d in (0..v.len()).rev() { + v[d] += 1; + if v[d] < lim[d] { + return true; + } + v[d] = 0; + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + fn collect(sel: &Selection, dims: &[u64]) -> Vec<(Vec, u64, u64)> { + let mut out = Vec::new(); + for_each_run(sel, dims, |c, n, s| { + out.push((c.to_vec(), n, s)); + Ok(()) + }) + .unwrap(); + out + } + + #[test] + fn runs() { + assert_eq!( + collect(&Selection::All, &[2, 3]), + vec![(vec![0, 0], 3, 0), (vec![1, 0], 3, 3)] + ); + let h = Selection::Hyperslab { + start: vec![1, 0], + stride: vec![2, 3], + count: vec![2, 2], + block: vec![1, 2], + }; + assert_eq!( + collect(&h, &[5, 6]), + vec![ + (vec![1, 0], 2, 0), + (vec![1, 3], 2, 2), + (vec![3, 0], 2, 4), + (vec![3, 3], 2, 6) + ] + ); + assert_eq!(collect(&Selection::All, &[]), vec![(vec![], 1, 0)]); + assert_eq!(collect(&Selection::All, &[0, 4]), vec![]); + } +} diff --git a/crates/clawhdf5/src/error.rs b/crates/clawhdf5/src/error.rs index de34355..110abe3 100644 --- a/crates/clawhdf5/src/error.rs +++ b/crates/clawhdf5/src/error.rs @@ -6,7 +6,10 @@ use clawhdf5_format::error::FormatError; use clawhdf5_format::message_type::MessageType; /// Errors that can occur when using the high-level API. +/// +/// Non-exhaustive: new kinds of failure may be added. #[derive(Debug)] +#[non_exhaustive] pub enum Error { /// I/O error from the filesystem. Io(std::io::Error), @@ -36,6 +39,16 @@ pub enum Error { /// Actual alignment of the data pointer. actual: usize, }, + /// The requested change is valid but not supported (by + /// [`FileEditor`](crate::FileEditor): a chunk index, filter or header + /// layout it cannot modify). Nothing was written. + Unsupported(String), + /// An argument does not fit the object (a selection outside the + /// dataset, a buffer of the wrong length, a shrinking resize, ...). + InvalidArgument(String), + /// The file is locked by another writer (another [`FileEditor`](crate::FileEditor), + /// or libhdf5 with file locking on). + Locked(String), } impl fmt::Display for Error { @@ -58,6 +71,9 @@ impl fmt::Display for Error { "zero-copy type mismatch: expected {expected}, got {actual}" ) } + Error::Unsupported(msg) => write!(f, "unsupported: {msg}"), + Error::InvalidArgument(msg) => write!(f, "invalid argument: {msg}"), + Error::Locked(msg) => write!(f, "file is locked: {msg}"), Error::ZeroCopyUnaligned { required, actual } => { write!( f, diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 222e432..280f64b 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -23,8 +23,25 @@ //! builder.set_attr("version", AttrValue::I64(1)); //! builder.write("output.h5").unwrap(); //! ``` +//! +//! # Modifying a file in place +//! +//! ```no_run +//! use clawhdf5::{FileEditor, Selection}; +//! +//! let mut ed = FileEditor::open("data.h5").unwrap(); +//! ed.resize("series", &[1100]).unwrap(); // a chunked dataset, maxshape (None,) +//! let tail = Selection::Hyperslab { +//! start: vec![1000], +//! stride: vec![1], +//! count: vec![100], +//! block: vec![1], +//! }; +//! ed.write_values("series", &tail, &[0.5f64; 100]).unwrap(); +//! ``` mod cache_image; +mod edit; pub mod error; pub mod lazy; #[cfg(feature = "mmap")] @@ -34,6 +51,7 @@ pub mod types; pub mod vlen; pub mod writer; +pub use edit::FileEditor; pub use error::Error; pub use lazy::{LazyDataset, LazyFile, LazyGroup}; #[cfg(feature = "mmap")] diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 1b20dd6..3a11e20 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -1104,13 +1104,18 @@ impl<'f> Dataset<'f> { .ok_or(Error::MissingMessage(msg_type)) } - fn datatype(&self) -> Result { + /// The dataset's object header as parsed. + pub(crate) fn header(&self) -> &ObjectHeader { + &self.header + } + + pub(crate) fn datatype(&self) -> Result { let data = self.required_payload(MessageType::Datatype)?; let (dt, _) = Datatype::parse_in_header(&data, self.header.version)?; Ok(dt) } - fn dataspace(&self) -> Result { + pub(crate) fn dataspace(&self) -> Result { let data = self.required_payload(MessageType::Dataspace)?; let mut ds = Dataspace::parse(&data, self.file.length_size())?; // libhdf5 reports a virtual dataset with unlimited or printf-style @@ -1130,7 +1135,7 @@ impl<'f> Dataset<'f> { Ok(ds) } - fn data_layout(&self) -> Result { + pub(crate) fn data_layout(&self) -> Result { let msg = find_message(&self.header, MessageType::DataLayout)?; Ok(DataLayout::parse( &msg.data, @@ -1143,7 +1148,7 @@ impl<'f> Dataset<'f> { /// that is present but unparseable is an error: treating it as "no /// filters" would hand the caller the still-compressed bytes as if they /// were the data. - fn filter_pipeline(&self) -> Result, Error> { + pub(crate) fn filter_pipeline(&self) -> Result, Error> { self.message_payload(MessageType::FilterPipeline)? .map(|data| FilterPipeline::parse(&data).map_err(Error::Format)) .transpose() diff --git a/crates/clawhdf5/tests/edit_tests.rs b/crates/clawhdf5/tests/edit_tests.rs new file mode 100644 index 0000000..2467256 --- /dev/null +++ b/crates/clawhdf5/tests/edit_tests.rs @@ -0,0 +1,138 @@ +//! `FileEditor` on files clawhdf5 writes, read back with our own reader +//! (libhdf5 interop is in `clawhdf5-tools/tests/edit_interop.rs`, which can +//! also run `h5rs check`). + +use clawhdf5::{AttrValue, Error, File, FileBuilder, FileEditor, Selection}; + +fn block(start: u64, count: u64) -> Selection { + Selection::Hyperslab { + start: vec![start], + stride: vec![1], + count: vec![count], + block: vec![1], + } +} + +fn sample(dir: &std::path::Path) -> std::path::PathBuf { + let path = dir.join("f.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("ext") + .with_i32_data(&[0, 1, 2, 3, 4]) + .with_shape(&[5]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[4]) + .with_deflate(6); + b.create_dataset("raw") + .with_f64_data(&[0.5; 8]) + .with_shape(&[2, 4]) + .with_maxshape(&[u64::MAX, 4]) + .with_chunks(&[1, 4]); + b.create_dataset("flat").with_i64_data(&[1, 2, 3]); + b.set_attr("title", AttrValue::String("t".into())); + b.write(&path).unwrap(); + path +} + +#[test] +fn append_overwrite_and_attributes_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + let len_before = std::fs::metadata(&path).unwrap().len(); + let mut expect: Vec = (0..5).collect(); + let mut raw = vec![0.5f64; 8]; + { + let mut ed = FileEditor::open(&path).unwrap(); + for k in 0..300u64 { + let n = expect.len() as u64; + let add = 1 + k % 5; + ed.resize("ext", &[n + add]).unwrap(); + let vals: Vec = (0..add).map(|j| (n + j) as i32 * 2).collect(); + ed.write_values("ext", &block(n, add), &vals).unwrap(); + expect.extend(&vals); + } + // A filtered chunk rewritten with data that compresses worse moves. + let noisy: Vec = (0..4).map(|i| i * 7_919_993).collect(); + ed.write_values("ext", &block(0, 4), &noisy).unwrap(); + expect[..4].copy_from_slice(&noisy); + + ed.resize("raw", &[5, 4]).unwrap(); + raw.resize(20, 0.0); + let sel = Selection::Hyperslab { + start: vec![1, 1], + stride: vec![2, 2], + count: vec![2, 2], + block: vec![1, 1], + }; + ed.write_values("raw", &sel, &[1.0f64, 2.0, 3.0, 4.0]) + .unwrap(); + for (i, (r, c)) in [(1, 1), (1, 3), (3, 1), (3, 3)].iter().enumerate() { + raw[r * 4 + c] = i as f64 + 1.0; + } + ed.write_values("flat", &Selection::Points(vec![vec![2]]), &[30i64]) + .unwrap(); + ed.set_attr("/", "title", &AttrValue::String("a longer title".into())) + .unwrap(); + ed.set_attr("ext", "count", &AttrValue::I64(expect.len() as i64)) + .unwrap(); + } + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("ext").unwrap().read_i32().unwrap(), expect); + assert_eq!(f.dataset("raw").unwrap().shape().unwrap(), vec![5, 4]); + assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw); + assert_eq!( + f.dataset("flat").unwrap().read_i64().unwrap(), + vec![1, 2, 30] + ); + let root = f.root().attrs().unwrap(); + assert!(matches!(root.get("title"), Some(AttrValue::String(s)) if s == "a longer title")); + let ext = f.dataset("ext").unwrap().attrs().unwrap(); + assert!(matches!(ext.get("count"), Some(AttrValue::I64(n)) if *n == expect.len() as i64)); + assert!(std::fs::metadata(&path).unwrap().len() > len_before); +} + +#[test] +fn errors_leave_the_file_untouched() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + let before = std::fs::read(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); + assert!(ed.write_all("missing", &[0; 4]).is_err()); + // Wrong length, wrong type, outside the extent, beyond maxshape, + // shrinking, a rank change. + assert!(matches!( + ed.write_all("flat", &[0; 7]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.write_values("flat", &Selection::All, &[1i32, 2, 3]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.write_values("ext", &block(4, 2), &[1i32, 2]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.resize("raw", &[3, 5]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_)))); + assert!(matches!( + ed.resize("ext", &[4, 1]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.resize("flat", &[4]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.set_attr("/", "", &AttrValue::I64(1)), + Err(Error::InvalidArgument(_)) + )); + // No-ops write nothing. + ed.resize("ext", &[5]).unwrap(); + ed.write_values("ext", &Selection::None, &[] as &[i32]) + .unwrap(); + drop(ed); + assert!(std::fs::read(&path).unwrap() == before); +} -- 2.54.0 From 677dc5ec7c44be4c0a6ac402ddffc94537bbfbb8 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:35:06 -0500 Subject: [PATCH 29/48] =?UTF-8?q?docs:=20FileEditor=20=E2=80=94=20changelo?= =?UTF-8?q?g,=20limits=20and=20leaked=20space,=20README=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit known-issues records what the editor refuses, that freed space is never reused (append-workload file sizes measured 2026-09-26 on tank with the ignored measure_append_waste test; sizes are deterministic), and that there is no journal. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 46 ++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 7 +++++++ README.md | 17 ++++++++++++++++ docs/known-issues.md | 35 +++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f4316..06069f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,52 @@ ## Unreleased +### In-place modification (2026-09-26) +- **`clawhdf5::FileEditor` modifies an existing file where it lies.** + `FileBuilder` builds whole files in memory; the editor opens a file + written by libhdf5 (any `libver`, including HDF5 2.0's own format) or by + clawhdf5 and changes only what an edit touches, recomputing the checksum + of every structure it changes. It takes an exclusive `flock` on the file + (the lock libhdf5 takes), so a second editor gets `Error::Locked`. + - `write_selection` / `write_all` / `write_values`: overwrite values of a + compact, contiguous (also never-written, late-allocated) or chunked + dataset, in its own datatype, under any selection. Chunks are decoded, + updated and re-encoded through the dataset's filters; a chunk that no + longer fits moves to the end of the file. New chunks are added to + version-1 B-tree (every chunked dataset of h5py's default `libver`), + Extensible Array, Fixed Array and single-chunk indexes — creating the + index, its data blocks, super blocks and pages, and splitting B-tree + nodes, as libhdf5 does: after the same sequence of writes the B-tree has + the same number of nodes per level and the Extensible Array header the + same block statistics as libhdf5's (tested). + - `resize`: grow a chunked dataset up to its maximum dimensions (h5py's + `Dataset.resize`). + - `set_attr`: add or replace an attribute in an object header, in free + space or in a new continuation chunk at the end of the file. + - Each edit is planned in memory and refused as a whole + (`Error::Unsupported`, file untouched) when any part is not supported: + new chunks in a version-2 B-tree index (two or more unlimited + dimensions) or an implicit index, shrinking, variable-length and + reference data, attributes in dense storage, past an object's compact + limit or with tracked creation order, files with a metadata cache + image, paged or persistent free space, or marked open by another + writer. New error variants `Error::Unsupported`, + `Error::InvalidArgument`, `Error::Locked`, and `clawhdf5::Error` is now + `#[non_exhaustive]` — a breaking change for code that matches it + exhaustively (the Python bindings map the new variants to + `NotImplementedError`, `ValueError` and `OSError`). + - Durability: the new space (chunks, index blocks) is written and synced + before any existing byte changes, then the metadata that links it in, + then a second sync. There is no journal: a crash during the second + phase can leave the file inconsistent (as with libhdf5 without SWMR). + Freed space is not reused (see `docs/known-issues.md`). + - Tests: `crates/clawhdf5-tools/tests/edit_interop.rs` (h5py `earliest`, + `v114` and `latest` files and clawhdf5 files; after every round h5py + reads the expected values, h5dump and `h5rs check --data` accept the + file, and h5py `r+` modifies it further; random operations against a + model) and `crates/clawhdf5/tests/edit_tests.rs`. +- `clawhdf5_format::type_builders::build_attr_message` is public. + ### Chunked full reads (2026-09-26) - **Chunks are decoded straight into the output, into reused buffers.** A full read of a chunked dataset faulted in about three times its size in diff --git a/CLAUDE.md b/CLAUDE.md index b029c5d..2482401 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,6 +150,13 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F Alerts never block a save — drain them with `HDF5Memory::take_anomaly_alerts`. `MemorySource` for this bookkeeping is inferred from the caller-supplied `source_channel` string (a heuristic, not an authenticated trust boundary). +- In-place modification: `clawhdf5::FileEditor` (`crates/clawhdf5/src/edit/`) + overwrites values, grows chunked datasets and sets attributes in existing + files (h5py- or clawhdf5-written) without rewriting them; anything it + cannot do safely is `Error::Unsupported` before any write (limits in + `docs/known-issues.md`). Test changes with + `cargo test -p clawhdf5-tools --test edit_interop` (h5py, h5dump, + `h5rs check`). - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only - Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. diff --git a/README.md b/README.md index 25f43a7..b772985 100644 --- a/README.md +++ b/README.md @@ -433,6 +433,23 @@ b.write("groups.h5")?; A group holds at most 65 535 links; more is an error, as is a link over 65 515 bytes (a very long soft-link target) in a group of more than 8 links. +### Modifying an existing file + +```rust +use clawhdf5::{AttrValue, FileEditor, Selection}; + +// A file from h5py or clawhdf5, dataset "x" chunked with maxshape=(None,). +let mut ed = FileEditor::open("data.h5")?; // exclusive lock, like libhdf5 +ed.resize("x", &[1100])?; // h5py: ds.resize((1100,)) +let sel = Selection::Hyperslab { start: vec![1000], stride: vec![1], count: vec![100], block: vec![1] }; +ed.write_values("x", &sel, &[0.5f64; 100])?; // ds[1000:1100] = 0.5 +ed.set_attr("x", "units", &AttrValue::String("m/s".into()))?; +``` + +Each call changes the file in place (no rewrite) and syncs it. What it +cannot change safely is refused before anything is written; see +[known issues](docs/known-issues.md) for the limits. + ### Python `crates/clawhdf5-py` is a Python package (PyO3 + numpy) that reads HDF5 with diff --git a/docs/known-issues.md b/docs/known-issues.md index 287e501..4d49f0e 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,41 @@ deleting it. --- +## In-place modification (`FileEditor`) limits + +**Status:** open (documented 2026-09-26). `clawhdf5::FileEditor` refuses, +with `Error::Unsupported` and without writing anything: +- new, moved or resized chunks in a **version-2 B-tree** chunk index (what + libhdf5 uses for two or more unlimited dimensions) — existing unfiltered + chunks, and filtered ones that re-encode to the same size, are + overwritten in place; `resize` works — and new chunks in an **implicit** + index (it has all of its chunks from the start); +- **shrinking** a dataset; +- variable-length and reference data; +- attributes of an object in **dense storage**, past its compact limit (8 + by default) or with tracked **creation order**; +- partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external + raw data files, virtual datasets; +- files with a metadata cache image, paged or persistent free-space + management, a driver info block, or version-3 consistency flags set. + +**Space is never reused.** There is no free-space manager: the old bytes of +a filtered chunk that grows and has to move, and of an attribute that is +replaced by a larger one, are leaked (`h5repack` reclaims them). A chunk +that is the last thing in the file grows in place instead, which covers the +usual append. Measured 2026-09-26 on tank with +`cargo test --release -p clawhdf5-tools --test edit_interop -- --ignored +--nocapture measure_append_waste` (file sizes are deterministic): 1000 +appends of 100 `f8` values to a 1-D dataset with 1024-element chunks give +810 504 bytes unfiltered, as libhdf5's file, and 307 210 bytes with gzip +(libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values with +4096-element gzip chunks give 119 684 bytes against libhdf5's 50 292 +(`h5repack`: 49 930), because the chunk being appended to is followed by +new index blocks and moves each time it grows. + +**No journal.** A crash while an edit patches existing structures can leave +the file inconsistent; see the `FileEditor` documentation. + ## Selection reads that decode more than the selection **Status:** open (documented 2026-09-26). `Dataset::read_selection` (and so -- 2.54.0 From 85efde0b4af74dcbd413e3827cda36f405a0fa13 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:36:02 -0500 Subject: [PATCH 30/48] test: rustfmt the lookup tests Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/tests/dense_storage_interop.rs | 10 ++++++++-- crates/clawhdf5/tests/indexed_lookup_interop.rs | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index 3f741f6..f4c8a5b 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -230,8 +230,14 @@ fn many_huge_attributes_are_found_through_their_index() { for i in 0..300i64 { let name = format!("h{i:03}"); let want: Vec = (0..600).map(|v| v + i).collect(); - assert!(matches!(&attrs[&name], AttrValue::I64Array(v) if *v == want), "{name}"); - assert!(matches!(d.attr(&name).unwrap(), Some(AttrValue::I64Array(v)) if v == want), "{name}"); + assert!( + matches!(&attrs[&name], AttrValue::I64Array(v) if *v == want), + "{name}" + ); + assert!( + matches!(d.attr(&name).unwrap(), Some(AttrValue::I64Array(v)) if v == want), + "{name}" + ); } } diff --git a/crates/clawhdf5/tests/indexed_lookup_interop.rs b/crates/clawhdf5/tests/indexed_lookup_interop.rs index 518af1e..30adb0c 100644 --- a/crates/clawhdf5/tests/indexed_lookup_interop.rs +++ b/crates/clawhdf5/tests/indexed_lookup_interop.rs @@ -376,7 +376,10 @@ fn links_of_every_kind_resolve_by_name_as_in_h5py() { "group".to_string() } (Err(e), Err(e2)) => { - assert!(is_not_found(&e) && is_not_found(&e2), "{key}: {e:?} / {e2:?}"); + assert!( + is_not_found(&e) && is_not_found(&e2), + "{key}: {e:?} / {e2:?}" + ); "none".to_string() } (Err(e), Ok(_)) => panic!("{key}: dataset {e:?} but group ok"), -- 2.54.0 From d2b25f154f4fc02a723fd6dae537ecec375dc187 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:41:26 -0500 Subject: [PATCH 31/48] format: in-memory fast path in the Storage read helpers read_exact_at and read_upto (and so Window::read) ask as_contiguous() first and slice the file directly when the backend holds it in memory: one dynamic call per structure read instead of two or three (len, read_at, then len again for errors). Same results and errors. Provisional (busy machine, not for docs): a listing that walks 400 symbol-table groups through the facade went from about 18% to about 14% slower than before the Storage conversion; the extra cost is a few tens of nanoseconds per structure read, which the facade's per-lookup re-listing (range-reads.md M0) multiplies. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/storage.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index c0a1c37..7d68eeb 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -198,6 +198,14 @@ pub fn read_exact_at( .saturating_add(len), available: len_usize(file), }; + // In-memory fast path: one dynamic call, then plain slicing. + if let Some(all) = file.as_contiguous() { + return usize::try_from(offset) + .ok() + .and_then(|start| all.get(start..start.checked_add(len)?)) + .map(Cow::Borrowed) + .ok_or_else(eof); + } match offset.checked_add(len as u64) { Some(end) if end <= file.len() => {} _ => return Err(eof()), @@ -271,6 +279,11 @@ pub fn read_upto( offset: u64, max: usize, ) -> Result, FormatError> { + if let Some(all) = file.as_contiguous() { + let start = usize::try_from(offset).map_or(all.len(), |o| o.min(all.len())); + let end = start.saturating_add(max).min(all.len()); + return Ok(Cow::Borrowed(&all[start..end])); + } let avail = file.len().saturating_sub(offset); let len = usize::try_from(avail).map_or(max, |a| a.min(max)); let bytes = file.read_at(offset, len)?; -- 2.54.0 From 92c8285549c5c556cabe4d9506274b3a593495e7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:07:01 -0500 Subject: [PATCH 32/48] format: verify B-tree v2 internal node checksums Only leaves and the header were checked. Harmless while every lookup read the whole tree, but the indexed lookup prunes children by the keys stored in internal nodes, so one corrupted byte there could route a name to the wrong child and report it missing with no error. A BTIN whose lookup3 checksum does not match is now ChecksumMismatch on every read (lookups and full traversals), as in libhdf5. Test: one byte of the root BTIN of the 35 001-link h5py group's name index changed -> lookups, paths and listings through File, MmapFile and LazyFile all fail with ChecksumMismatch, and h5py refuses both. Before, lookups returned Ok. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/btree_v2.rs | 19 +++++ .../clawhdf5/tests/indexed_lookup_interop.rs | 84 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index 7582c05..d3f6250 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -355,6 +355,23 @@ fn read_internal_node( pos += total_nrec_width; // skip total records in subtree children.push((addr, child_nrec)); } + + // The checksum follows the child pointers and covers the node up to it. + // Lookups prune children by the keys in this node, so an unverified + // internal node could hide a record without any error: libhdf5 refuses + // a mismatch here, and so does this. + #[cfg(feature = "checksum")] + { + ensure_len(file_data, pos, 4)?; + let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]); + let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]); + if computed != stored { + return Err(FormatError::ChecksumMismatch { + expected: stored, + computed, + }); + } + } Ok((records_start, children)) } @@ -733,6 +750,8 @@ mod tests { buf.extend_from_slice(&child_nrec.to_le_bytes()[..nrec_width]); buf.resize(buf.len() + total_width, 0); } + let sum = crate::checksum::jenkins_lookup3(&buf); + buf.extend_from_slice(&sum.to_le_bytes()); buf } diff --git a/crates/clawhdf5/tests/indexed_lookup_interop.rs b/crates/clawhdf5/tests/indexed_lookup_interop.rs index 30adb0c..71f48f0 100644 --- a/crates/clawhdf5/tests/indexed_lookup_interop.rs +++ b/crates/clawhdf5/tests/indexed_lookup_interop.rs @@ -411,3 +411,87 @@ fn links_of_every_kind_resolve_by_name_as_in_h5py() { } } } + +/// The link name index (v2 B-tree, record type 5) of the big group: its +/// depth and root node address, read from the one type-5 `BTHD` in the file. +fn name_index_root(bytes: &[u8]) -> (u16, usize) { + let headers: Vec = bytes + .windows(4) + .enumerate() + .filter(|(i, w)| *w == b"BTHD" && bytes.get(i + 5) == Some(&5)) + .map(|(i, _)| i) + .collect(); + assert_eq!(headers.len(), 1, "type-5 B-tree headers at {headers:?}"); + let h = headers[0]; + // signature, version, type, node size (4), record size (2), depth (2), + // split and merge percent, root address (8). + let depth = u16::from_le_bytes([bytes[h + 12], bytes[h + 13]]); + let root = u64::from_le_bytes(bytes[h + 16..h + 24].try_into().unwrap()); + (depth, usize::try_from(root).unwrap()) +} + +/// One byte changed in a key of the name index's root (an internal node) +/// must be an error, not a name quietly routed to the wrong child and +/// reported missing: lookups prune children by those keys. libhdf5 checks +/// the internal node's checksum and refuses the group; so must we, for a +/// lookup and for a listing. +#[test] +fn a_corrupt_internal_index_node_is_an_error_not_a_missing_name() { + skip_if_no_python!(); + let fx = fixture(); + let mut bytes = std::fs::read(&fx.path).unwrap(); + let (depth, root) = name_index_root(&bytes); + assert!(depth >= 2, "want a deep index, got depth {depth}"); + assert_eq!(&bytes[root..root + 4], b"BTIN"); + // Signature, version, type, then record 0: its name hash comes first. + bytes[root + 6] ^= 0x5a; + let dir = tempfile::tempdir().unwrap(); + let bad = dir.path().join("bad.h5"); + std::fs::write(&bad, &bytes).unwrap(); + let bad = bad.display().to_string(); + + let is_checksum = |e: &clawhdf5::Error| { + matches!( + e, + clawhdf5::Error::Format(FormatError::ChecksumMismatch { .. }) + ) + }; + let f = File::open(&bad).unwrap(); + let g = f.group("g").unwrap(); + // Every name, present or not, goes through the root. + for name in fx.links.keys().step_by(97).chain(&fx.missing_links) { + let err = g.dataset(name).map(|_| ()).unwrap_err(); + assert!(is_checksum(&err), "dataset({name:?}): {err:?}"); + } + let err = f.dataset("/g/n0").map(|_| ()).unwrap_err(); + assert!(is_checksum(&err), "path: {err:?}"); + let err = g.datasets().unwrap_err(); + assert!(is_checksum(&err), "listing: {err:?}"); + let err = g.entries().unwrap_err(); + assert!(is_checksum(&err), "entries: {err:?}"); + + let m = MmapFile::open(&bad).unwrap(); + let mg = m.group("g").unwrap(); + assert!(mg.dataset("n0").is_err_and(|e| is_checksum(&e))); + assert!(mg.datasets().is_err_and(|e| is_checksum(&e))); + let l = LazyFile::open_mmap(&bad).unwrap(); + let lg = l.group("g").unwrap(); + assert!(lg.dataset("n0").is_err_and(|e| is_checksum(&e))); + assert!(lg.datasets().is_err_and(|e| is_checksum(&e))); + + // libhdf5 refuses both too. + let out = run_python(&format!( + "import h5py\n\ + r = []\n\ + with h5py.File(r'{bad}', 'r') as f:\n\ + \x20 g = f['g']\n\ + \x20 for op in (lambda: g['n0'], lambda: list(g)):\n\ + \x20 try:\n\ + \x20 op()\n\ + \x20 r.append('ok')\n\ + \x20 except Exception as e:\n\ + \x20 r.append('checksum' if 'checksum' in str(e) else repr(e))\n\ + print(' '.join(r))", + )); + assert_eq!(out, "checksum checksum"); +} -- 2.54.0 From b6cbd2319f9d156a19ef077d10e2647bab8d05b7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:08:57 -0500 Subject: [PATCH 33/48] format: checked chunk addresses on the parallel read path Three `chunk_info.address as usize` casts behind the `parallel` feature survived the conversion, because check-32bit-casts.sh linted only default features plus plugin-filters. On a 32-bit target with rayon a chunk address past 4 GiB still wrapped onto another part of the file. They go through addr::to_usize now, and the lane index (h % n, always < n) through saturating_usize. The script now lints no default features, default features, and every optional feature but szip (wasm32; the set with zstd, which does not build for wasm32, on the host, where the lint reports the same casts). With the old parallel_read.rs/lane_partition.rs it fails listing the four casts; the old script passed them. CHANGELOG and the design note give the exact count (119) and what is not covered. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 13 ++++- crates/clawhdf5-format/src/lane_partition.rs | 3 +- crates/clawhdf5-format/src/parallel_read.rs | 7 ++- docs/design/range-reads.md | 9 +-- scripts/check-32bit-casts.sh | 61 +++++++++++++++----- 5 files changed, 68 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3b0b6c..14f8053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,14 +28,21 @@ resolves it in dense and compact groups. ### Checked address conversion (2026-09-26) -- **No 64-bit file value is truncated on a 32-bit target.** Every - `u64 as usize` cast in `clawhdf5-format` (115) is gone: file addresses, +- **No 64-bit file value is truncated on a 32-bit target.** All 119 + truncating `u64 as usize` casts in `clawhdf5-format` that clippy's + `cast_possible_truncation` reports, under every feature the crate is + built with in CI except `szip` (115 with default features and + `plugin-filters`, 4 more behind `parallel`), are gone: file addresses, lengths and counts go through `addr::to_usize`, which fails with `FormatError::Overflow` where the value does not fit (wasm32 and other 32-bit targets; it used to wrap onto another part of the file), and in-memory counts through `addr::saturating_usize`. On 64-bit targets nothing changes. `scripts/check-32bit-casts.sh` (run by `ci-test.sh`) - lints the wasm32 build and fails on any new truncating cast. + lints the crate with no default features, with default features, and + with every optional feature but `szip` (for wasm32; the set with `zstd`, + which does not build for wasm32, for the host), and fails on any new + truncating cast. The facade, `clawhdf5-io` and `clawhdf5-ann` are not + covered. ### Chunked full reads (2026-09-26) - **Chunks are decoded straight into the output, into reused buffers.** A diff --git a/crates/clawhdf5-format/src/lane_partition.rs b/crates/clawhdf5-format/src/lane_partition.rs index 2b8b8da..0b86e6b 100644 --- a/crates/clawhdf5-format/src/lane_partition.rs +++ b/crates/clawhdf5-format/src/lane_partition.rs @@ -112,7 +112,8 @@ pub fn partition( for idx in 0..num_items { let h = fxhash_combine(seed, idx as u64); - let lane = (h % num_lanes as u64) as usize; + // Below `num_lanes`, so it fits. + let lane = crate::addr::saturating_usize(h % num_lanes as u64); lanes[lane].push(idx); } diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index d9927c5..92fb1fa 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -7,6 +7,7 @@ //! The lane assignment is seeded by dataset metadata so repeated reads of //! the same region produce identical partitions (cache-friendly, reproducible). +use crate::addr::to_usize; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; @@ -210,7 +211,7 @@ pub fn decompress_chunks_lane_partitioned( for &index in &indices { let chunk_info = &chunks[index]; - let c_addr = chunk_info.address as usize; + let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; if c_addr @@ -288,7 +289,7 @@ pub fn decompress_chunks_parallel( .par_iter() .enumerate() .map(|(index, chunk_info)| { - let c_addr = chunk_info.address as usize; + let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; if c_addr .checked_add(size) @@ -332,7 +333,7 @@ pub fn decompress_chunks_sequential( ) -> Result>, FormatError> { let mut result = Vec::with_capacity(chunks.len()); for chunk_info in chunks { - let c_addr = chunk_info.address as usize; + let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; if c_addr .checked_add(size) diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1edee1a..9629a21 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -383,10 +383,11 @@ fast path within benchmark noise. attribute names through the name indexes (`group_v2::resolve_child`, `attribute::find_attribute_in_file`; creation-order lookups by name do not exist in the API, so the creation-order index is still only listed), - `addr::to_usize`/`saturating_usize` for all 115 `u64 as usize` casts in - `clawhdf5-format` (the 133 above counted any `*addr*/*offset* as usize`, - mostly widening `u8`/`u32` casts; `scripts/check-32bit-casts.sh` lints - wasm32 for the truncating ones), and `Group::entries`/`File::group_at`. + `addr::to_usize`/`saturating_usize` for all 119 truncating `u64 as usize` + casts clippy finds in `clawhdf5-format` under any CI-built feature set but + `szip` (the 133 above counted any `*addr*/*offset* as usize`, mostly + widening `u8`/`u32` casts; `scripts/check-32bit-casts.sh` lints those + feature sets for new ones), and `Group::entries`/`File::group_at`. The facade, io and ann casts are not converted. **M1 — metadata over the trait, in-memory impl identical to today (2–3 weeks).** diff --git a/scripts/check-32bit-casts.sh b/scripts/check-32bit-casts.sh index cfed6f5..104bdcd 100755 --- a/scripts/check-32bit-casts.sh +++ b/scripts/check-32bit-casts.sh @@ -5,8 +5,18 @@ # of the file. File values go through `addr::to_usize` (a clean error) and # in-memory counts through `addr::saturating_usize`. # -# Lints wasm32-unknown-unknown with clippy's cast_possible_truncation and -# fails on any u64 -> usize finding (other truncations are not checked here). +# Lints with clippy's cast_possible_truncation and fails on any u64 -> usize +# finding (other truncations are not checked here), once per feature set +# below. Together the sets compile every feature-gated line of the crate that +# ci-test.sh builds: features only add code, except `not(feature = ...)` +# paths for std/checksum/fast-checksum/szip, which the no-default-features +# and default sets cover. szip is left out (it needs libaec), as in +# ci-test.sh. +# +# The sets are linted for wasm32 where they build there. zstd links a C +# library that does not build for wasm32, so the set with it is linted for +# the host: the lint reports u64 -> usize casts whatever the target's +# pointer width, and the crate has no pointer-width-dependent code. # # Usage: # ./scripts/check-32bit-casts.sh @@ -16,19 +26,42 @@ set -euo pipefail -TARGET="wasm32-unknown-unknown" -echo "==> Checking for truncating u64 -> usize casts in clawhdf5-format ($TARGET)" +WASM="wasm32-unknown-unknown" +ALL_BUT_ZSTD="parallel,lz4,pcodec,fast-checksum,blake3_hash,plugin-filters,lookup-stats" -out=$(cargo clippy -p clawhdf5-format --target "$TARGET" \ - --features plugin-filters --message-format short \ - -- -A clippy::all -W clippy::cast_possible_truncation 2>&1) || { - echo "$out" - echo "==> clippy failed" >&2 - exit 1 -} -found=$(grep -F 'casting `u64` to `usize`' <<<"$out" || true) -if [ -n "$found" ]; then - echo "$found" +# target|cargo feature arguments +SETS=( + "$WASM|--no-default-features" + "$WASM|--no-default-features --features std,checksum" + "$WASM|" + "$WASM|--features $ALL_BUT_ZSTD" + "host|--features $ALL_BUT_ZSTD,zstd" +) + +status=0 +for set in "${SETS[@]}"; do + target=${set%%|*} + args=${set#*|} + target_args=() + if [ "$target" != host ]; then + target_args=(--target "$target") + fi + echo "==> Checking for truncating u64 -> usize casts in clawhdf5-format ($target: ${args:-default features})" + # shellcheck disable=SC2086 # $args is a list of arguments + out=$(cargo clippy -p clawhdf5-format "${target_args[@]}" $args \ + --message-format short \ + -- -A clippy::all -W clippy::cast_possible_truncation 2>&1) || { + echo "$out" + echo "==> clippy failed" >&2 + exit 1 + } + found=$(grep -F 'casting `u64` to `usize`' <<<"$out" || true) + if [ -n "$found" ]; then + echo "$found" + status=1 + fi +done +if [ "$status" -ne 0 ]; then echo "==> use addr::to_usize (file values) or addr::saturating_usize (in-memory counts)" >&2 exit 1 fi -- 2.54.0 From f7e2ab12f27b10677de8385cd623d39b9b673944 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:09:34 -0500 Subject: [PATCH 34/48] clawhdf5: FileEditor skips optional filters that fail, as libhdf5 does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The editor stored every chunk through the whole pipeline with filter mask 0. For LZF that did not shrink a chunk, h5py instead stores it raw with the filter's mask bit set. A chunk the editor stored LZF-encoded at exactly the raw size was then rewritten raw by libhdf5 at the same size; libhdf5 does not touch the index entry when the size is unchanged, so the stale mask 0 stayed and h5py (and h5dump) could no longer read the dataset. clawhdf5_format::filters::compress_chunk_masked runs the pipeline as H5Z_pipeline does: an optional filter (H5Z_FLAG_OPTIONAL) that fails is skipped and its bit set, a mandatory one fails the write, and LZF/Blosc output no smaller than the input counts as failure, as in the reference filters (their output buffer is the input's size). Deflate, LZ4, Zstd, bitshuffle and bzip2 never fail on size in libhdf5 and are kept as before. Test: edit_interop optional_filters_that_fail_are_skipped — the reviewer's repro at every libver: the editor stores the chunk exactly as h5py does (mask 1, size 5; shuffle+LZF+fletcher32 mask 2), h5py r+ rewrites and extends the datasets, and h5py, h5dump and our reader read every value. Fails on the previous editor (mask 0; h5dump cannot read /u8). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 +- crates/clawhdf5-format/src/filters.rs | 120 ++++++++++++++++++ crates/clawhdf5-tools/tests/edit_interop.rs | 128 ++++++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 7 +- 4 files changed, 260 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06069f7..f070cd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,15 @@ index, its data blocks, super blocks and pages, and splitting B-tree nodes, as libhdf5 does: after the same sequence of writes the B-tree has the same number of nodes per level and the Extensible Array header the - same block statistics as libhdf5's (tested). + same block statistics as libhdf5's (tested). Filters run as libhdf5's + `H5Z_pipeline` runs them (new + `clawhdf5_format::filters::compress_chunk_masked`): an optional filter + that fails — LZF or Blosc output no smaller than the chunk — is skipped + and its filter-mask bit set, so the chunk is stored exactly as h5py + stores it; a mandatory filter that fails fails the edit. (Storing such + a chunk LZF-encoded at the raw size with a clear mask let a later + libhdf5 rewrite of it keep the stale mask, and h5py could no longer + read the dataset.) - `resize`: grow a chunked dataset up to its maximum dimensions (h5py's `Dataset.resize`). - `set_attr`: add or replace an attribute in an object header, in free diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 40d6cd9..f187746 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -346,6 +346,72 @@ pub fn compress_chunk( Ok(result) } +/// Filter flag bit 0: `H5Z_FLAG_OPTIONAL`. +const FILTER_FLAG_OPTIONAL: u16 = 0x0001; + +/// Filters whose reference HDF5 filter (h5py's `lzf_filter.c`, +/// hdf5-blosc's `blosc_filter.c`) gives the encoder an output buffer only as +/// large as its input, so output that is not smaller than the input is a +/// failure there. +const FAIL_UNLESS_SMALLER: &[u16] = &[ + crate::filter_pipeline::FILTER_LZF, + crate::filter_pipeline::FILTER_BLOSC, +]; + +/// Run a chunk through a filter pipeline for writing the way libhdf5's +/// `H5Z_pipeline` does, returning the bytes to store and the chunk's filter +/// mask (bit `i` set: filter `i` was skipped). +/// +/// A filter that fails is skipped if the pipeline marks it optional +/// (`H5Z_FLAG_OPTIONAL`): its mask bit is set and the next filter gets the +/// same input. A mandatory filter that fails fails the write. Failure +/// includes what the reference filter counts as failure: LZF and Blosc +/// output that is not smaller than the input (h5py then stores the chunk +/// unfiltered with the bit set; storing it filtered with a clear mask can +/// leave a stale mask once libhdf5 rewrites the chunk at the same size). +/// +/// A filter this build cannot encode is [`FormatError::UnsupportedFilter`] +/// even when optional: libhdf5 skips an optional filter only when its own +/// build lacks it, and every libhdf5 has the ones clawhdf5 cannot encode. +pub fn compress_chunk_masked( + data: &[u8], + pipeline: &FilterPipeline, + element_size: u32, +) -> Result<(Vec, u32), FormatError> { + if pipeline.filters.len() > 32 { + return Err(FormatError::CompressionError( + "more than 32 filters in a pipeline".into(), + )); + } + let mut result = data.to_vec(); + let mut mask = 0u32; + for (i, filter) in pipeline.filters.iter().enumerate() { + let ctx = FilterContext { + filter, + element_size: element_size as usize, + max_output: 0, + }; + let out = match filter_registry::encode(&result, &ctx) { + Ok(out) + if FAIL_UNLESS_SMALLER.contains(&filter.filter_id) && out.len() >= result.len() => + { + Err(FormatError::CompressionError(format!( + "filter {} did not shrink the chunk", + filter.filter_id + ))) + } + r => r, + }; + match out { + Ok(out) => result = out, + Err(e @ FormatError::UnsupportedFilter(_)) => return Err(e), + Err(_) if filter.flags & FILTER_FLAG_OPTIONAL != 0 => mask |= 1 << i, + Err(e) => return Err(e), + } + } + Ok((result, mask)) +} + /// The filters compiled into this build, sorted by ID (see /// [`crate::filter_registry`]). A filter whose cargo feature is off is left /// out, so it fails as [`FormatError::UnsupportedFilter`] like any unknown ID. @@ -2099,6 +2165,60 @@ mod tests { } } + /// `compress_chunk_masked` follows `H5Z_pipeline`: an optional LZF that + /// does not shrink the chunk is skipped with its mask bit set (h5py + /// stores `[182, 0, 0, 0, 0]` raw with mask 1), a mandatory one fails, + /// and filters that grow the data (deflate) are kept, as libhdf5 keeps + /// them. + #[test] + #[cfg(all(feature = "lzf", feature = "deflate"))] + fn masked_compression_skips_optional_filters_that_fail() { + use crate::filter_pipeline::FILTER_LZF; + let opt = |id: u16| FilterDescription { + flags: FILTER_FLAG_OPTIONAL, + ..filter(id) + }; + let pl = |filters: Vec| FilterPipeline { + version: 2, + filters, + }; + let raw = [182u8, 0, 0, 0, 0]; + let (out, mask) = compress_chunk_masked(&raw, &pl(vec![opt(FILTER_LZF)]), 1).unwrap(); + assert_eq!((out.as_slice(), mask), (&raw[..], 1)); + let (out, mask) = compress_chunk_masked( + &raw, + &pl(vec![ + opt(FILTER_SHUFFLE), + opt(FILTER_LZF), + filter(FILTER_FLETCHER32), + ]), + 1, + ) + .unwrap(); + assert_eq!((out.len(), mask), (raw.len() + 4, 2)); + assert_eq!( + decompress_chunk_masked( + &out, + &pl(vec![ + opt(FILTER_SHUFFLE), + opt(FILTER_LZF), + filter(FILTER_FLETCHER32) + ]), + raw.len(), + 1, + mask + ) + .unwrap(), + raw + ); + assert!(compress_chunk_masked(&raw, &pl(vec![filter(FILTER_LZF)]), 1).is_err()); + let zeros = [0u8; 256]; + let (out, mask) = compress_chunk_masked(&zeros, &pl(vec![opt(FILTER_LZF)]), 1).unwrap(); + assert!(out.len() < zeros.len() && mask == 0); + let (out, mask) = compress_chunk_masked(&raw, &pl(vec![opt(FILTER_DEFLATE)]), 1).unwrap(); + assert!(out.len() > raw.len() && mask == 0); + } + #[test] #[cfg(feature = "deflate")] fn filter_mask_skips_only_the_masked_filters() { diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 2df54d7..fe44aa8 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1187,3 +1187,131 @@ fn measure_append_waste() { ); } } + +/// Optional filters that fail are skipped as libhdf5 skips them: an LZF +/// that does not shrink a chunk leaves it stored unfiltered with its mask +/// bit set, exactly as h5py stores it. Storing the LZF stream with a clear +/// mask at the raw chunk's size once let a later libhdf5 rewrite of the +/// chunk (raw, same size) keep the stale mask, and h5py then failed to read +/// the dataset. After the editor, h5py r+ rewrites and extends the datasets +/// and h5py, h5dump (on the chunks it can decode: LZF is not in h5dump) and +/// our reader read every value. +#[test] +fn optional_filters_that_fail_are_skipped() { + if !tools_ok() { + return; + } + for (li, (lv, _)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("optional_{li}.h5")); + let p = path.to_str().unwrap(); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 f.create_dataset('u8', shape=(5,), dtype='u1', chunks=(5,), maxshape=(None,), compression='lzf')\n\ + \x20 f.create_dataset('mix', shape=(32,), dtype=' = (0..32) + .map(|i| { + if (i / 8) % 2 == 0 { + rng.next() as i32 + } else { + 7 + } + }) + .collect(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.write_all("u8", &[182, 0, 0, 0, 0]).unwrap(); + ed.write_values("mix", &Selection::All, &mix).unwrap(); + drop(ed); + // Stored as h5py stores it: raw, LZF's bit (0; 1 behind shuffle) set. + let masks = py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + i = lambda d, k: f[d].id.get_chunk_info(k)\n\ + print(i('u8', 0).filter_mask, i('u8', 0).size, i('ref', 0).filter_mask, i('ref', 0).size,\n\ + \x20 *[(i('mix', k).filter_mask, i('mix', k).size < 36) for k in range(4)])\n" + )); + assert_eq!( + masks, "1 5 1 5 (2, False) (0, True) (2, False) (0, True)", + "{lv}" + ); + let dump = |ds: &str| { + let o = Command::new("h5dump") + .args(["-d", ds, "-y", "-w", "0", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5dump -d {ds} {p}:\n{}", text(&o)); + let s = String::from_utf8_lossy(&o.stdout).into_owned(); + s.split_once("DATA {") + .and_then(|(_, r)| r.split_once('}')) + .map(|(d, _)| { + d.split(|c: char| c == ',' || c.is_whitespace()) + .filter(|t| !t.is_empty()) + .map(|t| t.parse::().unwrap()) + .collect::>() + }) + .unwrap_or_default() + }; + if *lv != "'latest'" { + assert_eq!(dump("/u8"), [182, 0, 0, 0, 0]); + } + // libhdf5 rewrites the chunk raw at the same size, then extends the + // datasets with more incompressible data. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['u8'][...] = [182, 0, 0, 0, 1]\n\ + \x20 f['u8'].resize((12,))\n\ + \x20 f['u8'][5:] = [9, 200, 3, 77, 1, 250, 42]\n\ + \x20 m = f['mix']\n\ + \x20 m[8:16] = np.arange(8, dtype=' = vec![182, 0, 0, 0, 1, 9, 200, 3, 77, 1, 250, 42]; + let mut mix_want = mix.clone(); + mix_want[0..8].fill(5); + for (k, v) in mix_want[8..16].iter_mut().enumerate() { + *v = k as i32 * 104_729 + 12_345; + } + mix_want.extend([11; 8]); + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("u8") + .unwrap() + .read_selection(&Selection::All) + .unwrap(), + u8_want + ); + assert_eq!(f.dataset("mix").unwrap().read_i32().unwrap(), mix_want); + drop(f); + verify( + &path, + "mix", + &Model { + shape: vec![40], + data: mix_want, + }, + ); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + assert f['u8'][()].tolist() == {u8_want:?}, f['u8'][()]\n" + )); + if *lv != "'latest'" { + let got: Vec = dump("/u8").into_iter().map(|v| v as u8).collect(); + assert_eq!(got, u8_want); + } + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index b135a3f..c0a5d07 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -1116,11 +1116,10 @@ fn write_selection( Ok(()) })?; for (scaled, buf) in bufs { + // As libhdf5 does: an optional filter that fails (LZF that + // does not shrink the chunk) is skipped and its mask bit set. let (bytes, mask) = match &t.pipeline { - Some(p) => ( - clawhdf5_format::filters::compress_chunk(&buf, p, es as u32)?, - 0u32, - ), + Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, es as u32)?, None => (buf, 0u32), }; let len = bytes.len() as u64; -- 2.54.0 From 5b3d32b37d06c6929369c71296c18414da216b3b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:09:37 -0500 Subject: [PATCH 35/48] format: test the address overflow path on 64-bit hosts addr::to_usize's error branch only ran where usize is narrower than u64, and no such target runs tests in CI, so on x86_64 the test checked only that every u64 fits. to_usize and saturating_usize are now the usize instances of generic to_index/saturating_index; the test runs the same code with u32 standing in for a 32-bit usize: values past u32::MAX (including one an `as` cast would wrap to 0x1234) are Overflow, and the saturating form clamps. A mutant that truncates instead fails the test; the old addr.rs does not provide the helper the test needs. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/addr.rs | 55 +++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/crates/clawhdf5-format/src/addr.rs b/crates/clawhdf5-format/src/addr.rs index 09c2cef..3198f1f 100644 --- a/crates/clawhdf5-format/src/addr.rs +++ b/crates/clawhdf5-format/src/addr.rs @@ -20,7 +20,16 @@ use crate::error::FormatError; /// platform's `usize` (only possible on targets narrower than 64 bits). #[inline] pub fn to_usize(value: u64) -> Result { - usize::try_from(value).map_err(|_| too_large(value)) + to_index::(value) +} + +/// [`to_usize`] for an index type of any width. `usize` is 64 bits wide on +/// the hosts CI tests on, where the error path cannot be reached through +/// `usize`; tests run the same code with `u32` in its place, as on a 32-bit +/// target. +#[inline] +fn to_index>(value: u64) -> Result { + T::try_from(value).map_err(|_| too_large(value)) } /// A count or offset into an in-memory buffer (a codec's progress counter, @@ -33,7 +42,14 @@ pub fn to_usize(value: u64) -> Result { /// read from the file uses [`to_usize`]. #[inline] pub fn saturating_usize(value: u64) -> usize { - usize::try_from(value).unwrap_or(usize::MAX) + saturating_index(value, usize::MAX) +} + +/// [`saturating_usize`] for an index type of any width, whose largest +/// value is `max` (see [`to_index`]). +#[inline] +fn saturating_index>(value: u64, max: T) -> T { + T::try_from(value).unwrap_or(max) } #[cold] @@ -66,16 +82,31 @@ mod tests { #[test] fn values_past_usize_max_are_an_error_not_truncated() { - // Only reachable where usize is narrower than u64; on a 64-bit host - // every u64 fits, which the first branch checks instead. - if let Some(past) = (usize::MAX as u64).checked_add(1) { - let err = to_usize(past).unwrap_err(); - assert!(matches!(err, FormatError::Overflow(_)), "{err:?}"); - // The value an `as usize` cast would have produced is not returned. - assert!(to_usize(u64::MAX).is_err()); - assert!(to_usize(past + 0x10).is_err()); - } else { - assert_eq!(to_usize(u64::MAX), Ok(u64::MAX as usize)); + // Reachable through `usize` only where it is narrower than u64 (no + // such target runs tests in CI), so the same conversion is run with + // u32 standing in for a 32-bit usize. + let max = u64::from(u32::MAX); + assert_eq!(to_index::(max), Ok(u32::MAX)); + for past in [max + 1, max + 0x10, 0x1_0000_1234, u64::MAX] { + let err = to_index::(past).unwrap_err(); + assert!( + matches!(err, FormatError::Overflow(_)), + "{past:#x}: {err:?}" + ); + } + // Where an `as` cast would have wrapped to a small, valid-looking + // index, it is not returned. + assert_eq!(0x1_0000_1234_u64 as u32, 0x1234); + assert!(to_index::(0x1_0000_1234).is_err()); + + assert_eq!(saturating_index(max + 1, u32::MAX), u32::MAX); + assert_eq!(saturating_index(0x1_0000_1234, u32::MAX), u32::MAX); + assert_eq!(saturating_index(0x1234, u32::MAX), 0x1234); + + // And through `usize` itself, whichever width it has here. + match (usize::MAX as u64).checked_add(1) { + Some(past) => assert!(matches!(to_usize(past), Err(FormatError::Overflow(_)))), + None => assert_eq!(to_usize(u64::MAX), Ok(usize::MAX)), } } } -- 2.54.0 From 04a7f6f6c7bc40b2ecfbaffc3211c2cda680c10d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:11:47 -0500 Subject: [PATCH 36/48] read: of two links with one name, the first wins everywhere A valid group has one link per name, but a damaged or hand-made one can have two. resolve_child followed the first soft link of the name, the listing skipped a dangling one and listed the name via a later link, and path resolution followed the last symbolic link: three answers. All now take the first link of the name (header message order in a compact group, name index order in a dense one) and ignore the rest, even if the first dangles. That is libhdf5's rule for compact groups (H5G__compact_lookup stops at the first Link message); h5py opens nothing for a dangling first link although a later one resolves. For a dense group libhdf5 binary-searches the index and may land on another of several exact duplicates; documented on first_link_named. find_symbolic_link's v2 branch was dead (only v1 groups reach it) and is now v1-only. Test: an h5py compact group with soft links dup_A (dangling, or to /d) and dup_B (the other), dup_B renamed to dup_A in the header and re-checksummed. Lookup, path and listing through all three readers match h5py for both orders. With the old group_v2.rs the path lookup returned 42 where h5py opens nothing. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 ++ crates/clawhdf5-format/src/group_v2.rs | 154 ++++++++++-------- .../clawhdf5/tests/indexed_lookup_interop.rs | 115 +++++++++++++ 3 files changed, 211 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14f8053..70cd926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,16 @@ types): one attribute, found in dense storage through its name index (record type 8) instead of reading every attribute (`clawhdf5_format::attribute::find_attribute_in_file`). +- **Two links of one name: the first wins everywhere.** A group cannot + validly hold two links of one name, but a damaged or hand-made one can. + The listing, `resolve_child` (`Group::dataset`/`group`) and path + resolution now all use only the first link of a name (header message + order in a compact group, name index order in a dense one) and ignore + the rest, even if the first dangles — libhdf5's rule for compact groups + (h5py fails to open a dangling first link although a later one + resolves). Before, the listing skipped a dangling first link and listed + the name via a later one that lookup did not follow, and path resolution + followed the last. - **`Group::entries()` and `File::group_at(address)`**: a listing's `(name, address)` pairs, to open children without looking names up again. - Test: `crates/clawhdf5/tests/indexed_lookup_interop.rs` — every child of diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index e1bd9f5..0edf972 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -6,6 +6,11 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; +#[cfg(not(feature = "std"))] +use alloc::collections::BTreeSet; +#[cfg(feature = "std")] +use std::collections::BTreeSet; + use crate::addr::to_usize; use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records, find_btree_v2_records}; use crate::checksum::jenkins_lookup3; @@ -159,45 +164,34 @@ fn resolve_dense_entries( Ok(entries) } -/// The soft or external link called `name` in this group, if there is one. -/// Hard links are what `resolve_group_entries` returns; this is consulted only -/// when a path component isn't among them. -fn find_symbolic_link( +/// The soft link called `name` in a v1 (symbol table) group, if there is +/// one. Hard links are what `resolve_group_entries` returns; this is +/// consulted only when a path component isn't among them. +fn find_v1_symbolic_link( file_data: &[u8], object_header: &ObjectHeader, name: &str, offset_size: u8, length_size: u8, ) -> Result, FormatError> { - if is_v1_group(object_header) { - let Some(sym_msg) = object_header - .messages - .iter() - .find(|m| m.msg_type == MessageType::SymbolTable) - else { - return Ok(None); - }; - let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - return group_v1::find_v1_soft_link(file_data, &stm, name, offset_size, length_size) - .map(|target| target.map(|target_path| LinkTarget::Soft { target_path })); - } - if !is_v2_group(object_header) { + let Some(sym_msg) = object_header + .messages + .iter() + .find(|m| m.msg_type == MessageType::SymbolTable) + else { return Ok(None); - } - // As when all links were scanned: the last symbolic link of that name. - Ok( - links_named(file_data, object_header, name, offset_size, length_size)? - .into_iter() - .rev() - .map(|link| link.link_target) - .find(|t| !matches!(t, LinkTarget::Hard { .. })), - ) + }; + let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; + group_v1::find_v1_soft_link(file_data, &stm, name, offset_size, length_size) + .map(|target| target.map(|target_path| LinkTarget::Soft { target_path })) } /// B-tree v2 record type of a dense group's link name index. const LINK_NAME_INDEX: u8 = 5; -/// The links called `name` in a v2 group (a valid group has at most one). +/// The links called `name` in a v2 group (a valid group has at most one), +/// in storage order: header message order for a compact group, name index +/// order for a dense one. /// /// In dense storage the link name index (a v2 B-tree of lookup3 name /// hashes, record type 5) is descended to the records with the name's hash, @@ -272,6 +266,32 @@ fn links_named( Ok(found) } +/// The link called `name` in a v2 group, if any. +/// +/// A valid group has at most one; libhdf5 cannot create two. If a damaged +/// or hand-made group has several, the first wins and the rest are +/// ignored, whatever their kind and even if the first cannot be followed. +/// That is libhdf5's rule for a compact group (`H5G__compact_lookup` stops +/// at the first Link message of that name; h5py then fails to open a +/// dangling first link although a later one resolves). For a dense group +/// "first" is first in name index order; libhdf5 binary-searches the index +/// and may land on another of several exact duplicates. The listing +/// ([`resolve_group_children`]), [`resolve_child`] and path resolution all +/// apply this rule, so they agree. +fn first_link_named( + file_data: &[u8], + object_header: &ObjectHeader, + name: &str, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + Ok( + links_named(file_data, object_header, name, offset_size, length_size)? + .into_iter() + .next(), + ) +} + /// The link [`resolve_path_any`] follows for one path component `name` of /// the group with header `object_header`: a hard link (as `Hard`), else a /// soft or external link of that name, else `None`. Fails with @@ -293,29 +313,25 @@ fn lookup_link( object_header_address: e.object_header_address, })); } - return find_symbolic_link(file_data, object_header, name, offset_size, length_size); + return find_v1_symbolic_link(file_data, object_header, name, offset_size, length_size); } if !is_v2_group(object_header) { return Err(FormatError::PathNotFound(String::from( "object header is not a group", ))); } - let links = links_named(file_data, object_header, name, offset_size, length_size)?; - if let Some(addr) = links.iter().find_map(|l| match l.link_target { - LinkTarget::Hard { - object_header_address, - } if object_header_address != u64::MAX => Some(object_header_address), - _ => None, - }) { - return Ok(Some(LinkTarget::Hard { - object_header_address: addr, - })); - } - Ok(links - .into_iter() - .rev() - .map(|link| link.link_target) - .find(|t| !matches!(t, LinkTarget::Hard { .. }))) + Ok( + first_link_named(file_data, object_header, name, offset_size, length_size)? + .map(|link| link.link_target) + .filter(|t| { + !matches!( + t, + LinkTarget::Hard { + object_header_address: u64::MAX + } + ) + }), + ) } /// The object header address of the child called `name` of the group at @@ -342,19 +358,14 @@ pub fn resolve_child( .map(|e| e.object_header_address) .ok_or_else(not_found); } - let links = links_named(file_data, &header, name, os, ls)?; - // The listing puts hard links before resolved soft links. - if let Some(addr) = links.iter().find_map(|l| match l.link_target { - LinkTarget::Hard { + // The first link of that name only, as the listing (see + // `first_link_named`). + match first_link_named(file_data, &header, name, os, ls)?.map(|l| l.link_target) { + Some(LinkTarget::Hard { object_header_address, - } => Some(object_header_address), - _ => None, - }) { - return Ok(addr); - } - for link in links { - if let LinkTarget::Soft { target_path } = link.link_target { - return match resolve_path_from(file_data, superblock, group_address, &target_path) { + }) => Ok(object_header_address), + Some(LinkTarget::Soft { target_path }) => { + match resolve_path_from(file_data, superblock, group_address, &target_path) { // Left out of the listing: dangling, cyclic, or in another file. Err( FormatError::PathNotFound(_) @@ -362,10 +373,10 @@ pub fn resolve_child( | FormatError::ExternalLinkUnsupported { .. }, ) => Err(not_found()), other => other, - }; + } } + Some(LinkTarget::External { .. }) | None => Err(not_found()), } - Err(not_found()) } /// Find and parse the Link Info message from an object header. @@ -471,16 +482,23 @@ pub fn resolve_group_children( } entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e))); } else if is_v2_group(&header) { - let mut visit = |link: LinkMessage| match link.link_target { - LinkTarget::Hard { - object_header_address, - } => entries.push(GroupEntry { - name: link.name, - object_header_address, - cache_type: 0, - }), - LinkTarget::Soft { target_path } => soft.push((link.name, target_path)), - LinkTarget::External { .. } => {} + // Only the first link of each name counts (see `first_link_named`). + let mut seen = BTreeSet::new(); + let mut visit = |link: LinkMessage| { + if !seen.insert(link.name.clone()) { + return; + } + match link.link_target { + LinkTarget::Hard { + object_header_address, + } => entries.push(GroupEntry { + name: link.name, + object_header_address, + cache_type: 0, + }), + LinkTarget::Soft { target_path } => soft.push((link.name, target_path)), + LinkTarget::External { .. } => {} + } }; let link_info = find_link_info(&header, os)?; if let Some(fh_addr) = link_info.fractal_heap_address { diff --git a/crates/clawhdf5/tests/indexed_lookup_interop.rs b/crates/clawhdf5/tests/indexed_lookup_interop.rs index 71f48f0..98b04f0 100644 --- a/crates/clawhdf5/tests/indexed_lookup_interop.rs +++ b/crates/clawhdf5/tests/indexed_lookup_interop.rs @@ -495,3 +495,118 @@ fn a_corrupt_internal_index_node_is_an_error_not_a_missing_name() { )); assert_eq!(out, "checksum checksum"); } + +/// Rename the one link called `from` to `to` (same length) in `bytes`, and +/// re-checksum the object header chunk holding it: two links of one name, +/// which libhdf5 cannot write. +fn rename_link_in_header(bytes: &mut [u8], from: &[u8], to: &[u8]) { + assert_eq!(from.len(), to.len()); + let find = |hay: &[u8], needle: &[u8]| hay.windows(needle.len()).position(|w| w == needle); + let at = find(bytes, from).expect("link name"); + assert!(find(&bytes[at + 1..], from).is_none(), "name not unique"); + bytes[at..at + to.len()].copy_from_slice(to); + // The v2 object header (chunk 0) holding it. + let ohdr = bytes[..at] + .windows(4) + .rposition(|w| w == b"OHDR") + .expect("OHDR"); + let flags = bytes[ohdr + 5]; + let mut pos = ohdr + 6; + if flags & 0x20 != 0 { + pos += 16; // times + } + if flags & 0x10 != 0 { + pos += 4; // attribute phase change + } + let width = 1usize << (flags & 3); + let mut size = [0u8; 8]; + size[..width].copy_from_slice(&bytes[pos..pos + width]); + let end = pos + width + usize::try_from(u64::from_le_bytes(size)).unwrap(); + assert!(at < end, "name outside chunk 0"); + let sum = jenkins_lookup3(&bytes[ohdr..end]); + bytes[end..end + 4].copy_from_slice(&sum.to_le_bytes()); +} + +/// Two soft links of one name (a damaged or hand-made group; libhdf5 +/// cannot create one), one dangling: only the first counts, as in libhdf5, +/// which opens the first Link message of a name and fails if it dangles. +/// Lookup, path and listing agree — before, the listing skipped a dangling +/// first link and listed the name via the second, which lookup did not +/// follow, and path resolution followed the last. +#[test] +fn of_two_links_with_one_name_the_first_wins_everywhere() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + for dangling_first in [true, false] { + let path = dir + .path() + .join(format!("dup_{dangling_first}.h5")) + .display() + .to_string(); + let (first, second) = if dangling_first { + ("/nowhere_xyz", "/d") + } else { + ("/d", "/nowhere_xyz") + }; + run_python(&format!( + "import h5py, numpy as np\n\ + with h5py.File(r'{path}', 'w', libver='latest') as f:\n\ + \x20 f.create_dataset('d', data=np.int64(42))\n\ + \x20 s = f.create_group('s')\n\ + \x20 s['dup_A'] = h5py.SoftLink('{first}')\n\ + \x20 s['dup_B'] = h5py.SoftLink('{second}')", + )); + let mut bytes = std::fs::read(&path).unwrap(); + rename_link_in_header(&mut bytes, b"dup_B", b"dup_A"); + std::fs::write(&path, &bytes).unwrap(); + + // What libhdf5 opens under that name: both names listed, first link + // followed. + let out = run_python(&format!( + "import h5py\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 s = f['s']\n\ + \x20 assert list(s) == ['dup_A', 'dup_A'], list(s)\n\ + \x20 try:\n\ + \x20 print(int(s['dup_A'][()]))\n\ + \x20 except KeyError:\n\ + \x20 print('none')", + )); + let want = if dangling_first { "none" } else { "42" }; + assert_eq!(out, want, "h5py, dangling first: {dangling_first}"); + let want = (!dangling_first).then_some(42i64); + + let f = File::open(&path).unwrap(); + let s = f.group("s").unwrap(); + let got = |r: Result, clawhdf5::Error>| match r { + Ok(ds) => Some(ds.read_i64().unwrap()[0]), + Err(e) => { + assert!(is_not_found(&e), "{e:?}"); + None + } + }; + assert_eq!(got(s.dataset("dup_A")), want, "lookup, {dangling_first}"); + assert_eq!(got(f.dataset("/s/dup_A")), want, "path, {dangling_first}"); + let listed = s.datasets().unwrap(); + let listed_n = listed.iter().filter(|n| *n == "dup_A").count(); + assert_eq!(listed_n, usize::from(want.is_some()), "{listed:?}"); + let entries = s.entries().unwrap(); + assert_eq!(entries.len(), listed_n, "{entries:?}"); + + let m = MmapFile::open(&path).unwrap(); + let l = LazyFile::open_mmap(&path).unwrap(); + let (mg, lg) = (m.group("s").unwrap(), l.group("s").unwrap()); + match want { + Some(v) => { + assert_eq!(mg.dataset("dup_A").unwrap().read_i64().unwrap(), vec![v]); + assert_eq!(lg.dataset("dup_A").unwrap().read_i64().unwrap(), vec![v]); + } + None => { + assert!(mg.dataset("dup_A").is_err_and(|e| is_not_found(&e))); + assert!(lg.dataset("dup_A").is_err_and(|e| is_not_found(&e))); + } + } + assert_eq!(mg.datasets().unwrap(), listed); + assert_eq!(lg.datasets().unwrap(), listed); + } +} -- 2.54.0 From 1ea9132e10135e20c55536497bffc520814d0fdf Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:11:47 -0500 Subject: [PATCH 37/48] docs: changelog note for B-tree v2 internal node checksums Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70cd926..33e6bb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,10 @@ resolves). Before, the listing skipped a dangling first link and listed the name via a later one that lookup did not follow, and path resolution followed the last. +- **B-tree v2 internal nodes are checksum-verified.** Lookups prune + children by internal-node keys, so a corrupted internal node could hide + a name with no error; a mismatch is now `ChecksumMismatch`, as in + libhdf5, for lookups and listings alike. - **`Group::entries()` and `File::group_at(address)`**: a listing's `(name, address)` pairs, to open children without looking names up again. - Test: `crates/clawhdf5/tests/indexed_lookup_interop.rs` — every child of -- 2.54.0 From 485bea0f4f3444f414bdb65224014661c62a6310 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:12:35 -0500 Subject: [PATCH 38/48] clawhdf5: set_attr adds the Attribute Info message a version-2 header needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libhdf5 counts a version-2 object header's attributes through its Attribute Info message (0x15) and reports none when the header has none. set_attr gave v110/latest groups, the root group and datasets without attributes an attribute message only, so h5py listed the attribute but len(obj.attrs) and H5Oget_info's num_attrs said 0, and stayed wrong after h5py r+ added more. Like H5O__attr_create, the edit now adds the message when a version-2 header lacks it, in the same planned edit: version 0, the header's creation-order track/index flags, maximum creation index 0, undefined fractal heap and B-tree addresses, message flag DONTSHARE — byte for byte what libhdf5 writes. It goes before the attribute (libhdf5's order) when free space holds both, else after it, so a continuation chunk made for the attribute also takes it. Test: edit_interop attribute_count_in_version_2_headers — v110 and latest files, attributes set on the root group, groups and datasets with and without existing attributes: h5py's len/num_attrs/list/values, h5dump -A and our reader agree, also after h5py r+ adds attributes up to and past the compact limit. Fails on the previous editor (h5py len 0). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 +- crates/clawhdf5-tools/tests/edit_interop.rs | 87 +++++++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 39 +++++++++ crates/clawhdf5/src/edit/ohdr.rs | 5 ++ 4 files changed, 136 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f070cd5..2516491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,11 @@ - `resize`: grow a chunked dataset up to its maximum dimensions (h5py's `Dataset.resize`). - `set_attr`: add or replace an attribute in an object header, in free - space or in a new continuation chunk at the end of the file. + space or in a new continuation chunk at the end of the file. A + version-2 header (h5py `libver='v110'` and later) without an Attribute + Info message gets one, as libhdf5's `H5O__attr_create` adds it: libhdf5 + counts such a header's attributes through that message, and without it + h5py reported `len(obj.attrs) == 0` while listing them. - Each edit is planned in memory and refused as a whole (`Error::Unsupported`, file untouched) when any part is not supported: new chunks in a version-2 B-tree index (two or more unlimited diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index fe44aa8..7587126 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1315,3 +1315,90 @@ fn optional_filters_that_fail_are_skipped() { assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); } } + +/// libhdf5 counts a version-2 object header's attributes through its +/// Attribute Info message and reports none without one. The editor adds +/// that message, as `H5O__attr_create` does, when it gives a version-2 +/// header (h5py `libver='v110'`/`'latest'`) its first attribute: afterwards +/// h5py lists, counts and reads every attribute, new and existing, h5dump +/// agrees, and h5py `r+` can add more (past the compact limit, into dense +/// storage) with the count still right. +#[test] +fn attribute_count_in_version_2_headers() { + if !tools_ok() { + return; + } + for (lv, dump) in [("'v110'", true), ("'latest'", false)] { + let dir = tmpdir(); + let path = dir.path().join("acount.h5"); + let p = path.to_str().unwrap(); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 f.create_group('g')\n\ + \x20 f.create_group('has').attrs['old'] = 7\n\ + \x20 f.create_dataset('d', data=np.arange(3, dtype=' f.root().attrs().unwrap(), + "d" | "e" => f.dataset(o).unwrap().attrs().unwrap(), + _ => f.group(o).unwrap().attrs().unwrap(), + }; + assert_eq!(attrs.len(), n + extra, "{o}"); + assert!(matches!(attrs.get("a1"), Some(AttrValue::F64(x)) if *x == 1.5)); + } + if dump { + // Every object's attributes: 3 set by the editor on each of + // the five, 2 existing, `extra` from h5py on each. + let out = Command::new("h5dump").args(["-A", p]).output().unwrap(); + assert!(out.status.success(), "h5dump -A:\n{}", text(&out)); + let s = String::from_utf8_lossy(&out.stdout); + assert_eq!( + s.matches("ATTRIBUTE \"").count(), + 5 * (3 + extra) + 2, + "h5dump -A:\n{s}" + ); + } + }; + check(0); + // libhdf5 adds more: to 7 or 8 (compact), then past its limit. + for extra in [4usize, 10] { + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'g', 'has', 'd', 'e']:\n\ + \x20 for i in range({extra}): f[o].attrs[f'h{{i}}'] = i\n" + )); + check(extra); + check_tools(&path, dump); + } + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index c0a5d07..15237fc 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -47,6 +47,8 @@ const MSG_EXTERNAL: u16 = 0x07; const MSG_ATTR_INFO: u16 = 0x15; /// Message flag: the message is shared (stored elsewhere). const MSG_FLAG_SHARED: u8 = 0x02; +/// Message flag: the message must not be shared (`H5O_MSG_FLAG_DONTSHARE`). +const MSG_FLAG_DONTSHARE: u8 = 0x04; /// An HDF5 file opened for in-place modification. /// @@ -815,7 +817,24 @@ impl FileEditor { if let Some(i) = existing { hdr.delete(img, i)?; } + // libhdf5 counts a version-2 header's attributes through its + // Attribute Info message and reports none without one; like + // H5O__attr_create, add it when missing: before the attribute + // when free space holds both (libhdf5's order), else after it, + // so that a new continuation chunk made for the attribute has + // room for it too. + let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none()) + .then(|| attr_info_message(hdr.flags, img.os)); + let ainfo_first = ainfo + .as_ref() + .is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len())); + if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; + } hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; + } hdr.finish(img) }) } @@ -900,6 +919,26 @@ fn attr_name(d: &[u8]) -> Result<&[u8], Error> { Ok(name.split(|&b| b == 0).next().unwrap_or(name)) } +/// A new Attribute Info message for a version-2 header with flags +/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order +/// tracked / indexed as the header's flags say, maximum creation index 0, +/// and no dense storage (undefined fractal heap and B-tree addresses). +fn attr_info_message(hdr_flags: u8, os: u8) -> Vec { + let track = hdr_flags & 0x04 != 0; + let index = hdr_flags & 0x08 != 0; + let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; + if track { + b.extend_from_slice(&0u16.to_le_bytes()); + } + let undef_addr = vec![0xffu8; os as usize]; + b.extend_from_slice(&undef_addr); + b.extend_from_slice(&undef_addr); + if index { + b.extend_from_slice(&undef_addr); + } + b +} + /// A version-2 header's limit on compact attributes: stored when its flags /// say so, else libhdf5's default of 8. fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs index 5446702..f3abfe0 100644 --- a/crates/clawhdf5/src/edit/ohdr.rs +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -326,6 +326,11 @@ impl Header { .map(|(i, _)| i) } + /// Whether free space in the header can take a body of `len` bytes. + pub(crate) fn has_free(&self, len: usize) -> bool { + self.best_nil(self.padded(len)).is_some() + } + /// Put a message into slot `i` (a NIL message, or a message being /// moved away), splitting off the rest as a NIL message. fn place( -- 2.54.0 From b668878129dfe2245099ac80b4475c46949fb769 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:13:45 -0500 Subject: [PATCH 39/48] clawhdf5: FileEditor reports filters it cannot run as Error::Unsupported MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dataset whose filter this build cannot encode (scale-offset, N-Bit, SZIP; a plugin filter the build lacks) failed with Error::Format("unsupported filter: 6"), although the editor documents every refused edit as Error::Unsupported, and the Python bindings raised ValueError rather than NotImplementedError. Every edit now maps FormatError::UnsupportedFilter to Error::Unsupported; the file is left untouched as before. Test: edit_interop unencodable_filters_are_unsupported — h5py scale-offset datasets (integer with chunks, integer never written, float D-scale): Error::Unsupported naming the filter, and the file byte for byte unchanged. Fails on the previous editor (Format(UnsupportedFilter(6))). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 ++-- crates/clawhdf5-tools/tests/edit_interop.rs | 33 +++++++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 16 +++++++++- docs/known-issues.md | 7 ++++- 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2516491..f959f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,8 +40,10 @@ (`Error::Unsupported`, file untouched) when any part is not supported: new chunks in a version-2 B-tree index (two or more unlimited dimensions) or an implicit index, shrinking, variable-length and - reference data, attributes in dense storage, past an object's compact - limit or with tracked creation order, files with a metadata cache + reference data, chunks through a filter this build cannot encode + (scale-offset, N-Bit, SZIP), attributes in dense storage, past an + object's compact limit or with tracked creation order, files with a + metadata cache image, paged or persistent free space, or marked open by another writer. New error variants `Error::Unsupported`, `Error::InvalidArgument`, `Error::Locked`, and `clawhdf5::Error` is now diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 7587126..85d4d0c 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1402,3 +1402,36 @@ fn attribute_count_in_version_2_headers() { } } } + +/// A dataset whose filters this build cannot encode (scale-offset) +/// is `Error::Unsupported`, as the editor documents, not a format error, +/// and the file is left byte for byte as it was. +#[test] +fn unencodable_filters_are_unsupported() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("unencodable.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 f.create_dataset('so', data=np.arange(16, dtype=' assert!(msg.contains("filter"), "{ds}: {msg}"), + other => panic!("{ds}: {other:?}"), + } + } + drop(ed); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 15237fc..f6a590e 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -622,7 +622,7 @@ impl FileEditor { check_editable(&f)?; let sb = f.superblock().clone(); let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); - let r = op(&f, &mut img)?; + let r = op(&f, &mut img).map_err(unsupported_filter)?; if img.is_dirty() { if img.eoa() != img.old_eoa() { set_superblock_eof(&mut img, &sb)?; @@ -840,6 +840,20 @@ impl FileEditor { } } +/// A filter this build cannot run (scale-offset, N-Bit and SZIP have no +/// encoder; a plugin filter may be missing) is something the editor does not +/// support, not a malformed file. +fn unsupported_filter(e: Error) -> Error { + match e { + Error::Format(clawhdf5_format::error::FormatError::UnsupportedFilter(id)) => { + Error::Unsupported(format!( + "datasets with filter {id}, which this build cannot run" + )) + } + e => e, + } +} + /// libhdf5's checks before it opens a file for writing, and what this /// editor cannot keep consistent. fn check_editable(f: &File) -> Result<(), Error> { diff --git a/docs/known-issues.md b/docs/known-issues.md index 4d49f0e..818a487 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -13,11 +13,16 @@ deleting it. with `Error::Unsupported` and without writing anything: - new, moved or resized chunks in a **version-2 B-tree** chunk index (what libhdf5 uses for two or more unlimited dimensions) — existing unfiltered - chunks, and filtered ones that re-encode to the same size, are + chunks, and filtered ones that re-encode to the same size and filter + mask, are overwritten in place; `resize` works — and new chunks in an **implicit** index (it has all of its chunks from the start); - **shrinking** a dataset; - variable-length and reference data; +- chunks through a filter this build cannot encode (scale-offset, N-Bit, + SZIP, or a plugin filter it lacks), even an optional one: libhdf5 skips + an optional filter only when its own build lacks it, which none does for + these; - attributes of an object in **dense storage**, past its compact limit (8 by default) or with tracked **creation order**; - partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external -- 2.54.0 From fe377266e1cdbb8923af2260da6bab295f8e8194 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:14:58 -0500 Subject: [PATCH 40/48] clawhdf5: FileEditor unmaps the file before an edit writes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each edit planned over the reader's memory map of the file and committed while that File, and the Image's &[u8] over the mapping, were still alive, writing the same file through the editor's descriptor. Nothing read the mapping during the writes, but a shared slice whose memory changes underneath it is undefined behaviour under Rust's aliasing rules. Image::into_plan now detaches the edit's writes (patches, end of allocation) into a Plan that owns all of its bytes and borrows nothing; edit() takes the user-block size, drops the File — unmapping the file — and only then commits the Plan. The invariant is documented in the image module and the editor's module docs. Test: edit::tests::file_is_not_mapped_while_an_edit_writes_it checks /proc/self/maps at the moment each commit starts (write, resize, set_attr): never mapped. With the commit moved back before the reader is dropped (the previous order) it reports all three commits with the file mapped. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/src/edit/image.rs | 32 +++++++++++- crates/clawhdf5/src/edit/mod.rs | 82 +++++++++++++++++++++++++++++-- 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs index a100b66..254abcd 100644 --- a/crates/clawhdf5/src/edit/image.rs +++ b/crates/clawhdf5/src/edit/image.rs @@ -4,9 +4,17 @@ //! An edit never writes to the file while it is being planned. Every change //! is recorded here first (reads see them), so an edit that fails half-way — //! a filter that cannot encode, a chunk index this code does not handle — -//! leaves the file exactly as it was. [`Image::commit`] then writes the -//! changes in an order that keeps the old metadata valid for as long as +//! leaves the file exactly as it was. [`Image::into_plan`] then detaches the +//! changes from the bytes they were planned over, and [`Plan::commit`] +//! writes them in an order that keeps the old metadata valid for as long as //! possible (see there). +//! +//! **Invariant:** the base bytes an image reads are the reader's view of the +//! file — a memory map when the `mmap` feature is on. Nothing may write the +//! file while that view is alive: a write through another descriptor would +//! change memory behind a live `&[u8]`, which Rust's aliasing rules forbid. +//! So a [`Plan`] owns everything it writes and borrows nothing, and the +//! editor drops the reader (unmapping the file) before it commits. use std::collections::BTreeMap; use std::io::{Seek, SeekFrom, Write}; @@ -178,6 +186,26 @@ impl<'a> Image<'a> { Ok(()) } + /// The edit's writes, detached from the base bytes (see the module's + /// invariant: the reader that owns them can then be dropped before + /// anything is written). + pub(crate) fn into_plan(self) -> Plan { + Plan { + patches: self.patches, + eoa: self.eoa, + old_eoa: self.old_eoa, + } + } +} + +/// The writes of a planned edit, owning all of their bytes. +pub(crate) struct Plan { + patches: BTreeMap>, + eoa: u64, + old_eoa: u64, +} + +impl Plan { /// Write the edit to `file`, whose superblock is at `user_block`. /// /// Order: first everything in newly allocated space (new chunks, new diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index f6a590e..4903672 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -10,7 +10,10 @@ //! Each operation is planned in memory first ([`image::Image`]): if any part //! of it is unsupported, nothing is written. The plan is then committed with //! the new space (new chunks, new index blocks) written and synced before -//! the existing bytes that link it in, then synced again. +//! the existing bytes that link it in, then synced again. The plan owns the +//! bytes it writes; the reader it was planned with (a memory map of the +//! file) is dropped before the first write, so no `&[u8]` over the mapping +//! is alive while the file changes (see `image`). mod btree1; mod earray; @@ -614,6 +617,12 @@ impl FileEditor { &self.path } + /// Plan an edit over the file's current bytes, then commit it. + /// + /// The reader (a memory map of the file, with the `mmap` feature) is + /// dropped before anything is written: the plan owns every byte it + /// writes, so no slice over the mapping is alive while the file changes + /// underneath it (see `image`'s invariant). fn edit( &mut self, op: impl FnOnce(&File, &mut Image<'_>) -> Result, @@ -621,13 +630,22 @@ impl FileEditor { let f = File::open(&self.path)?; check_editable(&f)?; let sb = f.superblock().clone(); + let user_block = f.user_block_size(); let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); let r = op(&f, &mut img).map_err(unsupported_filter)?; - if img.is_dirty() { + let plan = if img.is_dirty() { if img.eoa() != img.old_eoa() { set_superblock_eof(&mut img, &sb)?; } - img.commit(&mut self.file, f.user_block_size())?; + Some(img.into_plan()) + } else { + None + }; + drop(f); + if let Some(plan) = plan { + #[cfg(test)] + tests::note_commit(&self.path); + plan.commit(&mut self.file, user_block)?; } Ok(r) } @@ -1263,3 +1281,61 @@ fn decode_chunk( } Ok(out) } + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::path::Path; + + use super::FileEditor; + use crate::{AttrValue, FileBuilder}; + + thread_local! { + /// Commits seen on this thread, and how many found the file mapped. + static COMMITS: Cell<(usize, usize)> = const { Cell::new((0, 0)) }; + } + + /// Called just before an edit writes the file: whether this process + /// still maps it (`/proc/self/maps` lists every mapping by path). + pub(super) fn note_commit(path: &Path) { + let path = std::fs::canonicalize(path).unwrap(); + let maps = std::fs::read_to_string("/proc/self/maps").unwrap_or_default(); + let mapped = maps + .lines() + .any(|l| l.ends_with(&format!(" {}", path.display()))); + COMMITS.with(|c| { + let (n, m) = c.get(); + c.set((n + 1, m + usize::from(mapped))); + }); + } + + /// No edit writes the file while the reader's memory map of it (and so + /// a `&[u8]` over it) is alive. + #[test] + #[cfg(all(target_os = "linux", feature = "mmap"))] + fn file_is_not_mapped_while_an_edit_writes_it() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("mapped.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("x") + .with_i32_data(&[1, 2, 3, 4]) + .with_shape(&[4]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[2]) + .with_deflate(4); + b.write(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.write_values("x", &crate::Selection::All, &[5i32, 6, 7, 8]) + .unwrap(); + ed.resize("x", &[6]).unwrap(); + ed.set_attr("x", "a", &AttrValue::I64(1)).unwrap(); + drop(ed); + let (commits, mapped) = COMMITS.with(Cell::get); + assert_eq!((commits, mapped), (3, 0)); + let f = crate::File::open(&path).unwrap(); + assert_eq!( + f.dataset("x").unwrap().read_i32().unwrap(), + [5, 6, 7, 8, 0, 0] + ); + } +} -- 2.54.0 From 052098bf36940b3eacc4e9b3742133cec75d2342 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:22:37 -0500 Subject: [PATCH 41/48] format: monomorphise the Storage parsers so local files stay as fast Every `*_in` core and the read helpers take `file: &S` with `S: Storage + ?Sized` instead of `&dyn Storage`, and the `&[u8]` wrappers pass the slice itself, so they compile to a `[u8]` instance: `as_contiguous()` inlines to `Some(self)` and each structure read is the slice code's bounds check again, with no indirect call. `&dyn Storage` still works (`S = dyn Storage`); there is one parser implementation. Also, so the structure reads cost no more than the slice checks did: - ObjectHeader::parse_in reads the prefix once (signature included) instead of the signature and then the prefix: two reads for a one-chunk header instead of three on a range backend; - the symbol-table node and group B-tree (v1) loops walk their entries with chunks_exact over the bytes read, and the node's redundant second bounds check is gone (the entries' read is the check, same error); - a version-1 header's message list is sized from its (capped) count. Same results and errors; the unit and equivalence tests are unchanged. New Criterion bench `clawhdf5/benches/local_metadata_bench.rs` over a 400-group version-1 file written by h5py (new fixture `v1_groups_400.h5`): ObjectHeader::parse, symbol-table nodes, the group B-tree walk and a facade listing, using only APIs that exist at f2ff2c4 so it builds there for an A/B. Provisional A/B against f2ff2c4 (busy machine, not for docs): both builds linked into one binary and timed in alternation, 200 rounds; median ratio new/old: facade listing -0.5% to -3.5% (was +14%), ObjectHeader::parse +1% to +2% (was +25%), symbol-table nodes -18%, group B-tree walk -18%, local-heap names and resolve_group_children within +-1.5%. An old-vs-old-copy run shows +-2% from code layout alone. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/attribute.rs | 44 ++++----- crates/clawhdf5-format/src/btree_v1.rs | 40 ++++---- crates/clawhdf5-format/src/data_layout.rs | 6 +- .../clawhdf5-format/src/extensible_array.rs | 18 ++-- crates/clawhdf5-format/src/fixed_array.rs | 10 +- crates/clawhdf5-format/src/fractal_heap.rs | 34 ++++--- crates/clawhdf5-format/src/global_heap.rs | 16 +-- crates/clawhdf5-format/src/local_heap.rs | 18 ++-- crates/clawhdf5-format/src/object_header.rs | 75 +++++++-------- crates/clawhdf5-format/src/shared_message.rs | 48 ++++----- crates/clawhdf5-format/src/signature.rs | 2 +- crates/clawhdf5-format/src/storage.rs | 37 ++++--- crates/clawhdf5-format/src/superblock.rs | 13 ++- crates/clawhdf5-format/src/superblock_ext.rs | 27 +++--- crates/clawhdf5-format/src/symbol_table.rs | 51 +++------- .../tests/fixtures/v1_groups_400.h5 | Bin 0 -> 355840 bytes crates/clawhdf5/Cargo.toml | 4 + .../clawhdf5/benches/local_metadata_bench.rs | 91 ++++++++++++++++++ 18 files changed, 315 insertions(+), 219 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/v1_groups_400.h5 create mode 100644 crates/clawhdf5/benches/local_metadata_bench.rs diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 815719e..8724d7e 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -51,7 +51,7 @@ impl AttributeMessage { /// /// `length_size` is needed for dataspace dimension parsing. pub fn parse(data: &[u8], length_size: u8) -> Result { - Self::parse_impl(data, length_size, None) + Self::parse_impl(data, length_size, None::<(&[u8], u8)>) } /// [`AttributeMessage::parse`] with access to the rest of the file, which @@ -66,24 +66,24 @@ impl AttributeMessage { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in_storage(data, &file_data, offset_size, length_size) + Self::parse_in_storage(data, file_data, offset_size, length_size) } /// [`AttributeMessage::parse_in_file`] with the file behind any /// [`Storage`]. - pub fn parse_in_storage( + pub fn parse_in_storage( data: &[u8], - file: &dyn Storage, + file: &S, offset_size: u8, length_size: u8, ) -> Result { Self::parse_impl(data, length_size, Some((file, offset_size))) } - fn parse_impl( + fn parse_impl( data: &[u8], length_size: u8, - file: Option<(&dyn Storage, u8)>, + file: Option<(&S, u8)>, ) -> Result { ensure_len(data, 0, 2)?; let version = data[0]; @@ -98,12 +98,12 @@ impl AttributeMessage { /// The bytes of an embedded datatype/dataspace message, following the /// shared-message reference when `shared` is set. - fn embedded_message<'a>( + fn embedded_message<'a, S: Storage + ?Sized>( bytes: &'a [u8], shared: bool, msg_type: MessageType, length_size: u8, - file: Option<(&dyn Storage, u8)>, + file: Option<(&S, u8)>, ) -> Result, FormatError> { if !shared { return Ok(Cow::Borrowed(bytes)); @@ -155,10 +155,10 @@ impl AttributeMessage { }) } - fn parse_v2( + fn parse_v2( data: &[u8], length_size: u8, - file: Option<(&dyn Storage, u8)>, + file: Option<(&S, u8)>, ) -> Result { // Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared. let flags = data.get(1).copied().unwrap_or(0); @@ -209,10 +209,10 @@ impl AttributeMessage { }) } - fn parse_v3( + fn parse_v3( data: &[u8], length_size: u8, - file: Option<(&dyn Storage, u8)>, + file: Option<(&S, u8)>, ) -> Result { // Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared. let flags = data.get(1).copied().unwrap_or(0); @@ -427,15 +427,15 @@ pub fn extract_attributes_full( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - extract_attributes_full_in(&file_data, header, offset_size, length_size) + extract_attributes_full_in(file_data, header, offset_size, length_size) } /// [`extract_attributes_full`] over any [`Storage`]. Dense attribute /// storage is indexed by a v2 B-tree, which is not read over [`Storage`] /// yet: on a backend without the whole file in memory an object with dense /// attributes is [`FormatError::ContiguousStorageRequired`]. -pub fn extract_attributes_full_in( - file: &dyn Storage, +pub fn extract_attributes_full_in( + file: &S, header: &ObjectHeader, offset_size: u8, length_size: u8, @@ -457,13 +457,13 @@ pub fn extract_attributes_tolerant( offset_size: u8, length_size: u8, ) -> Result<(Vec, Vec), FormatError> { - extract_attributes_tolerant_in(&file_data, header, offset_size, length_size) + extract_attributes_tolerant_in(file_data, header, offset_size, length_size) } /// [`extract_attributes_tolerant`] over any [`Storage`] (see /// [`extract_attributes_full_in`] for dense storage). -pub fn extract_attributes_tolerant_in( - file_data: &dyn Storage, +pub fn extract_attributes_tolerant_in( + file_data: &S, header: &ObjectHeader, offset_size: u8, length_size: u8, @@ -478,8 +478,8 @@ pub fn extract_attributes_tolerant_in( /// Read every attribute; each one that fails goes to `on_error`, which /// either stops the read (returns the error) or skips that attribute. -fn extract_attributes_with( - file_data: &dyn Storage, +fn extract_attributes_with( + file_data: &S, header: &ObjectHeader, offset_size: u8, length_size: u8, @@ -572,8 +572,8 @@ fn find_attribute_info( /// Extract attributes from dense storage (fractal heap + B-tree v2), and /// each one's creation order into `orders`. #[allow(clippy::too_many_arguments)] -fn extract_dense_attributes( - file_data: &dyn Storage, +fn extract_dense_attributes( + file_data: &S, attr_info: &AttributeInfoMessage, fh_addr: u64, offset_size: u8, diff --git a/crates/clawhdf5-format/src/btree_v1.rs b/crates/clawhdf5-format/src/btree_v1.rs index 3378131..4853ca1 100644 --- a/crates/clawhdf5-format/src/btree_v1.rs +++ b/crates/clawhdf5-format/src/btree_v1.rs @@ -77,13 +77,13 @@ impl BTreeV1Node { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in(&file_data, offset as u64, offset_size, length_size) + Self::parse_in(file_data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]: one read of the node's header, /// one of its keys and children. - pub fn parse_in( - file: &dyn Storage, + pub fn parse_in( + file: &S, offset: u64, offset_size: u8, _length_size: u8, @@ -126,24 +126,22 @@ impl BTreeV1Node { let needed = eu * (key_size + os) + key_size; // eu children + (eu+1) keys let body = read_exact_at(file, body_start, needed)?; let file_data: &[u8] = &body; - let mut pos = 0usize; let mut keys = Vec::with_capacity(eu + 1); let mut children = Vec::with_capacity(eu); - for _i in 0..eu { - // key[i] - let key = read_offset(file_data, pos, offset_size)?; - keys.push(key); - pos += key_size; - // child[i] - let child = read_offset(file_data, pos, offset_size)?; - children.push(child); - pos += os; + if os == 0 { + // What reading the first key reports (and keeps `chunks_exact` + // below from being given a zero size). + return Err(FormatError::InvalidOffsetSize(offset_size)); } - // final key - let key = read_offset(file_data, pos, offset_size)?; - keys.push(key); + // `needed` bytes: key[0], child[0], ..., child[eu - 1], key[eu]. + let (pairs, last) = file_data.split_at(eu * (key_size + os)); + for pair in pairs.chunks_exact(key_size + os) { + keys.push(read_offset(pair, 0, offset_size)?); + children.push(read_offset(pair, key_size, offset_size)?); + } + keys.push(read_offset(last, 0, offset_size)?); Ok(BTreeV1Node { node_type, @@ -167,12 +165,12 @@ pub fn collect_symbol_table_nodes( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - collect_symbol_table_nodes_in(&file_data, btree_address, offset_size, length_size) + collect_symbol_table_nodes_in(file_data, btree_address, offset_size, length_size) } /// [`collect_symbol_table_nodes`] over any [`Storage`]: two reads per node. -pub fn collect_symbol_table_nodes_in( - file: &dyn Storage, +pub fn collect_symbol_table_nodes_in( + file: &S, btree_address: u64, offset_size: u8, length_size: u8, @@ -180,8 +178,8 @@ pub fn collect_symbol_table_nodes_in( collect_symbol_table_nodes_inner(file, btree_address, offset_size, length_size, 0) } -fn collect_symbol_table_nodes_inner( - file: &dyn Storage, +fn collect_symbol_table_nodes_inner( + file: &S, btree_address: u64, offset_size: u8, length_size: u8, diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 480f459..e4873c5 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -311,14 +311,14 @@ impl DataLayout { file_data: &[u8], length_size: u8, ) -> Result<(), FormatError> { - self.resolve_vds_mappings_in(&file_data, length_size) + self.resolve_vds_mappings_in(file_data, length_size) } /// [`Self::resolve_vds_mappings`] over any [`Storage`]: one read of the /// global heap collection holding the mappings. - pub fn resolve_vds_mappings_in( + pub fn resolve_vds_mappings_in( &mut self, - file_data: &dyn Storage, + file_data: &S, length_size: u8, ) -> Result<(), FormatError> { if let DataLayout::Virtual { diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index 529955d..a95c124 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -121,12 +121,12 @@ impl ExtensibleArrayHeader { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in(&file_data, offset as u64, offset_size, length_size) + Self::parse_in(file_data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]: one read of the header. - pub fn parse_in( - file: &dyn Storage, + pub fn parse_in( + file: &S, offset: u64, offset_size: u8, length_size: u8, @@ -304,8 +304,8 @@ fn page_nelmts(header: &ExtensibleArrayHeader) -> Option { /// paged. The bitmap lives in the super block, not here — a paged data block /// stores only its prefix, then one slot per page. #[allow(clippy::too_many_arguments)] -fn read_data_block_elements( - file: &dyn Storage, +fn read_data_block_elements( + file: &S, db_offset: u64, nelmts: usize, header: &ExtensibleArrayHeader, @@ -448,8 +448,8 @@ pub fn read_extensible_array_chunks( /// index block's prefix, one of the whole index block, and the same for /// every super block and data block it references. #[allow(clippy::too_many_arguments)] -pub fn read_extensible_array_chunks_in( - file: &dyn Storage, +pub fn read_extensible_array_chunks_in( + file: &S, header: &ExtensibleArrayHeader, dataset_dims: &[u64], max_dims: Option<&[u64]>, @@ -642,8 +642,8 @@ pub fn read_extensible_array_chunks_in( /// + block offset + the page-init bitmap for every data block it owns /// + one address per data block + checksum. #[allow(clippy::too_many_arguments)] -fn read_super_block( - file: &dyn Storage, +fn read_super_block( + file: &S, sb_offset: u64, ndblks: usize, dblk_nelmts: usize, diff --git a/crates/clawhdf5-format/src/fixed_array.rs b/crates/clawhdf5-format/src/fixed_array.rs index de936a7..e6be63b 100644 --- a/crates/clawhdf5-format/src/fixed_array.rs +++ b/crates/clawhdf5-format/src/fixed_array.rs @@ -92,12 +92,12 @@ impl FixedArrayHeader { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in(&file_data, offset as u64, offset_size, length_size) + Self::parse_in(file_data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]: one read of the header. - pub fn parse_in( - file: &dyn Storage, + pub fn parse_in( + file: &S, offset: u64, offset_size: u8, length_size: u8, @@ -174,8 +174,8 @@ pub fn read_fixed_array_chunks( /// [`read_fixed_array_chunks`] over any [`Storage`]: one read of the data /// block's prefix, one of the whole data block (pages included). #[allow(clippy::too_many_arguments)] -pub fn read_fixed_array_chunks_in( - file: &dyn Storage, +pub fn read_fixed_array_chunks_in( + file: &S, header: &FixedArrayHeader, dataset_dims: &[u64], max_dims: Option<&[u64]>, diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index 1c6a77c..e47c2c9 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -139,13 +139,13 @@ impl FractalHeapHeader { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in(&file_data, offset as u64, offset_size, length_size) + Self::parse_in(file_data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]: one read of the header (two /// when it holds an I/O filter pipeline). - pub fn parse_in( - file: &dyn Storage, + pub fn parse_in( + file: &S, offset: u64, offset_size: u8, length_size: u8, @@ -382,15 +382,15 @@ impl FractalHeapHeader { id_bytes: &[u8], offset_size: u8, ) -> Result, FormatError> { - self.read_managed_object_in(&file_data, id_bytes, offset_size) + self.read_managed_object_in(file_data, id_bytes, offset_size) } /// [`Self::read_managed_object`] over any [`Storage`]. A huge object /// found through the huge-object v2 B-tree still needs the whole file /// in memory ([`FormatError::ContiguousStorageRequired`] otherwise). - pub fn read_managed_object_in( + pub fn read_managed_object_in( &self, - file_data: &dyn Storage, + file_data: &S, id_bytes: &[u8], offset_size: u8, ) -> Result, FormatError> { @@ -423,7 +423,11 @@ impl FractalHeapHeader { } /// Read a huge object (heap ID type 1). - fn read_huge_object(&self, file: &dyn Storage, id: &[u8]) -> Result, FormatError> { + fn read_huge_object( + &self, + file: &S, + id: &[u8], + ) -> Result, FormatError> { let os = usize::from(self.offset_size); let ls = usize::from(self.length_size); // (address, stored length, filter mask, decoded length); the last two @@ -475,9 +479,9 @@ impl FractalHeapHeader { /// Look up huge object `key` in the huge-object v2 B-tree, returning /// (address, stored length, filter mask, decoded length). - fn find_huge_record( + fn find_huge_record( &self, - file: &dyn Storage, + file: &S, key: u64, ) -> Result<(u64, u64, u32, u64), FormatError> { if is_undefined(self.huge_btree_address, self.offset_size) { @@ -554,9 +558,9 @@ impl FractalHeapHeader { } /// Read a managed object (heap ID type 0). - fn read_heap_managed( + fn read_heap_managed( &self, - file_data: &dyn Storage, + file_data: &S, id_bytes: &[u8], offset_size: u8, ) -> Result, FormatError> { @@ -604,9 +608,9 @@ impl FractalHeapHeader { /// header), so we just add it to the block address minus the block's heap /// offset. A filtered heap stores each direct block (header included) /// through its filter pipeline, so the block is decoded first. - fn read_from_direct_block( + fn read_from_direct_block( &self, - file: &dyn Storage, + file: &S, block: DirectBlock, target_offset: u64, length: usize, @@ -645,9 +649,9 @@ impl FractalHeapHeader { /// Read an object by traversing an indirect block to find the right direct block. #[allow(clippy::too_many_arguments)] - fn read_from_indirect_block( + fn read_from_indirect_block( &self, - file: &dyn Storage, + file: &S, iblock_addr: usize, nrows: u16, iblock_heap_offset: u64, diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index e635d4f..9861e8f 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -99,13 +99,13 @@ impl GlobalHeapCollection { offset: usize, length_size: u8, ) -> Result { - Self::parse_in(&file_data, offset as u64, length_size) + Self::parse_in(file_data, offset as u64, length_size) } /// [`Self::parse`] over any [`Storage`]: one read of the header, one of /// the collection. - pub fn parse_in( - file: &dyn Storage, + pub fn parse_in( + file: &S, offset: u64, length_size: u8, ) -> Result { @@ -136,13 +136,13 @@ impl GlobalHeapCollection { offset: usize, length_size: u8, ) -> Result { - Self::parse_index_in(&file_data, offset as u64, length_size) + Self::parse_index_in(file_data, offset as u64, length_size) } /// [`Self::parse_index`] over any [`Storage`]: one read of the header, /// one of the collection. The object offsets are file offsets. - pub fn parse_index_in( - file: &dyn Storage, + pub fn parse_index_in( + file: &S, offset: u64, length_size: u8, ) -> Result { @@ -152,8 +152,8 @@ impl GlobalHeapCollection { /// Read the collection at `offset` and index its objects: the /// collection's bytes, its offset as a `usize`, and the index (with /// file offsets). - fn read_collection( - file: &dyn Storage, + fn read_collection( + file: &S, offset: u64, length_size: u8, ) -> Result<(Cow<'_, [u8]>, usize, GlobalHeapIndex), FormatError> { diff --git a/crates/clawhdf5-format/src/local_heap.rs b/crates/clawhdf5-format/src/local_heap.rs index a4e3480..952a9d4 100644 --- a/crates/clawhdf5-format/src/local_heap.rs +++ b/crates/clawhdf5-format/src/local_heap.rs @@ -44,12 +44,12 @@ impl LocalHeap { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in(&file_data, offset as u64, offset_size, length_size) + Self::parse_in(file_data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]: one read of the header. - pub fn parse_in( - file: &dyn Storage, + pub fn parse_in( + file: &S, offset: u64, offset_size: u8, length_size: u8, @@ -97,14 +97,14 @@ impl LocalHeap { /// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the /// undefined address) is accepted as "no free list" too. pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> { - self.validate_free_list_in(&file_data, length_size) + self.validate_free_list_in(file_data, length_size) } /// [`Self::validate_free_list`] over any [`Storage`]: two small reads /// per free block. - pub fn validate_free_list_in( + pub fn validate_free_list_in( &self, - file: &dyn Storage, + file: &S, length_size: u8, ) -> Result<(), FormatError> { const FREE_NULL: u64 = 1; @@ -149,14 +149,14 @@ impl LocalHeap { /// Read a null-terminated string from the heap's data segment at the given byte offset. pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result { - self.read_string_in(&file_data, string_offset) + self.read_string_in(file_data, string_offset) } /// [`Self::read_string`] over any [`Storage`]: one read, from the /// string to the end of the data segment. - pub fn read_string_in( + pub fn read_string_in( &self, - file: &dyn Storage, + file: &S, string_offset: u64, ) -> Result { let file_len = len_usize(file); diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 1fa4b41..4c21eb1 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -7,7 +7,7 @@ use byteorder::{ByteOrder, LittleEndian}; use crate::error::FormatError; use crate::message_type::MessageType; -use crate::storage::{Storage, len_usize, read_exact_at, read_upto}; +use crate::storage::{Storage, Window, len_usize, read_exact_at}; /// OHDR signature for v2 object headers. const OHDR_SIGNATURE: [u8; 4] = *b"OHDR"; @@ -119,36 +119,43 @@ impl ObjectHeader { offset_size: u8, length_size: u8, ) -> Result { - Self::parse_in(&data, offset as u64, offset_size, length_size) + Self::parse_in(data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]. /// - /// Reads the signature, the prefix (at most [`V2_PREFIX_MAX`] bytes), - /// then each chunk as one bounded read, continuation chunks included. - pub fn parse_in( - file: &dyn Storage, + /// Reads the prefix (at most [`V2_PREFIX_MAX`] bytes, signature + /// included), then each chunk as one bounded read, continuation chunks + /// included. + pub fn parse_in( + file: &S, offset: u64, offset_size: u8, length_size: u8, ) -> Result { - let sig = read_exact_at(file, offset, 4)?; - if *sig == OHDR_SIGNATURE { - Self::parse_v2(file, offset, offset_size, length_size) + // The longest prefix of either version, in one read. It holds the + // whole prefix or ends at the end of the file, so its bounds checks + // are the whole-file ones. + let prefix = Window::read(file, offset, V2_PREFIX_MAX)?; + prefix.ensure(0, 4)?; + if prefix.bytes[..4] == OHDR_SIGNATURE { + Self::parse_v2(file, offset, &prefix, offset_size, length_size) } else { - Self::parse_v1(file, offset, offset_size, length_size) + Self::parse_v1(file, offset, &prefix, offset_size, length_size) } } - fn parse_v1( - file: &dyn Storage, + fn parse_v1( + file: &S, offset: u64, + prefix: &Window<'_>, offset_size: u8, length_size: u8, ) -> Result { // version(1) + reserved(1) + num_messages(2) + ref_count(4) + header_size(4) = 12 // then pad to 8-byte alignment from start of header - let prefix = read_exact_at(file, offset, 12)?; + prefix.ensure(0, 12)?; + let prefix = &prefix.bytes[..12]; let version = prefix[0]; if version != 1 { @@ -179,7 +186,9 @@ impl ObjectHeader { })?; // parse_v1_chunk reads the chunk, with the bounds check that was here. - let mut messages = Vec::new(); + // The prefix's count (NIL messages included, capped: it is untrusted) + // sizes the list once instead of growing it message by message. + let mut messages = Vec::with_capacity(num_messages.min(64)); let chunk0_count = Self::parse_v1_chunk( file, msg_start, @@ -221,8 +230,8 @@ impl ObjectHeader { /// end of the chunk, or leftover bytes too few for a message header (a /// "gap", which only version 2 allows). #[allow(clippy::too_many_arguments)] - fn parse_v1_chunk( - file: &dyn Storage, + fn parse_v1_chunk( + file: &S, offset: u64, length: usize, offset_size: u8, @@ -295,31 +304,21 @@ impl ObjectHeader { Ok(count) } - fn parse_v2( - file: &dyn Storage, + fn parse_v2( + file: &S, offset: u64, + prefix: &Window<'_>, offset_size: u8, length_size: u8, ) -> Result { - // The prefix, read as one window. The window holds the whole prefix - // or ends at the end of the file, so a position past the window is - // past the end of the file: `ensure_len` checks positions relative - // to the header against it and reports them as the whole-file check - // did, with absolute positions and the file's length. - let window = read_upto(file, offset, V2_PREFIX_MAX)?; - let data: &[u8] = &window; + // `ensure_len` checks positions relative to the header against the + // prefix window and reports them as the whole-file check did, with + // absolute positions and the file's length. + let data: &[u8] = &prefix.bytes; let file_len = len_usize(file); let base = usize::try_from(offset).unwrap_or(usize::MAX); let abs = |rel: usize| base.saturating_add(rel); - let ensure_len = |_: &[u8], rel: usize, needed: usize| -> Result<(), FormatError> { - match rel.checked_add(needed) { - Some(end) if end <= data.len() => Ok(()), - _ => Err(FormatError::UnexpectedEof { - expected: abs(rel).saturating_add(needed), - available: file_len, - }), - } - }; + let ensure_len = |_: &[u8], rel: usize, needed: usize| prefix.ensure(rel, needed); let offset = 0usize; // signature(4) + version(1) + flags(1) = 6 ensure_len(data, offset, 6)?; @@ -533,8 +532,8 @@ impl ObjectHeader { } #[allow(clippy::too_many_arguments)] - fn parse_v2_continuation( - file: &dyn Storage, + fn parse_v2_continuation( + file: &S, offset: u64, length: usize, has_creation_order: bool, @@ -1328,7 +1327,7 @@ mod tests { /// Every header, and every truncation of it, parses to the same result /// (or the same error) through a `read_at`-only storage as from a slice; - /// a header in one chunk takes three reads (signature, prefix, chunk). + /// a header in one chunk takes two reads (prefix, chunk). #[test] fn parse_in_matches_slice_parse() { use crate::storage::CountingStorage; @@ -1375,6 +1374,6 @@ mod tests { let one_chunk = build_v2_header(0x00, &[(0x01, &[42], 0)], None); let storage = CountingStorage::new(one_chunk); ObjectHeader::parse_in(&storage, 0, 8, 8).unwrap(); - assert_eq!(storage.reads(), 3); + assert_eq!(storage.reads(), 2); } } diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 5c8b1cb..05ee649 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -254,13 +254,13 @@ pub fn parse_sohm_table( nindexes: u8, offset_size: u8, ) -> Result { - parse_sohm_table_in(&file_data, table_addr as u64, nindexes, offset_size) + parse_sohm_table_in(file_data, table_addr as u64, nindexes, offset_size) } /// [`parse_sohm_table`] over any [`Storage`]: one read of the signature, /// one of every index entry. -pub fn parse_sohm_table_in( - file: &dyn Storage, +pub fn parse_sohm_table_in( + file: &S, table_addr: u64, nindexes: u8, offset_size: u8, @@ -384,13 +384,13 @@ pub fn parse_sohm_list( num_messages: u16, offset_size: u8, ) -> Result, FormatError> { - parse_sohm_list_in(&file_data, list_addr as u64, num_messages, offset_size) + parse_sohm_list_in(file_data, list_addr as u64, num_messages, offset_size) } /// [`parse_sohm_list`] over any [`Storage`]: one read of the signature, one /// of every entry. -pub fn parse_sohm_list_in( - file: &dyn Storage, +pub fn parse_sohm_list_in( + file: &S, list_addr: u64, num_messages: u16, offset_size: u8, @@ -420,14 +420,14 @@ pub fn parse_sohm_btree_entries( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - parse_sohm_btree_entries_in(&file_data, btree_addr as u64, offset_size, length_size) + parse_sohm_btree_entries_in(file_data, btree_addr as u64, offset_size, length_size) } /// [`parse_sohm_btree_entries`] over any [`Storage`]. The v2 B-tree is not /// read over [`Storage`] yet, so this needs the whole file in memory /// ([`FormatError::ContiguousStorageRequired`] otherwise). -pub fn parse_sohm_btree_entries_in( - file: &dyn Storage, +pub fn parse_sohm_btree_entries_in( + file: &S, btree_addr: u64, offset_size: u8, length_size: u8, @@ -455,12 +455,12 @@ pub fn load_sohm_table( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - load_sohm_table_in(&file_data, offset_size, length_size) + load_sohm_table_in(file_data, offset_size, length_size) } /// [`load_sohm_table`] over any [`Storage`]. -pub fn load_sohm_table_in( - file_data: &dyn Storage, +pub fn load_sohm_table_in( + file_data: &S, offset_size: u8, length_size: u8, ) -> Result, FormatError> { @@ -498,12 +498,12 @@ pub fn message_data_with_sohm<'a>( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - message_data_with_sohm_in(&file_data, msg, offset_size, length_size) + message_data_with_sohm_in(file_data, msg, offset_size, length_size) } /// [`message_data_with_sohm`] over any [`Storage`]. -pub fn message_data_with_sohm_in<'a>( - file_data: &dyn Storage, +pub fn message_data_with_sohm_in<'a, S: Storage + ?Sized>( + file_data: &S, msg: &'a crate::object_header::HeaderMessage, offset_size: u8, length_size: u8, @@ -568,8 +568,8 @@ pub fn resolve_sohm_message( } /// [`resolve_sohm_message`] over any [`Storage`]. -pub fn resolve_sohm_message_in( - file_data: &dyn Storage, +pub fn resolve_sohm_message_in( + file_data: &S, heap_id: &[u8; FHEAP_ID_LEN], sohm_table: &SohmTable, target_msg_type: MessageType, @@ -603,12 +603,12 @@ pub fn message_data<'a>( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - message_data_in(&file_data, msg, offset_size, length_size) + message_data_in(file_data, msg, offset_size, length_size) } /// [`message_data`] over any [`Storage`]. -pub fn message_data_in<'a>( - file_data: &dyn Storage, +pub fn message_data_in<'a, S: Storage + ?Sized>( + file_data: &S, msg: &'a crate::object_header::HeaderMessage, offset_size: u8, length_size: u8, @@ -650,8 +650,8 @@ pub fn resolve_shared_message( } /// [`resolve_shared_message`] over any [`Storage`]. -pub fn resolve_shared_message_in( - file_data: &dyn Storage, +pub fn resolve_shared_message_in( + file_data: &S, shared_ref: &SharedMessageRef, target_msg_type: MessageType, offset_size: u8, @@ -692,8 +692,8 @@ pub fn resolve_shared_message_with_sohm( } /// [`resolve_shared_message_with_sohm`] over any [`Storage`]. -pub fn resolve_shared_message_with_sohm_in( - file_data: &dyn Storage, +pub fn resolve_shared_message_with_sohm_in( + file_data: &S, shared_ref: &SharedMessageRef, target_msg_type: MessageType, offset_size: u8, diff --git a/crates/clawhdf5-format/src/signature.rs b/crates/clawhdf5-format/src/signature.rs index 4b152ee..3a2e5a1 100644 --- a/crates/clawhdf5-format/src/signature.rs +++ b/crates/clawhdf5-format/src/signature.rs @@ -42,7 +42,7 @@ pub fn find_signature(data: &[u8]) -> Result { /// [`find_signature`] over any [`Storage`]: one 8-byte read per candidate /// offset. -pub fn find_signature_in(file: &dyn Storage) -> Result { +pub fn find_signature_in(file: &S) -> Result { let len = file.len(); let mut offset = 0u64; while offset.checked_add(8).is_some_and(|end| end <= len) { diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 7d68eeb..6d09e7e 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -10,8 +10,15 @@ //! `impl Storage for [u8]` serves the in-memory case with no copy, and //! [`Storage::as_contiguous`] lets a hot loop borrow the whole file at once //! when the backend has it. Modules are converted one at a time: a converted -//! parser has an `*_in(file: &dyn Storage, ..)` core and keeps its old -//! `&[u8]` signature as a thin wrapper, so callers do not change. +//! parser has an `*_in(file: &S, ..)` core and keeps +//! its old `&[u8]` signature as a thin wrapper, so callers do not change. +//! +//! The cores are generic rather than taking `&dyn Storage` so that the +//! wrappers monomorphise for `[u8]`: the bounds check of each structure read +//! inlines to what the slice code did, with no indirect call and no copy, +//! which keeps local files as fast as before the migration. A `&dyn Storage` +//! still works (`S = dyn Storage`), and a remote backend pays one indirect +//! call per structure read. //! //! The trait is synchronous and `no_std`: parsing is CPU work, and a remote //! backend bridges to its own I/O. @@ -176,7 +183,7 @@ impl Storage for std::sync::Arc { /// `storage.len()` as the `usize` the parsers' end-of-file errors report /// (saturating on targets where the file is larger than the address space). #[inline] -pub(crate) fn len_usize(file: &dyn Storage) -> usize { +pub(crate) fn len_usize(file: &S) -> usize { usize::try_from(file.len()).unwrap_or(usize::MAX) } @@ -187,8 +194,8 @@ pub(crate) fn len_usize(file: &dyn Storage) -> usize { /// `available = storage length` — the error the `&[u8]` parsers give for /// the same bounds check (`offset + len > file_data.len()`). #[inline] -pub fn read_exact_at( - file: &dyn Storage, +pub fn read_exact_at( + file: &S, offset: u64, len: usize, ) -> Result, FormatError> { @@ -198,7 +205,8 @@ pub fn read_exact_at( .saturating_add(len), available: len_usize(file), }; - // In-memory fast path: one dynamic call, then plain slicing. + // In-memory fast path: plain slicing (for `S = [u8]` this inlines to + // the slice code's bounds check). if let Some(all) = file.as_contiguous() { return usize::try_from(offset) .ok() @@ -219,6 +227,8 @@ pub fn read_exact_at( Ok(bytes) } +#[cold] +#[inline(never)] fn short_read() -> FormatError { FormatError::Storage( "short read inside the file (the storage shrank or the backend failed)".into(), @@ -240,7 +250,11 @@ pub(crate) struct Window<'a> { impl<'a> Window<'a> { /// Read up to `max` bytes at `base`. - pub fn read(file: &'a dyn Storage, base: u64, max: usize) -> Result { + pub fn read( + file: &'a S, + base: u64, + max: usize, + ) -> Result { Ok(Window { bytes: read_upto(file, base, max)?, base: usize::try_from(base).unwrap_or(usize::MAX), @@ -259,6 +273,7 @@ impl<'a> Window<'a> { } /// Check that `[rel, rel + needed)` (relative to `base`) is in the file. + #[inline] pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> { match rel.checked_add(needed) { Some(end) if end <= self.bytes.len() => Ok(()), @@ -274,8 +289,8 @@ impl<'a> Window<'a> { /// storage. For structures whose size is only known once their prefix has /// been parsed and whose parsers bound-check what they are given. #[inline] -pub fn read_upto( - file: &dyn Storage, +pub fn read_upto( + file: &S, offset: u64, max: usize, ) -> Result, FormatError> { @@ -297,8 +312,8 @@ pub fn read_upto( /// [`Storage`] yet. On a backend without a contiguous view this is the /// clean [`FormatError::ContiguousStorageRequired`] error, never a guess. #[inline] -pub fn require_contiguous<'a>( - file: &'a dyn Storage, +pub fn require_contiguous<'a, S: Storage + ?Sized>( + file: &'a S, what: &'static str, ) -> Result<&'a [u8], FormatError> { file.as_contiguous() diff --git a/crates/clawhdf5-format/src/superblock.rs b/crates/clawhdf5-format/src/superblock.rs index 318c945..dc21f7c 100644 --- a/crates/clawhdf5-format/src/superblock.rs +++ b/crates/clawhdf5-format/src/superblock.rs @@ -166,13 +166,13 @@ impl Superblock { file_data: &[u8], signature_offset: usize, ) -> Result { - self.refresh_eof_in(&file_data, signature_offset as u64) + self.refresh_eof_in(file_data, signature_offset as u64) } /// [`Self::refresh_eof`] over any [`Storage`]. - pub fn refresh_eof_in( + pub fn refresh_eof_in( &mut self, - file: &dyn Storage, + file: &S, signature_offset: u64, ) -> Result { let refreshed = Superblock::parse_in(file, signature_offset)?; @@ -233,13 +233,16 @@ impl Superblock { /// [`FormatError::UserBlockNotStripped`] because the addresses in the /// returned superblock would otherwise be applied to the wrong bytes. pub fn parse(data: &[u8], signature_offset: usize) -> Result { - Self::parse_in(&data, signature_offset as u64) + Self::parse_in(data, signature_offset as u64) } /// [`Self::parse`] over any [`Storage`]: one read of the first /// [`SUPERBLOCK_READ_LEN`] bytes (fewer when the file is shorter, which /// is then refused with the same end-of-file errors as a short slice). - pub fn parse_in(file: &dyn Storage, signature_offset: u64) -> Result { + pub fn parse_in( + file: &S, + signature_offset: u64, + ) -> Result { if signature_offset != 0 { return Err(FormatError::UserBlockNotStripped(signature_offset)); } diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs index 1a684f6..8f09aff 100644 --- a/crates/clawhdf5-format/src/superblock_ext.rs +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -171,13 +171,13 @@ pub fn read_superblock_extension( data: &[u8], sb: &Superblock, ) -> Result, FormatError> { - read_superblock_extension_in(&data, sb) + read_superblock_extension_in(data, sb) } /// [`read_superblock_extension`] over any [`Storage`]; its length is the /// end of file. -pub fn read_superblock_extension_in( - file: &dyn Storage, +pub fn read_superblock_extension_in( + file: &S, sb: &Superblock, ) -> Result, FormatError> { let os = sb.offset_size; @@ -351,12 +351,12 @@ impl CacheImage { location: CacheImageLocation, sb: &Superblock, ) -> Result { - Self::decode_in(&data, location, sb) + Self::decode_in(data, location, sb) } /// [`Self::decode`] over any [`Storage`]: one read of the image block. - pub fn decode_in( - file: &dyn Storage, + pub fn decode_in( + file: &S, location: CacheImageLocation, sb: &Superblock, ) -> Result { @@ -483,7 +483,10 @@ impl CacheImage { } /// [`Self::block`] over any [`Storage`]. - pub fn block_in<'a>(&self, file: &'a dyn Storage) -> Result, FormatError> { + pub fn block_in<'a, S: Storage + ?Sized>( + &self, + file: &'a S, + ) -> Result, FormatError> { image_block_in(file, self.location) } @@ -511,8 +514,8 @@ fn image_block(data: &[u8], location: CacheImageLocation) -> Result<&[u8], Forma Ok(&data[start as usize..start as usize + len]) } -fn image_block_in( - file: &dyn Storage, +fn image_block_in( + file: &S, location: CacheImageLocation, ) -> Result, FormatError> { let (start, len) = image_block_range(file.len(), location)?; @@ -540,12 +543,12 @@ fn image_block_range( /// ([`CacheImage::decode`]). `data` is the file from the superblock on, up /// to its recorded end of file. pub fn cache_image_state(data: &[u8], sb: &Superblock) -> Result { - cache_image_state_in(&data, sb) + cache_image_state_in(data, sb) } /// [`cache_image_state`] over any [`Storage`]. -pub fn cache_image_state_in( - file: &dyn Storage, +pub fn cache_image_state_in( + file: &S, sb: &Superblock, ) -> Result { match read_superblock_extension_in(file, sb)? { diff --git a/crates/clawhdf5-format/src/symbol_table.rs b/crates/clawhdf5-format/src/symbol_table.rs index 8fb4453..467f43b 100644 --- a/crates/clawhdf5-format/src/symbol_table.rs +++ b/crates/clawhdf5-format/src/symbol_table.rs @@ -4,7 +4,7 @@ use alloc::vec::Vec; use crate::error::FormatError; -use crate::storage::{Storage, len_usize, read_exact_at}; +use crate::storage::{Storage, read_exact_at}; /// Symbol Table message (type 0x0011) found in v1 group object headers. #[derive(Debug, Clone, PartialEq)] @@ -80,17 +80,16 @@ impl SymbolTableNode { offset: usize, offset_size: u8, ) -> Result { - Self::parse_in(&file_data, offset as u64, offset_size) + Self::parse_in(file_data, offset as u64, offset_size) } /// [`Self::parse`] over any [`Storage`]: one read of the node's header, /// one of its entries. - pub fn parse_in( - file: &dyn Storage, + pub fn parse_in( + file: &S, offset: u64, offset_size: u8, ) -> Result { - let file_len = len_usize(file); // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8 let header = read_exact_at(file, offset, 8)?; @@ -108,42 +107,22 @@ impl SymbolTableNode { let os = offset_size as usize; // Each entry: link_name_offset(os) + obj_hdr_addr(os) + cache_type(4) + reserved(4) + scratch(16) let entry_size = os + os + 4 + 4 + 16; - // `offset + 8` fits: the header's read checked it. - let entries_start = offset as usize + 8; - let needed = entries_start.checked_add(num_symbols * entry_size).ok_or( - FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_len, - }, - )?; - if needed > file_len { - return Err(FormatError::UnexpectedEof { - expected: needed, - available: file_len, - }); - } - let body = read_exact_at(file, entries_start as u64, num_symbols * entry_size)?; + // `offset + 8` fits: the header's read checked it. The entries' + // read is the bounds check (`offset + 8 + entries > file length`, + // which cannot overflow: at most 65535 entries of 40 bytes). + let body = read_exact_at(file, offset + 8, num_symbols * entry_size)?; let file_data: &[u8] = &body; let mut entries = Vec::with_capacity(num_symbols); - let mut pos = 0usize; - for _ in 0..num_symbols { - let link_name_offset = read_offset(file_data, pos, offset_size)?; - pos += os; - let object_header_address = read_offset(file_data, pos, offset_size)?; - pos += os; - let cache_type = u32::from_le_bytes([ - file_data[pos], - file_data[pos + 1], - file_data[pos + 2], - file_data[pos + 3], - ]); - pos += 4; + for entry in file_data.chunks_exact(entry_size) { + let link_name_offset = read_offset(entry, 0, offset_size)?; + let object_header_address = read_offset(entry, os, offset_size)?; + let pos = 2 * os; + let cache_type = + u32::from_le_bytes([entry[pos], entry[pos + 1], entry[pos + 2], entry[pos + 3]]); // reserved 4 bytes - pos += 4; let mut scratch_pad = [0u8; 16]; - scratch_pad.copy_from_slice(&file_data[pos..pos + 16]); - pos += 16; + scratch_pad.copy_from_slice(&entry[pos + 8..pos + 24]); entries.push(SymbolTableEntry { link_name_offset, diff --git a/crates/clawhdf5-format/tests/fixtures/v1_groups_400.h5 b/crates/clawhdf5-format/tests/fixtures/v1_groups_400.h5 new file mode 100644 index 0000000000000000000000000000000000000000..5237559583ebd35892389fba2a6cdfff0a87461b GIT binary patch literal 355840 zcmeI534p6fRqrzkLuMGD*&}O%AYsokcP4`%VHpS@Lo*BzK_J80Ff2h98$k$rgDgRO z(4c5g(H0N_qV{tQD%uE21jQ(dK~bV8HmDdDoi}sq_n-H=-qbBBH(&QR_L=vjx{~wN zsXFKU&Z(2`+g&Kwe@EE@uWjA${?|V1re__uL9tjky|CiUajF|GJ9nTEWJvvjtig+)~y4V;nc#}g<|1Z?&ZQtv2gk9ivOC&|5x$rUOWC0ub;LrepiU!GI8fNRVW;f-^-s+ z+-m&4E7?~^*#e5+HbzjKZMw92rD+t0imNN%Sn!L7U-;1bopO0rm%df2(pv`Jnvb+bcD{=$NMzt~#$Y zu%G;Ic|{hp@v1Bk$}3Yi)idHwai!N69mRv67gBl9zCt{-$P?PP@vx}Jj>dB#9`?LG zKyjzII$7(82fzNM@<7{k`JMBs>qC3R?Ns|V9!}PC|FfSD@t`(er_YK9|DKo11J~0a z59>ED9^^wC;}!_V+DQRsmy$P#eE^W_x+)cE4#OLE6l#rjj4G{-_8p0(7nA|SNk>|7WLTCcrL_4i8zuKvf^O^ck}OZno@b- z#{`1++TI=Ay2gV%Y2%435aOXs9O>Dtc$mRm5q+sVT#F5c>cix&PQZAOCv7~D1wuSD z?y~=>W%HoEsfC&upL1oDI3J98Tnx!71Juk?u{ZBpDRuJrn%qj*@mn_HoazEmFg zalGKYbAFCnU;8#57WLTCcrL`lfH;yBvf`mra~gW+OXcA@Y%o+GihN*cJjjzap2z|r z9){=be<~{;y2qTJKKfF5&_Cx0@zA-4TivjPAZ)e%U;z4_imgm_9;! z{Jx@#T$%foHp~ue&Xy4X{E$XqO@mz?9 zGI1m;WW~b_?uzJ3<$)ik4}LDO&Cll;5AvjqC$d0@hXy})rDwC^Vem$$VbphHDi3#H zgTea1&v}W5(0R+qWqscHTbza>`clv9eVbC-{r^U*ga4mN?1lfw0wE46{5+&Ym(9KU z%}o@v>6fI(@1IBug(3Gy$@%xnE!G=RUnS?)+W~T8>aXPdpEpG;w?io3^&jH4tbAvA zAh#~e#l}K;pg|mIlj1~irPmi7)gR6u|Gtffg zeW^U$gbjvxD7Kw|I5r;SQ5&zy0wEsOAGu$8**vIk+D6Ts&-tXs|MzS||lJo1(6uAv}l$_5eV!0i3 z?;Kv@wm8ObsdVQm;pc7YI!BMp^&SMh4M;^IMQato#INbFFJ~c;#1rTW%Q-; zz@LQ)@-V#At*?DsAGWB+j>dB#9@@l_tdJECrKdR!4fLh*a7#8Ax_6GAHy-3k8&70`5Dy#A+5gnCc~BoVL(S$T z?hEPh`-(l}rhLvIIsd*jM{bY%m*o69bU8roKX}yC=M$;i4!UTLN1PQdaChO*&=76|2)@$>gTChinh``=BUiHAOQaEQKC9&U~55D$$PIsxt5`ml6g z9i=BiJWMzCTOlhR2ITJ%`ciqg4ciRy(0Z{GFdp{%(`Hv~fe;USFWN7k6%Rx5_ZWSt zJbWkH4Dm4NIRWEgzdvnu)fNcxPFhOcnzT3*Rt4CHY~p}f*2 zjQ-|co@Cf325Kehb`){qw!pbhjrpeR>+En@++N& z7Wz_o_%1dW;$i$+Cty6tlQy2n0wEqYh$B6l6%UQqI1O#|rShOZgAwAP{6@FF@gPsy zcp?jgc-VaX{->7BgZi*C3vba!NRQuFRCr^r=k_Hz|Gw4G8`K?5{W{d-jTLy5oX_We zE97scduQu)Zg1=OEj6!{Ia(;M^!fctmRIr@19{w0D6dTW`*$nu6j%G-O`nN}`EA|G zrQ5qPm4`>5I>f`~UEI3bxAkG^zB)=zgyN`6VY4_oO3|?RSwnF&*>Y=I94&6!qx58` zKKFu^_flywV{nY-)#GT@5|GVik@vu!D+(ln14|hRzh=;YGaRS=6^}*@B zI!aH3c$ok6ek)|f!_L6znV>I~hr6=P5D%T7a{|T#hO*&=76|cBB#!iORy^#Izi01u zno@b-&uU%%&x=mG{QnAt5*uDAtQ`Mp=RNKP;{n6ia6$`&cqskC{>PThgZi*_7B16A zT&c(FE4sYV<9;PM|Gu@s8~uUXzvTQn)Z>i-JW9^zbH5exH`BdyLjIa>yFI4nl?q1- z<(1uEbOM%F@)rYn+)*g6bcrMFS==eE^!lQsc&NPBt+0l^R355qAjHFD$oj}xABG`p zxJnCzc<2#FdMGO%n(ucSI_OK~fj`q4ym!ui$*nJrjR$$u#;dYGh=)FLq-V0?VeNxX zLl=FiJk;4>s6I4zoPhBlPuh4Q3xs$W{L22Pmd%6uuuT>o(nm;--&gc`qsaYAa{hg5 zi#JNt9m)B1Xuum~c$A#a=YA{XZ>D=^=fiGqJ^YrMSI*&Rp}ey8>rTM(O8#OXk2?zG zmHDsj-|dy$?SFaUnRwWt4o=XQ%EK{s5aOZpTTZ}uSlCrZ>cJ2XMdC=t$cl$u^7rgF zou*VC&Sirk9=acK0>*Dtc$mQ5eB?Bx@^Bs-4Dqo0Q72$LU?>|- zXn_z9<=@-?*s^(0AGXcH4f+V_@%xG)Z}hoeNzT7-?eNBcx+6Ki4vlzY2#=EU`P^@X z{LOUloPEUYt%%=J^GbuGh4RW|*9ll&$zKfQaYv!N(j$(vXK|;v((8+k;-UHbZiNo| zQh7L^4TN}@ecTCX-}d?cq8>XM&xLsC6GyT_Ry?eI%xUPNFO`S8v%wG#^FMY1#)CX* z>DjD!=zPLy=%Fu_hwo;CAs*H~?F5VmdD6xcSs=v2@K5$XwQL^LhwZZP z=p#;_^!RvjQ)AQ{IYpaA6BFv8q-HekKb3UAXomp+n410`&J3L26abrejQpxt_6>h z^ZDFwh5XHQ@9a?r2ly>DuiTHLh4M=GD^9@jO8#OXk2?zGl@f8JJ&QZVm0n+T6b}>X z;C#=ml*$9YZZ-IO33~tT*44hP4_nk@N8`B=4`t#=R>+En8Qc}om&(Ka*_s-(n?X8U8Qu9iS zqlNOy{J))m<(2%!KpuA#$}0onNP8A{iYvXo=qMgK-*PMT(3i@?1KB``hvKPj=i0aR zVT*d~Xgn9f3Zd)a8H&i2lB7E(U6duP_3 zy?U=7$}45!NT(BbiYvXo=qMg$)WIV9Qh9g?8wl~xzpnAHsE>}~b0HoY#F1>06%TVb zD4{QvhYKS-46biH$cHw@EfC_NMI32`tavD{xRuN3OXcCAY%o+G<~KGTQF1xQDALYhW9`xs?Lw(rbj&A4LxAAbYp8KEue29k@air5{ z#Y3^|RxYD2m52AU!4MDqJMHHoblwJXS)aH0M8|yzeW~aDK;*o=JAZrUEhCrpc{@*b z=PjZy^}NH#c{`W7^_A~*9)Aq^y>b^twtGUe*n0l@p}F$@kbBAs`cm!iNEBE;4_Vq? zzrq5cI4@Nc+0IUyEj!L9#P|GePD5&(KMFU7^7-yLZe8u$c)(ybT%`p!y!Un&oej_@$Qr|}>k+8DP$h=&1j zq!qH_q4V8tMSxpFvt`|v)Q9cT4~^&}q{r_o z#>kbpUrEltZ%vRZQ+Fii*P$tL4S1BC&u83%hby6bXZL&D3HtagHLrX>#|Y(>&IN9L z`QQ4m!}WXKwTJRbkvP)+#hv0xuP-`^hh6I6?0!x|Di4ok10f!|E#qNPA05T#LOhg+ zBiSS?9wu-%f52iqJTAgR@4?1{d}w3b0wEsC#F19Wiia88715VkA0E#JL-k?uP~$;9 zv@vdh5D$$DMSxpFvt`|v)Q8RJhg$Rz(&P6Pd&mvAUrEltZ_SY#Qg#( zxCKHy4BH~Wt)bbn>qGaEZsk7uQh9hHM-0`6&SQ;-codj~ED+*h^cWG~BsE(W59-5; z^h0C%2{;-&a(S8*;yroPXbHAUC4!NY1ZA zP2|S#C^?_cxCIYaLif(%Q{4&5_$@WBtaFS|UYS4BtuOyuA9lEY&%5?eUKtQa+P}C{ zTBcO z9!k$M9*$yMFM8$>52NRbfNXi_k-rD%OXcCx2oK#2<6+U*IEv4Oco@GRTOQPhwdjYY zd@doaeqXVMT!Z_SLAyKN6Gno<~=5K@9a|thxjcuuRJ4?S9&jU z7SjE|`mnr5#^LTSlvm2ck#;BU6jyqE(NR3isDnlHrSkAhHW2E=`Y$ma7WL6ld@jU8 zgE*2+vf^P52PO2S^6;z(4}+H(5AvanaSMcaXc0$RAuAq=FLf)I(U;1@v)N#%KFnWb zJjjPO#w`%yq5Vn`;MUM=S@$LNVe9lm>+})QW6p&oKm z>XPJqKI0ZVTnXJfOD}gPXyCWhywc?up}bOjty^FIw?6D}{hoL2p}aEO6lv}RnklYM z_FUqj`)apxAAPAjJco^hcqqNzcsN;K9i`_(JdA!s1UPGGwrn1HXVG=R-V9-<&NE>cckahxYhf zLR$U4qK{mQ`<3MU`_>k6ZR(EX{5mv1ZXF&a=kuBOSkS$5Kph<6x757yyhvW@ztvet z_XF$0@*Ww7yT4FgX%I)+ow!q6>GefN@i32__>L@)Q;$i%ABEVTgvt{$pCw~vom&(HrMR;hu z$9OnO@qDsp4e>Dj1rd-f4+HY|2z{wMyePs$|NX|p$;QZ0dOpO%-utrUL4DXR{ZN5E zLR$U4VvJmy`<3MU`_=@xb?T1f{5mv6ZUY`A=kuBOSkS$5NF5yGx757y;z(W@e9&1) z_XF$0@*Ww7yT4FgX%R=-ow!q6>GefN@lgDLTe*zBR33hq4TSoz;fIZfMSXM>p9}HO zCXQs2tavDW$gSK!Un&nTiSRJ`RpUWEv@vdh5D)9bkyglxhw?AGm0Rdb<>94lFjOB( zzhOMchc?D75aMCu*F=C@L$hVwm(+*N=!Z7xBc#XgEB26^@;QU#{QK4%xjpV*lJo0O z;dE>RkCOBGj9c(?6?!O@mp$M>2ZuuUMc^!TVMXSKJ0M)o_Fn`yfXeRk>*aI znd0hX&m|uE)WIS8Qh9h88wv5y_#NZnWPNp%o)7Ub{fG!~*3fL(JPgR+BlM;6@bU-` zt&bWHM=739_N*Zu_I^(UWXr>l{5?irDi5!S@G$)& z_)+(|{yp4?pIg+=Www6Ny|3>Nj@5r-pEhwId#(PZlW+3=_7~l}!@TkzZr)?QbE^Bp z+5_gz6*nI--@S#KPnoaX$;}JbaF(Am+E?wqoc`f^+;#%ts$}^A7XYr`^2Ad^~mY0rU2k-F(D+ z`b{^VGG9NF4-&4qoXAsrT*iNck2(Knb)9>a7utN$WZu7v`=Z0V^xf`@9`nryx-SOI zM~`%0jF`8c4ukPj} z=EZxv`ILG8LN_nq=F<1O`6~0l)7-qty!?Z1-eEp`rJMJdbDD(*^{pfB(bGS0>!;pJ zUdawZee2%Xt*idRKF3+uWk=${(7j}fIFb=uR22$}E4{wxC?48>>{i}DUn&o;Vgn%_ z3V&)mEb619_*{sG?N5sUXBEws&BOYixRp22m&(JdBRs79x$$rm<9g9Ehj`fej0nh< zhYj-g7Wz_ocujrc$_$@WBY)10R#^;=cbU&~@EbozVxcdv`l~v+M zyAyYcE4{wxC?2NN!2h7^!Rfj3cQh9hY8wmAbTi-Gs7WL6ld@jU8lQ@!1vf`oe4Y%?t`cirL z(FhOQml+T8p^b41gm_pZj2 z9=?q*FcB;e;$d?3!SJv}{@y`fDi3dq@UVG3g?7H zEw3Dohp(PmLwTi19O+czZrOcU;rcFKSJ9Ws!`mY~Y~RFqkPmH)TOh>48gZl*vf^Ro zMlN0}=u73{9c(aEA9ikPJjjPO#w`%yp+g*Lg{*j3ExCAYqA!&Pe*IkV-*K$m%6O0u zZH!wW#6y?ggQ*pk&4c=|UHYLOeT4M*eZ?5L37<1a&cAO>kehM;lAK?MrpV2yOOo^X z)cNx|st>E&4DRt;YF_yX4j8JltGBVdlGhl>Zkn5A!=_%Y*u`8U0X^K0^00zpvOsuEYIGa{hg5j$D_zBRRhg z75F;`J$RIy&u83%hby5zY==5H!EdQ~Wxz2)ec1NhEUz4{r>|bxLwRM5IMQjv-Lm_z zmAkljt)MTJhj&GI*r^#0@}Z4!3xs&+5Jy@eD;`!WE?%4HOXcCG*>e{7f3ZXE|UfuT<`7 zc_ptgkjH(6^2+2ME{?QkakuO~Y>PU$gT7QA-W}ng*)$&HLmT532=Oqx*TL|xP5$0R zUn&nj7vW*;zQ#j53QR&42=Oq#&%yApL;jwiFO`R%kMOYl0OLWvwlQvj5D!KE-nLd) z_I*ixSebsPL?0nNeqT{RuFL&Oa{hg*fn1NeBRRhgHIeJXqvU)(b^g4L>ce)agR=`9 z|EYOpn*)aWu$>24Udd|=^^L@<>Ix8zEmFgbMwKU|KELx@gN`C z7`H%(hc0oX6|&-?@?hc>eW^UWhYg16!(`2PkPmH)TOhzTYH%ER~Nsf=9Twxz))W4-`-hB_htJ%ta*)yeC#WfS7wjO zHm`_>5x>}H`XTp&%Beg&mV^-EVeK(iXY&zhSC3tUc$h!>V0hSRJ3SNhrSkB&2oIgd z84p*FNX*ABLp&6TBkfZ6GTl4$`l6%i!!G%I_E;CUsXRPB!o$uJjE6;ibQGTp@ld+N z#gS~XY#!8y&CYTfn(uS>CFxmbzWILl`W|vy+^;0(-?!$-^{G3O^XpLIT2Ai(9wq1V zsq^P`R9=~ozvi9A@=6B-gzlYtA9sEThd&=&*(=&VeV)v8}V!+TnrTZ+muI?$u!(m5UzV(NAC=o~6y|TaJ zO0O?EiiZi@&7Zj#4 zY+7a0%op^b41gm@SbM_M5(9y%{{@!CUQDi6QH21E6s_)_CRKD04zfe;VF zm$*363R&^ceX-NiM_(!rzsd$fJak@RJjjPO#w`%yVf3Duk3J)P+sZ2+VaZbdiv_6 zJ(O2U#F0)T?v~w$O{jzOSGl-N<>7xvc<8;>c#scmj9VbYLzy_z3R&?mgS#U7QhE3_ zHW;c8{nr~0@}Z4!3xs%R5Jy@eD<0->P(oiS55LX^Lp;p-#)EulW84BE9$Ig5aikTN z&4c=|UHYLmeT4M*eZ?5LA@?iE`S+~}awF=Fu?2J&U_#_hFqkyLj!PFO`SijPOu=oADqY+8DP$ zh=<`@T^wnJta#}DnA6ioUn&p3#Rfz5q4W;pK|ZuGZh;UFqqiRn4?Xht0DY-E{C0$g z?oS#I@hC6}Ss=v2_$RXEL4DYaerU?)5~@RfU$KW=gZq`_{QK4%xfXRta(*2uoZ&3c zhDXWye9BIF9o2{Rse?oOmYP>a957U8d+)NmlGhl>7bOV5mL}e%^SH4{eNFAjCt9IMNDP z@lbrXi`O#xQhE4YHW=bz{$ArjKD04zfe;Vv_qaIH3d`m}eOQrxXq`Sndi=g(1-TLT zE6MrytrBu$>W<|6I<$)1l)5B2pHH1XucP{~(zf$g1HYx_mEYrlp}bQ3faR6E#y}qT z70N5a_h*||#6$OePEQ|wsXTl%!b9mp#zP+CF0XqE@i6+}!SK){e-F@?%ERwRcqsp} z@sQWp%j3R6Jd8hlFg*0h-$V4J^6&={9(w=Vc*tYi<#kUX9;UyVEf4C$%Jf5fd@iAW z$nPsE$hEj%NzT7-HIQplcO>W6p(b+c@F+Q-&%F9g`_jF0Kph<6x756{%Yvaktp6L9 zS7dV=;}!_zl?HL76~x`LzaKWI4wlfD%EQOlV2FpoZyOKtp^b41gm`EXM_M5(9*V!| z;1?xLvGCdN^<^vtBu^0`-lwwVL4DXd{ZN5ELj926S9FnUbH9?Df8W|bZk@U#Ilm6|klTPq z$@zTd)o6CoFvP>?uZ;)!(8jn0LOiS! zM_M5(9?G9}@!CRPDi5D#gCQPDe``F*hc?D75aMBD>f%T%ESm@QVVm?roAeRVr2K%USlth`wH=}_fH4I!;t(vMqerqe;(ms@UO;09^)>rdkXPT_;R*9s1MtwA6lW0 zP(S4N6+`6KxnD`nzi;gzw?W;JoL`4V$Zf)-No96_s$V@aQZLKZ>f1@!h)eb zZ1`2nE3&zbaSMdLnv2&q`cirLEE^2*Q2sCD zK|ZuGZh;UFo8NSCq!pIUgZi*t`k^iQ2*7c&ESm@QVP*QE3Vnq1 z_PD|wB9Jnk!$R|>?D_AKs}-G_~+gVSR!Zc};qmk1B* z=Nk|5p^b41gm_pXjOF>Zkn539tHR>+EnsklR5 zYJK=uHW=bz+%z8KLmT532=P$4my08E1SfT_AKs} z-G{aA?c#MEeW^TrCBnnr{f!6t(8jn0LOg5{M_M5(9@_VF@w$P&R35&{21E6s@F3$s zKD04zfe;Vd4|H*)6|&-C{Q*wTCi+r&_%}8f;$dUWc#scmj9VbY!_N0+%Y*u`P5Pl- z`Ur97_Z5BQR=HnE&cAPMAy=X9NY1ZA1LT_UC^?_cxCIYa!u{UDn->Njb0=8;%;|1J zoNw>O&X23iH~6{5Ci6ngz29NJNgvl^zVZ(?5Q!o3Ao&U+U&f z=F<%~?=W9~wVU^tyHytdU-ju5+>^H8I<@ZnJ6j0Voy~_kyQ-hE->12_&yK>Ap}JEc zj${MnWyO_VUvv}?d)$*&(3i@?f3SfN4_l8k9v1b{QG71MLz6g?O|s&la4~ffeW^Tr zEyBb0_ZbiJp^b41gm_pZj+%3Bg+n^3^p)Zw(uSa-TeUkAYAKDnV zK!}IkCmswBo8<3p^riCfjR+5wry39OC@=|GAjHGuDF?&D7WsPzeW^TrGs45>rN)DN zZDZU5As%K=%a#ZAVY~EAb3T{Qz0B_`#_HF&UrEltZ%vTvPuM3Zo^ZATh z@Ngy6hiy{_ckx?lUilWs2=!rG&$7I7xSqaxX%FRt&kNDE6;ZET0vhb5C6>uL-k?jdB%f$Xk*+0As#x!kyglx zht(f&@!CXRDi4>j!4MBCFEk$HLmT532=UO>&mUeHZ|;j_-Ivsd&FGtY^bz9D?<@9@ zoA5b<!8qFaP+r-6k&7dpM%*ncuZV|D>fko|Qh7Kf!b9by#)EulW84BE9wsk27#_CB-#h3_ z<>Ax_56zbw5Ai5430WY-!|Y`T!^1ZDdl!AFJe(HcVe8e#gM4jc+yWsU=C90_2lZh^ z{kt#v2;Iy4zG4Nr4)-g``S-07a$V|<rxAC{?!#7I)L#?pv`;z*w7X46(K01Ug;1=+OxP@ zb|1F-(=J|{=u73{+7TXhf8KbI4{eNFAjCtLIMNDP@lbg;@ru4w9p0!WW_^s+r?`KeW^TLH^M{Z1IB}VXk*+0As+htetE61Y#!8ytLFKxN6Gno>il^f)rYOU*Kyy)Z>f1@1p|cou;zy> zujDlb^0=>1UYUK+#gXs+f54+^=*{`~|P37SR5gvAa!+4MnZH!wW#6yX{ z>!lTz&4c=|P5PlSeZ-Y|yuPB3T#x&e@Jy60hh>t+O|Z@G$uu<3T>O zF>Zkn4?W^YD`dq(bL8T+gT7QAZXDrZ_EFdOU)}K3=rzW*8a%yN?v0ikNXPcmH8)J9BI#2a;Fzg zaug3c)WHe*QhE4}2oIgH@oMg{*j(z}@^)i}7%?2oJk|Zam0`HpVRw;-UN*7e`uQ**vHZ+od0B&__s* z-&c&0>vO-7oPXb%AUB}yNY1ZAQ{;y5C^?@`ojMg{*j(z}@_x7USVI5gvM9HXh_d8{-xT@lYm?v_e)q%;2twzSR2g zoe>@;Uo{@&LmT532=UPPii;zyuxuXGhZX6ETJ#ap`!cO>W6 zp;hEY@F+Q-Pn|!nqx!Hp`J{y3QuE4fF+ivfoBfC7mAuA49`_Z>D}CZfdlq-g?!(sp z&BbdMeW^U$F2cinZam0`HpVRw;$c7>X@#tK=zPt^YY%;?JbYJ#hvK)42l>#(xCKHy z48Q5(NGoK;L-*@WPal1$JlsCQL#J^1@gu0NU3}367hLheW!`@}r;SJ30wEqo|D7!l z>ch(PLu2{~ap(6H739j?uO#Q+w;ISbs5_GL>rfN97CcJM=QHoop+2lf9US1d)Vxy0 z0HHptdz$5yyvIVG_7}=4CE`eX6?dvvdVSGRJWQyA^HW{irt)xy2oJq8jE6;ibQGTp z@lYm?WRt9Tn895UeW^U$F~URtOyfa5v@vdh5DyLFNGoK;!yFDu=u73{P7xkv*ESyH zLmT532=UN5+r^PqST+yp!&>x1ZTbl5@%xH3eoM_Ocg6srK5V{Xc_ptgkjH(6^2&fX(w@cLviq>kbzHpm(3i@? zT_QXbZ)iNohc?D75aMBY0~beHAuArb*K>OM=u73{t`QzeH!&XMLmT532=Oqw@xk!W zBYzLjm&(K4B0O|&W<12Bz$9dW5D(+;$d(87Ve9lmQ$Cka9rF8%E^-a-SCaGZTN}u= zs5_GL>rfB5Hatqs=Tmmd>!?1gPaPcMx755+!2qE?tanSxD|wB9Jnk!$SIWeZ_AKs} z-G|MngGKbE@=%TN(7(0uARpQow?K%8263bnvf^P52PO2S@=%NLFu1MpARpQow?K%8 z7ICB%vf`n58yBx-^riApkMJ-r8xQiKjd2Tvcxd0=#gSH6HV^8il^f)rXaC=ls>cZ>f3Z91IZZ!-{vdypq=# z$m703d1ZK~Z1akE=-$EU>7y@|hhq^QN_R6J@)&n{-BXB%(OnOQhaUNRfWA~7&W-R; zt{D$`jlDeXE5yUNdN4fn$=^ftrSfoIgooa_#zP+CF0XqE@i09nTOQPhZPO3!@wo)y zUGV#gA#yG5SCaGZTRX_LsXLPM>(B_fb$FDV&u3o!rhVz&IiL=X@LOtLX<&d*AJ#wL z@``M3W84CvywV_!w1T)>_V>f))WH(^Qh7K(!o%Pm#)EulW84BE9$LhaR>+En;@w@m zmeH5W!`&l14DV$;$cHw@EfC_NO&n>3tavEh)5U88eW^TrcZ7%H{fr0s(8jn0LOiVB z$HkFWST+yp!*=P1Hs~Xy$L}k~$c?#QNzT7-O^};%|B{?vho;EwQI{m=^QrUabyOc# zZaROp@LOtLxd#Ra^f3|r=JoKo81N5cxaL))2_<;w*L!bOTL|-Zo-xJ}X@et!7ud$cMeT8_Ke(%BXFd%=A(3i@?y&^pHA7(t{ zG4ArZrw|W&56zYb^ksN9arC;rmnzoIc(8Ej6#SFhHme8$R3eifnFU z+ybGz(k70yg1B4u_rprhbn)6iUn&m|jPNk}0pmeFv@vdh5D)9bkyglxhjQ1&YYTm; zJUl4E!}$5egM4UX+yWsUHi#pwkQEP&=el@pqc4?*2S<1)|B&$@AKDnVK!}IU4HrjR zVc9&W4{Om6ZP7!?1g^#bRw zb^Ml^SH2emg!-_?4_jWzYYgOZU!lA*eQ~yVMLZ0sgCq2%^6-!d53Qc@kjJ>o>z+b9 z?7j40co>qu$LLGt;lc&N2CG?^TQCl{E|y>cd8_v%GS+p1yi%59O70;z*|vcgwD`4263bnvf`ofBQ9Rs=u73{q6iPuzVRR*+8DP$h=)z$NGoK; zL+gz$Uf0o=%EQAWJT(57@gN`C7`H%(hpo4`IMNEs=0SbfCjHPheT4M*eMKL+0{1J) z`S-0Y;md%6u zuxW6p%HT1@F+Q-Pn|!nqwbwkGA-)(s%uQ8CveTDMM263c4i@RmtI~zaa;Z#>*U^{C!=ocS?7i1`kPmH)TOh>47ICB%vf`or3oc$a(3i@?V(yAV#(Fi@tFc~<_3ErwXT3V>)mg93dUe*TvtFI`>a161 zy>nRa9M(IB_0D0vb6D>j);ovw&SAZCSnnLxJH~p)Snn9?9b>&?taps{j} z$5`)N);pK=&SkxGS?^rdJD2s&WxaD*?_Ab9m-WtLz4KV_Jk~pp_0D6x^H}dZ);o{& z&SSmvSg*l)4c2S0UW4@-tk+<@2J1Cgufci^);pi|&S$;zS?_$-JD>H=XT9@T?|jxf zpY_gn>s8^S3Ln*qTd!Ji>s2dmy=uj+SFO19suj0hwc^&RR#*={s_;>Tk1Bjr;iC#4 zRrsjFM-@J*@KJ@2DtuJoqY58Y_^8516+Wu)QH75xd{p713LjPYsKQ4TKC19hg^wzH zRNTk1Bjr;iC#4RrsjFM-@J*@KJ@2DtuJoqY58Y_^8516+Wu) zQH75xd{p713LjPYsKQ4TKC19hg^wzHRNTk1Bjr;iC#4RrsjF zM-@J*@KJ@2DtuJoqY58Y_^8516+Wu)QH75xd{p713LjPYsKQ4TKC19hgO3_~)Zn8A zA2s->!AA`~YVc8mj~aZ`;G+f~HTbB(M-4t|@KJ-08hq5?qXr)}_^8204L)k{QG<^f zeAM8h1|K!}sKG}KK5Fn$gO3_~)Zn8AA2s->!AA`~YVc8mj~aZ`;G+f~HTbB(M-4t| z@KJ-08hq5?qXr)}_^8204L)k{QG<^feAM8h1|K!}sKG}KK5Fn$gO3_~)Zn8AA2s-> z!AA`~YVc8mj~aZ`;G+f~HTbB(M-4t|@KJ-08hq5?qXr)}_^8204L)k{QG<^feAM8h z1|K!}sKG}KK5Fn$gO3_~)Zn8AA9eVs!$%!H>hMvAk2-wR;iC>8b@-^mM;$)u@KJ}4 zI(*dOqYfW+_^8829X{&tQHPH@eAMBi4j*;+sKZAcKI-sMhmSgZ)ZwEJA9eVs!$%!H z>hMvAk2-wR;iC>8b@-^mM;$)u@KJ}4I(*dOqYfW+_^8829X{&tQHPH@eAMBi4j*;+ zsKZAcKI-sMhmSgZ)ZwEJA9eVs!$%!H>hMvAk2-wR;iC>8b@-^mM;$)u@KJ}4I(*dO zqYfW+_^8829X{&tQHPH@eAMBi4j*;+sKZAcKI-sMhmSgZ)ZwEJA9eVs!$%!H>c{!m zxV&e$;y-<^(AD4Pb1&Qfngz@PW&yK+S->n{7BCAe$pZF$i6t5Gf9Nuy?@MfbJlpR} z==*f-k2yUX=u7=R-6fIl(-l7H)?dSa#>4;6Kwi?ZLOg8$@xk!0{t2gN6Md;XJR!ov z%BPHnC56h7SSG~7&Yv6%4;$p~E%c@G@WcoYn-k;VNCfhd4jJNM_fNCsK|dc>yp}u2 z@Nw?vkMw!!g!#PVUSC13_{9Bk{(Y;2T$8$^_53=tirgAJO3vqVNoLM$8F&9%DYP#< z?YIv8xR!n|-yXj|LBAhjgMY`Pf2UOVfb+9{UaotIvu^?Wtp1sMU-sJlf}1yGzpuJ^ zhxz0(H}5g;T>Bcw_cdTXyQ!Oxn0L!=K4m^X=H`X7oymIlb@Nr`#fQ0hlX?GfZr)+O z{&Y9rWS*JfuO~Tuy0i6J=XdK%t|ozT$lDCnohEUl?I|xSuJrn% zqj)I%g^Sl!^riCfeI{W=jbD( z$L}jD$W8bhN^<^vtAX5%`lh&P@3>a~ z(eg@OV<3;Zlf=iho?n&sQk0>ARpQow?K%8$(Ig>hb{8= z4*F7gczT3~=D!*b@hC6}Ss=v2?8^tk!#4SQ7k#NbTpHnF>)(tA`P#;~1wuT`zmhEv z>cd*}Lq+-s-OK#GVhyTu)!Uw1@J_8gZo4h`VL?VJrXc;+En)&F$y+C*O}56_D5u=_3JK|ZuGZh;UFUE)Y9WW__}8^kO6Qh9iGgoo8r z+|O;PuGzn9kVkF2Dhq^o=<)N1dS=-?s1IAGAL`RbNRQuFbdj6!IfLZ<`_=|>bM9Y~ z^XpI#xgtDD&gWC-&+Di@todIquXONRYF_DLfKVS+Io1jMI*QMQc$l4WFg$FNzjx7>%EJ#tcvw5jcsPo2 zz37=kJj~BL7#?=W-xKtu^6=aU58Kx<9u|#_qxf8iha&&(OEy{deMx=TCjC%}K0E3;d=IMSZQ-Lm_zZR+4I`cip#VT6aZ+ZYe> zp^b41gm{?W`e1n2A%9QMm&(HrMR@4k&UlDNfl0^$As&jvk>s+f54+^=*==3irtBA91A~udf&**W-RAIsd*jL9S2Tk(^(L zrpOK8QF1fKVT{duPild5wWQ?kkj6y2O$8Ebf-whgI%K zyrM6)&i-(OhsoWH2l>#(xCKHy^oS#^kQEQjySRAmpf8n&mqd7&)r<%E(8jn0LOk?| zBdw4X4{H?{uU+(|^6=6K56$z82l>#(xCKHy4D|cuSz_5ds1KXb4-M%fq{r_o_K+)b zzmlAP-Hq58bBmARpQow?K%85^F<>d;ZbrvpE`eDN8LMT zBI) zLmT532=UM-j4S1BC&!^6x*HL{~_d@5dK7LEhE1MV~)Q5E*VRy^_tazBf-TYCD@$k9`553125AvanaSMca zC=*9oAuAqca92cMYJK>T2oICT8xQiKjd2TvcxXJ%#gSH6HV^8 z2HdYC=ij&5$PKAGlJo0O2e}bEO3vp~=g;e?K5R}tDdD%&yz+Vs5bDEbPq4g_*BHp- zzCwAWPaJ8_;%?b}*jmTMYZrZ~JiH;o!~Ds{gM4UX+yWsU2E>t8$cl%~6J5Oa(3i@? z8zVdvpJqJBhc?D75aMCD?&3%*WW_`GDNauxeW^UW=}I21Q8@nPmGjc^t)Fs=d*Lh& zqVcrC%5nLXVxe&P|G)gd=JEf%#^tjso;dZ?Q;$DbEVy+}J7sw3ezJ~>U;aOReUYot z-RrMWSm%B{=3a_<3jRH{i(Hv{Ejhn_Zy?tok4etYYdz#z_*Zhizc-O, objs: &mut usize) { + for name in g.datasets().unwrap_or_default() { + *objs += 1; + if let Ok(ds) = g.dataset(&name) { + let _ = black_box(ds.shape()); + let _ = black_box(ds.dtype()); + let _ = black_box(ds.attrs()); + } + } + for name in g.groups().unwrap_or_default() { + *objs += 1; + if let Ok(sub) = g.group(&name) { + walk(&sub, objs); + } + } +} + +fn bench_local_metadata(c: &mut Criterion) { + let bytes = std::fs::read(FIXTURE).unwrap(); + let (_, f) = clawhdf5_format::signature::split_user_block(&bytes).unwrap(); + let sb = Superblock::parse(f, 0).unwrap(); + let (os, ls) = (sb.offset_size, sb.length_size); + let root = ObjectHeader::parse(f, sb.root_group_address as usize, os, ls).unwrap(); + let stm = root + .messages + .iter() + .find(|m| m.msg_type == MessageType::SymbolTable) + .map(|m| SymbolTableMessage::parse(&m.data, os).unwrap()) + .unwrap(); + let nodes = collect_symbol_table_nodes(f, stm.btree_address, os, ls).unwrap(); + let headers: Vec = nodes + .iter() + .flat_map(|&a| SymbolTableNode::parse(f, a as usize, os).unwrap().entries) + .map(|e| e.object_header_address) + .collect(); + assert_eq!(headers.len(), 401); + + let mut g = c.benchmark_group("local_metadata"); + g.bench_function("object_header_parse_x401", |b| { + b.iter(|| { + for &a in &headers { + black_box(ObjectHeader::parse(f, a as usize, os, ls).unwrap()); + } + }) + }); + g.bench_function("snod_parse_all", |b| { + b.iter(|| { + for &a in &nodes { + black_box(SymbolTableNode::parse(f, a as usize, os).unwrap()); + } + }) + }); + g.bench_function("btree_v1_walk", |b| { + b.iter(|| black_box(collect_symbol_table_nodes(f, stm.btree_address, os, ls).unwrap())) + }); + let file = File::open(FIXTURE).unwrap(); + g.bench_function("facade_list_400_groups", |b| { + b.iter(|| { + let mut n = 0; + walk(&file.root(), &mut n); + assert_eq!(n, 401); + }) + }); + g.finish(); +} + +criterion_group!(benches, bench_local_metadata); +criterion_main!(benches); -- 2.54.0 From e5359354b7ce2468d4544932b244c635ecda22c7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:22:42 -0500 Subject: [PATCH 42/48] format: equivalence harness only accepts the known whole-file fallbacks The harness counted ContiguousStorageRequired from any parser as an allowed difference, so a converted module that wrongly fell back to the whole file would still pass. It now accepts the error only from the three sites that are not converted yet (dense attribute storage, a SOHM B-tree index, huge fractal-heap objects, all found through a v2 B-tree) and only in the checks that can reach them; anything else fails with the check and the site named. Checked by making LocalHeap::parse_in return the error first: the fixture and h5py runs fail ("local heap fell back to the whole file"). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/storage_equivalence.rs | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index ada6274..2f31a8b 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -11,9 +11,11 @@ //! be identical, value for value and error for error. //! //! The one allowed difference is [`FormatError::ContiguousStorageRequired`] -//! from the storage path: the structures still indexed by a v2 B-tree (dense -//! attributes, a SOHM B-tree index, huge fractal-heap objects), which fail -//! cleanly instead of reading the whole file. Those are counted. +//! from the storage path, and only from the structures still indexed by a v2 +//! B-tree (dense attributes, a SOHM B-tree index, huge fractal-heap objects; +//! see `CONTIGUOUS_REQUIRED`), which fail cleanly instead of reading the +//! whole file. Those are counted; the error from any other site or check +//! fails the harness. //! //! Milestones M2/M3 extend `check_object` with the raw-data and group //! parsers as they are converted. @@ -71,6 +73,38 @@ use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode}; const MAX_OBJECTS: usize = 1500; const MAX_HEAP_IDS: usize = 200; +/// The structures that still need the whole file in memory, because they +/// are found through a version-2 B-tree (not converted yet), and the checks +/// that can reach each of them. Anything else answering +/// [`FormatError::ContiguousStorageRequired`] is a converted parser falling +/// back to the whole file, and fails the harness. +const CONTIGUOUS_REQUIRED: &[(&str, &[&str])] = &[ + ( + "dense attribute storage (a v2 B-tree)", + &["attributes", "attributes (tolerant)"], + ), + ( + "a shared-message B-tree index", + &[ + "SOHM B-tree", + "shared message", + "fill value", + "attributes", + "attributes (tolerant)", + ], + ), + ( + "a huge fractal-heap object's B-tree", + &["heap object", "attributes", "attributes (tolerant)"], + ), +]; + +fn may_require_contiguous(check: &str, site: &str) -> bool { + CONTIGUOUS_REQUIRED + .iter() + .any(|(s, checks)| *s == site && checks.contains(&check)) +} + #[derive(Default, Debug)] struct Tally { files: usize, @@ -98,7 +132,13 @@ impl Walk<'_> { got: &Result, ) { self.tally.checks += 1; - if let Err(FormatError::ContiguousStorageRequired(_)) = got { + if let Err(FormatError::ContiguousStorageRequired(site)) = got { + assert!( + may_require_contiguous(what, site), + "{}: {what} fell back to the whole file ({site}), which only the \ + v2-B-tree-indexed structures may do", + self.name + ); self.tally.contiguous_required += 1; return; } -- 2.54.0 From 76c97f6c94405b7d00969160dfcf347becd7c38d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:31:46 -0500 Subject: [PATCH 43/48] format: bound Storage reads that hostile size fields could stretch On a backend without the file in memory, a structure read whose length comes from untrusted header fields was clamped only by the end of the file, so a crafted size made one read (and copy) of up to the rest of the file. Each such read now covers what the parser actually uses: - local heap names: read in growing pieces (64 bytes first, then 4x) up to the end of the data segment, instead of the rest of the segment per name (quadratic for a big symbol-table group); - fractal heap indirect blocks: the doubling-table geometry locates the entry covering the object, and the first read ends at that entry; only if it is unallocated does the walk read the rest of the block (it visits every entry then). One walk implementation serves both; - paged fixed/extensible array data blocks over 1 MiB: the prefix and page bitmap, then each page in use on its own (smaller blocks are still one read); - blocks under one checksum (non-paged array data blocks, extensible array index and super blocks): the bounds check that comes first (the checksum's; the page bitmap's for a super block) is made against the file length before reading (Window::check_extent), so a block claimed past the end of the file costs no read. With the checksum feature off the parser has no such first check and the old read stands. Other windows were already bounded (the superblock and object header prefixes, the fractal heap header by a u16, SOHM tables by u8/u16 counts) or are exact reads checked against the file length first. In memory nothing changes: the pieces are borrowed slices. Tests: CountingStorage over a crafted heap (16 MiB file, width and rows 0xFFFF: under 1 KiB read, 16.7 MB before), a heap segment claiming 64 MiB (one 64-byte read per short name), long names at every piece boundary, a fixed array block claimed past the end of a 16 MiB file (under 64 bytes read), and in the equivalence harness an h5py file with a 2.4 MB fixed array block and a >1 MiB extensible array block, whole and cut at 97 points: every chunk index agrees with the slice read and the largest takes 205 KB (2.4 MB and 1.2 MB when read whole). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../clawhdf5-format/src/extensible_array.rs | 68 +++-- crates/clawhdf5-format/src/fixed_array.rs | 80 ++++-- crates/clawhdf5-format/src/fractal_heap.rs | 241 +++++++++++++++--- crates/clawhdf5-format/src/local_heap.rs | 86 ++++++- crates/clawhdf5-format/src/storage.rs | 28 ++ .../tests/storage_equivalence.rs | 59 ++++- 6 files changed, 467 insertions(+), 95 deletions(-) diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index a95c124..c2f23f8 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -12,7 +12,7 @@ use alloc::{format, vec, vec::Vec}; use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; -use crate::storage::{Storage, Window, read_exact_at}; +use crate::storage::{PAGED_BLOCK_ONE_READ_MAX, Storage, Window, read_exact_at}; /// Verify the Jenkins lookup3 checksum stored immediately after /// `data[start..end]`, as every Extensible Array structure carries one. `w` @@ -373,6 +373,9 @@ fn read_data_block_elements( .checked_mul(elem_bytes) .and_then(|b| pos.checked_add(b)) .ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?; + // The checksum's bounds check comes first: make it before reading. + #[cfg(feature = "checksum")] + Window::check_extent(file, db_offset, end, 4)?; let w = Window::read(file, db_offset, end.saturating_add(4))?; verify_checksum(&w, 0, end)?; read_run(&w, pos, nelmts, start_index, &mut chunks)?; @@ -384,13 +387,26 @@ fn read_data_block_elements( // clear were never written; their slot still occupies the file, so stride // over it rather than reading zeros as addresses. let npages = nelmts.div_ceil(page); - // The whole data block in one window: every position checked below lies - // inside it (or past the end of the file). + // The whole data block in one window when it is small: every position + // checked below lies inside it (or past the end of the file). A larger + // block is read as its prefix, then each page in use on its own. let block_len = pos .saturating_add(4) .saturating_add(npages.saturating_mul(page.saturating_mul(elem_bytes).saturating_add(4))); - let w = Window::read(file, db_offset, block_len)?; - verify_checksum(&w, 0, pos)?; + let whole = if block_len <= PAGED_BLOCK_ONE_READ_MAX { + Some(Window::read(file, db_offset, block_len)?) + } else { + None + }; + let head_w; + let head = match &whole { + Some(w) => w, + None => { + head_w = Window::read(file, db_offset, pos + 4)?; + &head_w + } + }; + verify_checksum(head, 0, pos)?; pos += 4; let page_stride = page .checked_mul(elem_bytes) @@ -405,10 +421,20 @@ fn read_data_block_elements( .is_some_and(|byte| byte & (0x80 >> (bit % 8)) != 0); if initialised { let count = core::cmp::min(page, nelmts - p * page); + // `w` holds the page from `base` on (positions below are + // relative to it, and `pos` to the data block). + let page_w; + let (w, base) = match &whole { + Some(w) => (w, 0), + None => { + page_w = Window::read(file, db_offset.saturating_add(pos as u64), page_stride)?; + (&page_w, pos) + } + }; // Each page carries its own checksum, over a full page's worth of // slots even when the last one holds fewer live elements. - verify_checksum(&w, pos, pos + page * elem_bytes)?; - read_run(&w, pos, count, start_index + p * page, &mut chunks)?; + verify_checksum(w, pos - base, pos - base + page * elem_bytes)?; + read_run(w, pos - base, count, start_index + p * page, &mut chunks)?; } pos = pos .checked_add(page_stride) @@ -544,6 +570,9 @@ pub fn read_extensible_array_chunks_in( .ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?; // The whole index block in one window: every position read below is // before `ib_end`. + // The checksum's bounds check comes first: make it before reading. + #[cfg(feature = "checksum")] + Window::check_extent(file, ib_offset, ib_end, 4)?; let w = Window::read(file, ib_offset, ib_end.saturating_add(4))?; verify_checksum(&w, 0, ib_end)?; @@ -682,26 +711,25 @@ fn read_super_block( // Positions below are relative to the super block, whose bytes (up to // its checksum) are all in one window. let bitmap_start = sb_header_size; - let w = Window::read( - file, - sb_offset, - bitmap_start - .saturating_add(bitmap_bytes) - .saturating_add(ndblks.saturating_mul(os)) - .saturating_add(4), - )?; - w.ensure(bitmap_start, bitmap_bytes)?; - let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes]; - + // The bitmap's bounds check, then (with checksums) the checksum's, come + // before anything else is read from the block: make them before reading + // it, so size fields stretching it past the end of the file cost no read. + Window::check_extent(file, sb_offset, bitmap_start, bitmap_bytes)?; let mut pos = bitmap_start + bitmap_bytes; - let mut chunks = Vec::new(); - let mut global_idx = start_index; // One checksum covers the prefix, the bitmap and every data block address. let sb_end = ndblks .checked_mul(os) .and_then(|b| pos.checked_add(b)) .ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?; + #[cfg(feature = "checksum")] + Window::check_extent(file, sb_offset, sb_end, 4)?; + let w = Window::read(file, sb_offset, sb_end.saturating_add(4))?; + w.ensure(bitmap_start, bitmap_bytes)?; + let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes]; + + let mut chunks = Vec::new(); + let mut global_idx = start_index; verify_checksum(&w, 0, sb_end)?; for i in 0..ndblks { diff --git a/crates/clawhdf5-format/src/fixed_array.rs b/crates/clawhdf5-format/src/fixed_array.rs index e6be63b..711aa53 100644 --- a/crates/clawhdf5-format/src/fixed_array.rs +++ b/crates/clawhdf5-format/src/fixed_array.rs @@ -9,7 +9,7 @@ use alloc::{format, vec, vec::Vec}; use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; -use crate::storage::{Storage, Window, len_usize, read_exact_at}; +use crate::storage::{PAGED_BLOCK_ONE_READ_MAX, Storage, Window, len_usize, read_exact_at}; /// Verify the Jenkins lookup3 checksum stored immediately after /// `data[start..end]`, as every Fixed Array structure carries one. `w` is @@ -278,6 +278,9 @@ pub fn read_fixed_array_chunks_in( // then a checksum over both. One window holds all of it (or ends at // the end of the file), so its bounds checks are the whole-file ones. let end = elem_at(elements_start, num_elements)?; + // The checksum's bounds check comes first: make it before reading. + #[cfg(feature = "checksum")] + Window::check_extent(file, db_offset as u64, end - db_offset, 4)?; let w = Window::read(file, db_offset as u64, end.saturating_add(4) - db_offset)?; verify_checksum(&w, 0, end - db_offset)?; for i in 0..num_elements { @@ -310,21 +313,34 @@ pub fn read_fixed_array_chunks_in( available: file_len, }); } - // The whole data block in one window: every page slot is at most - // `page_stride` bytes, so every position checked below lies inside it - // (or past the end of the file). + // The whole data block in one window when it is small: every page slot + // is at most `page_stride` bytes, so every position checked below lies + // inside it (or past the end of the file). A larger block is read as its + // prefix and bitmap, then each page in use on its own. let block_len = (pages_start - db_offset).saturating_add(npages.saturating_mul(page_stride)); - let w = Window::read(file, db_offset as u64, block_len)?; + let whole = if block_len <= PAGED_BLOCK_ONE_READ_MAX { + Some(Window::read(file, db_offset as u64, block_len)?) + } else { + None + }; + let head_w; + let head = match &whole { + Some(w) => w, + None => { + head_w = Window::read(file, db_offset as u64, pages_start - db_offset)?; + &head_w + } + }; // The prefix and page bitmap are covered by their own checksum, and each // initialised page by one of its own. - verify_checksum(&w, 0, bitmap_start + bitmap_size - db_offset)?; + verify_checksum(head, 0, bitmap_start + bitmap_size - db_offset)?; for p in 0..npages { let page_first = p * page_nelmts; // < num_elements, cannot overflow let page_count = core::cmp::min(page_nelmts, num_elements - page_first); // Check the page-init bit (MSB-first within each byte). - let bit_byte = w.bytes[bitmap_start + p / 8 - db_offset]; + let bit_byte = head.bytes[bitmap_start + p / 8 - db_offset]; let bit_mask = 1u8 << (7 - (p % 8)); if bit_byte & bit_mask == 0 { continue; // entire page unallocated @@ -334,18 +350,21 @@ pub fn read_fixed_array_chunks_in( .checked_mul(page_stride) .and_then(|o| pages_start.checked_add(o)) .ok_or_else(stride_overflow)?; - verify_checksum( - &w, - page_off - db_offset, - elem_at(page_off, page_count)? - db_offset, - )?; + let page_end = elem_at(page_off, page_count)?; + // `w` holds the page from `base` on (positions below are relative + // to it). + let page_w; + let (w, base) = match &whole { + Some(w) => (w, db_offset), + None => { + page_w = + Window::read(file, page_off as u64, page_end.saturating_add(4) - page_off)?; + (&page_w, page_off) + } + }; + verify_checksum(w, page_off - base, page_end - base)?; for e in 0..page_count { - push_element( - &w, - page_first + e, - elem_at(page_off, e)? - db_offset, - &mut chunks, - )?; + push_element(w, page_first + e, elem_at(page_off, e)? - base, &mut chunks)?; } } @@ -949,4 +968,29 @@ mod tests { } } } + + /// A header whose element count stretches its data block (one checksum + /// over the whole block) far past the end of a 16 MiB file: the + /// checksum's bounds check fails before the block is read, with the + /// slice read's error. + #[cfg(feature = "checksum")] + #[test] + fn oversized_block_fails_before_reading() { + use crate::storage::CountingStorage; + let mut f = build_fixed_array(3, false, 10); + f.resize(16 << 20, 0); + let mut h = FixedArrayHeader::parse(&f, 0x100, 8, 8).unwrap(); + h.max_nelmts_bits = 30; + h.num_elements = 4 << 20; + let dims = [h.num_elements * 20]; + let want = read_fixed_array_chunks(&f, &h, &dims, None, &[20], 8, 8, 8); + assert!( + matches!(want, Err(FormatError::UnexpectedEof { .. })), + "{want:?}" + ); + let storage = CountingStorage::new(f); + let got = read_fixed_array_chunks_in(&storage, &h, &dims, None, &[20], 8, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + assert!(storage.bytes_read() < 64, "{} bytes", storage.bytes_read()); + } } diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index e47c2c9..e1f88ca 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -669,15 +669,20 @@ impl FractalHeapHeader { let iblock_header = 5 + offset_size as usize + block_offset_bytes; let tw = self.table_width as u64; let nrows_usize = nrows as usize; - let mut current_heap_offset = iblock_heap_offset; // Rows below max_direct_rows hold direct blocks; rows at/above hold // child indirect blocks. (NOT the FRHP "starting rows" field.) let start_indirect = self.max_direct_rows(); let max_direct_rows = nrows_usize.min(start_indirect); - // The block up to its last child entry, in one window: every - // position read below lies inside it, so its bounds checks are the + // The block up to its last child entry. The walk below reads + // entries in order and stops at the one covering the target, which + // the geometry alone locates, so the first window ends there: a + // header claiming a huge table costs a read of the entries in front + // of the target, not of the rest of the file. Only when that entry + // is unallocated (or none covers the target) does the walk go on, + // over the whole block. Either window holds what it was asked for or + // ends at the end of the file, so its bounds checks are the // whole-file ones. let direct_entry = usize::from(offset_size) + if self.filter_pipeline.is_some() { @@ -685,21 +690,129 @@ impl FractalHeapHeader { } else { 0 }; - let entries = - |rows: usize, entry: usize| rows.saturating_mul(tw as usize).saturating_mul(entry); - let block_len = iblock_header - .saturating_add(entries(max_direct_rows, direct_entry)) - .saturating_add(entries( - nrows_usize.saturating_sub(start_indirect), - usize::from(offset_size), - )); - let w = Window::read(file, iblock_addr as u64, block_len)?; + let direct_entries = max_direct_rows.saturating_mul(tw as usize); + let entries_len = |n: usize| { + n.min(direct_entries) + .saturating_mul(direct_entry) + .saturating_add( + n.saturating_sub(direct_entries) + .saturating_mul(usize::from(offset_size)), + ) + }; + let all_entries = direct_entries.saturating_add( + nrows_usize + .saturating_sub(start_indirect) + .saturating_mul(tw as usize), + ); + let block_len = iblock_header.saturating_add(entries_len(all_entries)); + let target_entry = self.indirect_entry_for(nrows_usize, iblock_heap_offset, target_offset); + let first_len = target_entry.map_or(block_len, |i| { + iblock_header + .saturating_add(entries_len(i.saturating_add(1))) + .min(block_len) + }); + let mut next = self.walk_indirect_block( + &Window::read(file, iblock_addr as u64, first_len)?, + nrows_usize, + iblock_heap_offset, + target_offset, + offset_size, + target_entry.map_or(usize::MAX, |i| i.saturating_add(1)), + )?; + if next.is_none() && first_len < block_len { + next = self.walk_indirect_block( + &Window::read(file, iblock_addr as u64, block_len)?, + nrows_usize, + iblock_heap_offset, + target_offset, + offset_size, + usize::MAX, + )?; + } + match next { + Some(IndirectChild::Direct(block)) => { + self.read_from_direct_block(file, block, target_offset, length) + } + Some(IndirectChild::Indirect { + addr, + nrows, + heap_offset, + }) => self.read_from_indirect_block( + file, + addr, + nrows, + heap_offset, + target_offset, + length, + offset_size, + depth_remaining - 1, + ), + None => Err(FormatError::UnexpectedEof { + expected: target_offset as usize + length, + available: len_usize(file), + }), + } + } + + /// Which child entry of an indirect block (numbered in walk order: + /// direct rows, then indirect rows) covers `target_offset`, from the + /// doubling-table geometry alone — the entry + /// [`Self::walk_indirect_block`] stops at if it is allocated. `None` + /// when no entry does. + fn indirect_entry_for(&self, nrows: usize, heap_offset: u64, target: u64) -> Option { + // The walk adds block sizes with saturation; in u128 the same test + // is `cur <= target < cur + size` without it (a target of u64::MAX + // is never inside a saturated range). + if target == u64::MAX { + return None; + } + let (tw, target) = (u128::from(self.table_width), u128::from(target)); + let mut cur = u128::from(heap_offset); + let mut before = 0usize; + for row in 0..nrows { + if target < cur { + return None; + } + // Direct and indirect rows alike span this row's block size per + // entry. + let size = u128::from(self.block_size_for_row(row)); + let span = size * tw; + if size > 0 && target < cur + span { + let col = usize::try_from((target - cur) / size).ok()?; + return before.checked_add(col); + } + cur += span; + before = before.saturating_add(self.table_width as usize); + } + None + } + + /// Walk an indirect block's child entries in order, in the window `w` + /// (the block from its signature on), and return the allocated child + /// covering `target_offset`, or `None` when no entry among the first + /// `limit` does. + fn walk_indirect_block( + &self, + w: &Window<'_>, + nrows: usize, + iblock_heap_offset: u64, + target_offset: u64, + offset_size: u8, + limit: usize, + ) -> Result, FormatError> { let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed); let read_offset = |_: &[u8], pos: usize, size: u8| { w.ensure(pos, usize::from(size))?; read_offset(&w.bytes, pos, size) }; let file_data: &[u8] = &w.bytes; + let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8); + let iblock_header = 5 + offset_size as usize + block_offset_bytes; + let tw = self.table_width as u64; + let mut current_heap_offset = iblock_heap_offset; + let start_indirect = self.max_direct_rows(); + let max_direct_rows = nrows.min(start_indirect); + let mut walked = 0usize; // Parse indirect block header ensure_len(file_data, 0, 4)?; @@ -712,6 +825,10 @@ impl FractalHeapHeader { let block_size = self.block_size_for_row(row); for _col in 0..tw { + if walked == limit { + return Ok(None); + } + walked += 1; let child_addr = read_offset(file_data, pos, offset_size)?; pos += offset_size as usize; @@ -738,18 +855,13 @@ impl FractalHeapHeader { && target_offset >= current_heap_offset && target_offset < block_end { - return self.read_from_direct_block( - file, - DirectBlock { - addr: child_addr as usize, - size: block_size, - heap_offset: current_heap_offset, - filtered_size, - filter_mask, - }, - target_offset, - length, - ); + return Ok(Some(IndirectChild::Direct(DirectBlock { + addr: child_addr as usize, + size: block_size, + heap_offset: current_heap_offset, + filtered_size, + filter_mask, + }))); } current_heap_offset = block_end; } @@ -758,11 +870,15 @@ impl FractalHeapHeader { // Rows at and above `start_indirect` hold child indirect blocks. A // child in row r spans exactly that row's block size of heap space, // so it has as many rows as a table of that total size needs. - for row in start_indirect..nrows_usize { + for row in start_indirect..nrows { let child_space = self.block_size_for_row(row); let child_nrows = self.rows_for_size(child_space); for _col in 0..tw { + if walked == limit { + return Ok(None); + } + walked += 1; let child_addr = read_offset(file_data, pos, offset_size)?; pos += offset_size as usize; @@ -771,25 +887,16 @@ impl FractalHeapHeader { && target_offset >= current_heap_offset && target_offset < block_end { - return self.read_from_indirect_block( - file, - child_addr as usize, - child_nrows, - current_heap_offset, - target_offset, - length, - offset_size, - depth_remaining - 1, - ); + return Ok(Some(IndirectChild::Indirect { + addr: child_addr as usize, + nrows: child_nrows, + heap_offset: current_heap_offset, + })); } current_heap_offset = block_end; } } - - Err(FormatError::UnexpectedEof { - expected: target_offset as usize + length, - available: len_usize(file), - }) + Ok(None) } /// Number of rows in the doubling table whose block size is at most the @@ -833,6 +940,16 @@ impl FractalHeapHeader { /// A managed direct block's location, extent and (for a filtered heap) its /// stored size and filter mask. +/// The child of an indirect block that covers a heap offset. +enum IndirectChild { + Direct(DirectBlock), + Indirect { + addr: usize, + nrows: u16, + heap_offset: u64, + }, +} + struct DirectBlock { addr: usize, size: u64, @@ -1151,6 +1268,50 @@ mod tests { } } + /// A header claiming a huge doubling table (width 0xFFFF, 0xFFFF rows in + /// the root indirect block) in a 16 MiB file: reading an object from the + /// table's first block reads the entries up to it, not the rest of the + /// file, and gives what the slice read gives. When the covering entry is + /// unallocated the walk goes on over the whole block, still identically. + #[test] + fn huge_table_claims_read_only_what_the_walk_needs() { + use crate::storage::CountingStorage; + let (mut file, _) = build_simple_heap(8, 8); + file.resize(16 << 20, 0); + file[600..604].copy_from_slice(b"FHIB"); + let first_entry = 600 + 5 + 8 + 2; + file[first_entry..first_entry + 8].copy_from_slice(&256u64.to_le_bytes()); + let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap(); + hdr.table_width = 0xFFFF; + hdr.root_block_address = 600; + hdr.current_rows_in_root_indirect_block = 0xFFFF; + let managed_id = |offset: u64, len: u64| { + let payload = offset | (len << 16); + let mut id = vec![0u8]; + id.extend_from_slice(&payload.to_le_bytes()[..6]); + id + }; + let storage = CountingStorage::new(file.clone()); + let id = managed_id(15, 13); + let want = hdr.read_managed_object(&file, &id, 8); + assert!(want.is_ok(), "{want:?}"); + storage.reset(); + assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want); + assert!( + storage.bytes_read() < 1024, + "{} bytes in {} reads", + storage.bytes_read(), + storage.reads() + ); + // The second entry (heap offsets 128..256) is unallocated (zero is + // not the undefined address, so make it all ones). + file[first_entry + 8..first_entry + 16].fill(0xFF); + let storage = CountingStorage::new(file.clone()); + let id = managed_id(130, 4); + let want = hdr.read_managed_object(&file, &id, 8); + assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want); + } + /// A huge object found through the huge-object B-tree needs the whole /// file in memory until the B-tree reader is converted: a clean error /// on other storage. diff --git a/crates/clawhdf5-format/src/local_heap.rs b/crates/clawhdf5-format/src/local_heap.rs index 952a9d4..1215562 100644 --- a/crates/clawhdf5-format/src/local_heap.rs +++ b/crates/clawhdf5-format/src/local_heap.rs @@ -36,6 +36,10 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { }) } +/// First read of a name on a backend without the file in memory: most link +/// names are shorter than this. +const NAME_READ_START: usize = 64; + impl LocalHeap { /// Parse a local heap header at the given offset in the file data. pub fn parse( @@ -152,8 +156,9 @@ impl LocalHeap { self.read_string_in(file_data, string_offset) } - /// [`Self::read_string`] over any [`Storage`]: one read, from the - /// string to the end of the data segment. + /// [`Self::read_string`] over any [`Storage`]: one read of up to 64 + /// bytes for a short name, more (each four times the last) up to the end + /// of the data segment for a longer one. pub fn read_string_in( &self, file: &S, @@ -180,19 +185,33 @@ impl LocalHeap { }); } - // Find null terminator + // Find the null terminator, which lies before the end of the data + // segment (or of the file). In memory that is one borrowed slice; + // otherwise the bytes are read in growing pieces, so a name costs a + // read of about its own length, not of the rest of the segment + // (whose size is an untrusted header field). let search_end = seg_end.min(file_len); - let rest = read_exact_at(file, str_start as u64, search_end - str_start)?; - let Some(len) = rest.iter().position(|&b| b == 0) else { - return Err(FormatError::UnexpectedEof { - expected: search_end + 1, - available: search_end, - }); + let total = search_end - str_start; + let mut want = if file.as_contiguous().is_some() { + total + } else { + total.min(NAME_READ_START) }; - - let s = core::str::from_utf8(&rest[..len]) - .map_err(|_| FormatError::InvalidLocalHeapSignature)?; - Ok(String::from(s)) + loop { + let rest = read_exact_at(file, str_start as u64, want)?; + if let Some(len) = rest.iter().position(|&b| b == 0) { + let s = core::str::from_utf8(&rest[..len]) + .map_err(|_| FormatError::InvalidLocalHeapSignature)?; + return Ok(String::from(s)); + } + if want == total { + return Err(FormatError::UnexpectedEof { + expected: search_end + 1, + available: search_end, + }); + } + want = want.saturating_mul(4).min(total); + } } } @@ -393,4 +412,45 @@ mod tests { } } } + + /// Names of every length around the first read's size, and one with no + /// terminator, read identically through a `read_at`-only storage; a + /// short name in a heap whose header claims a huge data segment costs + /// one small read, not a read of the rest of the file. + #[test] + fn long_names_and_hostile_segment_sizes() { + use crate::storage::CountingStorage; + let names: Vec = [0usize, 1, 63, 64, 65, 255, 256, 257, 1000, 5000] + .iter() + .map(|&n| "n".repeat(n)) + .collect(); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + let mut file = build_heap_file(0, 64, &refs, 8, 8); + let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); + let storage = CountingStorage::new(file.clone()); + let mut off = 0u64; + for name in &names { + let got = heap.read_string_in(&storage, off); + assert_eq!(got, heap.read_string(&file, off)); + assert_eq!(got.unwrap(), *name); + off += name.len() as u64 + 1; + } + // The last name loses its terminator: both report the same error. + let seg_end = 64 + heap.data_segment_size as usize; + file[seg_end - 1] = b'n'; + let storage = CountingStorage::new(file.clone()); + let last = off - names[names.len() - 1].len() as u64 - 1; + let want = heap.read_string(&file, last); + assert!(want.is_err()); + assert_eq!(heap.read_string_in(&storage, last), want); + + // A 64 MiB file whose heap claims a data segment reaching its end. + let mut big = build_heap_file(0, 64, &["short", "names"], 8, 8); + big.resize(64 << 20, 0); + big[8..16].copy_from_slice(&((64u64 << 20) - 64).to_le_bytes()); + let heap = LocalHeap::parse(&big, 0, 8, 8).unwrap(); + let storage = CountingStorage::new(big.clone()); + assert_eq!(heap.read_string_in(&storage, 6).unwrap(), "names"); + assert_eq!((storage.reads(), storage.bytes_read()), (1, 64)); + } } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 6d09e7e..555ac78 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -235,6 +235,12 @@ fn short_read() -> FormatError { ) } +/// Largest paged data block (fixed or extensible array) read in one piece. +/// A bigger one is read as its prefix and then page by page, only the pages +/// in use, so a block whose size fields claim more than the file holds +/// costs no more than the pages it really has. +pub(crate) const PAGED_BLOCK_ONE_READ_MAX: usize = 1 << 20; + /// A window of the file: up to `max` bytes read at `base`, fewer only at /// the end of the file. Its [`Window::ensure`] reports a bounds failure /// exactly as the whole-file check `ensure_len(file_data, base + rel, n)` @@ -272,6 +278,28 @@ impl<'a> Window<'a> { } } + /// [`Window::ensure`] for a window at `base` that has not been read: + /// whether `[rel, rel + needed)` lies in the file, with the same error. + /// Lets a parser whose first step is to check a structure's whole extent + /// (a checksum at its end) fail before reading a structure that a + /// hostile size field has stretched past the end of the file. + pub fn check_extent( + file: &S, + base: u64, + rel: usize, + needed: usize, + ) -> Result<(), FormatError> { + let base = usize::try_from(base).unwrap_or(usize::MAX); + let file_len = len_usize(file); + match base.checked_add(rel).and_then(|p| p.checked_add(needed)) { + Some(end) if end <= file_len => Ok(()), + _ => Err(FormatError::UnexpectedEof { + expected: base.saturating_add(rel).saturating_add(needed), + available: file_len, + }), + } + } + /// Check that `[rel, rel + needed)` (relative to `base`) is in the file. #[inline] pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> { diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs index 2f31a8b..5cc71b9 100644 --- a/crates/clawhdf5-format/tests/storage_equivalence.rs +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -113,6 +113,10 @@ struct Tally { contiguous_required: usize, reads: u64, bytes: u64, + /// Chunk indexes (fixed and extensible arrays) read, and the most bytes + /// one of them took through the storage. + chunk_indexes: usize, + max_chunk_index_bytes: u64, } struct Walk<'a> { @@ -152,6 +156,12 @@ impl Walk<'_> { ); } + fn index_read(&mut self, bytes_before: u64) { + self.tally.chunk_indexes += 1; + let bytes = self.storage.bytes_read() - bytes_before; + self.tally.max_chunk_index_bytes = self.tally.max_chunk_index_bytes.max(bytes); + } + fn st(&self) -> &dyn Storage { self.storage } @@ -403,6 +413,7 @@ impl Walk<'_> { let Ok(h) = h else { return }; let want = read_fixed_array_chunks(slice, &h, &ds.dimensions, max, dims, es, os, ls); + let before = self.storage.bytes_read(); let got = read_fixed_array_chunks_in( self.st(), &h, @@ -413,6 +424,7 @@ impl Walk<'_> { os, ls, ); + self.index_read(before); self.same("fixed array chunks", &want, &got); } else { let h = ExtensibleArrayHeader::parse(slice, *addr as usize, os, ls); @@ -429,6 +441,7 @@ impl Walk<'_> { os, ls, ); + let before = self.storage.bytes_read(); let got = read_extensible_array_chunks_in( self.st(), &h, @@ -439,6 +452,7 @@ impl Walk<'_> { os, ls, ); + self.index_read(before); self.same("extensible array chunks", &want, &got); } } @@ -451,7 +465,11 @@ fn check_file(path: &Path, tally: &mut Tally) { let Ok(bytes) = std::fs::read(path) else { return; }; - let Ok((_, hdf5)) = split_user_block(&bytes) else { + check_bytes(&path.display().to_string(), &bytes, tally); +} + +fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) { + let Ok((_, hdf5)) = split_user_block(bytes) else { return; }; let storage = CountingStorage::new(hdf5.to_vec()); @@ -459,7 +477,7 @@ fn check_file(path: &Path, tally: &mut Tally) { let mut walk = Walk { slice: hdf5, storage: &storage, - name: path.display().to_string(), + name: name.to_string(), tally, }; walk.run(); @@ -473,7 +491,7 @@ fn check_file(path: &Path, tally: &mut Tally) { tally.checks - before.0, storage.reads(), storage.bytes_read(), - path.display() + name ); } } @@ -581,6 +599,16 @@ with h5py.File(p('v1_groups.h5'), 'w', libver='earliest', userblock_size=512) as g.attrs['n'] = i f.create_dataset('x', data=np.arange(10)) +# Paged chunk indexes whose data blocks are bigger than the storage reads +# in one piece (1 MiB), with two chunks written: a fixed array of 300 000 +# chunks (a 2.4 MB data block) and an extensible array grown to 1.2e9 (its +# last data block holds 131 072 chunks: over 1 MiB). +with h5py.File(p('big_paged.h5'), 'w', libver='latest') as f: + d = f.create_dataset('fa', shape=(300000,), chunks=(1,), dtype='u1') + d[5] = 1; d[250000] = 2 + e = f.create_dataset('ea', shape=(1,), maxshape=(None,), chunks=(1,), dtype='u1') + e.resize((1200000000,)); e[10] = 1; e[1100000000] = 3 + libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*')) if libs: lib = ctypes.CDLL(libs[0]) @@ -633,7 +661,7 @@ fn h5py_files_parse_identically_through_storage() { .iter() .map(|f| f.file_name().unwrap().to_string_lossy().into_owned()) .collect(); - for want in ["ea.h5", "v1_groups.h5"] { + for want in ["ea.h5", "v1_groups.h5", "big_paged.h5"] { assert!(names.iter().any(|n| n == want), "{names:?}"); } if interop_required() { @@ -641,6 +669,29 @@ fn h5py_files_parse_identically_through_storage() { } let mut tally = Tally::default(); for f in &files { + if f.ends_with("big_paged.h5") { + // Only the pages in use are read, not the whole data blocks: + // read in one piece, the fixed array took 2.4 MB and the + // extensible array 1.2 MB (its super block's page bitmap and + // block addresses are most of what remains). + let mut big = Tally::default(); + check_file(f, &mut big); + eprintln!("big paged blocks: {big:?}"); + assert_eq!(big.chunk_indexes, 2, "{big:?}"); + assert!(big.max_chunk_index_bytes < 256 << 10, "{big:?}"); + // Truncated anywhere, the page-by-page reads still agree with + // the slice reads (errors included). + let bytes = std::fs::read(f).unwrap(); + for cut in (0..bytes.len()).step_by(bytes.len() / 97) { + check_bytes( + &format!("big_paged.h5 cut at {cut}"), + &bytes[..cut], + &mut big, + ); + } + eprintln!("big paged blocks, truncated: {big:?}"); + assert!(big.chunk_indexes > 100, "{big:?}"); + } check_file(f, &mut tally); } eprintln!("h5py files: {tally:?}"); -- 2.54.0 From 895c79a2fee29eecc1fcee4d08c6ddcef085b7cf Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:34:35 -0500 Subject: [PATCH 44/48] docs: M1 changelog after review: probe identity, speed, bounded reads - Per-file probe output is identical for 696 of 697 files, not all: cve-2025-2310.h5's error string depends on which parallel chunk decode fails first, at f2ff2c4 as on this branch. - The parser cores are generic (S: Storage + ?Sized); provisional A/B numbers against f2ff2c4, including the one bench that still shows ObjectHeader::parse slower when old and new are separate binaries. - Reads sized by untrusted fields are bounded; the harness only accepts the known whole-file fallbacks. - range-reads.md records why M1 went generic rather than &dyn. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 52 +++++++++++++++++++++++++++++++------- docs/design/range-reads.md | 14 ++++++++-- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f33cba0..203553d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,11 @@ and (with `std`) `Arc`s of a `Storage`. Slices and `Vec`s serve borrowed bytes, so parsing an in-memory file costs no copy. - **The metadata parsers read through `Storage`.** Each converted parser has - an `*_in(&dyn Storage, ..)` core, and its `&[u8]` function is now a thin - wrapper over it, so no caller changes: the superblock + an `*_in(&S, ..)` core (a `&dyn Storage` works too), + and its `&[u8]` function is now a thin wrapper over it, so no caller + changes. The wrappers compile to a `[u8]` instance of the same code, so a + structure read in memory is a bounds check and a borrowed slice, with no + indirect call and no copy. Converted: the superblock (`Superblock::parse_in`), its extension and cache image (`read_superblock_extension_in`, `cache_image_state_in`), object headers with their continuation chunks (`ObjectHeader::parse_in`), local and @@ -32,6 +35,17 @@ (a prefix, then the structure) instead of slicing the whole file; the open-ended `&file_data[addr..]` slices in these modules are gone. Bounds errors keep their values (absolute position, file length). +- Reads sized by untrusted fields are bounded by what the parser uses, so + a crafted size cannot turn one structure into a read of the rest of the + file on a range backend: local-heap names are read in growing pieces + (64 bytes first) rather than to the end of the data segment; a fractal + heap indirect block is read up to the entry covering the object (the + whole block only when that entry is unallocated); paged fixed and + extensible array data blocks over 1 MiB are read page by page, only the + pages in use; and a block under one checksum whose claimed size runs + past the end of the file fails its bounds check before any read (with + the `checksum` feature). An object header's prefix is one read (was + two). - Structures still indexed by a v2 B-tree — dense attribute storage, a SOHM B-tree index and huge fractal-heap objects found through their B-tree — are not converted yet (the @@ -42,11 +56,27 @@ - **No behaviour change**, checked three ways (2026-09-26, tank): every existing test passes unchanged; the conformance sweep (`conformance/run.sh --no-fetch`) gives a byte-identical `results.json` - and identical per-file probe output for all 697 files at `f2ff2c4` and on - this branch; and a transcript of every converted `&[u8]` function's - result over the fixtures, the conformance corpus and the h5py-written - files below (748 files, 7 603 object headers) is byte-identical between - the two builds. + at `f2ff2c4` and on this branch, and identical per-file probe output for + 696 of the 697 files — the exception, `cve-2025-2310.h5`, reports one of + two errors depending on which parallel chunk decode fails first, at + `f2ff2c4` as on this branch; and a transcript of every converted `&[u8]` + function's result over the fixtures, the conformance corpus and the + h5py-written files below (748 files, 7 603 object headers) is + byte-identical between the two builds. +- **Speed on local files** (provisional: tank was shared with other jobs; + both builds linked into one binary and timed alternately, 200 rounds; + new Criterion bench `clawhdf5/benches/local_metadata_bench.rs` over a + 400-group version-1 file, `clawhdf5-format/tests/fixtures/v1_groups_400.h5`): + against `f2ff2c4`, listing the file through the facade is 2.7% faster, + `ObjectHeader::parse` is within ±1%, symbol-table nodes and the group + B-tree walk are about 19% faster (their entry loops were tightened), + local-heap names and `resolve_group_children` 1.5–3% faster. The same + harness run on two copies of the old code differs by up to 2%. The + Criterion bench itself, old and new as separate binaries run alternately + (3 rounds), agrees except for `ObjectHeader::parse`, which it puts about + 7% slower (25.9 vs 24.1 µs for 401 headers) while the listing that + parses those headers is 5–8% faster; that one remains unexplained and is + to be rechecked on an idle machine. - New equivalence harness `clawhdf5-format/tests/storage_equivalence.rs`: every converted parser runs over the file as a slice and over `storage::CountingStorage` — a `Storage` that serves an in-memory buffer @@ -55,8 +85,12 @@ the fixtures, files h5py writes for it (extensible arrays with super blocks and paged data blocks, paged fixed arrays, large v1 and dense groups, a user block, SOHM list and B-tree indexes, dense, shared and - committed-type attributes), and with `CLAWHDF5_STORAGE_CORPUS=` a - corpus (all 653 HDF5 files of the conformance corpus pass). Milestones M2 + committed-type attributes, fixed and extensible array data blocks over + 1 MiB, whole and truncated), and with `CLAWHDF5_STORAGE_CORPUS=` a + corpus (all 653 HDF5 files of the conformance corpus pass). Only the + structures listed above as not converted may answer + `ContiguousStorageRequired`, and only in the checks that reach them; a + converted parser falling back to the whole file fails it. Milestones M2 and M3 extend it. ### Chunked full reads (2026-09-26) diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 488db01..4747897 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -261,6 +261,11 @@ every `file_data[a..b]` becomes `file.read_at(a, b - a)?`. (binary size matters for wasm); `&dyn Storage` costs one indirect call per structure read, negligible next to parsing. Hot raw-data loops keep their speed through `as_contiguous()`. + *M1 outcome:* `&dyn` was not negligible for small structures — with it, + `ObjectHeader::parse` was ~25% and a 400-group facade listing ~14% slower + than the slice code (provisional, shared machine). The cores are now + generic (`S: Storage + ?Sized`), so the `&[u8]` wrappers get a `[u8]` + instance and `dyn Storage` is one more instance, not one per backend. ### (b) A page-cache "virtual slice" @@ -392,7 +397,7 @@ fast path within benchmark noise. B-tree v1/v2, fractal heap, fixed/extensible array, symbol table, group v1/v2, shared messages, attributes, fill value, data layout — one commit each. The old `&[u8]` signature stays as a thin wrapper over the new one - (`fn parse(data: &[u8], ..) { parse_in(data as &dyn Storage, ..) }`), so + (`fn parse(data: &[u8], ..) { parse_in(data, ..) }`, generic core), so callers and the other crates don't move yet. - Replace the 5 open-ended slices and 38 `len()` checks with bounded reads. @@ -433,8 +438,13 @@ Total: roughly 6–10 engineer-weeks for M0–M4 (estimate, not measured). - `impl Storage for [u8]` returns `Cow::Borrowed` — no copy, no allocation. - Hot loops (raw-data copies, contiguous typed reads, `read_selection_native`) branch once on `as_contiguous()` and then run today's code. -- `&dyn` dispatch is per structure, not per byte; parse code keeps working on +- The parser cores are generic over `S: Storage + ?Sized`, so the in-memory + instance has no dispatch at all; a remote backend behind `&dyn Storage` + pays one indirect call per structure read. Parse code keeps working on the returned slice. +- Reads sized by untrusted fields cover what the parser uses (see + `CHANGELOG.md`, M1), so a hostile size costs no read of the rest of the + file. - Gate: `crates/clawhdf5/benches/mmap_bench.rs`, the concurrent-read benches in `clawhdf5-bench`, and the conformance run time, before and after each M1/M2 commit, on an otherwise idle machine. Anything outside noise blocks the -- 2.54.0 From 0e8522cfad34fd12ae6109abf6f94951b470d62f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:45:51 -0500 Subject: [PATCH 45/48] clawhdf5-format: the writer skips optional filters that fail, as libhdf5 does FileBuilder stored every chunk of an LZF or Blosc dataset through the filter with filter mask 0. libhdf5 counts LZF and Blosc output no smaller than the chunk as a failure of the optional filter and stores the chunk raw with the filter's mask bit set. For an LZF chunk whose stream was exactly the chunk's size, the first libhdf5 rewrite stored raw data at the same size and kept our stale mask 0 in the index, and h5py could no longer read the dataset. precompress_chunks now runs chunks through compress_chunk_masked (as FileEditor does since f7e2ab1), sequentially and on the parallel path, and build_chunked_data_from_precompressed records each chunk's real mask in every index the writer builds: single chunk (layout field), Fixed Array and Extensible Array filtered elements, and version-2 B-tree type 11 records (create_datasets_parallel goes through the same path). The writer builds no version-1 B-tree or implicit index. PrecompressedChunks::chunks gains the mask. Files whose chunks all compress are byte-identical. Latent only in the unreleased LZF/Blosc writer (added 2026-09-26); no tagged release writes either filter. Tests: - plugin_filters_interop skipped_optional_filters_are_masked_as_libhdf5_masks_them: LZF, shuffle+LZF+fletcher32 and Blosc over random, compressible and alternating chunks in every index; masks equal an h5py-written twin's; h5py r+ rewrites and extends them; h5py, h5dump and our reader read every value. Before: 20 of 24 datasets had masks other than h5py's, and with that check disabled h5py failed to read the rewritten datasets ("filter returned failure during read"). - plugin_filters_interop files_whose_chunks_all_compress_are_unchanged: pins the pre-fix bytes of five all-compressing files. - chunked_write skipped_lzf_chunks_are_masked_in_every_index (fails before: mask 0, want 2). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 31 ++ crates/clawhdf5-format/src/chunked_write.rs | 123 ++++- .../clawhdf5/tests/plugin_filters_interop.rs | 454 ++++++++++++++++++ docs/known-issues.md | 17 + 4 files changed, 613 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f959f39..eea7a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1062,6 +1062,37 @@ and fails their objects (see below). - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- **`FileBuilder` stored LZF and Blosc chunks with filter mask 0 even when + the filter had not shrunk them** (fixed 2026-09-26). Latent in the + unreleased LZF/Blosc writer only (added 2026-09-26, "Plugin filters"): + no tagged release writes LZF or Blosc, so v2.7.0 and earlier are + unaffected. libhdf5 treats LZF and Blosc output no smaller than the chunk + as a filter failure and, both being optional filters, stores such a chunk + unfiltered with the filter's mask bit set. clawhdf5 stored the filter's + output with a clear mask. For an LZF chunk whose stream was exactly the + chunk's size (h5py stores `[182, 0, 0, 0, 0]` in a 5-byte chunk raw), + the first libhdf5 rewrite of that chunk stored the new data raw at the + same size and, the size being unchanged, kept the stale mask 0: h5py + then failed to read the dataset ("filter returned failure during read"). + The whole-file writer now runs chunks through the pipeline as libhdf5 + does (`clawhdf5_format::filters::compress_chunk_masked`, as `FileEditor` + already did) and records each chunk's real mask in every chunk index it + builds (single chunk, Fixed Array, Extensible Array, version-2 B-tree; + it builds no version-1 B-tree or implicit index), in the sequential and + `parallel` paths and `create_datasets_parallel`. Files whose chunks all + compress are byte-identical to before. `PrecompressedChunks::chunks` is + now `(raw size, stored bytes, filter mask)` (**breaking** for code that + reads it). Files written before the fix read correctly; rewrite them + (with this build or `h5repack`) before modifying them with libhdf5. + Tests: `plugin_filters_interop` + `skipped_optional_filters_are_masked_as_libhdf5_masks_them` (LZF, + shuffle+LZF+fletcher32 and Blosc, random, compressible and alternating + chunks, every index: masks equal an h5py-written twin's; after h5py r+ + rewrites and extends the datasets, h5py, h5dump and our reader read every + value — before the fix 20 of 24 datasets had other masks than h5py's, + and h5py could not read the rewritten `[x, 0, 0, 0, 0]` datasets) and + `files_whose_chunks_all_compress_are_unchanged`; `chunked_write` + `skipped_lzf_chunks_are_masked_in_every_index`. - **Scale-offset data read wrong values in every release that decoded it (v2.2.0 to v2.7.0), silently, on ordinary h5py files** (fixed 2026-09-26). Of 1480 scale-offset datasets h5py writes across every diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 2a8bd09..b93142d 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -17,7 +17,7 @@ use crate::filter_pipeline::{ FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; -use crate::filters::compress_chunk; +use crate::filters::compress_chunk_masked; /// Round a file offset up to the next cache-line boundary. /// /// This ensures chunk data starts at an address that is a multiple of the @@ -489,7 +489,12 @@ pub fn split_into_chunks( #[cfg(feature = "parallel")] const PARALLEL_COMPRESS_THRESHOLD: usize = 2; -/// Compress all chunks, using parallel compression when beneficial. +/// Compress all chunks, using parallel compression when beneficial, and +/// return each chunk's stored bytes with its filter mask. +/// +/// Chunks run through the pipeline as libhdf5 runs them +/// ([`compress_chunk_masked`]): an optional filter that fails — LZF or Blosc +/// output no smaller than its input — is skipped and its mask bit set. /// /// With the `parallel` feature and more than [`PARALLEL_COMPRESS_THRESHOLD`] /// filtered chunks, compression runs across rayon threads; otherwise it is @@ -499,7 +504,7 @@ fn compress_all_chunks( chunks: &[(Vec, Vec)], pipeline: &Option, element_size: u32, -) -> Result>, FormatError> { +) -> Result, u32)>, FormatError> { #[cfg(feature = "parallel")] { if let Some(pl) = pipeline @@ -508,7 +513,7 @@ fn compress_all_chunks( use rayon::prelude::*; return chunks .par_iter() - .map(|(_offsets, chunk_bytes)| compress_chunk(chunk_bytes, pl, element_size)) + .map(|(_offsets, chunk_bytes)| compress_chunk_masked(chunk_bytes, pl, element_size)) .collect(); } } @@ -518,9 +523,9 @@ fn compress_all_chunks( .iter() .map(|(_offsets, chunk_bytes)| { if let Some(pl) = pipeline { - compress_chunk(chunk_bytes, pl, element_size) + compress_chunk_masked(chunk_bytes, pl, element_size) } else { - Ok(chunk_bytes.clone()) + Ok((chunk_bytes.clone(), 0)) } }) .collect() @@ -798,8 +803,10 @@ pub fn build_fixed_array_at( /// writer passes eliminates the double-compression that the two-pass layout /// algorithm previously performed. pub struct PrecompressedChunks { - /// Per-chunk: (raw_size_bytes, compressed_bytes). - pub chunks: Vec<(u64, Vec)>, + /// Per-chunk: (raw_size_bytes, stored_bytes, filter_mask). Bit `i` of + /// the mask is set when filter `i` was skipped (an optional filter that + /// failed); 0 for every chunk of an unfiltered dataset. + pub chunks: Vec<(u64, Vec, u32)>, pub has_filters: bool, pub element_size: usize, pub shape: Vec, @@ -834,7 +841,7 @@ pub fn precompress_chunks( let chunks = raw_chunks .into_iter() .zip(compressed) - .map(|((_offsets, raw_bytes), c)| (raw_bytes.len() as u64, c)) + .map(|((_offsets, raw_bytes), (c, mask))| (raw_bytes.len() as u64, c, mask)) .collect(); Ok(PrecompressedChunks { @@ -867,7 +874,7 @@ pub fn build_chunked_data_from_precompressed( let mut data_buf = Vec::new(); let mut written_chunks = Vec::with_capacity(num_chunks); - for (raw_size, compressed) in &pre.chunks { + for (raw_size, compressed, filter_mask) in &pre.chunks { let aligned_offset = align_to_cache_line(data_buf.len()); if aligned_offset > data_buf.len() { data_buf.resize(aligned_offset, 0u8); @@ -879,7 +886,7 @@ pub fn build_chunked_data_from_precompressed( address, compressed_size, raw_size: *raw_size, - filter_mask: 0, + filter_mask: *filter_mask, }); } @@ -916,7 +923,7 @@ pub fn build_chunked_data_from_precompressed( } else { None }; - let filter_mask = if pre.has_filters { Some(0u32) } else { None }; + let filter_mask = pre.has_filters.then_some(written_chunks[0].filter_mask); serialize_v4_single_chunk( &chunk_dims_u32, chunk_addr, @@ -1943,6 +1950,98 @@ mod tests { bytes_to_f64(&output) } + /// Every chunk index the writer builds records each chunk's real filter + /// mask: LZF output no smaller than the chunk is skipped (bit 1, behind + /// shuffle) and the chunk stored shuffled only; compressible chunks keep + /// mask 0. The data reads back through both kinds of chunk. + #[cfg(feature = "lzf")] + #[test] + fn skipped_lzf_chunks_are_masked_in_every_index() { + let c = 64usize; + // Chunks alternate: random bytes (LZF cannot shrink them), then 7s. + let mut state = 0x1234_5678_u64; + let data: Vec = (0..4 * c) + .map(|i| { + if (i / c).is_multiple_of(2) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + f64::from_bits(state) + } else { + 7.0 + } + }) + .collect(); + let raw = f64_to_bytes(&data); + let options = ChunkOptions { + plugin: Some(PluginFilter::Lzf), + ..Default::default() + }; + let c64 = c as u64; + #[allow(clippy::type_complexity)] + let cases: [(&[u64], &[u64], Option<&[u64]>, u8, &[u32]); 4] = [ + (&[c64], &[c64], None, 1, &[2]), + (&[4 * c64], &[c64], None, 3, &[2, 0, 2, 0]), + (&[4 * c64], &[c64], Some(&[u64::MAX]), 4, &[2, 0, 2, 0]), + ( + &[2, 2 * c64], + &[1, c64], + Some(&[u64::MAX, u64::MAX]), + 5, + &[2, 0, 2, 0], + ), + ]; + let base = 0x1000u64; + for (shape, chunks, maxshape, index_type, want_masks) in cases { + let n: u64 = shape.iter().product(); + let raw = &raw[..n as usize * 8]; + let result = + build_chunked_data_at_ext(raw, shape, chunks, 8, &options, base, maxshape).unwrap(); + let mut file = vec![0u8; base as usize]; + file.extend_from_slice(&result.data_bytes); + let layout = DataLayout::parse(&result.layout_message, 8, 8).unwrap(); + assert!( + matches!(&layout, DataLayout::Chunked { chunk_index_type, .. } + if *chunk_index_type == Some(index_type)), + "{layout:?}" + ); + let dataspace = Dataspace { + space_type: DataspaceType::Simple, + rank: shape.len() as u8, + dimensions: shape.to_vec(), + max_dimensions: maxshape.map(<[u64]>::to_vec), + }; + let (mut infos, _) = + crate::chunked_read::list_chunks(&file, &layout, &dataspace, 8, 8, 8).unwrap(); + infos.sort_by(|a, b| a.offsets.cmp(&b.offsets)); + let masks: Vec = infos.iter().map(|i| i.filter_mask).collect(); + assert_eq!(masks, want_masks, "index type {index_type}"); + for info in &infos { + // Skipped chunks are stored at the chunk's size (shuffled). + assert_eq!( + info.chunk_size == (c * 8) as u32, + info.filter_mask != 0, + "{info:?}" + ); + } + let pipeline = crate::filter_pipeline::FilterPipeline::parse( + result.pipeline_message.as_ref().unwrap(), + ) + .unwrap(); + let out = read_chunked_data( + &file, + &layout, + &dataspace, + &make_f64_type(), + Some(&pipeline), + 8, + 8, + ) + .unwrap(); + assert_eq!(out, raw, "index type {index_type}"); + } + } + #[test] fn ea_roundtrip_1d_inline_only() { let values: Vec = (0..10).map(|i| i as f64).collect(); diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 00911c6..ffe2fb1 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -586,3 +586,457 @@ with h5py.File(sys.argv[1], 'w') as f: let want: Vec = (0..16).collect(); assert_eq!(file.dataset("lzf_ok").unwrap().read_i32().unwrap(), want); } + +/// A family of datasets for the filter-mask tests: element type, chunk +/// length along the last dimension, the h5py `create_dataset` keywords of +/// the same filters, and how chunk `k` is filled. +#[cfg(feature = "lzf")] +struct MaskFamily { + name: &'static str, + /// 1 (`u1`) or 4 (` Vec<(&'static str, Vec, Vec, Option>)> { + vec![ + ("single", vec![c], vec![c], None), + ("fixed", vec![4 * c], vec![c], None), + ("ea", vec![4 * c], vec![c], Some(vec![u64::MAX])), + ( + "bt2", + vec![2, 2 * c], + vec![1, c], + Some(vec![u64::MAX, u64::MAX]), + ), + ] +} + +/// Raw little-endian bytes of the dataset `fam` fills over `shape`. +#[cfg(feature = "lzf")] +fn mask_data(fam: &MaskFamily, shape: &[u64], seed: u64) -> Vec { + let c = fam.chunk as usize; + let cols = *shape.last().unwrap() as usize; + let n: usize = shape.iter().product::() as usize; + let chunks_per_row = cols.div_ceil(c); + let mut state = seed; + let mut noise = move || { + state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + }; + let mut out = Vec::with_capacity(n * fam.elem); + for i in 0..n { + let (row, col) = (i / cols, i % cols); + let k = row * chunks_per_row + col / c; + let v: u64 = match fam.fill { + MaskFill::FiveBytes if col % c == 0 => 182 - k as u64, + MaskFill::FiveBytes => 0, + MaskFill::Alternating if k.is_multiple_of(2) => noise(), + MaskFill::Alternating | MaskFill::Compressible => 7, + }; + out.extend_from_slice(&v.to_le_bytes()[..fam.elem]); + } + out +} + +#[cfg(feature = "lzf")] +fn mask_families() -> Vec { + #[cfg_attr(not(feature = "blosc"), allow(unused_mut))] + let mut v = vec![ + MaskFamily { + name: "lzf5", + elem: 1, + chunk: 5, + h5py_kw: "dict(compression='lzf')", + build: |d| { + d.with_lzf().without_shuffle(); + }, + fill: MaskFill::FiveBytes, + }, + MaskFamily { + name: "lzf", + elem: 4, + chunk: 64, + h5py_kw: "dict(compression='lzf')", + build: |d| { + d.with_lzf().without_shuffle(); + }, + fill: MaskFill::Alternating, + }, + MaskFamily { + name: "mix", + elem: 4, + chunk: 8, + h5py_kw: "dict(compression='lzf', shuffle=True, fletcher32=True)", + build: |d| { + d.with_lzf().with_shuffle().with_fletcher32(); + }, + fill: MaskFill::Alternating, + }, + MaskFamily { + name: "lzfc", + elem: 4, + chunk: 64, + h5py_kw: "dict(compression='lzf')", + build: |d| { + d.with_lzf().without_shuffle(); + }, + fill: MaskFill::Compressible, + }, + ]; + #[cfg(feature = "blosc")] + { + use clawhdf5_format::chunked_write::{BloscCodec, BloscShuffle}; + v.push(MaskFamily { + name: "blosc", + elem: 4, + chunk: 64, + h5py_kw: "hdf5plugin.Blosc(cname='lz4', clevel=5, shuffle=hdf5plugin.Blosc.SHUFFLE)", + build: |d| { + d.with_blosc(BloscCodec::Lz4, 5, BloscShuffle::Byte); + }, + fill: MaskFill::Alternating, + }); + v.push(MaskFamily { + name: "blosc0", + elem: 4, + chunk: 64, + h5py_kw: "hdf5plugin.Blosc(cname='lz4', clevel=0, shuffle=hdf5plugin.Blosc.SHUFFLE)", + build: |d| { + d.with_blosc(BloscCodec::Lz4, 0, BloscShuffle::Byte); + }, + fill: MaskFill::Compressible, + }); + } + v +} + +/// Builds h5py twins of our datasets and prints, per dataset, the filter +/// masks by chunk offset in our file and in the twin. +const MASK_TWIN: &str = r#" +import sys, numpy as np, h5py +try: + import hdf5plugin +except ImportError: + hdf5plugin = None +ours, twin, spec = sys.argv[1], sys.argv[2], eval(sys.argv[3]) +def masks(ds): + return sorted((tuple(ds.id.get_chunk_info(k).chunk_offset), ds.id.get_chunk_info(k).filter_mask) + for k in range(ds.id.get_num_chunks())) +with h5py.File(ours, 'r') as o, h5py.File(twin, 'w', libver='v114') as t: + for name, dt, shape, chunks, maxshape, kw, raw in spec: + want = np.fromfile(raw, dtype=dt).reshape(shape) + assert np.array_equal(o[name][()], want), name + t.create_dataset(name, data=want, chunks=chunks, maxshape=maxshape, **eval(kw)) + print(name, masks(o[name]), '|', masks(t[name])) +"#; + +/// h5py (libhdf5) rewrites every chunk of our datasets — random chunks +/// become compressible and the other way round; the `[x,0,0,0,0]` chunks +/// change in place at the same size — then extends the resizable ones with +/// random data, and saves what each dataset must now hold. Prints the +/// datasets h5dump can decode (no chunk left LZF-encoded: h5dump has no +/// LZF filter). +const MASK_REWRITE: &str = r#" +import sys, numpy as np, h5py +try: + import hdf5plugin +except ImportError: + hdf5plugin = None +ours, spec = sys.argv[1], eval(sys.argv[2]) +rng = np.random.default_rng(3) +def noise(shape, dt): + return rng.integers(0, 256, int(np.prod(shape)) * np.dtype(dt).itemsize, + dtype=np.uint8).view(dt).reshape(shape) +dumpable = [] +with h5py.File(ours, 'r+') as f: + for name, dt, shape, chunks, maxshape, kw, raw in spec: + d = f[name] + want = d[()] + for s in d.iter_chunks(): + blk = want[s] + if dt == 'u1': + blk.flat[-1] = 1 + elif (blk == blk.flat[0]).all(): + blk[...] = noise(blk.shape, dt) + else: + blk[...] = 5 + d[...] = want + if maxshape is not None: + new = tuple(n + c for n, c in zip(shape, chunks)) + grown = noise(new, dt) + grown[tuple(slice(0, n) for n in shape)] = want + d.resize(new) + d[...] = grown + want = grown + want.tofile(raw + '.want') + pl = d.id.get_create_plist() + ids = [pl.get_filter(i)[0] for i in range(pl.get_nfilters())] + if 32000 in ids: + bit = 1 << ids.index(32000) + if not all(d.id.get_chunk_info(k).filter_mask & bit for k in range(d.id.get_num_chunks())): + continue + dumpable.append(name) +with h5py.File(ours, 'r') as f: + for name, dt, shape, chunks, maxshape, kw, raw in spec: + want = np.fromfile(raw + '.want', dtype=dt).reshape(f[name].shape) + assert np.array_equal(f[name][()], want), name +print(' '.join(dumpable)) +"#; + +/// Optional filters that fail are skipped in files `FileBuilder` writes, +/// exactly as libhdf5 skips them: an LZF or Blosc output no smaller than the +/// chunk leaves the chunk stored unfiltered with the filter's mask bit set. +/// The writer used to store every chunk filtered with mask 0. For LZF, a +/// chunk whose LZF stream is exactly the chunk's size (`[x,0,0,0,0]`) was +/// then corrupted by the first libhdf5 rewrite of it: libhdf5 stores the +/// new data raw at the same size and, the size being unchanged, leaves the +/// stale mask in the index, so h5py could no longer read the dataset. +/// +/// For every family × every chunk index the writer builds (single chunk, +/// Fixed Array, Extensible Array, version-2 B-tree): our masks equal those +/// of an h5py-written twin of the same data; then h5py r+ rewrites and +/// extends the datasets, and h5py, h5dump (where it has the filter) and our +/// reader read every value. +#[cfg(feature = "lzf")] +#[test] +fn skipped_optional_filters_are_masked_as_libhdf5_masks_them() { + let modules = if cfg!(feature = "blosc") { + "h5py, numpy, hdf5plugin" + } else { + "h5py, numpy" + }; + if !have_python(modules) { + return; + } + let dir = tempfile::tempdir().unwrap(); + let ours = dir.path().join("ours.h5"); + let twin = dir.path().join("twin.h5"); + let mut fb = clawhdf5::FileBuilder::new(); + let mut spec = Vec::new(); + let mut names = Vec::new(); + for (fi, fam) in mask_families().iter().enumerate() { + for (label, shape, chunks, maxshape) in mask_layouts(fam.chunk) { + let name = format!("{}_{label}", fam.name); + let data = mask_data(fam, &shape, fi as u64 * 31 + shape.len() as u64); + let raw = dir.path().join(format!("{name}.raw")); + std::fs::write(&raw, &data).unwrap(); + let ds = fb.create_dataset(&name); + if fam.elem == 1 { + ds.with_u8_data(&data); + } else { + let v: Vec = data + .as_chunks::<4>() + .0 + .iter() + .map(|&b| i32::from_le_bytes(b)) + .collect(); + ds.with_i32_data(&v); + } + ds.with_shape(&shape).with_chunks(&chunks); + if let Some(ms) = &maxshape { + ds.with_maxshape(ms); + } + (fam.build)(ds); + let py_tuple = |v: &[u64]| { + let items: Vec = v + .iter() + .map(|&d| { + if d == u64::MAX { + "None".into() + } else { + d.to_string() + } + }) + .collect(); + format!("({},)", items.join(",")) + }; + spec.push(format!( + "({name:?}, {:?}, {}, {}, {}, {:?}, {:?})", + if fam.elem == 1 { "u1" } else { "= 12, + "too few datasets with skipped filters:\n{out}" + ); + + // libhdf5 rewrites and extends them; everyone reads the new values. + let dumpable = run_python(MASK_REWRITE, &[ours.to_str().unwrap(), &spec]); + let plugin_path = run_python("import hdf5plugin; print(hdf5plugin.PLUGIN_PATH)", &[]); + let file = File::open(&ours).unwrap(); + for (name, elem) in &names { + let want = std::fs::read(dir.path().join(format!("{name}.raw.want"))).unwrap(); + let got = file + .dataset(name) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + assert!(got == want, "{name}: our reader after h5py r+"); + if !dumpable.split(' ').any(|d| d == name) || Command::new("h5dump").output().is_err() { + continue; + } + let o = Command::new("h5dump") + .env("HDF5_PLUGIN_PATH", &plugin_path) + .args(["-d", name, "-y", "-w", "0", ours.to_str().unwrap()]) + .output() + .unwrap(); + assert!( + o.status.success(), + "h5dump -d {name}: {}", + String::from_utf8_lossy(&o.stderr) + ); + let s = String::from_utf8_lossy(&o.stdout).into_owned(); + let vals: Vec = s + .split_once("DATA {") + .and_then(|(_, r)| r.split_once('}')) + .map(|(d, _)| { + d.split(|c: char| c == ',' || c.is_whitespace()) + .filter(|t| !t.is_empty()) + .map(|t| t.parse::().unwrap()) + .collect() + }) + .unwrap_or_default(); + let want_vals: Vec = want + .chunks_exact(*elem) + .map(|b| { + if *elem == 1 { + i64::from(b[0]) + } else { + i64::from(i32::from_le_bytes(b.try_into().unwrap())) + } + }) + .collect(); + assert_eq!(vals, want_vals, "h5dump -d {name}"); + } + assert!( + dumpable.split(' ').any(|d| d.starts_with("lzf5_")), + "{dumpable}" + ); +} + +/// Files whose chunks all compress are written exactly as before optional +/// filters could be skipped: every mask is 0 and nothing else changed. The +/// hashes are of the files the writer produced before that change. +#[cfg(feature = "lzf")] +#[test] +fn files_whose_chunks_all_compress_are_unchanged() { + use clawhdf5_format::checksum::jenkins_lookup3; + #[allow(clippy::type_complexity)] + #[cfg_attr(not(feature = "blosc"), allow(unused_mut))] + let mut cases: Vec<( + &str, + fn(&mut clawhdf5_format::type_builders::DatasetBuilder), + (usize, u32), + )> = vec![ + ( + "lzf_fixed", + |d| { + d.with_i32_data(&ramp_i32(4000)) + .with_chunks(&[500]) + .with_lzf(); + }, + (3965, 449169442), + ), + ( + "lzf_ea_noshuffle", + |d| { + d.with_i32_data(&ramp_i32(4000)) + .with_chunks(&[700]) + .with_maxshape(&[u64::MAX]) + .with_lzf() + .without_shuffle(); + }, + (7213, 4277403206), + ), + ( + "mix_bt2", + |d| { + d.with_f64_data(&ramp_f64(40 * 60)) + .with_shape(&[40, 60]) + .with_chunks(&[16, 16]) + .with_maxshape(&[u64::MAX, u64::MAX]) + .with_lzf() + .with_fletcher32(); + }, + (11495, 3340532700), + ), + ( + "lzf_single", + |d| { + d.with_u8_data(&ramp_u8(3000)) + .with_chunks(&[3000]) + .with_lzf(); + }, + (546, 690805477), + ), + ]; + #[cfg(feature = "blosc")] + cases.push(( + "blosc_fixed", + |d| { + use clawhdf5_format::chunked_write::{BloscCodec, BloscShuffle}; + d.with_i32_data(&ramp_i32(5000)) + .with_chunks(&[1024]) + .with_blosc(BloscCodec::Lz4, 5, BloscShuffle::Byte); + }, + (2776, 4278611376), + )); + for (name, build, want) in &cases { + let mut fb = clawhdf5::FileBuilder::new(); + build(fb.create_dataset("d")); + let bytes = fb.finish().unwrap(); + assert_eq!( + (bytes.len(), jenkins_lookup3(&bytes)), + *want, + "{name}: (length, lookup3 hash) of the file" + ); + } +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 818a487..44a5344 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,23 @@ deleting it. --- +## LZF/Blosc chunks written with a stale filter mask + +**Status:** fixed 2026-09-26, before any release (the LZF and Blosc writers +were added the same day; v2.7.0 and earlier write neither). + +`FileBuilder` stored every chunk of an LZF or Blosc dataset through the +filter with filter mask 0. libhdf5 counts LZF and Blosc output no smaller +than the chunk as a failure of the (optional) filter and stores the chunk +raw with the filter's mask bit set. When a chunk's LZF stream was exactly +the chunk's size, the first libhdf5 rewrite of it stored raw data at the +same size and left our mask 0 in the index, so h5py could no longer read +the dataset. `FileEditor` had the same bug, fixed earlier the same day. +Both now use `clawhdf5_format::filters::compress_chunk_masked`, and every +chunk index the writer builds records the real mask (see `CHANGELOG.md`). +Files written before the fix read correctly; rewrite them before letting +libhdf5 modify them. + ## In-place modification (`FileEditor`) limits **Status:** open (documented 2026-09-26). `clawhdf5::FileEditor` refuses, -- 2.54.0 From b49ec39aff2c61268de086a4e5af4cfbac10eb32 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 15:06:18 -0500 Subject: [PATCH 46/48] docs: conformance report after range-read M0/M1, ZFP and in-place editing (600 of 697 ok) Regenerated on tank: ok 599 -> 600 (h5ex_d_zfp.h5), our-error 4 -> 3, mismatch 2, no panics, hangs, crashes or OOM. Baseline raised. Co-Authored-By: Claude Opus 5.5 (1M context) --- CONFORMANCE.md | 13 ++++++------- conformance/baseline.json | 15 ++++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 1a8e2b7..3a1f0ac 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -13,15 +13,15 @@ fatal. This file is generated by `conformance/run.sh`; do not edit it by hand. | | | |---|---| -| date | 2026-09-26 17:10 UTC | -| clawhdf5 commit | `d0e3beb3aa8290aae523ce280b4380e75484b6bc` | +| date | 2026-09-26 20:06 UTC | +| clawhdf5 commit | `8fadb9f4242a35323262701328d380806d379140` | | machine | `tank`: AMD Ryzen 7 7800X3D 8-Core Processor, 16 CPUs, 61 GiB, Linux 7.0.0-34-generic x86_64 | | command | `conformance/run.sh --no-fetch --update-baseline` | | rustc | rustc 1.98.1 (48a229cea 2026-09-01) | | reference | h5py 3.16.0, HDF5 2.0.0, numpy 2.5.3, hdf5plugin 7.1.0, Python 3.14.4 | | h5dump | Version 1.14.6 (CVE corpus only) | | limits | 20 s timeout (SIGKILL), 4096 MiB address space, per process; 16 files in parallel | -| runtime | 24 s probing + comparing (0 s fetch/build before it) | +| runtime | 21 s probing + comparing (0 s fetch/build before it) | ## Results @@ -38,16 +38,16 @@ A file's class is the first that applies: | NCAS-CMS_pyfive | 33 | 32 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | | cve_hdf5 | 147 | 113 | 2 | 0 | 32 | 0 | 0 | 0 | 0 | | h5py_data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hdf5 | 466 | 403 | 2 | 1 | 60 | 0 | 0 | 0 | 0 | +| hdf5 | 466 | 404 | 1 | 1 | 60 | 0 | 0 | 0 | 0 | | netcdf-c | 20 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | netcdf4-python | 18 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | usnistgov_h5wasm | 5 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | xarray-data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| **all** | **697** | **599** | **4** | **2** | **92** | **0** | **0** | **0** | **0** | +| **all** | **697** | **600** | **3** | **2** | **92** | **0** | **0** | **0** | **0** | 2 of the 2 mismatches are a known h5py bug, not ours (see *Known not-our-bug*). -3 of the 4 our-errors are corrupt data that HDF5 2.0 reads only through a bug and clawhdf5 refuses (see *Known not-our-bug*). +3 of the 3 our-errors are corrupt data that HDF5 2.0 reads only through a bug and clawhdf5 refuses (see *Known not-our-bug*). Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`): @@ -73,7 +73,6 @@ Grouped by normalised error message. *files* counts files whose class this cause | files | objects | error | examples | |---:|---:|---|---| | 3 | 3 | `ChunkedReadError("…")` | `cve_hdf5/cvefiles/cve-2025-2308.h5`, `cve_hdf5/cvefiles/cve-2025-44904.h5`, `hdf5/test/testfiles/bad_nbit_parms_walk.h5` | -| 1 | 1 | `UnsupportedFilter(N)` | `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zfp.h5` | ## Mismatch root causes diff --git a/conformance/baseline.json b/conformance/baseline.json index 4c496f9..420a75b 100644 --- a/conformance/baseline.json +++ b/conformance/baseline.json @@ -1,15 +1,15 @@ { "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. Regenerate with `conformance/run.sh --update-baseline` after an intended change.", - "commit": "d0e3beb3aa8290aae523ce280b4380e75484b6bc", - "date": "2026-09-26 17:10 UTC", + "commit": "8fadb9f4242a35323262701328d380806d379140", + "date": "2026-09-26 20:06 UTC", "reference": "h5py 3.16.0 / HDF5 2.0.0", "files": 697, - "ok": 599, + "ok": 600, "counts": { "h5py-cannot-read": 92, "mismatch": 2, - "ok": 599, - "our-error": 4 + "ok": 600, + "our-error": 3 }, "per_corpus": { "NCAS-CMS_pyfive": { @@ -27,8 +27,8 @@ "hdf5": { "h5py-cannot-read": 60, "mismatch": 1, - "ok": 403, - "our-error": 2 + "ok": 404, + "our-error": 1 }, "netcdf-c": { "ok": 20 @@ -202,6 +202,7 @@ "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_jpeg.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_lzf.h5", + "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zfp.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zstd.h5", "hdf5/HDF5Examples/C/H5G/16/h5ex_g_iterate.h5", "hdf5/HDF5Examples/C/H5G/16/h5ex_g_traverse.h5", -- 2.54.0 From cadd27df5b7429d362f6cab2bce72885997b9c99 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 15:15:32 -0500 Subject: [PATCH 47/48] docs: ObjectHeader::parse A/B rechecked on an idle machine +6-7% (about 4 ns per header) is real; symbol-table nodes -17%, group B-tree walk -16%, facade listing -2.4%: local metadata reads are net slightly faster than main. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba7b828..fe377f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,8 +131,15 @@ Criterion bench itself, old and new as separate binaries run alternately (3 rounds), agrees except for `ObjectHeader::parse`, which it puts about 7% slower (25.9 vs 24.1 µs for 401 headers) while the listing that - parses those headers is 5–8% faster; that one remains unexplained and is - to be rechecked on an idle machine. + parses those headers is 5–8% faster. **Rechecked on an idle tank + (2026-09-26, load 1.95 at the start, 3–4 during; Criterion, `main` + `479d8b4` vs this branch `b49ec39` as separate binaries, 2 alternating + rounds):** `ObjectHeader::parse` for 401 headers 24.0–24.1 µs → 25.5–25.8 + µs (+6–7%, about 4 ns per header — real, not noise); symbol-table nodes + 2.39 → 1.94–2.04 µs (−17%); group B-tree walk 405–411 → 322–351 ns + (−16%); listing the 400-group file through the facade 8.56–8.71 → + 8.40–8.42 ms (−2.4%). Net, local metadata reads are slightly faster; the + per-header cost is a known, small regression. - New equivalence harness `clawhdf5-format/tests/storage_equivalence.rs`: every converted parser runs over the file as a slice and over `storage::CountingStorage` — a `Storage` that serves an in-memory buffer -- 2.54.0 From 0645dcf1736cc8398c5d7626c52039eacad35f6a Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 15:22:54 -0500 Subject: [PATCH 48/48] docs: data-read throughput unchanged by the Storage conversion Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe377f8..fa0a30f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,7 +139,12 @@ 2.39 → 1.94–2.04 µs (−17%); group B-tree walk 405–411 → 322–351 ns (−16%); listing the 400-group file through the facade 8.56–8.71 → 8.40–8.42 ms (−2.4%). Net, local metadata reads are slightly faster; the - per-header cost is a known, small regression. + per-header cost is a known, small regression. Data reads are unchanged: + `concurrent_read --decode-threads 1` (the `BENCHMARKS.md` "Concurrent + reads" workload), `main` and this branch alternating, 2 rounds each, on + the same idle start: every deflate and contiguous full-read row within + ±2.4% of `main` (the cache-bound contiguous hyperslab rows vary by up to + ±40% between `main`'s own rounds and are not comparable). - New equivalence harness `clawhdf5-format/tests/storage_equivalence.rs`: every converted parser runs over the file as a slice and over `storage::CountingStorage` — a `Storage` that serves an in-memory buffer -- 2.54.0