fix(format): apply the base address of files with a user block

A file may start with a user block (h5py userblock_size, h5jam), putting
the superblock at 512, 1024, ...; every address in the file is then
relative to the superblock. The signature search found it, but every
reader passed the whole file to the parsers, so addresses landed
userblock bytes early and the root group failed with
InvalidObjectHeaderVersion (twithub.h5, twithub513.h5,
h5clear_fsm_persist_user_*.h5).

Readers now view the file from the superblock on, taking the signature's
position as the base address as libhdf5 does: File (mmap, buffered,
from_bytes), MmapFile, LazyFile, AsyncHDF5File, the VOL and MPI VOL
readers, the HNSW loader and external VDS source files. File, MmapFile
and LazyFile gain user_block_size(). The new signature::split_user_block
returns the two parts, and Superblock::parse refuses a non-zero offset
(UserBlockNotStripped) so a format-level caller cannot silently apply
superblock-relative addresses to the whole file.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/data_read.rs

@@ -575,11 +575,13 @@ fn read_named_dataset_raw(
     use crate::group_v2::resolve_path_any;
     use crate::message_type::MessageType;
     use crate::object_header::ObjectHeader;
-    use crate::signature::find_signature;
+    use crate::signature::split_user_block;
     use crate::superblock::Superblock;
 
-    let sig = find_signature(file_data)?;
-    let sb = Superblock::parse(file_data, sig)?;
+    // An external source file is handed over whole, user block included;
+    // its addresses are relative to its superblock.
+    let (_, file_data) = split_user_block(file_data)?;
+    let sb = Superblock::parse(file_data, 0)?;
     let addr = resolve_path_any(file_data, &sb, path)?;
     let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?;
 

crates/clawhdf5-format/src/error.rs

@@ -120,6 +120,11 @@ pub enum FormatError {
     /// A shared-message reference points at an object header that holds no
     /// (unshared) message of the referenced type (raw message type id).
     SharedMessageTargetMissing(u16),
+    /// A superblock was parsed at a non-zero offset of the buffer (the file
+    /// has a user block of this many bytes). HDF5 addresses are relative to
+    /// the superblock, so the buffer must start there: see
+    /// `signature::split_user_block`.
+    UserBlockNotStripped(u64),
     /// A selection does not fit the dataset it was applied to (wrong rank, or
     /// it reaches past a dimension's extent).
     SelectionOutOfBounds(String),
@@ -342,6 +347,11 @@ impl fmt::Display for FormatError {
             FormatError::SelectionOutOfBounds(msg) => {
                 write!(f, "selection out of bounds: {msg}")
             }
+            FormatError::UserBlockNotStripped(n) => write!(
+                f,
+                "file has a {n}-byte user block: parse the bytes from the superblock on \
+                 (signature::split_user_block)"
+            ),
             FormatError::SharedMessageTargetMissing(t) => write!(
                 f,
                 "shared message reference points at an object header with no message of type \

crates/clawhdf5-format/src/lib.rs

@@ -26,12 +26,13 @@
 //! use clawhdf5_format::{signature, superblock, object_header, group_v2,
 //!     datatype, dataspace, data_layout, data_read, message_type::MessageType};
 //!
-//! let file_data = std::fs::read("output.h5").unwrap();
-//! let sig = signature::find_signature(&file_data).unwrap();
-//! let sb  = superblock::Superblock::parse(&file_data, sig).unwrap();
-//! let addr = group_v2::resolve_path_any(&file_data, &sb, "data").unwrap();
+//! let bytes = std::fs::read("output.h5").unwrap();
+//! // Addresses are relative to the superblock: skip any user block.
+//! let (_user_block, file_data) = signature::split_user_block(&bytes).unwrap();
+//! let sb  = superblock::Superblock::parse(file_data, 0).unwrap();
+//! let addr = group_v2::resolve_path_any(file_data, &sb, "data").unwrap();
 //! let hdr = object_header::ObjectHeader::parse(
-//!     &file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
+//!     file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
 //! ```
 //!
 //! # Features

crates/clawhdf5-format/src/shared_message.rs

@@ -408,8 +408,8 @@ pub fn load_sohm_table(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Option<SohmTable>, FormatError> {
-    let sig = crate::signature::find_signature(file_data)?;
-    let sb = crate::superblock::Superblock::parse(file_data, sig)?;
+    // `file_data` starts at the superblock (see `signature::split_user_block`).
+    let sb = crate::superblock::Superblock::parse(file_data, 0)?;
     let Some(ext_addr) = sb
         .superblock_extension_address
         .filter(|&a| !is_undefined(a, offset_size))

crates/clawhdf5-format/src/signature.rs

@@ -11,6 +11,16 @@ pub const HDF5_SIGNATURE: [u8; 8] = [0x89, b'H', b'D', b'F', b'\r', b'\n', 0x1A,
 /// (powers of two starting at 512, plus offset 0).
 ///
 /// Returns the byte offset where the signature was found.
+///
+/// A non-zero offset means the file starts with a *user block*, and every
+/// address inside the file is relative to the superblock's position, not to
+/// byte 0 (libhdf5 uses the signature's position as the base address even
+/// when the stored base-address field disagrees). The parsers in this crate
+/// take addresses as indices into `file_data`, so they must be handed the
+/// bytes from the signature on — use [`split_user_block`]. [`Superblock::parse`]
+/// refuses a non-zero offset for this reason.
+///
+/// [`Superblock::parse`]: crate::superblock::Superblock::parse
 pub fn find_signature(data: &[u8]) -> Result<usize, FormatError> {
     // Check offset 0
     if data.len() >= 8 && data[..8] == HDF5_SIGNATURE {
@@ -29,6 +39,17 @@ pub fn find_signature(data: &[u8]) -> Result<usize, FormatError> {
     Err(FormatError::SignatureNotFound)
 }
 
+/// Split a file into its user block and its HDF5 bytes.
+///
+/// Returns `(user_block, hdf5)`: `user_block` is everything before the
+/// superblock signature (empty for most files) and `hdf5` is the rest, in
+/// which every HDF5 address is a plain index. Pass `hdf5` as `file_data` to
+/// every parser in this crate, and parse the superblock at offset 0 of it.
+pub fn split_user_block(data: &[u8]) -> Result<(&[u8], &[u8]), FormatError> {
+    let offset = find_signature(data)?;
+    Ok(data.split_at(offset))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -88,6 +109,21 @@ mod tests {
         assert_eq!(find_signature(&data), Err(FormatError::SignatureNotFound));
     }
 
+    #[test]
+    fn split_user_block_rebases_at_the_signature() {
+        let mut data = vec![7u8; 1024];
+        data[512..520].copy_from_slice(&HDF5_SIGNATURE);
+        let (ub, hdf5) = split_user_block(&data).unwrap();
+        assert_eq!(ub.len(), 512);
+        assert_eq!(hdf5.len(), 512);
+        assert_eq!(&hdf5[..8], &HDF5_SIGNATURE);
+
+        data[..8].copy_from_slice(&HDF5_SIGNATURE);
+        let (ub, hdf5) = split_user_block(&data).unwrap();
+        assert!(ub.is_empty());
+        assert_eq!(hdf5.len(), 1024);
+    }
+
     #[test]
     fn signature_prefers_earliest() {
         // Signature at both 0 and 512, should return 0

crates/clawhdf5-format/src/superblock.rs

@@ -174,8 +174,18 @@ impl Superblock {
 
     /// Parse a superblock from `data` starting at `signature_offset`.
     ///
-    /// The signature must be present at the given offset.
+    /// The signature must be present at the given offset, and that offset
+    /// must be 0: every address in an HDF5 file is relative to the
+    /// superblock, so when a file has a user block (signature at 512, 1024,
+    /// …) the caller must pass the bytes from the signature on — see
+    /// [`crate::signature::split_user_block`] — and use that slice as
+    /// `file_data` everywhere. A non-zero offset is refused with
+    /// [`FormatError::UserBlockNotStripped`] because the addresses in the
+    /// returned superblock would otherwise be applied to the wrong bytes.
     pub fn parse(data: &[u8], signature_offset: usize) -> Result<Superblock, FormatError> {
+        if signature_offset != 0 {
+            return Err(FormatError::UserBlockNotStripped(signature_offset as u64));
+        }
         let d = data
             .get(signature_offset..)
             .ok_or(FormatError::UnexpectedEof {
@@ -676,7 +686,16 @@ mod tests {
         let mut data = vec![0u8; 1024];
         let v0 = build_v0_bytes(8);
         data[512..512 + v0.len()].copy_from_slice(&v0);
-        let sb = Superblock::parse(&data, 512).unwrap();
+        // Addresses are relative to the superblock, so parsing in place
+        // (where they would be applied to the whole buffer) is refused...
+        assert_eq!(
+            Superblock::parse(&data, 512),
+            Err(FormatError::UserBlockNotStripped(512))
+        );
+        // ...and the caller parses the bytes from the signature on.
+        let (ub, hdf5) = crate::signature::split_user_block(&data).unwrap();
+        assert_eq!(ub.len(), 512);
+        let sb = Superblock::parse(hdf5, 0).unwrap();
         assert_eq!(sb.version, 0);
         assert_eq!(sb.root_group_address, 96);
     }