format: read object headers with long continuation chains

libhdf5 follows any number of continuation chunks, and a header that is
full gains one per message added (each new chunk holding the next
continuation message), so a version-1 header with a few dozen attributes
added one at a time is a chain dozens of chunks long. The reader recursed
once per chunk and refused a chain deeper than 32 (NestingDepthExceeded):
h5py read such files, we did not. Version-2 headers stopped at 256
continuation chunks.

Version-1 chunks are now followed with an explicit stack (the same
depth-first message order as before), version-2 ones as before; both
refuse a chunk address seen twice (a cycle, what the limits guarded
against) and more than 65 536 chunks.

Regression: long_v1_continuation_chains_are_read (a 200-chunk chain),
v1_continuation_cycles_are_refused; the dense-attribute interop test's
'earliest' case produces such a chain.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/object_header.rs

@@ -1,7 +1,9 @@
 //! HDF5 Object Header parsing (v1 and v2).
 
 #[cfg(not(feature = "std"))]
-use alloc::vec::Vec;
+use alloc::{collections::BTreeSet, vec, vec::Vec};
+#[cfg(feature = "std")]
+use std::collections::BTreeSet;
 
 use byteorder::{ByteOrder, LittleEndian};
 
@@ -196,7 +198,6 @@ impl ObjectHeader {
             header_data_size,
             offset_size,
             length_size,
-            MAX_V1_CONTINUATION_DEPTH,
             &mut messages,
         )?;
         // libhdf5 reads every message in the first chunk and refuses a header
@@ -230,49 +231,55 @@ impl ObjectHeader {
     /// 8; libhdf5 refuses a message that is not aligned, that runs past the
     /// end of the chunk, or leftover bytes too few for a message header (a
     /// "gap", which only version 2 allows).
-    #[allow(clippy::too_many_arguments)]
+    ///
+    /// Continuation chunks are followed depth-first, as met, with an
+    /// explicit stack: a chain of continuation chunks as long as libhdf5
+    /// writes (each new chunk holding the next continuation message, one
+    /// per attribute added to a full header) is read without recursion.
+    /// A chunk address seen twice is a cycle and refused, as is a header of
+    /// more than [`MAX_V1_CHUNKS`] chunks.
     fn parse_v1_chunk<S: Storage + ?Sized>(
         file: &S,
         offset: u64,
         length: usize,
         offset_size: u8,
         length_size: u8,
-        depth_remaining: u16,
         messages: &mut Vec<HeaderMessage>,
     ) -> Result<usize, FormatError> {
-        if depth_remaining == 0 {
+        let mut stack = vec![(read_exact_at(file, offset, length)?, 0usize)];
-            return Err(FormatError::NestingDepthExceeded);
+        let mut seen = BTreeSet::new();
-        }
+        seen.insert(offset);
-        let chunk = read_exact_at(file, offset, length)?;
-        let data: &[u8] = &chunk;
-        let end = length;
-        let mut pos = 0usize;
         let mut count = 0usize;
 
-        while pos < end {
+        while let Some((chunk, pos)) = stack.last_mut() {
-            if end - pos < V1_MSG_HEADER_SIZE {
+            let data: &[u8] = chunk;
+            let end = data.len();
+            if *pos >= end {
+                stack.pop();
+                continue;
+            }
+            if end - *pos < V1_MSG_HEADER_SIZE {
                 return Err(FormatError::InvalidObjectHeader(
                     "gap found in early version of file format",
                 ));
             }
-            let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
+            let p = *pos;
-            let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
+            let msg_type_raw = LittleEndian::read_u16(&data[p..p + 2]);
-            let msg_flags = data[pos + 4];
+            let msg_data_size = LittleEndian::read_u16(&data[p + 2..p + 4]) as usize;
-            // reserved(3) at pos+5..pos+8
+            let msg_flags = data[p + 4];
-            pos += V1_MSG_HEADER_SIZE;
+            // reserved(3) at p+5..p+8
+            let p = p + V1_MSG_HEADER_SIZE;
 
             if !msg_data_size.is_multiple_of(8) {
                 return Err(FormatError::InvalidObjectHeader("message not aligned"));
             }
-            if msg_data_size > end - pos {
+            if msg_data_size > end - p {
                 return Err(FormatError::InvalidObjectHeader(
                     "message size exceeds buffer end",
                 ));
             }
-            let body = &data[pos..pos + msg_data_size];
+            let body = &data[p..p + msg_data_size];
             check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
-            count += 1;
-
             let msg_type = MessageType::from_u16(msg_type_raw);
             if msg_type != MessageType::Nil {
                 messages.push(HeaderMessage {
@@ -283,22 +290,26 @@ impl ObjectHeader {
                     data: body.to_vec(),
                 });
             }
-            pos += msg_data_size;
-
             // Follow continuations (v1 continuation chunks are just raw
             // messages, no signature); check_message has checked the body.
-            if msg_type == MessageType::ObjectHeaderContinuation {
+            let cont = if msg_type == MessageType::ObjectHeaderContinuation {
-                let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?;
+                Some((
-                let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
+                    read_offset(body, 0, offset_size)?,
-                Self::parse_v1_chunk(
+                    to_usize(read_offset(body, offset_size as usize, length_size)?)?,
-                    file,
+                ))
-                    cont_offset as u64,
+            } else {
-                    cont_length,
+                None
-                    offset_size,
+            };
-                    length_size,
+            *pos = p + msg_data_size;
-                    depth_remaining - 1,
+            // Only the first chunk's messages are held to the prefix count.
-                    messages,
+            if stack.len() == 1 {
-                )?;
+                count += 1;
+            }
+            if let Some((cont_offset, cont_length)) = cont {
+                if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS {
+                    return Err(FormatError::NestingDepthExceeded);
+                }
+                stack.push((read_exact_at(file, cont_offset, cont_length)?, 0));
             }
         }
 
@@ -425,13 +436,15 @@ impl ObjectHeader {
             &mut continuations,
         )?;
 
-        // Follow continuations (limit to prevent cycles in malformed data)
+        // Follow continuations. A chunk address seen twice is a cycle in
-        let mut cont_remaining = 256u16;
+        // malformed data; a valid header can have many chunks (libhdf5 adds
+        // one whenever a message no longer fits), up to the same bound as a
+        // version-1 header.
+        let mut seen = BTreeSet::new();
         while let Some((cont_offset, cont_length)) = continuations.pop() {
-            if cont_remaining == 0 {
+            if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS {
                 return Err(FormatError::NestingDepthExceeded);
             }
-            cont_remaining -= 1;
             Self::parse_v2_continuation(
                 file,
                 cont_offset as u64,
@@ -594,8 +607,10 @@ const V2_PREFIX_MAX: usize = 34;
 /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3).
 const V1_MSG_HEADER_SIZE: usize = 8;
 
-/// How deep version-1 continuation chunks may chain (malformed-data guard).
+/// Most chunks a version-1 object header may have (malformed-data guard;
-const MAX_V1_CONTINUATION_DEPTH: u16 = 32;
+/// libhdf5 has no limit, and a header that gains one continuation chunk per
+/// attribute added can have many).
+const MAX_V1_CHUNKS: usize = 1 << 16;
 
 /// Every defined version-2 object header status flag (libhdf5
 /// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation
@@ -901,6 +916,71 @@ mod tests {
         assert_eq!(hdr.messages[1].data[..2], [5, 6]);
     }
 
+    /// A version-1 header whose continuation chunks form a chain: chunk k
+    /// holds a Dataspace message `[k]` and the continuation to chunk k + 1.
+    /// With `cycle`, the last chunk points back at the first continuation
+    /// chunk.
+    fn v1_chain(n: usize, cycle: bool) -> Vec<u8> {
+        // Each continuation chunk: dataspace (8 + 8) + continuation (8 + 16).
+        let chunk_len = 40u64;
+        let first = 64u64;
+        let cont = |addr: u64| {
+            let mut b = addr.to_le_bytes().to_vec();
+            b.extend_from_slice(&chunk_len.to_le_bytes());
+            b
+        };
+        let mut data = build_v1_header(&[(0x0010, &cont(first)[..], 0)], 8, 8);
+        data.resize(first as usize, 0);
+        for k in 0..n {
+            let mut c = Vec::new();
+            c.extend_from_slice(&1u16.to_le_bytes());
+            c.extend_from_slice(&8u16.to_le_bytes());
+            c.extend_from_slice(&[0; 4]);
+            c.extend_from_slice(&(k as u64).to_le_bytes());
+            let next = if k + 1 < n {
+                first + (k as u64 + 1) * chunk_len
+            } else if cycle {
+                first
+            } else {
+                // The last chunk ends in a NIL message instead.
+                c.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]);
+                c.extend_from_slice(&[0; 16]);
+                data.extend_from_slice(&c);
+                continue;
+            };
+            c.extend_from_slice(&0x10u16.to_le_bytes());
+            c.extend_from_slice(&16u16.to_le_bytes());
+            c.extend_from_slice(&[0; 4]);
+            c.extend_from_slice(&cont(next));
+            data.extend_from_slice(&c);
+        }
+        data
+    }
+
+    /// libhdf5 reads any chain of continuation chunks (a header grows one
+    /// per attribute added when full); the reader used to stop at 32.
+    #[test]
+    fn long_v1_continuation_chains_are_read() {
+        let data = v1_chain(200, false);
+        let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
+        let spaces: Vec<u8> = hdr
+            .messages
+            .iter()
+            .filter(|m| m.msg_type == MessageType::Dataspace)
+            .map(|m| m.data[0])
+            .collect();
+        assert_eq!(spaces, (0..200).map(|k| k as u8).collect::<Vec<_>>());
+    }
+
+    #[test]
+    fn v1_continuation_cycles_are_refused() {
+        let data = v1_chain(5, true);
+        assert!(matches!(
+            ObjectHeader::parse(&data, 0, 8, 8),
+            Err(FormatError::NestingDepthExceeded)
+        ));
+    }
+
     #[test]
     fn parse_v1_unknown_message_ok() {
         let messages = [(0x00FFu16, &[0xAA, 0xBB][..], 0u8)];