quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): refuse object header messages over 64 KiB

Browse Source 
A v2 object header message has a 2-byte size field. The writer truncated
larger sizes to 16 bits, so an attribute over ~64 KiB (or a compact
dataset of 65532-65535 bytes, whose layout message adds 4 bytes) produced
a file libhdf5 rejects ("message of unshareable class flagged as
shareable", "bad flag combination").

ObjectHeaderWriter::serialize now returns a Result and fails on any message
over MAX_MESSAGE_SIZE; FileWriter::finish propagates it. Compact storage
falls back to contiguous above 65531 bytes, the real limit. Dense storage
for large attributes remains future work.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:04:07 -05:00
co-authored by Claude Opus 5.5
parent 46203ea761
commit b36998ef01
4 changed files with 239 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/object_header_writer.rs
+59 -13
View File
@@ -1,11 +1,17 @@
//! Object header writer for v2 format.
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use alloc::{format, vec::Vec};
use crate::checksum::jenkins_lookup3;
use crate::error::FormatError;
use crate::message_type::MessageType;
/// Largest message payload a v2 object header can describe: the per-message
/// size field is 2 bytes. A bigger message cannot be encoded at all — writing
/// its size truncated to 16 bits produced files libhdf5 refuses.
pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize;
/// Writer for v2 object headers with proper checksums.
pub struct ObjectHeaderWriter {
messages: Vec<(MessageType, Vec<u8>, u8)>, // (type, data, msg_flags)
@@ -30,7 +36,22 @@ impl ObjectHeaderWriter {
}
/// Serialize the complete v2 object header (OHDR + messages + checksum).
pub fn serialize(&self) -> Vec<u8> {
///
/// Fails with [`FormatError::SerializationError`] when a message is larger
/// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would
/// need dense attribute storage), rather than writing a corrupt header.
pub fn serialize(&self) -> Result<Vec<u8>, FormatError> {
if let Some((msg_type, data, _)) = self
.messages
.iter()
.find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE)
{
return Err(FormatError::SerializationError(format!(
"{msg_type:?} message is {} bytes; an object header message holds at most \
{MAX_MESSAGE_SIZE} bytes",
data.len()
)));
}
// Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data
let msg_bytes_total: usize = self
.messages
@@ -80,7 +101,7 @@ impl ObjectHeaderWriter {
let checksum = jenkins_lookup3(&buf);
buf.extend_from_slice(&checksum.to_le_bytes());
buf
Ok(buf)
}
}
@@ -125,15 +146,22 @@ impl BatchObjectHeaderWriter {
/// Compute the serialized size of each header without actually serializing.
/// Returns sizes in the same order as headers were added.
pub fn compute_sizes(&self) -> Vec<usize> {
self.headers.iter().map(|h| h.serialize().len()).collect()
pub fn compute_sizes(&self) -> Result<Vec<usize>, FormatError> {
self.headers
.iter()
.map(|h| h.serialize().map(|b| b.len()))
.collect()
}
/// Serialize all headers into a single contiguous buffer.
/// Returns `(combined_bytes, offsets)` where `offsets[i]` is the byte
/// offset of header `i` within the combined buffer.
pub fn serialize_all(&self) -> (Vec<u8>, Vec<usize>) {
let serialized: Vec<Vec<u8>> = self.headers.iter().map(|h| h.serialize()).collect();
pub fn serialize_all(&self) -> Result<(Vec<u8>, Vec<usize>), FormatError> {
let serialized: Vec<Vec<u8>> = self
.headers
.iter()
.map(|h| h.serialize())
.collect::<Result<_, _>>()?;
let total: usize = serialized.iter().map(|s| s.len()).sum();
let mut buf = Vec::with_capacity(total);
let mut offsets = Vec::with_capacity(serialized.len());
@@ -141,7 +169,7 @@ impl BatchObjectHeaderWriter {
offsets.push(buf.len());
buf.extend_from_slice(s);
}
(buf, offsets)
Ok((buf, offsets))
}
}
@@ -159,7 +187,7 @@ mod tests {
#[test]
fn empty_header_roundtrip() {
let writer = ObjectHeaderWriter::new();
let bytes = writer.serialize();
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.version, 2);
assert_eq!(hdr.messages.len(), 0);
@@ -170,7 +198,7 @@ mod tests {
let mut writer = ObjectHeaderWriter::new();
writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]);
writer.add_message(MessageType::Datatype, vec![5, 6]);
let bytes = writer.serialize();
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 2);
assert_eq!(hdr.messages[0].msg_type, MessageType::Dataspace);
@@ -184,12 +212,30 @@ mod tests {
let mut writer = ObjectHeaderWriter::new();
// Add a message with >255 bytes of payload
writer.add_message(MessageType::Datatype, vec![0xAA; 300]);
let bytes = writer.serialize();
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 1);
assert_eq!(hdr.messages[0].data.len(), 300);
}
#[test]
fn oversized_message_is_an_error_not_a_truncated_size() {
// 65535 bytes is the largest encodable payload.
let mut writer = ObjectHeaderWriter::new();
writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE]);
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages[0].data.len(), MAX_MESSAGE_SIZE);
// One byte more used to be written with its size wrapped to 0.
let mut writer = ObjectHeaderWriter::new();
writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE + 1]);
assert!(matches!(
writer.serialize(),
Err(FormatError::SerializationError(_))
));
}
#[test]
fn batch_writer_serialize_all() {
let mut batch = BatchObjectHeaderWriter::new();
@@ -204,7 +250,7 @@ mod tests {
batch.add(w2);
assert_eq!(batch.len(), 2);
let (buf, offsets) = batch.serialize_all();
let (buf, offsets) = batch.serialize_all().unwrap();
assert_eq!(offsets.len(), 2);
assert_eq!(offsets[0], 0);
@@ -222,7 +268,7 @@ mod tests {
fn batch_writer_empty() {
let batch = BatchObjectHeaderWriter::new();
assert!(batch.is_empty());
let (buf, offsets) = batch.serialize_all();
let (buf, offsets) = batch.serialize_all().unwrap();
assert!(buf.is_empty());
assert!(offsets.is_empty());
}