h5rs tools, browser reader, libhdf5 header checks, plugin filters, concurrency benchmark #14

Merged
osobh merged 60 commits from feat/p1-proof into main 2026-09-26 13:14:39 +00:00
2 changed files with 504 additions and 130 deletions
Showing only changes of commit f4dee1cd08 - Show all commits
+8
View File
@@ -201,6 +201,11 @@ pub enum FormatError {
DuplicateDatasetName(String), DuplicateDatasetName(String),
/// Integer overflow in size computation (malformed data protection). /// Integer overflow in size computation (malformed data protection).
Overflow(String), Overflow(String),
/// An object header that libhdf5 refuses to load (the reason is
/// libhdf5's own error text): a misaligned or overrunning message, a
/// wrong message count, contradictory message flags, a message of a
/// class that cannot be shared flagged shareable, …
InvalidObjectHeader(&'static str),
} }
impl fmt::Display for FormatError { impl fmt::Display for FormatError {
@@ -445,6 +450,9 @@ impl fmt::Display for FormatError {
FormatError::Overflow(msg) => { FormatError::Overflow(msg) => {
write!(f, "integer overflow: {msg}") write!(f, "integer overflow: {msg}")
} }
FormatError::InvalidObjectHeader(why) => {
write!(f, "corrupt object header: {why}")
}
} }
} }
} }
+496 -130
View File
@@ -108,10 +108,20 @@ impl ObjectHeader {
return Err(FormatError::InvalidObjectHeaderVersion(version)); return Err(FormatError::InvalidObjectHeaderVersion(version));
} }
let num_messages = LittleEndian::read_u16(&data[offset + 2..offset + 4]); let num_messages = LittleEndian::read_u16(&data[offset + 2..offset + 4]) as usize;
let reference_count = LittleEndian::read_u32(&data[offset + 4..offset + 8]); let reference_count = LittleEndian::read_u32(&data[offset + 4..offset + 8]);
let header_data_size = LittleEndian::read_u32(&data[offset + 8..offset + 12]) as usize; let header_data_size = LittleEndian::read_u32(&data[offset + 8..offset + 12]) as usize;
// libhdf5 (H5O__prefix_deserialize): a header with messages needs room
// for at least one message header, and one without has an empty chunk.
if (num_messages > 0 && header_data_size < V1_MSG_HEADER_SIZE)
|| (num_messages == 0 && header_data_size > 0)
{
return Err(FormatError::InvalidObjectHeader(
"bad object header chunk size",
));
}
// Pad to 8-byte alignment: header prefix is 12 bytes, pad to 16 // Pad to 8-byte alignment: header prefix is 12 bytes, pad to 16
let padding = 4; // pad 12-byte prefix to 16-byte alignment let padding = 4; // pad 12-byte prefix to 16-byte alignment
let msg_start = offset let msg_start = offset
@@ -124,64 +134,23 @@ impl ObjectHeader {
ensure_len(data, msg_start, header_data_size)?; ensure_len(data, msg_start, header_data_size)?;
let mut messages = Vec::new(); let mut messages = Vec::new();
let mut pos = msg_start; let chunk0_count = Self::parse_v1_chunk(
let msg_end = data,
msg_start msg_start,
.checked_add(header_data_size) header_data_size,
.ok_or(FormatError::UnexpectedEof { offset_size,
expected: usize::MAX, length_size,
available: data.len(), MAX_V1_CONTINUATION_DEPTH,
})?; &mut messages,
)?;
for _ in 0..num_messages { // libhdf5 reads every message in the first chunk and refuses a header
if pos + 8 > msg_end { // whose prefix claims fewer than that (continuation chunks are read
break; // later and not held to the count). Stopping after the claimed number
} // silently dropped the rest.
let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); if chunk0_count > num_messages {
let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; return Err(FormatError::InvalidObjectHeader(
let msg_flags = data[pos + 4]; "bad object header message count",
// reserved(3) at pos+5..pos+8 ));
pos += 8;
ensure_len(data, pos, msg_data_size)?;
let msg_type = MessageType::from_u16(msg_type_raw);
check_unknown_message(msg_type, msg_flags)?;
if msg_type != MessageType::Nil {
messages.push(HeaderMessage {
msg_type,
size: msg_data_size,
flags: msg_flags,
creation_order: None,
data: data[pos..pos + msg_data_size].to_vec(),
});
}
pos += msg_data_size;
// Follow continuations
if msg_type == MessageType::ObjectHeaderContinuation {
let cont_msg_data = &messages
.last()
.ok_or(FormatError::InvalidObjectHeaderSignature)?
.data;
if cont_msg_data.len() >= (offset_size as usize + length_size as usize) {
let cont_offset = read_offset(cont_msg_data, 0, offset_size)? as usize;
let cont_length =
read_offset(cont_msg_data, offset_size as usize, length_size)? as usize;
// Parse continuation block (v1: just raw messages, no signature)
let cont_msgs = Self::parse_v1_continuation(
data,
cont_offset,
cont_length,
offset_size,
length_size,
32, // max continuation depth
)?;
messages.extend(cont_msgs);
}
}
} }
Ok(ObjectHeader { Ok(ObjectHeader {
@@ -196,72 +165,87 @@ impl ObjectHeader {
}) })
} }
fn parse_v1_continuation( /// Parse the messages of one version-1 chunk (`length` bytes at
/// `offset`, no signature), following continuation messages as they are
/// met. Returns how many messages (NIL ones included) this chunk itself
/// holds.
///
/// A version-1 chunk is filled with messages whose sizes are multiples of
/// 8; libhdf5 refuses a message that is not aligned, that runs past the
/// end of the chunk, or leftover bytes too few for a message header (a
/// "gap", which only version 2 allows).
#[allow(clippy::too_many_arguments)]
fn parse_v1_chunk(
data: &[u8], data: &[u8],
offset: usize, offset: usize,
length: usize, length: usize,
offset_size: u8, offset_size: u8,
length_size: u8, length_size: u8,
depth_remaining: u16, depth_remaining: u16,
) -> Result<Vec<HeaderMessage>, FormatError> { messages: &mut Vec<HeaderMessage>,
) -> Result<usize, FormatError> {
if depth_remaining == 0 { if depth_remaining == 0 {
return Err(FormatError::NestingDepthExceeded); return Err(FormatError::NestingDepthExceeded);
} }
ensure_len(data, offset, length)?; ensure_len(data, offset, length)?;
let mut messages = Vec::new(); let end = offset + length;
let mut pos = offset; let mut pos = offset;
let end = offset.saturating_add(length); let mut count = 0usize;
while pos + 8 <= end { while pos < end {
if end - pos < V1_MSG_HEADER_SIZE {
return Err(FormatError::InvalidObjectHeader(
"gap found in early version of file format",
));
}
let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
let msg_flags = data[pos + 4]; let msg_flags = data[pos + 4];
pos += 8; // reserved(3) at pos+5..pos+8
pos += V1_MSG_HEADER_SIZE;
if pos + msg_data_size > end { if !msg_data_size.is_multiple_of(8) {
break; return Err(FormatError::InvalidObjectHeader("message not aligned"));
} }
if msg_data_size > end - pos {
return Err(FormatError::InvalidObjectHeader(
"message size exceeds buffer end",
));
}
let body = &data[pos..pos + msg_data_size];
check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
count += 1;
let msg_type = MessageType::from_u16(msg_type_raw); let msg_type = MessageType::from_u16(msg_type_raw);
check_unknown_message(msg_type, msg_flags)?;
if msg_type != MessageType::Nil { if msg_type != MessageType::Nil {
messages.push(HeaderMessage { messages.push(HeaderMessage {
msg_type, msg_type,
size: msg_data_size, size: msg_data_size,
flags: msg_flags, flags: msg_flags,
creation_order: None, creation_order: None,
data: data[pos..pos + msg_data_size].to_vec(), data: body.to_vec(),
}); });
} }
pos += msg_data_size; pos += msg_data_size;
// Recursive continuations // Follow continuations (v1 continuation chunks are just raw
// messages, no signature); check_message has checked the body.
if msg_type == MessageType::ObjectHeaderContinuation { if msg_type == MessageType::ObjectHeaderContinuation {
let cont_msg_data = &messages let cont_offset = read_offset(body, 0, offset_size)? as usize;
.last() let cont_length = read_offset(body, offset_size as usize, length_size)? as usize;
.ok_or(FormatError::InvalidObjectHeaderSignature)? Self::parse_v1_chunk(
.data; data,
if cont_msg_data.len() >= (offset_size as usize + length_size as usize) { cont_offset,
let cont_offset = read_offset(cont_msg_data, 0, offset_size)? as usize; cont_length,
let cont_length = offset_size,
read_offset(cont_msg_data, offset_size as usize, length_size)? as usize; length_size,
let cont_msgs = Self::parse_v1_continuation( depth_remaining - 1,
data, messages,
cont_offset, )?;
cont_length,
offset_size,
length_size,
depth_remaining - 1,
)?;
messages.extend(cont_msgs);
}
} }
} }
Ok(messages) Ok(count)
} }
fn parse_v2( fn parse_v2(
@@ -278,6 +262,11 @@ impl ObjectHeader {
return Err(FormatError::InvalidObjectHeaderVersion(version)); return Err(FormatError::InvalidObjectHeaderVersion(version));
} }
let flags = data[offset + 5]; let flags = data[offset + 5];
if flags & !V2_HDR_ALL_FLAGS != 0 {
return Err(FormatError::InvalidObjectHeader(
"unknown object header status flag(s)",
));
}
let mut pos = offset + 6; let mut pos = offset + 6;
@@ -297,7 +286,14 @@ impl ObjectHeader {
// Optional attribute storage thresholds (flags bit 4) // Optional attribute storage thresholds (flags bit 4)
if flags & 0x10 != 0 { if flags & 0x10 != 0 {
ensure_len(data, pos, 4)?; ensure_len(data, pos, 4)?;
// max_compact_attrs(2) + min_dense_attrs(2) — read but don't store for now // max_compact_attrs(2) + min_dense_attrs(2) — checked, not stored
let max_compact = LittleEndian::read_u16(&data[pos..pos + 2]);
let min_dense = LittleEndian::read_u16(&data[pos + 2..pos + 4]);
if max_compact < min_dense {
return Err(FormatError::InvalidObjectHeader(
"bad object header attribute phase change values",
));
}
pos += 4; pos += 4;
} }
@@ -312,6 +308,14 @@ impl ObjectHeader {
ensure_len(data, pos, chunk_size_width as usize)?; ensure_len(data, pos, chunk_size_width as usize)?;
let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize; let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize;
pos += chunk_size_width as usize; pos += chunk_size_width as usize;
// Bit 2: attribute creation order tracked → messages include creation order field
let has_creation_order = flags & 0x04 != 0;
let msg_header_size = if has_creation_order { 6 } else { 4 };
if chunk0_size > 0 && chunk0_size < msg_header_size {
return Err(FormatError::InvalidObjectHeader(
"bad object header chunk size",
));
}
let chunk0_msg_start = pos; let chunk0_msg_start = pos;
let chunk0_msg_end = pos let chunk0_msg_end = pos
@@ -335,9 +339,6 @@ impl ObjectHeader {
} }
} }
// Bit 2: attribute creation order tracked → messages include creation order field
let has_creation_order = flags & 0x04 != 0;
// Parse messages from chunk0 // Parse messages from chunk0
let mut messages = Vec::new(); let mut messages = Vec::new();
let mut continuations = Vec::new(); let mut continuations = Vec::new();
@@ -396,8 +397,20 @@ impl ObjectHeader {
) -> Result<(), FormatError> { ) -> Result<(), FormatError> {
let msg_header_size = if has_creation_order { 6 } else { 4 }; let msg_header_size = if has_creation_order { 6 } else { 4 };
let mut pos = start; let mut pos = start;
let mut null_count = 0usize;
while pos + msg_header_size <= end { while pos < end {
// Leftover bytes too few for a message header are a gap, which
// libhdf5 allows only in a chunk without NIL messages (a writer
// that leaves a gap had no NIL message to put the space in).
if end - pos < msg_header_size {
if null_count != 0 {
return Err(FormatError::InvalidObjectHeader(
"gap in chunk with no null messages",
));
}
break;
}
let msg_type_raw = data[pos] as u16; let msg_type_raw = data[pos] as u16;
let msg_data_size = LittleEndian::read_u16(&data[pos + 1..pos + 3]) as usize; let msg_data_size = LittleEndian::read_u16(&data[pos + 1..pos + 3]) as usize;
let msg_flags = data[pos + 3]; let msg_flags = data[pos + 3];
@@ -408,32 +421,29 @@ impl ObjectHeader {
}; };
pos += msg_header_size; pos += msg_header_size;
if pos + msg_data_size > end { if msg_data_size > end - pos {
// Could be padding at end of chunk return Err(FormatError::InvalidObjectHeader(
break; "message size exceeds buffer end",
));
} }
let body = &data[pos..pos + msg_data_size];
check_message(2, msg_type_raw, msg_flags, body, offset_size, length_size)?;
let msg_type = MessageType::from_u16(msg_type_raw); let msg_type = MessageType::from_u16(msg_type_raw);
check_unknown_message(msg_type, msg_flags)?;
let msg_data = data[pos..pos + msg_data_size].to_vec();
if msg_type == MessageType::ObjectHeaderContinuation { if msg_type == MessageType::ObjectHeaderContinuation {
// Parse continuation offset/length from message data // check_message has checked the body holds both fields.
if msg_data.len() >= (offset_size as usize + length_size as usize) { let cont_off = read_offset(body, 0, offset_size)? as usize;
let cont_off = read_offset(&msg_data, 0, offset_size)? as usize; let cont_len = read_offset(body, offset_size as usize, length_size)? as usize;
let cont_len = continuations.push((cont_off, cont_len));
read_offset(&msg_data, offset_size as usize, length_size)? as usize; } else if msg_type == MessageType::Nil {
continuations.push((cont_off, cont_len)); null_count += 1;
} } else {
} else if msg_type != MessageType::Nil {
messages.push(HeaderMessage { messages.push(HeaderMessage {
msg_type, msg_type,
size: msg_data_size, size: msg_data_size,
flags: msg_flags, flags: msg_flags,
creation_order, creation_order,
data: msg_data, data: body.to_vec(),
}); });
} }
@@ -496,22 +506,149 @@ impl ObjectHeader {
} }
} }
/// Header message flag bit 7: fail if the message is unknown, always. /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3).
const V1_MSG_HEADER_SIZE: usize = 8;
/// How deep version-1 continuation chunks may chain (malformed-data guard).
const MAX_V1_CONTINUATION_DEPTH: u16 = 32;
/// Every defined version-2 object header status flag (libhdf5
/// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation
/// order tracked/indexed, attribute phase-change values, times stored.
const V2_HDR_ALL_FLAGS: u8 = 0x3F;
// Header message flag bits (libhdf5 `H5O_MSG_FLAG_*`). Bit 0 (constant) needs
// no check. Bit 3 (fail if unknown and the file is opened for writing) never
// fails a read: the parser only ever reads, as libhdf5 ignores it for a
// read-only open.
const MSG_FLAG_SHARED: u8 = 0x02;
const MSG_FLAG_DONTSHARE: u8 = 0x04;
const MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE: u8 = 0x08;
const MSG_FLAG_MARK_IF_UNKNOWN: u8 = 0x10;
const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20;
const MSG_FLAG_SHAREABLE: u8 = 0x40;
/// Fail if the message is unknown, whatever the access mode.
const MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS: u8 = 0x80; const MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS: u8 = 0x80;
/// Refuse an unknown message the file says no reader may skip. /// Message type ids libhdf5 has a class for (`H5O_msg_class_g`): 0x00-0x18
/// except 0x09 (a test-only "bogus" message). Anything else is an unknown
/// message.
fn is_known_message(id: u16) -> bool {
id <= 0x18 && id != 0x09
}
/// Message classes that may be shared (`H5O_SHARE_IS_SHARABLE`): dataspace,
/// datatype, the two fill-value messages, filter pipeline and attribute.
fn is_shareable_message(id: u16) -> bool {
matches!(id, 0x01 | 0x03 | 0x04 | 0x05 | 0x0B | 0x0C)
}
/// Check one header message the way libhdf5 does while it loads an object
/// header (`H5O__chunk_deserialize`), so an object libhdf5 refuses to open is
/// refused here too instead of being read from a corrupt header:
/// ///
/// The parser only ever reads, so bit 3 (fail only when opened for writing) /// - contradictory flag combinations;
/// is ignored, as libhdf5 ignores it for a read-only open; bit 7 fails /// - an unknown message the file says no reader may skip (bit 7). This had
/// regardless of access mode. This had the two the wrong way round, failing /// bits 3 and 7 the wrong way round once, failing objects libhdf5 reads
/// objects libhdf5 reads and reading ones it refuses (`tbogus.h5`). /// and reading ones it refuses (`tbogus.h5`);
fn check_unknown_message(msg_type: MessageType, msg_flags: u8) -> Result<(), FormatError> { /// - a known message whose class cannot be shared, flagged shared or
match msg_type { /// shareable (`cve-2016-4332`);
MessageType::Unknown(id) if msg_flags & MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS != 0 => { /// - the messages libhdf5 decodes while loading the header, whose decode
Err(FormatError::UnsupportedMessage(id)) /// errors fail the load: continuation, reference count (which a version-1
} /// header cannot hold), and both modification-time messages.
_ => Ok(()), fn check_message(
header_version: u8,
id: u16,
flags: u8,
body: &[u8],
offset_size: u8,
length_size: u8,
) -> Result<(), FormatError> {
let bad_flags = FormatError::InvalidObjectHeader("bad flag combination for message");
if flags & MSG_FLAG_SHARED != 0 && flags & MSG_FLAG_DONTSHARE != 0 {
return Err(bad_flags);
} }
if flags & MSG_FLAG_WAS_UNKNOWN != 0
&& (flags & MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE != 0
|| flags & MSG_FLAG_MARK_IF_UNKNOWN == 0)
{
return Err(bad_flags);
}
if !is_known_message(id) {
if flags & MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS != 0 {
return Err(FormatError::UnsupportedMessage(id));
}
return Ok(());
}
if flags & (MSG_FLAG_SHARED | MSG_FLAG_SHAREABLE) != 0 && !is_shareable_message(id) {
return Err(FormatError::InvalidObjectHeader(
"message of unshareable class flagged as shareable",
));
}
let overrun = FormatError::InvalidObjectHeader("ran off end of input buffer while decoding");
match id {
// Continuation: address + length, and the chunk cannot be empty.
0x10 => {
if body.len() < offset_size as usize + length_size as usize {
return Err(overrun);
}
if read_offset(body, offset_size as usize, length_size)? == 0 {
return Err(FormatError::InvalidObjectHeader(
"invalid continuation chunk size (0)",
));
}
}
// Reference count: version-2 headers only; version 0 then a u32.
0x16 => {
if header_version == 1 {
return Err(FormatError::InvalidObjectHeader(
"object header version does not support reference count message",
));
}
match body.first() {
None => return Err(overrun),
Some(0) => {}
Some(_) => {
return Err(FormatError::InvalidObjectHeader(
"bad version number for reference count message",
));
}
}
if body.len() < 5 {
return Err(overrun);
}
}
// Old modification time: "YYYYMMDDhhmmss" and 2 reserved bytes.
0x0E => {
if body.len() < 16 {
return Err(overrun);
}
if !body[..14].iter().all(u8::is_ascii_digit) {
return Err(FormatError::InvalidObjectHeader(
"badly formatted modification time message",
));
}
}
// New modification time: version 1, 3 reserved bytes, u32 seconds.
0x12 => {
match body.first() {
None => return Err(overrun),
Some(1) => {}
Some(_) => {
return Err(FormatError::InvalidObjectHeader(
"bad version number for mtime message",
));
}
}
if body.len() < 8 {
return Err(overrun);
}
}
_ => {}
}
Ok(())
} }
#[cfg(test)] #[cfg(test)]
@@ -528,11 +665,14 @@ mod tests {
// Calculate total header message data size // Calculate total header message data size
let mut msg_bytes = Vec::new(); let mut msg_bytes = Vec::new();
for (mtype, mdata, mflags) in messages { for (mtype, mdata, mflags) in messages {
// v1 message sizes are multiples of 8 (the data is zero-padded).
let padded = mdata.len().div_ceil(8) * 8;
msg_bytes.extend_from_slice(&mtype.to_le_bytes()); // type(2) msg_bytes.extend_from_slice(&mtype.to_le_bytes()); // type(2)
msg_bytes.extend_from_slice(&(mdata.len() as u16).to_le_bytes()); // size(2) msg_bytes.extend_from_slice(&(padded as u16).to_le_bytes()); // size(2)
msg_bytes.push(*mflags); // flags(1) msg_bytes.push(*mflags); // flags(1)
msg_bytes.extend_from_slice(&[0u8; 3]); // reserved(3) msg_bytes.extend_from_slice(&[0u8; 3]); // reserved(3)
msg_bytes.extend_from_slice(mdata); // data msg_bytes.extend_from_slice(mdata); // data
msg_bytes.resize(msg_bytes.len() + padded - mdata.len(), 0);
} }
let mut buf = Vec::new(); let mut buf = Vec::new();
@@ -622,9 +762,10 @@ mod tests {
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 2); assert_eq!(hdr.messages.len(), 2);
assert_eq!(hdr.messages[0].msg_type, MessageType::Dataspace); assert_eq!(hdr.messages[0].msg_type, MessageType::Dataspace);
assert_eq!(hdr.messages[0].data, vec![1, 2, 3, 4]); // v1 message data is padded to a multiple of 8 bytes.
assert_eq!(hdr.messages[0].data, vec![1, 2, 3, 4, 0, 0, 0, 0]);
assert_eq!(hdr.messages[1].msg_type, MessageType::DataLayout); assert_eq!(hdr.messages[1].msg_type, MessageType::DataLayout);
assert_eq!(hdr.messages[1].data, vec![5, 6]); assert_eq!(hdr.messages[1].data[..2], [5, 6]);
} }
#[test] #[test]
@@ -650,7 +791,7 @@ mod tests {
// Bit 3 = fail if unknown *and the file is opened for writing*. This // Bit 3 = fail if unknown *and the file is opened for writing*. This
// parser only reads, so libhdf5 (read-only) opens such an object and // parser only reads, so libhdf5 (read-only) opens such an object and
// so must we. Bits 4/5 (mark if unknown / was unknown) never fail. // so must we. Bits 4/5 (mark if unknown / was unknown) never fail.
for flags in [0x08u8, 0x10, 0x20, 0x38] { for flags in [0x08u8, 0x10, 0x30] {
let messages = [(0x00FFu16, &[0xAA][..], flags)]; let messages = [(0x00FFu16, &[0xAA][..], flags)];
let data = build_v1_header(&messages, 8, 8); let data = build_v1_header(&messages, 8, 8);
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
@@ -658,6 +799,231 @@ mod tests {
} }
} }
#[test]
fn contradictory_message_flags_are_refused() {
// libhdf5: "bad flag combination for message" for shared + don't
// share, was-unknown without mark-if-unknown, and was-unknown with
// fail-if-unknown-on-write.
for flags in [0x06u8, 0x20, 0x38] {
let data = build_v1_header(&[(0x00FFu16, &[0xAA][..], flags)], 8, 8);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("bad flag combination for message"),
"flags {flags:#x}"
);
let data = build_v2_header(0x00, &[(0xF0, &[1, 2], flags)], None);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("bad flag combination for message"),
"flags {flags:#x}"
);
}
}
#[test]
fn unshareable_message_flagged_shareable_is_refused() {
// A layout (0x08) or modification time (0x12) message cannot be
// shared; bit 1 (shared) or bit 6 (shareable) on one is corruption
// (cve-2016-4332). A datatype (0x03) may be shareable.
let mtime = [1u8, 0, 0, 0, 0x10, 0x20, 0x30, 0x40];
for (id, flags) in [(0x08u16, 0x40u8), (0x08, 0x02), (0x12, 0x40)] {
let data = build_v1_header(&[(id, &mtime[..], flags)], 8, 8);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader(
"message of unshareable class flagged as shareable"
),
"id {id:#x} flags {flags:#x}"
);
}
let data = build_v1_header(&[(0x03, &[0u8; 8][..], 0x40)], 8, 8);
assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
// An unknown message is never checked for shareability.
let data = build_v1_header(&[(0x00FF, &[0u8; 8][..], 0x40)], 8, 8);
assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
}
#[test]
fn v1_message_must_be_aligned() {
// cve-2018-13873: a v1 message whose size is not a multiple of 8.
let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0)], 8, 8);
data[16 + 2] = 7; // size field of the only message
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("message not aligned")
);
}
#[test]
fn message_overrunning_its_chunk_is_refused() {
// It used to end the chunk quietly, dropping this message and any
// after it.
let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0)], 8, 8);
data[16 + 2] = 16;
data.resize(data.len() + 64, 0);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("message size exceeds buffer end")
);
let mut data = build_v2_header(0x00, &[(0x01, &[1, 2], 0)], None);
data[7 + 1] = 9; // size of the only message (after OHDR, ver, flags, chunk size)
let chk = crate::checksum::jenkins_lookup3(&data[..data.len() - 4]);
let n = data.len();
data[n - 4..].copy_from_slice(&chk.to_le_bytes());
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("message size exceeds buffer end")
);
}
#[test]
fn v1_gap_after_last_message_is_refused() {
// Fewer than 8 bytes left over: a gap, which only version 2 allows.
let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0)], 8, 8);
data[8] += 4; // header_data_size
data.extend_from_slice(&[0u8; 4]);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("gap found in early version of file format")
);
}
#[test]
fn v1_chunk_holding_more_messages_than_the_prefix_says_is_refused() {
// cve-2024-32619: the prefix says 1 message, the chunk holds 2. The
// second used to be dropped silently.
let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0), (0x03, &[0u8; 8][..], 0)], 8, 8);
data[2] = 1;
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("bad object header message count")
);
// Fewer in the chunk than the prefix says is fine (the rest may be in
// continuation chunks; libhdf5 only enforces that with strict checks).
data[2] = 3;
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap().messages.len(),
2
);
}
#[test]
fn v1_prefix_chunk_size_must_fit_the_message_count() {
let mut data = build_v1_header(&[], 8, 8);
data[8] = 8; // no messages but a non-empty chunk
data.extend_from_slice(&[0u8; 8]);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("bad object header chunk size")
);
}
#[test]
fn v1_header_cannot_hold_a_reference_count_message() {
// cve-2018-11204.
let data = build_v1_header(&[(0x16, &[0, 2, 0, 0, 0][..], 0)], 8, 8);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader(
"object header version does not support reference count message"
)
);
let data = build_v2_header(0x00, &[(0x16, &[0, 2, 0, 0, 0], 0)], None);
assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
}
#[test]
fn modification_time_messages_are_decoded_with_the_header() {
// cve-2024-33873 (version 0) and cve-2024-33874 (empty message).
for (body, why) in [
(
&[0u8, 0, 0, 0, 1, 2, 3, 4][..],
"bad version number for mtime message",
),
(&[][..], "ran off end of input buffer while decoding"),
] {
let data = build_v2_header(0x00, &[(0x12, body, 0)], None);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader(why)
);
}
let data = build_v2_header(0x00, &[(0x12, &[1, 0, 0, 0, 1, 2, 3, 4], 0)], None);
assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
// The old (0x0E) message is 14 ASCII digits and 2 reserved bytes.
let data = build_v1_header(&[(0x0E, &b"20110414214255\0\0"[..], 0)], 8, 8);
assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
let data = build_v1_header(&[(0x0E, &b"2011041421425x\0\0"[..], 0)], 8, 8);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("badly formatted modification time message")
);
}
#[test]
fn continuation_message_must_hold_a_nonempty_chunk() {
let mut cont = [0u8; 16];
cont[..8].copy_from_slice(&64u64.to_le_bytes());
let data = build_v2_header(0x00, &[(0x10, &cont, 0)], None);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("invalid continuation chunk size (0)")
);
let data = build_v2_header(0x00, &[(0x10, &cont[..8], 0)], None);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("ran off end of input buffer while decoding")
);
}
#[test]
fn v2_prefix_is_checked() {
let data = build_v2_header(0x40, &[(0x01, &[1], 0)], None);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("unknown object header status flag(s)")
);
// build_v2_header writes max_compact 8, min_dense 6; swap them.
let mut data = build_v2_header(0x10, &[(0x01, &[1], 0)], None);
data[6] = 6;
data[8] = 8;
let chk = crate::checksum::jenkins_lookup3(&data[..data.len() - 4]);
let n = data.len();
data[n - 4..].copy_from_slice(&chk.to_le_bytes());
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("bad object header attribute phase change values")
);
}
#[test]
fn v2_gap_is_allowed_only_without_nil_messages() {
// Three bytes after the last message: a gap (a message header is 4).
let mut data = build_v2_header(0x00, &[(0x01, &[1, 2, 3], 0), (0x03, &[], 0)], None);
// Turn the empty datatype message (4 header bytes) into a 3-byte gap
// by shrinking the chunk.
let n = data.len();
data.truncate(n - 5);
data[6] -= 1;
let chk = crate::checksum::jenkins_lookup3(&data);
data.extend_from_slice(&chk.to_le_bytes());
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap().messages.len(),
1
);
let mut data = build_v2_header(0x00, &[(0x00, &[1, 2, 3], 0), (0x03, &[], 0)], None);
let n = data.len();
data.truncate(n - 5);
data[6] -= 1;
let chk = crate::checksum::jenkins_lookup3(&data);
data.extend_from_slice(&chk.to_le_bytes());
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::InvalidObjectHeader("gap in chunk with no null messages")
);
}
#[test] #[test]
fn parse_v2_unknown_message_flags() { fn parse_v2_unknown_message_flags() {
let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x08)], None); let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x08)], None);