quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p1-userblock-shared' into fix/p1-read-gaps

Browse Source 
# Conflicts:
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/datatype.rs
#	crates/clawhdf5-format/src/shared_message.rs
#	docs/known-issues.md
This commit is contained in:
osobh
2026-09-25 22:40:55 -05:00
parent adf961c883 90e050944f
commit 45720fe5a6
21 changed files with 1023 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_read.rs
+5 -3
View File
@@ -575,11 +575,13 @@ fn read_named_dataset_raw(
use crate::group_v2::resolve_path_any;
use crate::message_type::MessageType;
use crate::object_header::ObjectHeader;
use crate::signature::find_signature;
use crate::signature::split_user_block;
use crate::superblock::Superblock;
let sig = find_signature(file_data)?;
let sb = Superblock::parse(file_data, sig)?;
// An external source file is handed over whole, user block included;
// its addresses are relative to its superblock.
let (_, file_data) = split_user_block(file_data)?;
let sb = Superblock::parse(file_data, 0)?;
let addr = resolve_path_any(file_data, &sb, path)?;
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?;
crates/clawhdf5-format/src/datatype.rs
+79 -73
View File
@@ -423,34 +423,42 @@ impl Datatype {
ensure_len(data, pos, 4)?;
let byte_offset = LittleEndian::read_u32(&data[pos..pos + 4]) as u64;
pos += 4;
// v1 members can be fixed-size arrays of their
// datatype (HDF5 before 1.4 had no array class):
// libhdf5 wraps such a member in an array type of the
// first `ndims` of the four stored dimensions and
// ignores the permutation.
let mut legacy_dims = Vec::new();
// v1 members can be fixed-size arrays of the member
// type (libhdf5 builds an array type from these
// fields; the permutation is ignored, as libhdf5
// does). Skipping them read a `[4] i32` member as
// one `i32`.
let mut array_dims = Vec::new();
if version == 1 {
ensure_len(data, pos, 28)?;
let ndims = data[pos] as usize;
if ndims > 4 {
// libhdf5 refuses more than four dimensions and,
// when building the array type, a zero-sized one.
let zero_dim = (0..ndims.min(4)).any(|j| {
let at = pos + 12 + 4 * j;
LittleEndian::read_u32(&data[at..at + 4]) == 0
});
if ndims > 4 || zero_dim {
return Err(FormatError::InvalidDatatypeVersion {
class: class_id,
version,
});
}
for i in 0..ndims {
let at = pos + 12 + 4 * i;
legacy_dims.push(LittleEndian::read_u32(&data[at..at + 4]));
}
array_dims = (0..ndims)
.map(|j| {
let at = pos + 12 + 4 * j;
LittleEndian::read_u32(&data[at..at + 4])
})
.collect();
pos += 28;
}
let (mut member_dt, consumed) =
Self::parse_with_depth(&data[pos..], depth + 1)?;
pos += consumed;
if !legacy_dims.is_empty() {
if !array_dims.is_empty() {
member_dt = Datatype::Array {
base_type: Box::new(member_dt),
dimensions: legacy_dims,
dimensions: array_dims,
};
}
members.push(CompoundMember {
@@ -1341,66 +1349,6 @@ mod tests {
assert_xyid_compound(dt);
}
/// A v1 compound member with legacy array dimensions (HDF5 before 1.4,
/// e.g. `tarrold.h5`): `{ i: i16, f: f32[2][3] }`. The member must become
/// an array type, not a scalar at the member's offset.
#[test]
fn test_compound_v1_legacy_array_member() {
let i16le: [u8; 12] = [
0x10, 0x08, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
];
let f32le: [u8; 20] = [
0x11, 0x20, 0x1f, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x17, 0x08,
0x00, 0x17, 0x7f, 0x00, 0x00, 0x00,
];
let mut b = vec![0x16, 0x02, 0x00, 0x00, 28, 0x00, 0x00, 0x00];
for (name, offset, ndims, dims, dt) in [
(&b"i"[..], 0u32, 0u8, [0u32; 4], &i16le[..]),
(&b"f"[..], 4, 2, [2, 3, 0, 0], &f32le[..]),
] {
let mut padded = name.to_vec();
padded.resize((name.len() + 1 + 7) & !7, 0);
b.extend_from_slice(&padded);
b.extend_from_slice(&offset.to_le_bytes());
b.extend_from_slice(&[ndims, 0, 0, 0]);
b.extend_from_slice(&[0, 1, 2, 3]); // dimension permutation
b.extend_from_slice(&[0; 4]);
for d in dims {
b.extend_from_slice(&d.to_le_bytes());
}
b.extend_from_slice(dt);
}
let (dt, consumed) = Datatype::parse(&b).unwrap();
assert_eq!(consumed, b.len());
let Datatype::Compound { size, members } = dt else {
panic!("expected Compound, got {dt:?}");
};
assert_eq!(size, 28);
assert!(matches!(
members[0].datatype,
Datatype::FixedPoint { size: 2, .. }
));
match &members[1].datatype {
Datatype::Array {
base_type,
dimensions,
} => {
assert_eq!(dimensions, &[2, 3]);
assert!(matches!(
**base_type,
Datatype::FloatingPoint { size: 4, .. }
));
}
other => panic!("expected an array member, got {other:?}"),
}
assert_eq!(members[1].datatype.type_size(), 24);
// More than four legacy dimensions is not a valid message.
let mut bad = b.clone();
bad[8 + 8 + 4] = 5; // first member's dimensionality
assert!(Datatype::parse(&bad).is_err());
}
#[test]
fn test_compound_v2_padded_names_no_array_fields() {
// v2 = v1 without the 28 bytes of per-member array fields; names are
@@ -1419,6 +1367,64 @@ mod tests {
assert_xyid_compound(dt);
}
#[test]
fn test_compound_v1_member_array_fields() {
// HDF5 1.6 wrote array members of a v1 compound through the legacy
// per-member fields (as in libhdf5's tools/test/testfiles/
// tcompound.h5 `type2`: `int_array` [4] i32, `float_array` [5][6]
// f32). They used to be skipped, reading each member as a scalar.
let i32le: [u8; 12] = [
0x10, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00,
];
let mut b = vec![0x16, 0x02, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00];
for (name, offset, dims) in [
(&b"int_array"[..], 0u32, &[4u32][..]),
(&b"xy"[..], 16, &[5u32, 6][..]),
] {
let mut padded = name.to_vec();
padded.resize((name.len() + 1 + 7) & !7, 0);
b.extend_from_slice(&padded);
b.extend_from_slice(&offset.to_le_bytes());
b.push(dims.len() as u8);
b.extend_from_slice(&[0u8; 3 + 4 + 4]); // reserved, permutation, reserved
for j in 0..4 {
b.extend_from_slice(&dims.get(j).copied().unwrap_or(0).to_le_bytes());
}
b.extend_from_slice(&i32le);
}
let (dt, consumed) = Datatype::parse(&b).unwrap();
assert_eq!(consumed, b.len());
let Datatype::Compound { members, .. } = dt else {
panic!("expected Compound, got {dt:?}");
};
let got: Vec<(&str, u64, u32, Option<Vec<u32>>)> = members
.iter()
.map(|m| {
let dims = match &m.datatype {
Datatype::Array { dimensions, .. } => Some(dimensions.clone()),
_ => None,
};
(m.name.as_str(), m.byte_offset, m.datatype.type_size(), dims)
})
.collect();
assert_eq!(
got,
vec![
("int_array", 0, 16, Some(vec![4])),
("xy", 16, 120, Some(vec![5, 6])),
]
);
// More than four dimensions cannot be encoded, and libhdf5 refuses a
// zero-sized dimension (a fuzzed tcompound.h5, cve-2024-32616.h5).
let mut bad = b.clone();
bad[8 + 16 + 4] = 5;
assert!(Datatype::parse(&bad).is_err());
let mut bad = b.clone();
bad[8 + 16 + 4] = 2; // [4, 0]
assert!(Datatype::parse(&bad).is_err());
}
#[test]
fn test_compound_v1_truncated_is_error_not_panic() {
let bytes = compound_v1_bytes();
crates/clawhdf5-format/src/error.rs
+24
View File
@@ -80,6 +80,9 @@ pub enum FormatError {
InvalidLocalHeapSignature,
/// Invalid local heap version.
InvalidLocalHeapVersion(u8),
/// A local heap's free list points outside its data segment (libhdf5:
/// "bad heap free list").
InvalidLocalHeapFreeList,
/// Invalid B-tree v1 signature.
InvalidBTreeSignature,
/// Invalid B-tree node type.
@@ -117,6 +120,14 @@ pub enum FormatError {
/// A message is marked shared but was parsed without access to the file,
/// so the reference to the real message could not be followed.
UnresolvedSharedMessage,
/// A shared-message reference points at an object header that holds no
/// (unshared) message of the referenced type (raw message type id).
SharedMessageTargetMissing(u16),
/// A superblock was parsed at a non-zero offset of the buffer (the file
/// has a user block of this many bytes). HDF5 addresses are relative to
/// the superblock, so the buffer must start there: see
/// `signature::split_user_block`.
UserBlockNotStripped(u64),
/// A selection does not fit the dataset it was applied to (wrong rank, or
/// it reaches past a dimension's extent).
SelectionOutOfBounds(String),
@@ -270,6 +281,9 @@ impl fmt::Display for FormatError {
FormatError::InvalidLocalHeapSignature => {
write!(f, "invalid local heap signature")
}
FormatError::InvalidLocalHeapFreeList => {
write!(f, "bad local heap free list")
}
FormatError::InvalidLocalHeapVersion(v) => {
write!(f, "invalid local heap version: {v}")
}
@@ -339,6 +353,16 @@ impl fmt::Display for FormatError {
FormatError::SelectionOutOfBounds(msg) => {
write!(f, "selection out of bounds: {msg}")
}
FormatError::UserBlockNotStripped(n) => write!(
f,
"file has a {n}-byte user block: parse the bytes from the superblock on \
(signature::split_user_block)"
),
FormatError::SharedMessageTargetMissing(t) => write!(
f,
"shared message reference points at an object header with no message of type \
{t:#06x}"
),
FormatError::UnresolvedSharedMessage => write!(
f,
"message is shared but no file data was available to resolve it"
crates/clawhdf5-format/src/group_v1.rs
+12
View File
@@ -45,9 +45,16 @@ pub fn resolve_v1_group_entries(
)?;
let mut entries = Vec::new();
let mut heap_checked = false;
for snod_addr in snod_addrs {
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
for entry in &snod.entries {
// Like libhdf5, look at the heap's free list only once a name is
// needed: an empty group with a damaged heap still lists.
if !heap_checked {
heap.validate_free_list(file_data, length_size)?;
heap_checked = true;
}
let name = heap.read_string(file_data, entry.link_name_offset)?;
entries.push(GroupEntry {
name,
@@ -85,12 +92,17 @@ pub fn find_v1_soft_link(
offset_size,
length_size,
)?;
let mut heap_checked = false;
for snod_addr in snod_addrs {
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
for entry in &snod.entries {
if entry.cache_type != CACHE_TYPE_SOFT_LINK {
continue;
}
if !heap_checked {
heap.validate_free_list(file_data, length_size)?;
heap_checked = true;
}
if heap.read_string(file_data, entry.link_name_offset)? != name {
continue;
}
crates/clawhdf5-format/src/lib.rs
+6 -5
View File
@@ -26,12 +26,13 @@
//! use clawhdf5_format::{signature, superblock, object_header, group_v2,
//! datatype, dataspace, data_layout, data_read, message_type::MessageType};
//!
//! let file_data = std::fs::read("output.h5").unwrap();
//! let sig = signature::find_signature(&file_data).unwrap();
//! let sb = superblock::Superblock::parse(&file_data, sig).unwrap();
//! let addr = group_v2::resolve_path_any(&file_data, &sb, "data").unwrap();
//! let bytes = std::fs::read("output.h5").unwrap();
//! // Addresses are relative to the superblock: skip any user block.
//! let (_user_block, file_data) = signature::split_user_block(&bytes).unwrap();
//! let sb = superblock::Superblock::parse(file_data, 0).unwrap();
//! let addr = group_v2::resolve_path_any(file_data, &sb, "data").unwrap();
//! let hdr = object_header::ObjectHeader::parse(
//! &file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
//! file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
//! ```
//!
//! # Features
crates/clawhdf5-format/src/local_heap.rs
+97 -2
View File
@@ -87,6 +87,57 @@ impl LocalHeap {
})
}
/// Walk the free list the way libhdf5 does when it loads a heap's data
/// (`H5HL__fl_deserialize`), rejecting a heap whose free list points
/// outside the data segment. libhdf5 refuses such a heap ("bad heap free
/// list"), and names read from it would be garbage.
///
/// libhdf5 only loads a heap when it needs a name from it (an empty
/// group's broken heap goes unnoticed), so call this before the first
/// [`Self::read_string`], not on parse.
///
/// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the
/// undefined address) is accepted as "no free list" too.
pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> {
const FREE_NULL: u64 = 1;
let ls = length_size as usize;
let undefined = if ls >= 8 {
u64::MAX
} else {
(1u64 << (8 * ls)) - 1
};
let size = self.data_segment_size;
let seg = self.data_segment_address;
let mut next = self.free_list_head_offset;
// Each free block holds two lengths, so a list longer than this
// revisits a block: a cycle.
let max_blocks = size / (2 * ls as u64) + 1;
let mut walked = 0u64;
while next != FREE_NULL && next != undefined {
if next >= size || walked >= max_blocks {
return Err(FormatError::InvalidLocalHeapFreeList);
}
walked += 1;
let at = seg
.checked_add(next)
.and_then(|a| usize::try_from(a).ok())
.ok_or(FormatError::InvalidLocalHeapFreeList)?;
let block_offset = next;
next = read_offset(file_data, at, length_size)?;
if next == 0 {
return Err(FormatError::InvalidLocalHeapFreeList);
}
let block_size = read_offset(file_data, at + ls, length_size)?;
if block_offset
.checked_add(block_size)
.is_none_or(|end| end > size)
{
return Err(FormatError::InvalidLocalHeapFreeList);
}
}
Ok(())
}
/// Read a null-terminated string from the heap's data segment at the given byte offset.
pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> {
let seg_addr = self.data_segment_address as usize;
@@ -162,8 +213,8 @@ mod tests {
// data_segment_size
write_val(&mut file, pos, data_seg_size as u64, length_size);
pos += length_size as usize;
// free_list_head_offset
write_val(&mut file, pos, 0xFFFFFFFF, length_size);
// free_list_head_offset: H5HL_FREE_NULL (no free space)
write_val(&mut file, pos, 1, length_size);
pos += length_size as usize;
// data_segment_address
write_val(&mut file, pos, data_seg_offset as u64, offset_size);
@@ -243,6 +294,50 @@ mod tests {
assert_eq!(s, "test");
}
/// Heap with data segment `[a, b, c, 0-padding]` whose free list starts
/// at `head` and has one block `(next, size)` at offset 8.
fn heap_with_free_block(head: u64, next: u64, size: u64) -> Vec<u8> {
let mut file = build_heap_file(0, 100, &["abcdefg"], 8, 8);
file.resize(200, 0);
write_val(&mut file, 8, 32, 8); // data segment size
write_val(&mut file, 16, head, 8);
write_val(&mut file, 108, next, 8);
write_val(&mut file, 116, size, 8);
file
}
#[test]
fn free_list_inside_the_segment_is_accepted() {
let file = heap_with_free_block(8, 1, 24);
let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
heap.validate_free_list(&file, 8).unwrap();
assert_eq!(heap.read_string(&file, 0).unwrap(), "abcdefg");
// An all-ones head is "no free list" too.
let file = heap_with_free_block(u64::MAX, 0, 0);
let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
assert!(heap.validate_free_list(&file, 8).is_ok());
}
#[test]
fn bad_free_list_is_rejected_like_libhdf5() {
for (head, next, size, why) in [
(40, 1, 8, "head past the segment"),
(8, 1, 25, "block runs past the segment"),
(8, 0, 8, "next offset of zero"),
(8, 8, 8, "cycle"),
(8, 999, 8, "next past the segment"),
] {
let file = heap_with_free_block(head, next, size);
// The header itself parses; the free list is checked on use.
let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
assert_eq!(
heap.validate_free_list(&file, 8).unwrap_err(),
FormatError::InvalidLocalHeapFreeList,
"{why}"
);
}
}
#[test]
fn invalid_version() {
let mut file = build_heap_file(0, 100, &["x"], 8, 8);
crates/clawhdf5-format/src/signature.rs
+36
View File
@@ -11,6 +11,16 @@ pub const HDF5_SIGNATURE: [u8; 8] = [0x89, b'H', b'D', b'F', b'\r', b'\n', 0x1A,
/// (powers of two starting at 512, plus offset 0).
///
/// Returns the byte offset where the signature was found.
///
/// A non-zero offset means the file starts with a *user block*, and every
/// address inside the file is relative to the superblock's position, not to
/// byte 0 (libhdf5 uses the signature's position as the base address even
/// when the stored base-address field disagrees). The parsers in this crate
/// take addresses as indices into `file_data`, so they must be handed the
/// bytes from the signature on — use [`split_user_block`]. [`Superblock::parse`]
/// refuses a non-zero offset for this reason.
///
/// [`Superblock::parse`]: crate::superblock::Superblock::parse
pub fn find_signature(data: &[u8]) -> Result<usize, FormatError> {
// Check offset 0
if data.len() >= 8 && data[..8] == HDF5_SIGNATURE {
@@ -29,6 +39,17 @@ pub fn find_signature(data: &[u8]) -> Result<usize, FormatError> {
Err(FormatError::SignatureNotFound)
}
/// Split a file into its user block and its HDF5 bytes.
///
/// Returns `(user_block, hdf5)`: `user_block` is everything before the
/// superblock signature (empty for most files) and `hdf5` is the rest, in
/// which every HDF5 address is a plain index. Pass `hdf5` as `file_data` to
/// every parser in this crate, and parse the superblock at offset 0 of it.
pub fn split_user_block(data: &[u8]) -> Result<(&[u8], &[u8]), FormatError> {
let offset = find_signature(data)?;
Ok(data.split_at(offset))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -88,6 +109,21 @@ mod tests {
assert_eq!(find_signature(&data), Err(FormatError::SignatureNotFound));
}
#[test]
fn split_user_block_rebases_at_the_signature() {
let mut data = vec![7u8; 1024];
data[512..520].copy_from_slice(&HDF5_SIGNATURE);
let (ub, hdf5) = split_user_block(&data).unwrap();
assert_eq!(ub.len(), 512);
assert_eq!(hdf5.len(), 512);
assert_eq!(&hdf5[..8], &HDF5_SIGNATURE);
data[..8].copy_from_slice(&HDF5_SIGNATURE);
let (ub, hdf5) = split_user_block(&data).unwrap();
assert!(ub.is_empty());
assert_eq!(hdf5.len(), 1024);
}
#[test]
fn signature_prefers_earliest() {
// Signature at both 0 and 512, should return 0
crates/clawhdf5-format/src/superblock.rs
+21 -2
View File
@@ -174,8 +174,18 @@ impl Superblock {
/// Parse a superblock from `data` starting at `signature_offset`.
///
/// The signature must be present at the given offset.
/// The signature must be present at the given offset, and that offset
/// must be 0: every address in an HDF5 file is relative to the
/// superblock, so when a file has a user block (signature at 512, 1024,
/// …) the caller must pass the bytes from the signature on — see
/// [`crate::signature::split_user_block`] — and use that slice as
/// `file_data` everywhere. A non-zero offset is refused with
/// [`FormatError::UserBlockNotStripped`] because the addresses in the
/// returned superblock would otherwise be applied to the wrong bytes.
pub fn parse(data: &[u8], signature_offset: usize) -> Result<Superblock, FormatError> {
if signature_offset != 0 {
return Err(FormatError::UserBlockNotStripped(signature_offset as u64));
}
let d = data
.get(signature_offset..)
.ok_or(FormatError::UnexpectedEof {
@@ -676,7 +686,16 @@ mod tests {
let mut data = vec![0u8; 1024];
let v0 = build_v0_bytes(8);
data[512..512 + v0.len()].copy_from_slice(&v0);
let sb = Superblock::parse(&data, 512).unwrap();
// Addresses are relative to the superblock, so parsing in place
// (where they would be applied to the whole buffer) is refused...
assert_eq!(
Superblock::parse(&data, 512),
Err(FormatError::UserBlockNotStripped(512))
);
// ...and the caller parses the bytes from the signature on.
let (ub, hdf5) = crate::signature::split_user_block(&data).unwrap();
assert_eq!(ub.len(), 512);
let sb = Superblock::parse(hdf5, 0).unwrap();
assert_eq!(sb.version, 0);
assert_eq!(sb.root_group_address, 96);
}
crates/clawhdf5-format/tests/fixtures/tcompound.h5
Vendored
BIN
View File
Binary file not shown.