quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p3-storage-trait' into feat/p3-range-zfp-edit

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/btree_v1.rs
#	crates/clawhdf5-format/src/data_layout.rs
#	crates/clawhdf5-format/src/extensible_array.rs
#	crates/clawhdf5-format/src/fixed_array.rs
#	crates/clawhdf5-format/src/fractal_heap.rs
#	crates/clawhdf5-format/src/local_heap.rs
#	crates/clawhdf5-format/src/shared_message.rs
This commit is contained in:
osobh
2026-09-26 14:51:51 -05:00
parent 234dd3e36c 895c79a2fe
commit 437e81cfff
26 changed files with 3480 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/object_header.rs
+138 -42
View File
@@ -8,6 +8,7 @@ use byteorder::{ByteOrder, LittleEndian};
use crate::addr::to_usize;
use crate::error::FormatError;
use crate::message_type::MessageType;
use crate::storage::{Storage, Window, len_usize, read_exact_at};
/// OHDR signature for v2 object headers.
const OHDR_SIGNATURE: [u8; 4] = *b"OHDR";
@@ -119,32 +120,52 @@ impl ObjectHeader {
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
ensure_len(data, offset, 4)?;
if data[offset..offset + 4] == OHDR_SIGNATURE {
Self::parse_v2(data, offset, offset_size, length_size)
Self::parse_in(data, offset as u64, offset_size, length_size)
}
/// [`Self::parse`] over any [`Storage`].
///
/// Reads the prefix (at most [`V2_PREFIX_MAX`] bytes, signature
/// included), then each chunk as one bounded read, continuation chunks
/// included.
pub fn parse_in<S: Storage + ?Sized>(
file: &S,
offset: u64,
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
// The longest prefix of either version, in one read. It holds the
// whole prefix or ends at the end of the file, so its bounds checks
// are the whole-file ones.
let prefix = Window::read(file, offset, V2_PREFIX_MAX)?;
prefix.ensure(0, 4)?;
if prefix.bytes[..4] == OHDR_SIGNATURE {
Self::parse_v2(file, offset, &prefix, offset_size, length_size)
} else {
Self::parse_v1(data, offset, offset_size, length_size)
Self::parse_v1(file, offset, &prefix, offset_size, length_size)
}
}
fn parse_v1(
data: &[u8],
offset: usize,
fn parse_v1<S: Storage + ?Sized>(
file: &S,
offset: u64,
prefix: &Window<'_>,
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
// version(1) + reserved(1) + num_messages(2) + ref_count(4) + header_size(4) = 12
// then pad to 8-byte alignment from start of header
ensure_len(data, offset, 12)?;
prefix.ensure(0, 12)?;
let prefix = &prefix.bytes[..12];
let version = data[offset];
let version = prefix[0];
if version != 1 {
return Err(FormatError::InvalidObjectHeaderVersion(version));
}
let num_messages = LittleEndian::read_u16(&data[offset + 2..offset + 4]) as usize;
let reference_count = LittleEndian::read_u32(&data[offset + 4..offset + 8]);
let header_data_size = LittleEndian::read_u32(&data[offset + 8..offset + 12]) as usize;
let num_messages = LittleEndian::read_u16(&prefix[2..4]) as usize;
let reference_count = LittleEndian::read_u32(&prefix[4..8]);
let header_data_size = LittleEndian::read_u32(&prefix[8..12]) as usize;
// libhdf5 (H5O__prefix_deserialize): a header with messages needs room
// for at least one message header, and one without has an empty chunk.
@@ -162,14 +183,15 @@ impl ObjectHeader {
.checked_add(12 + padding)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
available: data.len(),
available: len_usize(file),
})?;
ensure_len(data, msg_start, header_data_size)?;
let mut messages = Vec::new();
// parse_v1_chunk reads the chunk, with the bounds check that was here.
// The prefix's count (NIL messages included, capped: it is untrusted)
// sizes the list once instead of growing it message by message.
let mut messages = Vec::with_capacity(num_messages.min(64));
let chunk0_count = Self::parse_v1_chunk(
data,
file,
msg_start,
header_data_size,
offset_size,
@@ -209,9 +231,9 @@ impl ObjectHeader {
/// end of the chunk, or leftover bytes too few for a message header (a
/// "gap", which only version 2 allows).
#[allow(clippy::too_many_arguments)]
fn parse_v1_chunk(
data: &[u8],
offset: usize,
fn parse_v1_chunk<S: Storage + ?Sized>(
file: &S,
offset: u64,
length: usize,
offset_size: u8,
length_size: u8,
@@ -221,9 +243,10 @@ impl ObjectHeader {
if depth_remaining == 0 {
return Err(FormatError::NestingDepthExceeded);
}
ensure_len(data, offset, length)?;
let end = offset + length;
let mut pos = offset;
let chunk = read_exact_at(file, offset, length)?;
let data: &[u8] = &chunk;
let end = length;
let mut pos = 0usize;
let mut count = 0usize;
while pos < end {
@@ -268,8 +291,8 @@ impl ObjectHeader {
let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?;
let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
Self::parse_v1_chunk(
data,
cont_offset,
file,
cont_offset as u64,
cont_length,
offset_size,
length_size,
@@ -282,12 +305,22 @@ impl ObjectHeader {
Ok(count)
}
fn parse_v2(
data: &[u8],
offset: usize,
fn parse_v2<S: Storage + ?Sized>(
file: &S,
offset: u64,
prefix: &Window<'_>,
offset_size: u8,
length_size: u8,
) -> Result<ObjectHeader, FormatError> {
// `ensure_len` checks positions relative to the header against the
// prefix window and reports them as the whole-file check did, with
// absolute positions and the file's length.
let data: &[u8] = &prefix.bytes;
let file_len = len_usize(file);
let base = usize::try_from(offset).unwrap_or(usize::MAX);
let abs = |rel: usize| base.saturating_add(rel);
let ensure_len = |_: &[u8], rel: usize, needed: usize| prefix.ensure(rel, needed);
let offset = 0usize;
// signature(4) + version(1) + flags(1) = 6
ensure_len(data, offset, 6)?;
@@ -352,15 +385,20 @@ impl ObjectHeader {
}
let chunk0_msg_start = pos;
let chunk0_msg_end = pos
.checked_add(chunk0_size)
.ok_or(FormatError::UnexpectedEof {
let Some(chunk0_abs_end) = abs(pos).checked_add(chunk0_size) else {
return Err(FormatError::UnexpectedEof {
expected: usize::MAX,
available: data.len(),
})?;
available: file_len,
});
};
let chunk0_msg_end = chunk0_abs_end - base;
// The whole first chunk, prefix to checksum, in one read (its
// bounds check is the one on the checksum's 4 bytes).
let chunk0 = read_exact_at(file, base as u64, chunk0_msg_end.saturating_add(4))?;
let data: &[u8] = &chunk0;
// Validate checksum: from OHDR signature through all messages (before checksum)
ensure_len(data, chunk0_msg_end, 4)?;
#[cfg(feature = "checksum")]
{
let stored = LittleEndian::read_u32(&data[chunk0_msg_end..chunk0_msg_end + 4]);
@@ -395,8 +433,8 @@ impl ObjectHeader {
}
cont_remaining -= 1;
Self::parse_v2_continuation(
data,
cont_offset,
file,
cont_offset as u64,
cont_length,
has_creation_order,
offset_size,
@@ -495,9 +533,9 @@ impl ObjectHeader {
}
#[allow(clippy::too_many_arguments)]
fn parse_v2_continuation(
data: &[u8],
offset: usize,
fn parse_v2_continuation<S: Storage + ?Sized>(
file: &S,
offset: u64,
length: usize,
has_creation_order: bool,
offset_size: u8,
@@ -506,7 +544,9 @@ impl ObjectHeader {
continuations: &mut Vec<(usize, usize)>,
) -> Result<(), FormatError> {
// OCHK signature(4) + messages + checksum(4)
ensure_len(data, offset, length)?;
let chunk = read_exact_at(file, offset, length)?;
let data: &[u8] = &chunk;
let offset = 0usize;
if length < 8 {
return Err(FormatError::UnexpectedEof {
expected: 8,
@@ -547,6 +587,10 @@ impl ObjectHeader {
}
}
/// Longest version-2 object header prefix: signature(4) + version(1) +
/// flags(1) + times(16) + attribute phase change(4) + chunk-0 size(8).
const V2_PREFIX_MAX: usize = 34;
/// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3).
const V1_MSG_HEADER_SIZE: usize = 8;
@@ -755,13 +799,13 @@ mod tests {
let mut msg_bytes = Vec::new();
for (mtype, mdata, mflags) in messages {
// v1 message sizes are multiples of 8 (the data is zero-padded).
let padded = mdata.len().div_ceil(8) * 8;
let padded = <[u8]>::len(mdata).div_ceil(8) * 8;
msg_bytes.extend_from_slice(&mtype.to_le_bytes()); // type(2)
msg_bytes.extend_from_slice(&(padded as u16).to_le_bytes()); // size(2)
msg_bytes.push(*mflags); // flags(1)
msg_bytes.extend_from_slice(&[0u8; 3]); // reserved(3)
msg_bytes.extend_from_slice(mdata); // data
msg_bytes.resize(msg_bytes.len() + padded - mdata.len(), 0);
msg_bytes.resize(msg_bytes.len() + padded - <[u8]>::len(mdata), 0);
}
let mut buf = Vec::new();
@@ -1281,4 +1325,56 @@ mod tests {
let err = ObjectHeader::parse(&data, 0, 8, 8).unwrap_err();
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
}
/// Every header, and every truncation of it, parses to the same result
/// (or the same error) through a `read_at`-only storage as from a slice;
/// a header in one chunk takes two reads (prefix, chunk).
#[test]
fn parse_in_matches_slice_parse() {
use crate::storage::CountingStorage;
let mut headers = vec![
build_v1_header(&[], 8, 8),
build_v1_header(&[(0x0001, &[1, 2, 3], 0), (0x0003, &[9; 8], 0)], 8, 8),
build_v2_header(0x00, &[(0x01, &[42], 0)], None),
build_v2_header(0x03, &[(0x01, &[1, 2], 0), (0x03, &[3], 0)], None),
build_v2_header(0x24, &[(0x01, &[1], 0)], Some((1, 2, 3, 4))),
build_v2_header(0x35, &[(0x01, &[1], 0)], Some((5, 6, 7, 8))),
];
// A v2 header with a continuation chunk at 256.
let mut ochk = OCHK_SIGNATURE.to_vec();
ochk.extend_from_slice(&[0x03, 2, 0, 0, 0xDE, 0xAD]);
let sum = crate::checksum::jenkins_lookup3(&ochk);
ochk.extend_from_slice(&sum.to_le_bytes());
let mut cont = 256u64.to_le_bytes().to_vec();
cont.extend_from_slice(&(ochk.len() as u64).to_le_bytes());
let main = build_v2_header(0x00, &[(0x01, &[42], 0), (0x10, &cont, 0)], None);
let mut with_cont = vec![0u8; 256 + ochk.len()];
with_cont[..main.len()].copy_from_slice(&main);
with_cont[256..].copy_from_slice(&ochk);
headers.push(with_cont);
for h in headers {
for at in [0usize, 3] {
for cut in 0..=h.len() {
let mut f = vec![0u8; at];
f.extend_from_slice(&h[..cut]);
if at == 0 && cut == h.len() {
f.resize(f.len() + 64, 0);
}
let want = ObjectHeader::parse(&f, at, 8, 8);
let storage = CountingStorage::new(f.clone());
let got = ObjectHeader::parse_in(&storage, at as u64, 8, 8);
assert_eq!(
format!("{got:?}"),
format!("{want:?}"),
"at {at}, cut {cut}"
);
}
}
}
let one_chunk = build_v2_header(0x00, &[(0x01, &[42], 0)], None);
let storage = CountingStorage::new(one_chunk);
ObjectHeader::parse_in(&storage, 0, 8, 8).unwrap();
assert_eq!(storage.reads(), 2);
}
}