osobhandClaude Opus 5.5
7d7a7e75d4
The superblock records where the file's data ends. libhdf5 refuses to
open a file shorter than that ("truncated file", H5F__super_read) and
fails any read past it ("addr overflow" / "address plus size exceeds
file eoa"). clawhdf5 read whatever was left of a truncated file, and read
bytes after the recorded end as if they belonged to the file.
New Superblock::data_end: FormatError::TruncatedFile for a file shorter
than its recorded end, otherwise where the HDF5 data ends. As in libhdf5,
the recorded end moves with the superblock when its recorded base address
is not where it is (a user block added afterwards; cve-2021-36977, which
h5py reads, depends on it), and the check is skipped for a v3 superblock
still being written in SWMR mode. File, LazyFile, MmapFile and the
conformance probe refuse a truncated file and parse only up to the end.
Files clawhdf5 writes record their true length, and the v2.5.0 agent-store
fixture and files written by v2.7.0 (plain, paged, user-block free) pass
the check.
Interop test: h5py writes a file; a copy missing its last 8 bytes must be
refused by both, a copy with bytes appended and one moved behind a new
512-byte user block must read in both.
Conformance (cached corpus, tank): 570 -> 571 ok
(h5clear_fsm_persist_less.h5, whose data past the recorded end was being
read); ten files h5py refuses as truncated (cve-2018-13874,
cve-2018-13876, the family/multi/subfiling members, h5clear_fsm_persist_
greater/user_greater) are now refused at open instead of read.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
898 lines
33 KiB
Rust
898 lines
33 KiB
Rust
//! HDF5 Superblock parsing for versions 0, 1, 2, and 3.
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::vec::Vec;
|
||
|
||
use byteorder::{ByteOrder, LittleEndian};
|
||
|
||
use crate::error::FormatError;
|
||
use crate::signature::HDF5_SIGNATURE;
|
||
|
||
/// Parsed HDF5 superblock (all versions).
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct Superblock {
|
||
/// Superblock version (0–3).
|
||
pub version: u8,
|
||
/// Size of offsets in bytes (2, 4, or 8).
|
||
pub offset_size: u8,
|
||
/// Size of lengths in bytes (2, 4, or 8).
|
||
pub length_size: u8,
|
||
/// File base address.
|
||
pub base_address: u64,
|
||
/// End-of-file address.
|
||
pub eof_address: u64,
|
||
/// Root group object header address (v2/v3) or from symbol table entry (v0/v1).
|
||
pub root_group_address: u64,
|
||
/// Group leaf node K (v0/v1 only).
|
||
pub group_leaf_node_k: Option<u16>,
|
||
/// Group internal node K (v0/v1 only).
|
||
pub group_internal_node_k: Option<u16>,
|
||
/// Indexed storage internal node K (v1 only).
|
||
pub indexed_storage_internal_node_k: Option<u16>,
|
||
/// Free space address (v0/v1 only).
|
||
pub free_space_address: Option<u64>,
|
||
/// Driver info block address (v0/v1 only).
|
||
pub driver_info_address: Option<u64>,
|
||
/// File consistency flags.
|
||
pub consistency_flags: u32,
|
||
/// Superblock extension address (v2/v3 only).
|
||
pub superblock_extension_address: Option<u64>,
|
||
/// CRC32C checksum (v2/v3 only).
|
||
pub checksum: Option<u32>,
|
||
/// Page size of the non-standard "version 4" superblock layout (v4 only).
|
||
/// `None` for v0–v3.
|
||
///
|
||
/// HDF5 has no superblock version 4 — libhdf5 refuses it. A real paged
|
||
/// file is a v2/v3 superblock whose extension holds a File Space Info
|
||
/// message (what `FileWriter::with_page_size` writes). This field is kept
|
||
/// only so such files written by older clawhdf5 versions still parse.
|
||
pub page_size: Option<u32>,
|
||
}
|
||
|
||
/// Read an unsigned integer of `size` bytes (LE) from `data` at `pos`.
|
||
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||
let s = size as usize;
|
||
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: pos.saturating_add(s),
|
||
available: data.len(),
|
||
});
|
||
}
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match size {
|
||
2 => LittleEndian::read_u16(slice) as u64,
|
||
4 => LittleEndian::read_u32(slice) as u64,
|
||
8 => LittleEndian::read_u64(slice),
|
||
_ => unreachable!(), // validated before calling
|
||
})
|
||
}
|
||
|
||
fn validate_sizes(offset_size: u8, length_size: u8) -> Result<(), FormatError> {
|
||
if !matches!(offset_size, 2 | 4 | 8) {
|
||
return Err(FormatError::InvalidOffsetSize(offset_size));
|
||
}
|
||
if !matches!(length_size, 2 | 4 | 8) {
|
||
return Err(FormatError::InvalidLengthSize(length_size));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], needed: usize) -> Result<(), FormatError> {
|
||
if data.len() < needed {
|
||
Err(FormatError::UnexpectedEof {
|
||
expected: needed,
|
||
available: data.len(),
|
||
})
|
||
} else {
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// SWMR (Single Writer / Multiple Reader) consistency flag bits.
|
||
///
|
||
/// These flags are stored in the `consistency_flags` field of the superblock.
|
||
/// For v2/v3 superblocks, only the low byte is stored.
|
||
pub mod swmr_flags {
|
||
/// Bit 0: File is open for writing.
|
||
pub const WRITE_ACCESS: u32 = 0x01;
|
||
/// Bit 2: SWMR write access is active.
|
||
pub const SWMR_WRITE: u32 = 0x04;
|
||
}
|
||
|
||
impl Superblock {
|
||
/// Where the HDF5 data ends, relative to the superblock, for a file of
|
||
/// `file_len` bytes whose superblock is at `user_block` (both counted
|
||
/// from the start of the file), with libhdf5's truncation check
|
||
/// (`H5F__super_read`).
|
||
///
|
||
/// The superblock records the end of the file's data as an absolute
|
||
/// address. A file shorter than that was truncated, and libhdf5 refuses
|
||
/// to open it ("truncated file"); so does this, with
|
||
/// [`FormatError::TruncatedFile`]. Bytes past that address are not part
|
||
/// of the file: libhdf5 fails any read of them ("addr overflow" /
|
||
/// "address plus size exceeds file eoa"), so a reader should parse only
|
||
/// the data up to the returned end. As libhdf5 does for a SWMR reader,
|
||
/// the check is skipped for a version-3 superblock whose writer is still
|
||
/// writing it in SWMR mode (it extends the file as it goes); the data
|
||
/// then ends at the end of the file.
|
||
///
|
||
/// When the superblock's recorded base address differs from where the
|
||
/// superblock actually is (a user block added or removed after the file
|
||
/// was written), libhdf5 moves the recorded end of file by the same
|
||
/// amount, and so does this.
|
||
pub fn data_end(&self, user_block: u64, file_len: u64) -> Result<u64, FormatError> {
|
||
let eof =
|
||
i128::from(self.eof_address) - i128::from(self.base_address) + i128::from(user_block);
|
||
if eof < 0 || eof > i128::from(file_len) {
|
||
if self.version >= 3 && self.is_swmr_write() {
|
||
return Ok(file_len.saturating_sub(user_block));
|
||
}
|
||
return Err(FormatError::TruncatedFile {
|
||
stored_eof: u64::try_from(eof).unwrap_or(self.eof_address),
|
||
actual_len: file_len,
|
||
});
|
||
}
|
||
// 0 <= eof <= file_len, so it fits a u64.
|
||
Ok((eof as u64).saturating_sub(user_block))
|
||
}
|
||
|
||
/// Whether the file was opened with write access when the superblock was written.
|
||
pub fn is_write_access(&self) -> bool {
|
||
self.consistency_flags & swmr_flags::WRITE_ACCESS != 0
|
||
}
|
||
|
||
/// Whether SWMR write access is active.
|
||
///
|
||
/// When this flag is set, readers should periodically re-read the
|
||
/// superblock's EOF address and refresh metadata to see new data
|
||
/// written by the single writer.
|
||
pub fn is_swmr_write(&self) -> bool {
|
||
self.consistency_flags & swmr_flags::SWMR_WRITE != 0
|
||
}
|
||
|
||
/// Re-read the superblock's EOF address from the file data.
|
||
///
|
||
/// In SWMR mode, the writer updates the EOF address as new data is
|
||
/// appended. Readers should call this periodically to discover new data.
|
||
/// Returns the updated EOF address, or an error if the superblock
|
||
/// cannot be re-parsed.
|
||
pub fn refresh_eof(
|
||
&mut self,
|
||
file_data: &[u8],
|
||
signature_offset: usize,
|
||
) -> Result<u64, FormatError> {
|
||
let refreshed = Superblock::parse(file_data, signature_offset)?;
|
||
self.eof_address = refreshed.eof_address;
|
||
self.consistency_flags = refreshed.consistency_flags;
|
||
Ok(self.eof_address)
|
||
}
|
||
|
||
/// Serialize this superblock to bytes.
|
||
///
|
||
/// Writes v2/v3 format, or the non-standard v4 (with `page_size`) when
|
||
/// `self.version == 4` — which no HDF5 library opens; see
|
||
/// [`Self::page_size`]. Computes and appends Jenkins lookup3 checksum.
|
||
pub fn serialize(&self) -> Vec<u8> {
|
||
let mut buf = Vec::with_capacity(48);
|
||
buf.extend_from_slice(&HDF5_SIGNATURE);
|
||
buf.push(self.version);
|
||
buf.push(self.offset_size);
|
||
buf.push(self.length_size);
|
||
buf.push(self.consistency_flags as u8);
|
||
// base_address
|
||
Self::write_offset(&mut buf, self.base_address, self.offset_size);
|
||
// superblock extension address
|
||
let ext_addr = self.superblock_extension_address.unwrap_or(u64::MAX);
|
||
Self::write_offset(&mut buf, ext_addr, self.offset_size);
|
||
// eof_address
|
||
Self::write_offset(&mut buf, self.eof_address, self.offset_size);
|
||
// root_group_address
|
||
Self::write_offset(&mut buf, self.root_group_address, self.offset_size);
|
||
// page_size (v4 only)
|
||
if self.version >= 4 {
|
||
let ps = self.page_size.unwrap_or(0);
|
||
buf.extend_from_slice(&ps.to_le_bytes());
|
||
}
|
||
// checksum
|
||
let checksum = crate::checksum::jenkins_lookup3(&buf);
|
||
buf.extend_from_slice(&checksum.to_le_bytes());
|
||
buf
|
||
}
|
||
|
||
fn write_offset(buf: &mut Vec<u8>, val: u64, size: u8) {
|
||
match size {
|
||
2 => buf.extend_from_slice(&(val as u16).to_le_bytes()),
|
||
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
||
8 => buf.extend_from_slice(&val.to_le_bytes()),
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
/// Parse a superblock from `data` starting at `signature_offset`.
|
||
///
|
||
/// The signature must be present at the given offset, and that offset
|
||
/// must be 0: every address in an HDF5 file is relative to the
|
||
/// superblock, so when a file has a user block (signature at 512, 1024,
|
||
/// …) the caller must pass the bytes from the signature on — see
|
||
/// [`crate::signature::split_user_block`] — and use that slice as
|
||
/// `file_data` everywhere. A non-zero offset is refused with
|
||
/// [`FormatError::UserBlockNotStripped`] because the addresses in the
|
||
/// returned superblock would otherwise be applied to the wrong bytes.
|
||
pub fn parse(data: &[u8], signature_offset: usize) -> Result<Superblock, FormatError> {
|
||
if signature_offset != 0 {
|
||
return Err(FormatError::UserBlockNotStripped(signature_offset as u64));
|
||
}
|
||
let d = data
|
||
.get(signature_offset..)
|
||
.ok_or(FormatError::UnexpectedEof {
|
||
expected: signature_offset + 1,
|
||
available: data.len(),
|
||
})?;
|
||
ensure_len(d, 9)?; // signature(8) + version(1)
|
||
|
||
// Verify signature
|
||
if d[..8] != HDF5_SIGNATURE {
|
||
return Err(FormatError::SignatureNotFound);
|
||
}
|
||
|
||
let version = d[8];
|
||
match version {
|
||
0 => Self::parse_v0(d),
|
||
1 => Self::parse_v1(d),
|
||
2 | 3 => Self::parse_v2v3(d, version),
|
||
4 => Self::parse_v4(d),
|
||
v => Err(FormatError::UnsupportedVersion(v)),
|
||
}
|
||
}
|
||
|
||
fn parse_v0(d: &[u8]) -> Result<Superblock, FormatError> {
|
||
// sig(8) + version(1) + free_space_ver(1) + root_grp_ver(1) + reserved(1)
|
||
// + shared_hdr_ver(1) + offset_size(1) + length_size(1) + reserved(1)
|
||
// + group_leaf_k(2) + group_internal_k(2) + consistency_flags(4)
|
||
// = 24 bytes before variable-sized fields
|
||
ensure_len(d, 24)?;
|
||
|
||
let offset_size = d[13];
|
||
let length_size = d[14];
|
||
validate_sizes(offset_size, length_size)?;
|
||
|
||
let group_leaf_node_k = LittleEndian::read_u16(&d[16..18]);
|
||
let group_internal_node_k = LittleEndian::read_u16(&d[18..20]);
|
||
let consistency_flags = LittleEndian::read_u32(&d[20..24]);
|
||
|
||
let os = offset_size as usize;
|
||
// 4 addresses + root symbol table entry
|
||
let var_start = 24;
|
||
let sym_entry_size = os + os + 4 + 4 + 16; // link_name_off, obj_hdr_addr, cache_type, reserved, scratch
|
||
let total = var_start + 4 * os + sym_entry_size;
|
||
ensure_len(d, total)?;
|
||
|
||
let mut pos = var_start;
|
||
let base_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let free_space_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let eof_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let driver_info_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
|
||
// Root symbol table entry
|
||
let _link_name_offset = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let object_header_addr = read_offset(d, pos, offset_size)?;
|
||
|
||
Ok(Superblock {
|
||
version: 0,
|
||
offset_size,
|
||
length_size,
|
||
base_address,
|
||
eof_address,
|
||
root_group_address: object_header_addr,
|
||
group_leaf_node_k: Some(group_leaf_node_k),
|
||
group_internal_node_k: Some(group_internal_node_k),
|
||
indexed_storage_internal_node_k: None,
|
||
free_space_address: Some(free_space_address),
|
||
driver_info_address: Some(driver_info_address),
|
||
consistency_flags,
|
||
superblock_extension_address: None,
|
||
checksum: None,
|
||
page_size: None,
|
||
})
|
||
}
|
||
|
||
fn parse_v1(d: &[u8]) -> Result<Superblock, FormatError> {
|
||
// Same as v0 but adds indexed_storage_internal_node_k(2) + reserved(2) after group_internal_k
|
||
// sig(8) + version(1) + free_space_ver(1) + root_grp_ver(1) + reserved(1)
|
||
// + shared_hdr_ver(1) + offset_size(1) + length_size(1) + reserved(1)
|
||
// + group_leaf_k(2) + group_internal_k(2) + indexed_storage_k(2) + reserved(2)
|
||
// + consistency_flags(4) = 28
|
||
ensure_len(d, 28)?;
|
||
|
||
let offset_size = d[13];
|
||
let length_size = d[14];
|
||
validate_sizes(offset_size, length_size)?;
|
||
|
||
let group_leaf_node_k = LittleEndian::read_u16(&d[16..18]);
|
||
let group_internal_node_k = LittleEndian::read_u16(&d[18..20]);
|
||
let indexed_storage_internal_node_k = LittleEndian::read_u16(&d[20..22]);
|
||
// d[22..24] reserved
|
||
let consistency_flags = LittleEndian::read_u32(&d[24..28]);
|
||
|
||
let os = offset_size as usize;
|
||
let var_start = 28;
|
||
let sym_entry_size = os + os + 4 + 4 + 16;
|
||
let total = var_start + 4 * os + sym_entry_size;
|
||
ensure_len(d, total)?;
|
||
|
||
let mut pos = var_start;
|
||
let base_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let free_space_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let eof_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let driver_info_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
|
||
// Root symbol table entry
|
||
let _link_name_offset = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let object_header_addr = read_offset(d, pos, offset_size)?;
|
||
|
||
Ok(Superblock {
|
||
version: 1,
|
||
offset_size,
|
||
length_size,
|
||
base_address,
|
||
eof_address,
|
||
root_group_address: object_header_addr,
|
||
group_leaf_node_k: Some(group_leaf_node_k),
|
||
group_internal_node_k: Some(group_internal_node_k),
|
||
indexed_storage_internal_node_k: Some(indexed_storage_internal_node_k),
|
||
free_space_address: Some(free_space_address),
|
||
driver_info_address: Some(driver_info_address),
|
||
consistency_flags,
|
||
superblock_extension_address: None,
|
||
checksum: None,
|
||
page_size: None,
|
||
})
|
||
}
|
||
|
||
fn parse_v2v3(d: &[u8], version: u8) -> Result<Superblock, FormatError> {
|
||
// sig(8) + version(1) + offset_size(1) + length_size(1) + consistency_flags(1) = 12
|
||
ensure_len(d, 12)?;
|
||
|
||
let offset_size = d[9];
|
||
let length_size = d[10];
|
||
validate_sizes(offset_size, length_size)?;
|
||
let consistency_flags = d[11] as u32;
|
||
|
||
let os = offset_size as usize;
|
||
// 4 addresses + checksum(4)
|
||
let total = 12 + 4 * os + 4;
|
||
ensure_len(d, total)?;
|
||
|
||
let mut pos = 12;
|
||
let base_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let superblock_extension_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let eof_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let root_group_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
|
||
let stored_checksum = LittleEndian::read_u32(&d[pos..pos + 4]);
|
||
|
||
// Validate checksum if feature enabled
|
||
#[cfg(feature = "checksum")]
|
||
{
|
||
let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
|
||
if computed != stored_checksum {
|
||
return Err(FormatError::ChecksumMismatch {
|
||
expected: stored_checksum,
|
||
computed,
|
||
});
|
||
}
|
||
}
|
||
|
||
Ok(Superblock {
|
||
version,
|
||
offset_size,
|
||
length_size,
|
||
base_address,
|
||
eof_address,
|
||
root_group_address,
|
||
group_leaf_node_k: None,
|
||
group_internal_node_k: None,
|
||
indexed_storage_internal_node_k: None,
|
||
free_space_address: None,
|
||
driver_info_address: None,
|
||
consistency_flags,
|
||
superblock_extension_address: Some(superblock_extension_address),
|
||
checksum: Some(stored_checksum),
|
||
page_size: None,
|
||
})
|
||
}
|
||
|
||
fn parse_v4(d: &[u8]) -> Result<Superblock, FormatError> {
|
||
// Same layout as v2/v3, plus page_size(4) inserted before the checksum.
|
||
ensure_len(d, 12)?;
|
||
|
||
let offset_size = d[9];
|
||
let length_size = d[10];
|
||
validate_sizes(offset_size, length_size)?;
|
||
let consistency_flags = d[11] as u32;
|
||
|
||
let os = offset_size as usize;
|
||
// 4 addresses + page_size(4) + checksum(4)
|
||
let total = 12 + 4 * os + 4 + 4;
|
||
ensure_len(d, total)?;
|
||
|
||
let mut pos = 12;
|
||
let base_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let superblock_extension_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let eof_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
let root_group_address = read_offset(d, pos, offset_size)?;
|
||
pos += os;
|
||
|
||
let page_size = LittleEndian::read_u32(&d[pos..pos + 4]);
|
||
pos += 4;
|
||
|
||
let stored_checksum = LittleEndian::read_u32(&d[pos..pos + 4]);
|
||
pos += 4;
|
||
|
||
#[cfg(feature = "checksum")]
|
||
{
|
||
let computed = crate::checksum::jenkins_lookup3(&d[..pos - 4]);
|
||
if computed != stored_checksum {
|
||
return Err(FormatError::ChecksumMismatch {
|
||
expected: stored_checksum,
|
||
computed,
|
||
});
|
||
}
|
||
}
|
||
#[cfg(not(feature = "checksum"))]
|
||
{
|
||
let _ = pos;
|
||
}
|
||
|
||
Ok(Superblock {
|
||
version: 4,
|
||
offset_size,
|
||
length_size,
|
||
base_address,
|
||
eof_address,
|
||
root_group_address,
|
||
group_leaf_node_k: None,
|
||
group_internal_node_k: None,
|
||
indexed_storage_internal_node_k: None,
|
||
free_space_address: None,
|
||
driver_info_address: None,
|
||
consistency_flags,
|
||
superblock_extension_address: Some(superblock_extension_address),
|
||
checksum: Some(stored_checksum),
|
||
page_size: Some(page_size),
|
||
})
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Helper to build a v0 superblock byte buffer with 8-byte offsets.
|
||
fn build_v0_bytes(offset_size: u8) -> Vec<u8> {
|
||
let mut buf = Vec::new();
|
||
buf.extend_from_slice(&HDF5_SIGNATURE); // 0..8
|
||
buf.push(0); // version = 0
|
||
buf.push(0); // free_space_version
|
||
buf.push(0); // root_group_version
|
||
buf.push(0); // reserved
|
||
buf.push(0); // shared_header_version
|
||
buf.push(offset_size); // offset_size
|
||
buf.push(offset_size); // length_size (same for simplicity)
|
||
buf.push(0); // reserved
|
||
buf.extend_from_slice(&4u16.to_le_bytes()); // group_leaf_node_k
|
||
buf.extend_from_slice(&16u16.to_le_bytes()); // group_internal_node_k
|
||
buf.extend_from_slice(&0u32.to_le_bytes()); // consistency_flags
|
||
// base_address
|
||
write_offset(&mut buf, 0, offset_size);
|
||
// free_space_address
|
||
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size);
|
||
// eof_address
|
||
write_offset(&mut buf, 4096, offset_size);
|
||
// driver_info_address
|
||
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size);
|
||
// Root symbol table entry
|
||
write_offset(&mut buf, 0, offset_size); // link_name_offset
|
||
write_offset(&mut buf, 96, offset_size); // object_header_addr (root group)
|
||
buf.extend_from_slice(&0u32.to_le_bytes()); // cache_type
|
||
buf.extend_from_slice(&0u32.to_le_bytes()); // reserved
|
||
buf.extend_from_slice(&[0u8; 16]); // scratch pad
|
||
buf
|
||
}
|
||
|
||
fn write_offset(buf: &mut Vec<u8>, val: u64, size: u8) {
|
||
match size {
|
||
2 => buf.extend_from_slice(&(val as u16).to_le_bytes()),
|
||
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
|
||
8 => buf.extend_from_slice(&val.to_le_bytes()),
|
||
_ => panic!("bad test offset size"),
|
||
}
|
||
}
|
||
|
||
fn build_v1_bytes(offset_size: u8) -> Vec<u8> {
|
||
let mut buf = Vec::new();
|
||
buf.extend_from_slice(&HDF5_SIGNATURE);
|
||
buf.push(1); // version
|
||
buf.push(0); // free_space_version
|
||
buf.push(0); // root_group_version
|
||
buf.push(0); // reserved
|
||
buf.push(0); // shared_header_version
|
||
buf.push(offset_size);
|
||
buf.push(offset_size);
|
||
buf.push(0); // reserved
|
||
buf.extend_from_slice(&4u16.to_le_bytes()); // group_leaf_node_k
|
||
buf.extend_from_slice(&16u16.to_le_bytes()); // group_internal_node_k
|
||
buf.extend_from_slice(&32u16.to_le_bytes()); // indexed_storage_internal_node_k
|
||
buf.extend_from_slice(&0u16.to_le_bytes()); // reserved
|
||
buf.extend_from_slice(&0u32.to_le_bytes()); // consistency_flags
|
||
write_offset(&mut buf, 0, offset_size); // base
|
||
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size); // free space
|
||
write_offset(&mut buf, 8192, offset_size); // eof
|
||
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size); // driver info
|
||
// Root symbol table entry
|
||
write_offset(&mut buf, 0, offset_size);
|
||
write_offset(&mut buf, 200, offset_size); // root group addr
|
||
buf.extend_from_slice(&0u32.to_le_bytes());
|
||
buf.extend_from_slice(&0u32.to_le_bytes());
|
||
buf.extend_from_slice(&[0u8; 16]);
|
||
buf
|
||
}
|
||
|
||
fn build_v2_bytes(offset_size: u8, version: u8) -> Vec<u8> {
|
||
let mut buf = Vec::new();
|
||
buf.extend_from_slice(&HDF5_SIGNATURE);
|
||
buf.push(version);
|
||
buf.push(offset_size);
|
||
buf.push(offset_size); // length_size
|
||
buf.push(0); // consistency_flags
|
||
write_offset(&mut buf, 0, offset_size); // base_address
|
||
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size); // superblock ext
|
||
write_offset(&mut buf, 2048, offset_size); // eof
|
||
write_offset(&mut buf, 48, offset_size); // root group obj hdr
|
||
|
||
// Compute CRC32C of everything so far
|
||
let checksum = crate::checksum::jenkins_lookup3(&buf);
|
||
buf.extend_from_slice(&checksum.to_le_bytes());
|
||
buf
|
||
}
|
||
|
||
#[test]
|
||
fn data_end_refuses_truncated_files_like_libhdf5() {
|
||
// build_v2_bytes records base 0, end of file 2048.
|
||
let sb = Superblock::parse(&build_v2_bytes(8, 2), 0).unwrap();
|
||
assert_eq!(sb.data_end(0, 2048), Ok(2048));
|
||
// Bytes past the recorded end are not part of the file.
|
||
assert_eq!(sb.data_end(0, 4096), Ok(2048));
|
||
assert_eq!(
|
||
sb.data_end(0, 2047),
|
||
Err(FormatError::TruncatedFile {
|
||
stored_eof: 2048,
|
||
actual_len: 2047
|
||
})
|
||
);
|
||
// A user block added in front after the file was written (the
|
||
// recorded base address is still 0): the end moves with it.
|
||
assert_eq!(sb.data_end(512, 2560), Ok(2048));
|
||
assert!(sb.data_end(512, 2559).is_err());
|
||
// A v3 superblock still being written in SWMR mode is not checked.
|
||
let mut swmr = Superblock::parse(&build_v2_bytes(8, 3), 0).unwrap();
|
||
swmr.consistency_flags = swmr_flags::WRITE_ACCESS | swmr_flags::SWMR_WRITE;
|
||
assert_eq!(swmr.data_end(0, 1000), Ok(1000));
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v0_8byte_offsets() {
|
||
let data = build_v0_bytes(8);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.version, 0);
|
||
assert_eq!(sb.offset_size, 8);
|
||
assert_eq!(sb.base_address, 0);
|
||
assert_eq!(sb.eof_address, 4096);
|
||
assert_eq!(sb.root_group_address, 96);
|
||
assert_eq!(sb.group_leaf_node_k, Some(4));
|
||
assert_eq!(sb.group_internal_node_k, Some(16));
|
||
assert_eq!(sb.indexed_storage_internal_node_k, None);
|
||
assert_eq!(sb.free_space_address, Some(0xFFFFFFFFFFFFFFFF));
|
||
assert_eq!(sb.driver_info_address, Some(0xFFFFFFFFFFFFFFFF));
|
||
assert_eq!(sb.checksum, None);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v0_4byte_offsets() {
|
||
let data = build_v0_bytes(4);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.version, 0);
|
||
assert_eq!(sb.offset_size, 4);
|
||
assert_eq!(sb.eof_address, 4096);
|
||
assert_eq!(sb.root_group_address, 96);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v1_8byte_offsets() {
|
||
let data = build_v1_bytes(8);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.version, 1);
|
||
assert_eq!(sb.offset_size, 8);
|
||
assert_eq!(sb.eof_address, 8192);
|
||
assert_eq!(sb.root_group_address, 200);
|
||
assert_eq!(sb.indexed_storage_internal_node_k, Some(32));
|
||
assert_eq!(sb.group_leaf_node_k, Some(4));
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v1_4byte_offsets() {
|
||
let data = build_v1_bytes(4);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.version, 1);
|
||
assert_eq!(sb.offset_size, 4);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v2_8byte_offsets() {
|
||
let data = build_v2_bytes(8, 2);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.version, 2);
|
||
assert_eq!(sb.offset_size, 8);
|
||
assert_eq!(sb.eof_address, 2048);
|
||
assert_eq!(sb.root_group_address, 48);
|
||
assert!(sb.checksum.is_some());
|
||
assert_eq!(sb.group_leaf_node_k, None);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v2_4byte_offsets() {
|
||
let data = build_v2_bytes(4, 2);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.version, 2);
|
||
assert_eq!(sb.offset_size, 4);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v3() {
|
||
let data = build_v2_bytes(8, 3);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.version, 3);
|
||
}
|
||
|
||
#[test]
|
||
fn checksum_mismatch_v2() {
|
||
let mut data = build_v2_bytes(8, 2);
|
||
// Corrupt the checksum
|
||
let len = data.len();
|
||
data[len - 1] ^= 0xFF;
|
||
let err = Superblock::parse(&data, 0).unwrap_err();
|
||
matches!(err, FormatError::ChecksumMismatch { .. });
|
||
}
|
||
|
||
#[test]
|
||
fn unsupported_version() {
|
||
let mut data = vec![0u8; 64];
|
||
data[..8].copy_from_slice(&HDF5_SIGNATURE);
|
||
data[8] = 99;
|
||
assert_eq!(
|
||
Superblock::parse(&data, 0),
|
||
Err(FormatError::UnsupportedVersion(99))
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn truncated_data() {
|
||
let data = HDF5_SIGNATURE.to_vec(); // Just the signature, no version
|
||
// Only 8 bytes, need at least 9
|
||
assert!(matches!(
|
||
Superblock::parse(&data, 0),
|
||
Err(FormatError::UnexpectedEof { .. })
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn truncated_v0() {
|
||
let mut data = vec![0u8; 20]; // Too short for v0
|
||
data[..8].copy_from_slice(&HDF5_SIGNATURE);
|
||
data[8] = 0; // version 0
|
||
data[13] = 8; // offset_size
|
||
data[14] = 8; // length_size
|
||
assert!(matches!(
|
||
Superblock::parse(&data, 0),
|
||
Err(FormatError::UnexpectedEof { .. })
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn invalid_offset_size() {
|
||
let mut data = vec![0u8; 64];
|
||
data[..8].copy_from_slice(&HDF5_SIGNATURE);
|
||
data[8] = 0; // version 0
|
||
data[13] = 3; // invalid offset_size
|
||
data[14] = 8;
|
||
assert_eq!(
|
||
Superblock::parse(&data, 0),
|
||
Err(FormatError::InvalidOffsetSize(3))
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn invalid_length_size() {
|
||
let mut data = vec![0u8; 64];
|
||
data[..8].copy_from_slice(&HDF5_SIGNATURE);
|
||
data[8] = 0;
|
||
data[13] = 8;
|
||
data[14] = 5; // invalid length_size
|
||
assert_eq!(
|
||
Superblock::parse(&data, 0),
|
||
Err(FormatError::InvalidLengthSize(5))
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_at_nonzero_offset() {
|
||
let mut data = vec![0u8; 1024];
|
||
let v0 = build_v0_bytes(8);
|
||
data[512..512 + v0.len()].copy_from_slice(&v0);
|
||
// Addresses are relative to the superblock, so parsing in place
|
||
// (where they would be applied to the whole buffer) is refused...
|
||
assert_eq!(
|
||
Superblock::parse(&data, 512),
|
||
Err(FormatError::UserBlockNotStripped(512))
|
||
);
|
||
// ...and the caller parses the bytes from the signature on.
|
||
let (ub, hdf5) = crate::signature::split_user_block(&data).unwrap();
|
||
assert_eq!(ub.len(), 512);
|
||
let sb = Superblock::parse(hdf5, 0).unwrap();
|
||
assert_eq!(sb.version, 0);
|
||
assert_eq!(sb.root_group_address, 96);
|
||
}
|
||
|
||
#[test]
|
||
fn v2_2byte_offsets() {
|
||
let data = build_v2_bytes(2, 2);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert_eq!(sb.offset_size, 2);
|
||
assert_eq!(sb.eof_address, 2048);
|
||
}
|
||
|
||
#[test]
|
||
fn swmr_flags_default() {
|
||
let data = build_v2_bytes(8, 3);
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert!(!sb.is_write_access());
|
||
assert!(!sb.is_swmr_write());
|
||
}
|
||
|
||
#[test]
|
||
fn swmr_write_flag() {
|
||
let mut data = build_v2_bytes(8, 3);
|
||
// Set SWMR write flag (bit 2) in consistency_flags byte
|
||
// In v2/v3, consistency_flags is at offset 11
|
||
data[11] = 0x04; // SWMR_WRITE
|
||
// Recompute checksum
|
||
let cksum_offset = data.len() - 4;
|
||
let checksum = crate::checksum::jenkins_lookup3(&data[..cksum_offset]);
|
||
data[cksum_offset..].copy_from_slice(&checksum.to_le_bytes());
|
||
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert!(sb.is_swmr_write());
|
||
assert!(!sb.is_write_access());
|
||
}
|
||
|
||
#[test]
|
||
fn swmr_write_access_flag() {
|
||
let mut data = build_v2_bytes(8, 3);
|
||
data[11] = 0x05; // WRITE_ACCESS | SWMR_WRITE
|
||
let cksum_offset = data.len() - 4;
|
||
let checksum = crate::checksum::jenkins_lookup3(&data[..cksum_offset]);
|
||
data[cksum_offset..].copy_from_slice(&checksum.to_le_bytes());
|
||
|
||
let sb = Superblock::parse(&data, 0).unwrap();
|
||
assert!(sb.is_swmr_write());
|
||
assert!(sb.is_write_access());
|
||
}
|
||
|
||
#[test]
|
||
fn refresh_eof() {
|
||
let data = build_v2_bytes(8, 3);
|
||
let mut sb = Superblock::parse(&data, 0).unwrap();
|
||
let old_eof = sb.eof_address;
|
||
// Re-reading same data should keep same EOF
|
||
let new_eof = sb.refresh_eof(&data, 0).unwrap();
|
||
assert_eq!(new_eof, old_eof);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v4_with_page_size() {
|
||
// Superblock v4 = v2/v3 layout + page_size(4) before checksum.
|
||
let mut buf = Vec::new();
|
||
buf.extend_from_slice(&HDF5_SIGNATURE);
|
||
buf.push(4); // version = 4
|
||
buf.push(8); // offset_size
|
||
buf.push(8); // length_size
|
||
buf.push(0); // consistency_flags
|
||
write_offset(&mut buf, 0, 8); // base_address
|
||
write_offset(&mut buf, u64::MAX, 8); // superblock_extension_address = UNDEF
|
||
write_offset(&mut buf, 512, 8); // eof_address
|
||
write_offset(&mut buf, 96, 8); // root_group_address
|
||
buf.extend_from_slice(&4096u32.to_le_bytes()); // page_size (v4 addition)
|
||
let checksum = crate::checksum::jenkins_lookup3(&buf);
|
||
buf.extend_from_slice(&checksum.to_le_bytes());
|
||
|
||
let sb = Superblock::parse(&buf, 0).unwrap();
|
||
assert_eq!(sb.version, 4);
|
||
assert_eq!(sb.offset_size, 8);
|
||
assert_eq!(sb.eof_address, 512);
|
||
assert_eq!(sb.root_group_address, 96);
|
||
assert_eq!(sb.page_size, Some(4096));
|
||
}
|
||
|
||
#[test]
|
||
fn serialize_v4_roundtrip() {
|
||
let sb = Superblock {
|
||
version: 4,
|
||
offset_size: 8,
|
||
length_size: 8,
|
||
base_address: 0,
|
||
eof_address: 1024,
|
||
root_group_address: 96,
|
||
group_leaf_node_k: None,
|
||
group_internal_node_k: None,
|
||
indexed_storage_internal_node_k: None,
|
||
free_space_address: None,
|
||
driver_info_address: None,
|
||
consistency_flags: 0,
|
||
superblock_extension_address: Some(u64::MAX),
|
||
checksum: None,
|
||
page_size: Some(4096),
|
||
};
|
||
let bytes = sb.serialize();
|
||
let parsed = Superblock::parse(&bytes, 0).unwrap();
|
||
assert_eq!(parsed.version, 4);
|
||
assert_eq!(parsed.page_size, Some(4096));
|
||
assert_eq!(parsed.eof_address, 1024);
|
||
assert_eq!(parsed.root_group_address, 96);
|
||
}
|
||
|
||
#[test]
|
||
fn serialize_v3_unchanged_by_page_size_field() {
|
||
// v3 (page_size: None) must serialize identically to before this feature existed.
|
||
let sb = Superblock {
|
||
version: 3,
|
||
offset_size: 8,
|
||
length_size: 8,
|
||
base_address: 0,
|
||
eof_address: 2048,
|
||
root_group_address: 96,
|
||
group_leaf_node_k: None,
|
||
group_internal_node_k: None,
|
||
indexed_storage_internal_node_k: None,
|
||
free_space_address: None,
|
||
driver_info_address: None,
|
||
consistency_flags: 0,
|
||
superblock_extension_address: Some(u64::MAX),
|
||
checksum: None,
|
||
page_size: None,
|
||
};
|
||
let bytes = sb.serialize();
|
||
// sig(8) + version/offset/length/flags(4) + 4 addresses(8 each) + checksum(4)
|
||
assert_eq!(bytes.len(), 8 + 4 + 4 * 8 + 4);
|
||
let parsed = Superblock::parse(&bytes, 0).unwrap();
|
||
assert_eq!(parsed.version, 3);
|
||
assert_eq!(parsed.page_size, None);
|
||
}
|
||
}
|