quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request 'docs(clawhdf5): document DType variants, fix unresolved doc links' (#17) from sdlc-docs/clawhdf5-types-20260514-165210 into main

Browse Source
This commit is contained in:
redclawsystems
2026-05-14 23:54:48 +00:00
commit 3f222f6956
3030 changed files with 89917 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/superblock.rs
+655
View File
@@ -0,0 +1,655 @@
//! HDF5 Superblock parsing for versions 0, 1, 2, and 3.
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use byteorder::{ByteOrder, LittleEndian};
use crate::error::FormatError;
use crate::signature::HDF5_SIGNATURE;
/// Parsed HDF5 superblock (all versions).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Superblock {
/// Superblock version (0–3).
pub version: u8,
/// Size of offsets in bytes (2, 4, or 8).
pub offset_size: u8,
/// Size of lengths in bytes (2, 4, or 8).
pub length_size: u8,
/// File base address.
pub base_address: u64,
/// End-of-file address.
pub eof_address: u64,
/// Root group object header address (v2/v3) or from symbol table entry (v0/v1).
pub root_group_address: u64,
/// Group leaf node K (v0/v1 only).
pub group_leaf_node_k: Option<u16>,
/// Group internal node K (v0/v1 only).
pub group_internal_node_k: Option<u16>,
/// Indexed storage internal node K (v1 only).
pub indexed_storage_internal_node_k: Option<u16>,
/// Free space address (v0/v1 only).
pub free_space_address: Option<u64>,
/// Driver info block address (v0/v1 only).
pub driver_info_address: Option<u64>,
/// File consistency flags.
pub consistency_flags: u32,
/// Superblock extension address (v2/v3 only).
pub superblock_extension_address: Option<u64>,
/// CRC32C checksum (v2/v3 only).
pub checksum: Option<u32>,
}
/// Read an unsigned integer of `size` bytes (LE) from `data` at `pos`.
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
let s = size as usize;
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
return Err(FormatError::UnexpectedEof {
expected: pos.saturating_add(s),
available: data.len(),
});
}
let slice = &data[pos..pos + s];
Ok(match size {
2 => LittleEndian::read_u16(slice) as u64,
4 => LittleEndian::read_u32(slice) as u64,
8 => LittleEndian::read_u64(slice),
_ => unreachable!(), // validated before calling
})
}
fn validate_sizes(offset_size: u8, length_size: u8) -> Result<(), FormatError> {
if !matches!(offset_size, 2 | 4 | 8) {
return Err(FormatError::InvalidOffsetSize(offset_size));
}
if !matches!(length_size, 2 | 4 | 8) {
return Err(FormatError::InvalidLengthSize(length_size));
}
Ok(())
}
fn ensure_len(data: &[u8], needed: usize) -> Result<(), FormatError> {
if data.len() < needed {
Err(FormatError::UnexpectedEof {
expected: needed,
available: data.len(),
})
} else {
Ok(())
}
}
/// SWMR (Single Writer / Multiple Reader) consistency flag bits.
///
/// These flags are stored in the `consistency_flags` field of the superblock.
/// For v2/v3 superblocks, only the low byte is stored.
pub mod swmr_flags {
/// Bit 0: File is open for writing.
pub const WRITE_ACCESS: u32 = 0x01;
/// Bit 2: SWMR write access is active.
pub const SWMR_WRITE: u32 = 0x04;
}
impl Superblock {
/// Whether the file was opened with write access when the superblock was written.
pub fn is_write_access(&self) -> bool {
self.consistency_flags & swmr_flags::WRITE_ACCESS != 0
}
/// Whether SWMR write access is active.
///
/// When this flag is set, readers should periodically re-read the
/// superblock's EOF address and refresh metadata to see new data
/// written by the single writer.
pub fn is_swmr_write(&self) -> bool {
self.consistency_flags & swmr_flags::SWMR_WRITE != 0
}
/// Re-read the superblock's EOF address from the file data.
///
/// In SWMR mode, the writer updates the EOF address as new data is
/// appended. Readers should call this periodically to discover new data.
/// Returns the updated EOF address, or an error if the superblock
/// cannot be re-parsed.
pub fn refresh_eof(
&mut self,
file_data: &[u8],
signature_offset: usize,
) -> Result<u64, FormatError> {
let refreshed = Superblock::parse(file_data, signature_offset)?;
self.eof_address = refreshed.eof_address;
self.consistency_flags = refreshed.consistency_flags;
Ok(self.eof_address)
}
/// Serialize this superblock to bytes.
///
/// Always writes v2/v3 format. Computes and appends Jenkins lookup3 checksum.
pub fn serialize(&self) -> Vec<u8> {
let mut buf = Vec::with_capacity(48);
buf.extend_from_slice(&HDF5_SIGNATURE);
buf.push(self.version);
buf.push(self.offset_size);
buf.push(self.length_size);
buf.push(self.consistency_flags as u8);
// base_address
Self::write_offset(&mut buf, self.base_address, self.offset_size);
// superblock extension address
let ext_addr = self.superblock_extension_address.unwrap_or(u64::MAX);
Self::write_offset(&mut buf, ext_addr, self.offset_size);
// eof_address
Self::write_offset(&mut buf, self.eof_address, self.offset_size);
// root_group_address
Self::write_offset(&mut buf, self.root_group_address, self.offset_size);
// checksum
let checksum = crate::checksum::jenkins_lookup3(&buf);
buf.extend_from_slice(&checksum.to_le_bytes());
buf
}
fn write_offset(buf: &mut Vec<u8>, val: u64, size: u8) {
match size {
2 => buf.extend_from_slice(&(val as u16).to_le_bytes()),
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
8 => buf.extend_from_slice(&val.to_le_bytes()),
_ => {}
}
}
/// Parse a superblock from `data` starting at `signature_offset`.
///
/// The signature must be present at the given offset.
pub fn parse(data: &[u8], signature_offset: usize) -> Result<Superblock, FormatError> {
let d = data
.get(signature_offset..)
.ok_or(FormatError::UnexpectedEof {
expected: signature_offset + 1,
available: data.len(),
})?;
ensure_len(d, 9)?; // signature(8) + version(1)
// Verify signature
if d[..8] != HDF5_SIGNATURE {
return Err(FormatError::SignatureNotFound);
}
let version = d[8];
match version {
0 => Self::parse_v0(d),
1 => Self::parse_v1(d),
2 | 3 => Self::parse_v2v3(d, version),
v => Err(FormatError::UnsupportedVersion(v)),
}
}
fn parse_v0(d: &[u8]) -> Result<Superblock, FormatError> {
// sig(8) + version(1) + free_space_ver(1) + root_grp_ver(1) + reserved(1)
// + shared_hdr_ver(1) + offset_size(1) + length_size(1) + reserved(1)
// + group_leaf_k(2) + group_internal_k(2) + consistency_flags(4)
// = 24 bytes before variable-sized fields
ensure_len(d, 24)?;
let offset_size = d[13];
let length_size = d[14];
validate_sizes(offset_size, length_size)?;
let group_leaf_node_k = LittleEndian::read_u16(&d[16..18]);
let group_internal_node_k = LittleEndian::read_u16(&d[18..20]);
let consistency_flags = LittleEndian::read_u32(&d[20..24]);
let os = offset_size as usize;
// 4 addresses + root symbol table entry
let var_start = 24;
let sym_entry_size = os + os + 4 + 4 + 16; // link_name_off, obj_hdr_addr, cache_type, reserved, scratch
let total = var_start + 4 * os + sym_entry_size;
ensure_len(d, total)?;
let mut pos = var_start;
let base_address = read_offset(d, pos, offset_size)?;
pos += os;
let free_space_address = read_offset(d, pos, offset_size)?;
pos += os;
let eof_address = read_offset(d, pos, offset_size)?;
pos += os;
let driver_info_address = read_offset(d, pos, offset_size)?;
pos += os;
// Root symbol table entry
let _link_name_offset = read_offset(d, pos, offset_size)?;
pos += os;
let object_header_addr = read_offset(d, pos, offset_size)?;
Ok(Superblock {
version: 0,
offset_size,
length_size,
base_address,
eof_address,
root_group_address: object_header_addr,
group_leaf_node_k: Some(group_leaf_node_k),
group_internal_node_k: Some(group_internal_node_k),
indexed_storage_internal_node_k: None,
free_space_address: Some(free_space_address),
driver_info_address: Some(driver_info_address),
consistency_flags,
superblock_extension_address: None,
checksum: None,
})
}
fn parse_v1(d: &[u8]) -> Result<Superblock, FormatError> {
// Same as v0 but adds indexed_storage_internal_node_k(2) + reserved(2) after group_internal_k
// sig(8) + version(1) + free_space_ver(1) + root_grp_ver(1) + reserved(1)
// + shared_hdr_ver(1) + offset_size(1) + length_size(1) + reserved(1)
// + group_leaf_k(2) + group_internal_k(2) + indexed_storage_k(2) + reserved(2)
// + consistency_flags(4) = 28
ensure_len(d, 28)?;
let offset_size = d[13];
let length_size = d[14];
validate_sizes(offset_size, length_size)?;
let group_leaf_node_k = LittleEndian::read_u16(&d[16..18]);
let group_internal_node_k = LittleEndian::read_u16(&d[18..20]);
let indexed_storage_internal_node_k = LittleEndian::read_u16(&d[20..22]);
// d[22..24] reserved
let consistency_flags = LittleEndian::read_u32(&d[24..28]);
let os = offset_size as usize;
let var_start = 28;
let sym_entry_size = os + os + 4 + 4 + 16;
let total = var_start + 4 * os + sym_entry_size;
ensure_len(d, total)?;
let mut pos = var_start;
let base_address = read_offset(d, pos, offset_size)?;
pos += os;
let free_space_address = read_offset(d, pos, offset_size)?;
pos += os;
let eof_address = read_offset(d, pos, offset_size)?;
pos += os;
let driver_info_address = read_offset(d, pos, offset_size)?;
pos += os;
// Root symbol table entry
let _link_name_offset = read_offset(d, pos, offset_size)?;
pos += os;
let object_header_addr = read_offset(d, pos, offset_size)?;
Ok(Superblock {
version: 1,
offset_size,
length_size,
base_address,
eof_address,
root_group_address: object_header_addr,
group_leaf_node_k: Some(group_leaf_node_k),
group_internal_node_k: Some(group_internal_node_k),
indexed_storage_internal_node_k: Some(indexed_storage_internal_node_k),
free_space_address: Some(free_space_address),
driver_info_address: Some(driver_info_address),
consistency_flags,
superblock_extension_address: None,
checksum: None,
})
}
fn parse_v2v3(d: &[u8], version: u8) -> Result<Superblock, FormatError> {
// sig(8) + version(1) + offset_size(1) + length_size(1) + consistency_flags(1) = 12
ensure_len(d, 12)?;
let offset_size = d[9];
let length_size = d[10];
validate_sizes(offset_size, length_size)?;
let consistency_flags = d[11] as u32;
let os = offset_size as usize;
// 4 addresses + checksum(4)
let total = 12 + 4 * os + 4;
ensure_len(d, total)?;
let mut pos = 12;
let base_address = read_offset(d, pos, offset_size)?;
pos += os;
let superblock_extension_address = read_offset(d, pos, offset_size)?;
pos += os;
let eof_address = read_offset(d, pos, offset_size)?;
pos += os;
let root_group_address = read_offset(d, pos, offset_size)?;
pos += os;
let stored_checksum = LittleEndian::read_u32(&d[pos..pos + 4]);
// Validate checksum if feature enabled
#[cfg(feature = "checksum")]
{
let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
if computed != stored_checksum {
return Err(FormatError::ChecksumMismatch {
expected: stored_checksum,
computed,
});
}
}
Ok(Superblock {
version,
offset_size,
length_size,
base_address,
eof_address,
root_group_address,
group_leaf_node_k: None,
group_internal_node_k: None,
indexed_storage_internal_node_k: None,
free_space_address: None,
driver_info_address: None,
consistency_flags,
superblock_extension_address: Some(superblock_extension_address),
checksum: Some(stored_checksum),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Helper to build a v0 superblock byte buffer with 8-byte offsets.
fn build_v0_bytes(offset_size: u8) -> Vec<u8> {
let mut buf = Vec::new();
buf.extend_from_slice(&HDF5_SIGNATURE); // 0..8
buf.push(0); // version = 0
buf.push(0); // free_space_version
buf.push(0); // root_group_version
buf.push(0); // reserved
buf.push(0); // shared_header_version
buf.push(offset_size); // offset_size
buf.push(offset_size); // length_size (same for simplicity)
buf.push(0); // reserved
buf.extend_from_slice(&4u16.to_le_bytes()); // group_leaf_node_k
buf.extend_from_slice(&16u16.to_le_bytes()); // group_internal_node_k
buf.extend_from_slice(&0u32.to_le_bytes()); // consistency_flags
// base_address
write_offset(&mut buf, 0, offset_size);
// free_space_address
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size);
// eof_address
write_offset(&mut buf, 4096, offset_size);
// driver_info_address
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size);
// Root symbol table entry
write_offset(&mut buf, 0, offset_size); // link_name_offset
write_offset(&mut buf, 96, offset_size); // object_header_addr (root group)
buf.extend_from_slice(&0u32.to_le_bytes()); // cache_type
buf.extend_from_slice(&0u32.to_le_bytes()); // reserved
buf.extend_from_slice(&[0u8; 16]); // scratch pad
buf
}
fn write_offset(buf: &mut Vec<u8>, val: u64, size: u8) {
match size {
2 => buf.extend_from_slice(&(val as u16).to_le_bytes()),
4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
8 => buf.extend_from_slice(&val.to_le_bytes()),
_ => panic!("bad test offset size"),
}
}
fn build_v1_bytes(offset_size: u8) -> Vec<u8> {
let mut buf = Vec::new();
buf.extend_from_slice(&HDF5_SIGNATURE);
buf.push(1); // version
buf.push(0); // free_space_version
buf.push(0); // root_group_version
buf.push(0); // reserved
buf.push(0); // shared_header_version
buf.push(offset_size);
buf.push(offset_size);
buf.push(0); // reserved
buf.extend_from_slice(&4u16.to_le_bytes()); // group_leaf_node_k
buf.extend_from_slice(&16u16.to_le_bytes()); // group_internal_node_k
buf.extend_from_slice(&32u16.to_le_bytes()); // indexed_storage_internal_node_k
buf.extend_from_slice(&0u16.to_le_bytes()); // reserved
buf.extend_from_slice(&0u32.to_le_bytes()); // consistency_flags
write_offset(&mut buf, 0, offset_size); // base
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size); // free space
write_offset(&mut buf, 8192, offset_size); // eof
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size); // driver info
// Root symbol table entry
write_offset(&mut buf, 0, offset_size);
write_offset(&mut buf, 200, offset_size); // root group addr
buf.extend_from_slice(&0u32.to_le_bytes());
buf.extend_from_slice(&0u32.to_le_bytes());
buf.extend_from_slice(&[0u8; 16]);
buf
}
fn build_v2_bytes(offset_size: u8, version: u8) -> Vec<u8> {
let mut buf = Vec::new();
buf.extend_from_slice(&HDF5_SIGNATURE);
buf.push(version);
buf.push(offset_size);
buf.push(offset_size); // length_size
buf.push(0); // consistency_flags
write_offset(&mut buf, 0, offset_size); // base_address
write_offset(&mut buf, 0xFFFFFFFFFFFFFFFF, offset_size); // superblock ext
write_offset(&mut buf, 2048, offset_size); // eof
write_offset(&mut buf, 48, offset_size); // root group obj hdr
// Compute CRC32C of everything so far
let checksum = crate::checksum::jenkins_lookup3(&buf);
buf.extend_from_slice(&checksum.to_le_bytes());
buf
}
#[test]
fn parse_v0_8byte_offsets() {
let data = build_v0_bytes(8);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.version, 0);
assert_eq!(sb.offset_size, 8);
assert_eq!(sb.base_address, 0);
assert_eq!(sb.eof_address, 4096);
assert_eq!(sb.root_group_address, 96);
assert_eq!(sb.group_leaf_node_k, Some(4));
assert_eq!(sb.group_internal_node_k, Some(16));
assert_eq!(sb.indexed_storage_internal_node_k, None);
assert_eq!(sb.free_space_address, Some(0xFFFFFFFFFFFFFFFF));
assert_eq!(sb.driver_info_address, Some(0xFFFFFFFFFFFFFFFF));
assert_eq!(sb.checksum, None);
}
#[test]
fn parse_v0_4byte_offsets() {
let data = build_v0_bytes(4);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.version, 0);
assert_eq!(sb.offset_size, 4);
assert_eq!(sb.eof_address, 4096);
assert_eq!(sb.root_group_address, 96);
}
#[test]
fn parse_v1_8byte_offsets() {
let data = build_v1_bytes(8);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.version, 1);
assert_eq!(sb.offset_size, 8);
assert_eq!(sb.eof_address, 8192);
assert_eq!(sb.root_group_address, 200);
assert_eq!(sb.indexed_storage_internal_node_k, Some(32));
assert_eq!(sb.group_leaf_node_k, Some(4));
}
#[test]
fn parse_v1_4byte_offsets() {
let data = build_v1_bytes(4);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.version, 1);
assert_eq!(sb.offset_size, 4);
}
#[test]
fn parse_v2_8byte_offsets() {
let data = build_v2_bytes(8, 2);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.version, 2);
assert_eq!(sb.offset_size, 8);
assert_eq!(sb.eof_address, 2048);
assert_eq!(sb.root_group_address, 48);
assert!(sb.checksum.is_some());
assert_eq!(sb.group_leaf_node_k, None);
}
#[test]
fn parse_v2_4byte_offsets() {
let data = build_v2_bytes(4, 2);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.version, 2);
assert_eq!(sb.offset_size, 4);
}
#[test]
fn parse_v3() {
let data = build_v2_bytes(8, 3);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.version, 3);
}
#[test]
fn checksum_mismatch_v2() {
let mut data = build_v2_bytes(8, 2);
// Corrupt the checksum
let len = data.len();
data[len - 1] ^= 0xFF;
let err = Superblock::parse(&data, 0).unwrap_err();
matches!(err, FormatError::ChecksumMismatch { .. });
}
#[test]
fn unsupported_version() {
let mut data = vec![0u8; 64];
data[..8].copy_from_slice(&HDF5_SIGNATURE);
data[8] = 99;
assert_eq!(
Superblock::parse(&data, 0),
Err(FormatError::UnsupportedVersion(99))
);
}
#[test]
fn truncated_data() {
let data = HDF5_SIGNATURE.to_vec(); // Just the signature, no version
// Only 8 bytes, need at least 9
assert!(matches!(
Superblock::parse(&data, 0),
Err(FormatError::UnexpectedEof { .. })
));
}
#[test]
fn truncated_v0() {
let mut data = vec![0u8; 20]; // Too short for v0
data[..8].copy_from_slice(&HDF5_SIGNATURE);
data[8] = 0; // version 0
data[13] = 8; // offset_size
data[14] = 8; // length_size
assert!(matches!(
Superblock::parse(&data, 0),
Err(FormatError::UnexpectedEof { .. })
));
}
#[test]
fn invalid_offset_size() {
let mut data = vec![0u8; 64];
data[..8].copy_from_slice(&HDF5_SIGNATURE);
data[8] = 0; // version 0
data[13] = 3; // invalid offset_size
data[14] = 8;
assert_eq!(
Superblock::parse(&data, 0),
Err(FormatError::InvalidOffsetSize(3))
);
}
#[test]
fn invalid_length_size() {
let mut data = vec![0u8; 64];
data[..8].copy_from_slice(&HDF5_SIGNATURE);
data[8] = 0;
data[13] = 8;
data[14] = 5; // invalid length_size
assert_eq!(
Superblock::parse(&data, 0),
Err(FormatError::InvalidLengthSize(5))
);
}
#[test]
fn parse_at_nonzero_offset() {
let mut data = vec![0u8; 1024];
let v0 = build_v0_bytes(8);
data[512..512 + v0.len()].copy_from_slice(&v0);
let sb = Superblock::parse(&data, 512).unwrap();
assert_eq!(sb.version, 0);
assert_eq!(sb.root_group_address, 96);
}
#[test]
fn v2_2byte_offsets() {
let data = build_v2_bytes(2, 2);
let sb = Superblock::parse(&data, 0).unwrap();
assert_eq!(sb.offset_size, 2);
assert_eq!(sb.eof_address, 2048);
}
#[test]
fn swmr_flags_default() {
let data = build_v2_bytes(8, 3);
let sb = Superblock::parse(&data, 0).unwrap();
assert!(!sb.is_write_access());
assert!(!sb.is_swmr_write());
}
#[test]
fn swmr_write_flag() {
let mut data = build_v2_bytes(8, 3);
// Set SWMR write flag (bit 2) in consistency_flags byte
// In v2/v3, consistency_flags is at offset 11
data[11] = 0x04; // SWMR_WRITE
// Recompute checksum
let cksum_offset = data.len() - 4;
let checksum = crate::checksum::jenkins_lookup3(&data[..cksum_offset]);
data[cksum_offset..].copy_from_slice(&checksum.to_le_bytes());
let sb = Superblock::parse(&data, 0).unwrap();
assert!(sb.is_swmr_write());
assert!(!sb.is_write_access());
}
#[test]
fn swmr_write_access_flag() {
let mut data = build_v2_bytes(8, 3);
data[11] = 0x05; // WRITE_ACCESS | SWMR_WRITE
let cksum_offset = data.len() - 4;
let checksum = crate::checksum::jenkins_lookup3(&data[..cksum_offset]);
data[cksum_offset..].copy_from_slice(&checksum.to_le_bytes());
let sb = Superblock::parse(&data, 0).unwrap();
assert!(sb.is_swmr_write());
assert!(sb.is_write_access());
}
#[test]
fn refresh_eof() {
let data = build_v2_bytes(8, 3);
let mut sb = Superblock::parse(&data, 0).unwrap();
let old_eof = sb.eof_address;
// Re-reading same data should keep same EOF
let new_eof = sb.refresh_eof(&data, 0).unwrap();
assert_eq!(new_eof, old_eof);
}
}