quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): encode Time, BitField, Opaque and Reference datatypes

Browse Source 
Datatype::serialize returned an empty message for these four classes, so
any dataset or attribute of them (including a Raw attribute copied from
another file) was unreadable by libhdf5 ("ran off end of input buffer
while decoding"). They now encode exactly as libhdf5 does: legacy object
and region references as datatype version 1, H5T_STD_REF kinds as version
4 with their encoding version, opaque tags NUL-padded to 8 bytes.

Parsing an opaque tag now stops at its first NUL, so libhdf5's padding
no longer becomes part of the tag. Datatype::check_encodable rejects
what has no encoding (an opaque tag over 248 bytes); FileWriter::finish
calls it for every dataset and attribute type.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:06:51 -05:00
co-authored by Claude Opus 5.5
parent b36998ef01
commit be88e3fec7
3 changed files with 377 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/datatype.rs
+200 -3
View File
@@ -4,7 +4,7 @@
//! for compound, enumeration, variable-length, and array types.
#[cfg(not(feature = "std"))]
use alloc::{boxed::Box, string::String, vec, vec::Vec};
use alloc::{boxed::Box, format, string::String, vec, vec::Vec};
use byteorder::{ByteOrder, LittleEndian};
@@ -137,6 +137,17 @@ pub enum Datatype {
},
}
/// Longest opaque tag that can be stored: its NUL-padded length must fit
/// the 8-bit length in the datatype's class bits.
pub const MAX_OPAQUE_TAG_LEN: usize = 248;
/// An opaque tag up to (not including) its first NUL.
fn opaque_tag_text(tag: &[u8]) -> &[u8] {
tag.iter()
.position(|&b| b == 0)
.map_or(tag, |end| &tag[..end])
}
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
match offset.checked_add(needed) {
Some(end) if end <= data.len() => Ok(()),
@@ -361,7 +372,10 @@ impl Datatype {
// Opaque
let tag_len = bf0 as usize;
ensure_len(data, pos, tag_len)?;
let tag = data[pos..pos + tag_len].to_vec();
// The stored tag is NUL-padded to a multiple of 8 bytes; the
// tag itself ends at the first NUL (libhdf5 reads it with
// `strndup`).
let tag = opaque_tag_text(&data[pos..pos + tag_len]).to_vec();
// Tags are padded to multiple of 8 bytes
let padded = (tag_len + 7) & !7;
let pos = 8 + padded; // from start of properties
@@ -767,7 +781,74 @@ impl Datatype {
buf.extend_from_slice(&base_type.serialize());
buf
}
_ => Vec::new(),
Datatype::Time {
size,
bit_precision,
} => {
// Byte order is not modelled for time types; write little-endian.
let mut buf = Self::build_header(2, 1, [0, 0, 0], *size);
buf.extend_from_slice(&bit_precision.to_le_bytes());
buf
}
Datatype::BitField {
size,
byte_order,
bit_offset,
bit_precision,
} => {
let bf0 = u8::from(matches!(byte_order, DatatypeByteOrder::BigEndian));
let mut buf = Self::build_header(4, 1, [bf0, 0, 0], *size);
buf.extend_from_slice(&bit_offset.to_le_bytes());
buf.extend_from_slice(&bit_precision.to_le_bytes());
buf
}
Datatype::Opaque { size, tag } => {
// The tag is stored NUL-padded to a multiple of 8 bytes and the
// padded length goes in the class bits, as libhdf5 writes it.
// A tag longer than MAX_OPAQUE_TAG_LEN cannot be encoded;
// `check_encodable` rejects it before a file is written.
let tag = opaque_tag_text(tag);
let tag = &tag[..tag.len().min(MAX_OPAQUE_TAG_LEN)];
let padded = tag.len().div_ceil(8) * 8;
let mut buf = Self::build_header(5, 1, [padded as u8, 0, 0], *size);
buf.extend_from_slice(tag);
buf.resize(8 + padded, 0);
buf
}
Datatype::Reference { size, ref_type } => {
// Legacy references are datatype version 1; the H5T_STD_REF
// kinds only exist from version 4, which also carries their
// encoding version (1) in the high nibble.
let (version, bf0) = match ref_type {
ReferenceType::Object => (1, 0),
ReferenceType::DatasetRegion => (1, 1),
ReferenceType::Object2 => (4, 0x12),
ReferenceType::DatasetRegion2 => (4, 0x13),
ReferenceType::Attribute => (4, 0x14),
};
Self::build_header(7, version, [bf0, 0, 0], *size)
}
}
}
/// Check that this datatype can be written: every part of it has an
/// on-disk encoding. [`Self::serialize`] cannot report errors, so the
/// writer calls this first.
pub fn check_encodable(&self) -> Result<(), FormatError> {
match self {
Datatype::Opaque { tag, .. } if opaque_tag_text(tag).len() > MAX_OPAQUE_TAG_LEN => {
Err(FormatError::SerializationError(format!(
"opaque tag is {} bytes; at most {MAX_OPAQUE_TAG_LEN} can be stored",
opaque_tag_text(tag).len()
)))
}
Datatype::Compound { members, .. } => members
.iter()
.try_for_each(|m| m.datatype.check_encodable()),
Datatype::Enumeration { base_type, .. }
| Datatype::VariableLength { base_type, .. }
| Datatype::Array { base_type, .. } => base_type.check_encodable(),
_ => Ok(()),
}
}
@@ -1625,6 +1706,122 @@ mod tests {
);
}
fn hex(s: &str) -> Vec<u8> {
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
.collect()
}
/// `serialize` used to return an empty message for these four classes,
/// which libhdf5 rejects ("ran off end of input buffer while decoding").
/// Expected bytes are libhdf5's own encoding (HDF5 2.0 `H5Tencode`, or the
/// datatype message of an HDF5 2.0 file for `H5T_STD_REF`).
#[test]
fn serialize_matches_libhdf5_for_time_bitfield_opaque_reference() {
let cases = [
(
Datatype::Reference {
size: 8,
ref_type: ReferenceType::Object,
},
"1700000008000000",
),
(
Datatype::Reference {
size: 12,
ref_type: ReferenceType::DatasetRegion,
},
"170100000c000000",
),
(
Datatype::Reference {
size: 18,
ref_type: ReferenceType::Object2,
},
"4712000012000000",
),
(
Datatype::BitField {
size: 1,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 8,
},
"140000000100000000000800",
),
(
Datatype::BitField {
size: 2,
byte_order: DatatypeByteOrder::BigEndian,
bit_offset: 0,
bit_precision: 16,
},
"140100000200000000001000",
),
(
Datatype::Opaque {
size: 4,
tag: b"mytag".to_vec(),
},
"15080000040000006d79746167000000",
),
(
Datatype::Opaque {
size: 4,
tag: b"12345678".to_vec(),
},
"15080000040000003132333435363738",
),
(
Datatype::Opaque {
size: 4,
tag: vec![],
},
"1500000004000000",
),
(
Datatype::Time {
size: 4,
bit_precision: 32,
},
"12000000040000002000",
),
];
for (dt, expected) in cases {
let bytes = dt.serialize();
assert_eq!(bytes, hex(expected), "{dt:?}");
let (parsed, consumed) = Datatype::parse(&bytes).unwrap();
assert_eq!(parsed, dt);
assert_eq!(consumed, bytes.len());
}
}
#[test]
fn opaque_tag_padding_is_not_part_of_the_tag() {
// libhdf5 pads "mytag" to 8 bytes; parsing must not return the NULs,
// or copying the type would grow the tag.
let (dt, _) = Datatype::parse(&hex("15080000040000006d79746167000000")).unwrap();
assert_eq!(
dt,
Datatype::Opaque {
size: 4,
tag: b"mytag".to_vec()
}
);
let long = Datatype::Opaque {
size: 1,
tag: vec![b'x'; MAX_OPAQUE_TAG_LEN + 1],
};
assert!(long.check_encodable().is_err());
let ok = Datatype::Opaque {
size: 1,
tag: vec![b'x'; MAX_OPAQUE_TAG_LEN],
};
assert!(ok.check_encodable().is_ok());
assert_eq!(Datatype::parse(&ok.serialize()).unwrap().0, ok);
}
#[test]
fn test_error_invalid_reference_type() {
let buf = build_dt_header(7, 1, [5, 0, 0], 8);
crates/clawhdf5-format/src/file_writer.rs
+11
View File
@@ -1125,6 +1125,17 @@ impl FileWriter {
root_attrs.push(build_attr_message(n, v));
}
// Every datatype must have an on-disk encoding before anything is laid
// out: `Datatype::serialize` itself cannot report a failure.
let group_attrs = groups.iter().flat_map(|g| &g.attrs);
let ds_attrs = all_ds.iter().flat_map(|d| &d.attrs);
for a in root_attrs.iter().chain(group_attrs).chain(ds_attrs) {
a.datatype.check_encodable()?;
}
for d in &all_ds {
d.dt.check_encodable()?;
}
let is_vds: Vec<bool> = all_ds.iter().map(|d| d.virtual_sources.is_some()).collect();
let is_chunked: Vec<bool> = all_ds
.iter()