quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions 3 Packages Projects Releases 1 Wiki Activity

writer: track attribute creation order with track_order

Browse Source 
h5py's track_order=True orders attributes as well as links; the writer
tracked links only. A tracking object's header now sets the attribute
creation order tracked/indexed flags and carries per-message creation
orders, an Attribute Info message holds the next order (inline too),
and dense storage gets a type-9 creation-order index. The file default
applies to datasets, with DatasetBuilder::track_order per dataset; more
than 65 535 attributes on a tracking object is an error (libhdf5's
counter is 2 bytes). The reader lists such attributes in creation
order.

h5py lists them in order (inline, dense, 20 000 on one dataset) and
keeps numbering in r+ mode, including its inline-to-dense move.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 10:17:37 -05:00
co-authored by Claude Opus 5.5
parent d63c76e7ab
commit 193a5f8a82
9 changed files with 515 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/object_header_writer.rs
+57 -8
View File
@@ -12,9 +12,16 @@ use crate::message_type::MessageType;
/// its size truncated to 16 bits produced files libhdf5 refuses.
pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize;
/// Object header flags: attribute creation order tracked (each message
/// then carries a 2-byte creation order) and indexed.
const OHDR_ATTR_CRT_ORDER_TRACKED: u8 = 0x04;
const OHDR_ATTR_CRT_ORDER_INDEXED: u8 = 0x08;
/// Writer for v2 object headers with proper checksums.
pub struct ObjectHeaderWriter {
messages: Vec<(MessageType, Vec<u8>, u8)>, // (type, data, msg_flags)
messages: Vec<(MessageType, Vec<u8>, u8, u16)>, // (type, data, msg_flags, creation order)
/// Attribute creation order tracked and indexed.
attr_order: bool,
}
impl ObjectHeaderWriter {
@@ -22,17 +29,33 @@ impl ObjectHeaderWriter {
pub fn new() -> Self {
Self {
messages: Vec::new(),
attr_order: false,
}
}
/// Track and index attribute creation order, as libhdf5 does for an
/// object created with `H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED`
/// (h5py's `track_order=True`): the header's flags say so, and every
/// message carries a creation order (an attribute's own; 0 for the
/// others). libhdf5 reads the setting back from these flags.
pub fn track_attr_order(&mut self) {
self.attr_order = true;
}
/// Add a message to the header with default flags (0).
pub fn add_message(&mut self, msg_type: MessageType, data: Vec<u8>) {
self.messages.push((msg_type, data, 0));
self.messages.push((msg_type, data, 0, 0));
}
/// Add a message with specific flags.
pub fn add_message_with_flags(&mut self, msg_type: MessageType, data: Vec<u8>, flags: u8) {
self.messages.push((msg_type, data, flags));
self.messages.push((msg_type, data, flags, 0));
}
/// Add a message with its creation order, which is written only when
/// attribute creation order is tracked ([`Self::track_attr_order`]).
pub fn add_message_with_order(&mut self, msg_type: MessageType, data: Vec<u8>, order: u16) {
self.messages.push((msg_type, data, 0, order));
}
/// Serialize the complete v2 object header (OHDR + messages + checksum).
@@ -41,10 +64,10 @@ impl ObjectHeaderWriter {
/// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would
/// need dense attribute storage), rather than writing a corrupt header.
pub fn serialize(&self) -> Result<Vec<u8>, FormatError> {
if let Some((msg_type, data, _)) = self
if let Some((msg_type, data, _, _)) = self
.messages
.iter()
.find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE)
.find(|(_, data, _, _)| data.len() > MAX_MESSAGE_SIZE)
{
return Err(FormatError::SerializationError(format!(
"{msg_type:?} message is {} bytes; an object header message holds at most \
@@ -52,11 +75,13 @@ impl ObjectHeaderWriter {
data.len()
)));
}
// Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data
// Calculate total message bytes: each message has type(1) + size(2) +
// flags(1) [+ creation order(2)] + data
let msg_header = if self.attr_order { 6 } else { 4 };
let msg_bytes_total: usize = self
.messages
.iter()
.map(|(_, data, _)| 4 + data.len())
.map(|(_, data, _, _)| msg_header + data.len())
.sum();
// Determine chunk size field width based on msg_bytes_total
@@ -68,6 +93,12 @@ impl ObjectHeaderWriter {
(0x02u8, 4)
};
let flags = if self.attr_order {
flags | OHDR_ATTR_CRT_ORDER_TRACKED | OHDR_ATTR_CRT_ORDER_INDEXED
} else {
flags
};
let mut buf = Vec::new();
// OHDR signature
@@ -85,7 +116,7 @@ impl ObjectHeaderWriter {
}
// Messages
for (msg_type, data, msg_flags) in &self.messages {
for (msg_type, data, msg_flags, order) in &self.messages {
let type_id = msg_type.to_u16();
assert!(
type_id <= 255,
@@ -94,6 +125,9 @@ impl ObjectHeaderWriter {
buf.push(type_id as u8); // type (1 byte in v2)
buf.extend_from_slice(&(data.len() as u16).to_le_bytes()); // size (2 bytes)
buf.push(*msg_flags); // flags
if self.attr_order {
buf.extend_from_slice(&order.to_le_bytes()); // creation order
}
buf.extend_from_slice(data);
}
@@ -193,6 +227,21 @@ mod tests {
assert_eq!(hdr.messages.len(), 0);
}
#[test]
fn tracked_attribute_order_is_in_the_flags_and_every_message() {
let mut writer = ObjectHeaderWriter::new();
writer.track_attr_order();
writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]);
writer.add_message_with_order(MessageType::Attribute, vec![5, 6], 7);
let bytes = writer.serialize().unwrap();
assert_eq!(bytes[5] & 0x0C, 0x0C);
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 2);
assert_eq!(hdr.messages[0].creation_order, Some(0));
assert_eq!(hdr.messages[1].creation_order, Some(7));
assert_eq!(hdr.messages[1].data, vec![5, 6]);
}
#[test]
fn two_messages_roundtrip() {
let mut writer = ObjectHeaderWriter::new();