quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

writer: track attribute creation order with track_order

Browse Source 
h5py's track_order=True orders attributes as well as links; the writer
tracked links only. A tracking object's header now sets the attribute
creation order tracked/indexed flags and carries per-message creation
orders, an Attribute Info message holds the next order (inline too),
and dense storage gets a type-9 creation-order index. The file default
applies to datasets, with DatasetBuilder::track_order per dataset; more
than 65 535 attributes on a tracking object is an error (libhdf5's
counter is 2 bytes). The reader lists such attributes in creation
order.

h5py lists them in order (inline, dense, 20 000 on one dataset) and
keeps numbering in r+ mode, including its inline-to-dense move.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 10:17:37 -05:00
co-authored by Claude Opus 5.5
parent d63c76e7ab
commit 193a5f8a82
9 changed files with 515 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/file_writer.rs
+281 -79
View File
@@ -75,14 +75,57 @@ const DENSE_LINK_THRESHOLD: usize = 8;
// ---- OH builders ----
/// An object's attributes as its header stores them: inline Attribute
/// messages, or (`dense`) the Attribute Info message of dense storage; with
/// `track_order`, their creation order tracked and indexed.
#[derive(Clone, Copy)]
pub(crate) struct AttrStorage<'a> {
pub(crate) attrs: &'a [AttributeMessage],
pub(crate) dense: Option<&'a DenseAttrBlob>,
pub(crate) track_order: bool,
}
impl AttrStorage<'_> {
/// Add the attribute messages to the header being built. Tracking
/// creation order, as libhdf5 does it: the header's flags say so, an
/// Attribute Info message is written even for inline attributes (it
/// holds the next creation order), and each inline attribute's message
/// carries its creation order.
fn add_to(&self, w: &mut ObjectHeaderWriter) {
if self.track_order {
w.track_attr_order();
}
if let Some(blob) = self.dense {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
return;
}
if self.track_order {
w.add_message(
MessageType::AttributeInfo,
serialize_attribute_info(
u64::MAX,
u64::MAX,
Some((self.attrs.len() as u16, u64::MAX)),
),
);
}
for (i, attr) in self.attrs.iter().enumerate() {
w.add_message_with_order(
MessageType::Attribute,
attr.serialize(LENGTH_SIZE),
i as u16,
);
}
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn build_chunked_dataset_oh(
dt: &Datatype,
ds: &Dataspace,
layout_message: &[u8],
pipeline_message: Option<&[u8]>,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -94,13 +137,7 @@ pub(crate) fn build_chunked_dataset_oh(
if let Some(pm) = pipeline_message {
w.add_message(MessageType::FilterPipeline, pm.to_vec());
}
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -111,8 +148,7 @@ pub(crate) fn build_dataset_oh(
ds: &Dataspace,
data_addr: u64,
data_size: u64,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -132,13 +168,7 @@ pub(crate) fn build_dataset_oh(
dl.extend_from_slice(&data_addr.to_le_bytes());
dl.extend_from_slice(&data_size.to_le_bytes());
w.add_message(MessageType::DataLayout, dl);
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -148,8 +178,7 @@ pub(crate) fn build_compact_dataset_oh(
dt: &Datatype,
ds: &Dataspace,
data: &[u8],
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -164,13 +193,7 @@ pub(crate) fn build_compact_dataset_oh(
dl.extend_from_slice(&(data.len() as u16).to_le_bytes());
dl.extend_from_slice(data);
w.add_message(MessageType::DataLayout, dl);
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -182,8 +205,7 @@ pub(crate) fn build_group_oh(
links: &[LinkMessage],
link_info: &[u8],
dense_links: bool,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
let mut w = ObjectHeaderWriter::new();
@@ -198,13 +220,7 @@ pub(crate) fn build_group_oh(
w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE));
}
}
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -891,11 +907,31 @@ fn write_frhp(p: WriteFrhp) -> Vec<u8> {
frhp
}
/// libhdf5 numbers the attributes of an object that tracks their creation
/// order with a 2-byte counter.
fn check_tracked_attr_count(track_order: bool, n: usize) -> Result<(), FormatError> {
if track_order && n > usize::from(u16::MAX) {
return Err(FormatError::SerializationError(format!(
"{n} attributes on one object with creation order tracked: libhdf5 \
numbers at most {} (set fewer, or turn off track_order)",
u16::MAX
)));
}
Ok(())
}
/// Build dense attribute storage for a set of attributes.
///
/// With `track_order` the Attribute Info message tracks creation order (an
/// attribute's creation order is its position in `attrs`) and a type-9
/// creation-order index follows the name index, as libhdf5 writes for h5py's
/// `track_order=True`. libhdf5 numbers at most 65 535 attributes.
pub(crate) fn build_dense_attrs(
attrs: &[AttributeMessage],
base_address: u64,
track_order: bool,
) -> Result<DenseAttrBlob, FormatError> {
check_tracked_attr_count(track_order, attrs.len())?;
// Dense attrs use v3 attribute messages (adds character set encoding byte).
let serialized: Vec<Vec<u8>> = attrs.iter().map(|a| a.serialize_v3(LENGTH_SIZE)).collect();
@@ -936,7 +972,30 @@ pub(crate) fn build_dense_attrs(
let mut blob = heap.blob;
blob.extend_from_slice(&dense_v2_btree(8, record_size, &records, bthd_addr)?);
let attr_info = serialize_attribute_info(frhp_addr, bthd_addr);
let order = if track_order {
// Type 9 records: heap ID, message flags, creation order (the key).
let records: Vec<Vec<u8>> = heap_ids
.iter()
.enumerate()
.map(|(i, heap_id)| {
let mut rec = heap_id.clone();
rec.push(0); // msg_flags
rec.extend_from_slice(&(i as u32).to_le_bytes());
rec
})
.collect();
let corder_addr = base_address + blob.len() as u64;
blob.extend_from_slice(&dense_v2_btree(
9,
heap_id_length + 1 + 4,
&records,
corder_addr,
)?);
Some((attrs.len() as u16, corder_addr))
} else {
None
};
let attr_info = serialize_attribute_info(frhp_addr, bthd_addr, order);
Ok(DenseAttrBlob {
attr_info_message: attr_info,
@@ -1139,12 +1198,25 @@ fn encode_managed_id(offset: u64, length: u64, max_heap_size: u16, id_length: u1
id
}
fn serialize_attribute_info(fh_addr: u64, btree_name_addr: u64) -> Vec<u8> {
/// Serialize an Attribute Info message (version 0). `order` — the next
/// creation order to assign and the creation-order index's address — is
/// present when creation order is tracked and indexed.
fn serialize_attribute_info(
fh_addr: u64,
btree_name_addr: u64,
order: Option<(u16, u64)>,
) -> Vec<u8> {
let mut data = Vec::new();
data.push(0); // version
data.push(0x00); // flags
data.push(if order.is_some() { 0x03 } else { 0x00 }); // flags: tracked, indexed
if let Some((next, _)) = order {
data.extend_from_slice(&next.to_le_bytes());
}
data.extend_from_slice(&fh_addr.to_le_bytes());
data.extend_from_slice(&btree_name_addr.to_le_bytes());
if let Some((_, corder_addr)) = order {
data.extend_from_slice(&corder_addr.to_le_bytes());
}
data
}
@@ -1219,8 +1291,7 @@ pub(crate) fn build_vds_dataset_oh(
dt: &Datatype,
ds: &Dataspace,
global_heap_addr: u64,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
attrs: AttrStorage<'_>,
fill_message: &[u8],
refcount: u32,
) -> Result<Vec<u8>, FormatError> {
@@ -1235,13 +1306,7 @@ pub(crate) fn build_vds_dataset_oh(
dl.extend_from_slice(&global_heap_addr.to_le_bytes());
dl.extend_from_slice(&1u32.to_le_bytes()); // object index 1 in the collection
w.add_message(MessageType::DataLayout, dl);
if let Some(blob) = dense_blob {
w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone());
} else {
for attr in attrs {
w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE));
}
}
attrs.add_to(&mut w);
add_refcount(&mut w, refcount);
w.serialize()
}
@@ -1276,7 +1341,8 @@ fn write_undef_offset(buf: &mut Vec<u8>, offset_size: u8) {
pub struct FileWriter {
/// The root group's contents (its name is unused).
root: GroupBuilder,
/// Default for groups that do not call [`GroupBuilder::track_order`].
/// Default for groups and datasets that do not set their own
/// `track_order`.
track_order: bool,
/// Global alignment threshold: datasets with raw data >= this many bytes
/// will have their data aligned to `alignment_bytes`.
@@ -1311,11 +1377,18 @@ struct DsFlat {
virtual_sources: Option<Vec<VdsMapping>>,
/// Number of hard links to the dataset.
refcount: u32,
/// Track (and index) attribute creation order.
track_order: bool,
}
/// Convert a DatasetBuilder into a DsFlat, handling VDS (which does not
/// require a `data` field).
fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result<DsFlat, FormatError> {
fn flatten_ds(
db: DatasetBuilder,
refcount: u32,
default_track_order: bool,
) -> Result<DsFlat, FormatError> {
let track_order = db.track_order.unwrap_or(default_track_order);
let dt = db.datatype.ok_or(FormatError::DatasetMissingData)?;
let shape = db.shape.ok_or(FormatError::DatasetMissingShape)?;
let is_vds = db.virtual_sources.is_some();
@@ -1365,6 +1438,7 @@ fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result<DsFlat, FormatError>
alignment: db.alignment,
virtual_sources: db.virtual_sources,
refcount,
track_order,
})
}
@@ -1421,10 +1495,12 @@ impl FileWriter {
self
}
/// Track (and index) link creation order in every group that does not
/// set its own [`GroupBuilder::track_order`], the root included — as
/// h5py's `track_order=True`: libhdf5 then lists members in the order
/// they were added. Off by default (members are listed by name).
/// Track (and index) creation order — of links and attributes in every
/// group that does not set its own [`GroupBuilder::track_order`], the
/// root included, and of attributes on every dataset that does not set
/// its own [`DatasetBuilder::track_order`] — as h5py's
/// `track_order=True`: libhdf5 then lists members and attributes in the
/// order they were added. Off by default (they are listed by name).
pub fn track_order(&mut self, track: bool) -> &mut Self {
self.track_order = track;
self
@@ -1495,7 +1571,7 @@ impl FileWriter {
let all_ds: Vec<DsFlat> = tree
.datasets
.into_iter()
.map(|(db, refcount)| flatten_ds(db, refcount))
.map(|(db, refcount)| flatten_ds(db, refcount, self.track_order))
.collect::<Result<_, _>>()?;
let groups: Vec<GrpFlat> = tree
.groups
@@ -1512,6 +1588,15 @@ impl FileWriter {
})
.collect();
// Refuse up front what dense storage would refuse after the work.
let tracked = groups
.iter()
.map(|g| (g.track_order, g.attrs.len()))
.chain(all_ds.iter().map(|d| (d.track_order, d.attrs.len())));
for (track, n) in tracked {
check_tracked_attr_count(track, n)?;
}
// Every datatype must have an on-disk encoding before anything is laid
// out: `Datatype::serialize` itself cannot report a failure.
let group_attrs = groups.iter().flat_map(|g| &g.attrs);
@@ -1566,7 +1651,7 @@ impl FileWriter {
.map(|(gi, g)| {
let dummy_links = g.link_messages(&[], &[]);
let attr_blob = group_dense[gi]
.then(|| build_dense_attrs(&g.attrs, 0))
.then(|| build_dense_attrs(&g.attrs, 0, g.track_order))
.transpose()?;
let li = if group_links_dense[gi] {
serialize_link_info(
@@ -1582,8 +1667,11 @@ impl FileWriter {
&dummy_links,
&li,
group_links_dense[gi],
&g.attrs,
attr_blob.as_ref(),
AttrStorage {
attrs: &g.attrs,
dense: attr_blob.as_ref(),
track_order: g.track_order,
},
g.refcount,
)
.map(|oh| oh.len())
@@ -1602,7 +1690,7 @@ impl FileWriter {
let mut dummy_cursor = 0u64;
for (i, d) in all_ds.iter().enumerate() {
let dense_blob = ds_dense[i]
.then(|| build_dense_attrs(&d.attrs, 0))
.then(|| build_dense_attrs(&d.attrs, 0, d.track_order))
.transpose()?;
if is_vds[i] {
// VDS: dummy OH with address 0 to get the OH size. The global
@@ -1611,8 +1699,11 @@ impl FileWriter {
&d.dt,
&d.ds,
0, // dummy address
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1651,8 +1742,11 @@ impl FileWriter {
&d.ds,
&result.layout_message,
result.pipeline_message.as_deref(),
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1666,8 +1760,11 @@ impl FileWriter {
&d.dt,
&d.ds,
&d.raw,
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1682,8 +1779,11 @@ impl FileWriter {
&d.ds,
0,
d.raw.len() as u64,
&d.attrs,
dense_blob.as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: dense_blob.as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1727,7 +1827,7 @@ impl FileWriter {
group_link_blob_addrs.push(None);
}
if group_dense[gi] {
let blob = build_dense_attrs(&g.attrs, cursor2 as u64)?;
let blob = build_dense_attrs(&g.attrs, cursor2 as u64, g.track_order)?;
cursor2 += blob.blob.len();
group_dense_blobs.push(Some(blob));
} else {
@@ -1744,7 +1844,8 @@ impl FileWriter {
let addr = cursor2 as u64;
cursor2 += sz;
if ds_dense[i] {
let blob = build_dense_attrs(&all_ds[i].attrs, cursor2 as u64)?;
let blob =
build_dense_attrs(&all_ds[i].attrs, cursor2 as u64, all_ds[i].track_order)?;
cursor2 += blob.blob.len();
ds_dense_blobs.push(Some(blob));
} else {
@@ -1768,8 +1869,11 @@ impl FileWriter {
&d.dt,
&d.ds,
heap_addr,
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1796,8 +1900,11 @@ impl FileWriter {
&d.ds,
&result.layout_message,
result.pipeline_message.as_deref(),
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1812,8 +1919,11 @@ impl FileWriter {
&d.dt,
&d.ds,
&d.raw,
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1838,8 +1948,11 @@ impl FileWriter {
&d.ds,
cursor2 as u64,
d.raw.len() as u64,
&d.attrs,
ds_dense_blobs[i].as_ref(),
AttrStorage {
attrs: &d.attrs,
dense: ds_dense_blobs[i].as_ref(),
track_order: d.track_order,
},
&d.fill_message,
d.refcount,
)?;
@@ -1907,8 +2020,11 @@ impl FileWriter {
&links,
&li,
link_blob.is_some(),
&g.attrs,
group_dense_blobs[gi].as_ref(),
AttrStorage {
attrs: &g.attrs,
dense: group_dense_blobs[gi].as_ref(),
track_order: g.track_order,
},
g.refcount,
)?;
debug_assert_eq!(oh.len(), group_oh_sizes[gi]);
@@ -2139,6 +2255,92 @@ mod tests {
assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0, 3.0]);
}
/// Attribute names of the object at `path`, in the order the reader
/// lists them.
fn attr_names(bytes: &[u8], path: &str) -> Vec<String> {
let sig = signature::find_signature(bytes).unwrap();
let sb = Superblock::parse(bytes, sig).unwrap();
let addr = if path == "/" {
sb.root_group_address
} else {
resolve_path_any(bytes, &sb, path).unwrap()
};
let hdr =
ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
crate::attribute::extract_attributes_full(bytes, &hdr, sb.offset_size, sb.length_size)
.unwrap()
.into_iter()
.map(|a| a.name)
.collect()
}
#[test]
fn tracked_attributes_are_read_in_creation_order() {
let set = |names: &[String]| -> Vec<(String, AttrValue)> {
names
.iter()
.enumerate()
.map(|(i, n)| (n.clone(), AttrValue::I64(i as i64)))
.collect()
};
let compact: Vec<String> = ["zeta", "alpha", "mid"].map(String::from).to_vec();
let dense: Vec<String> = (0..30).rev().map(|i| format!("a{i:02}")).collect();
let mut fw = FileWriter::new();
fw.track_order(true);
for (n, v) in set(&compact) {
fw.set_root_attr(&n, v);
}
let ds = fw.create_dataset("dense");
ds.with_i32_data(&[1]);
for (n, v) in set(&dense) {
ds.set_attr(&n, v);
}
let ds = fw.create_dataset("untracked");
ds.with_i32_data(&[1]).track_order(false);
for (n, v) in set(&dense) {
ds.set_attr(&n, v);
}
let mut g = fw.create_group("g");
g.track_order(false);
for (n, v) in set(&compact) {
g.set_attr(&n, v);
}
fw.add_group(g.finish());
let bytes = fw.finish().unwrap();
assert_eq!(attr_names(&bytes, "/"), compact);
assert_eq!(attr_names(&bytes, "dense"), dense);
// Without tracking: storage order (inline: as added; dense: hash).
assert_eq!(attr_names(&bytes, "g"), compact);
let mut by_hash = dense.clone();
by_hash.sort_by_key(|n| crate::checksum::jenkins_lookup3(n.as_bytes()));
assert_eq!(attr_names(&bytes, "untracked"), by_hash);
}
#[test]
fn too_many_tracked_attributes_is_an_error() {
// libhdf5 numbers at most 65 535 attributes on an object that
// tracks their creation order (a 2-byte field). (`set_attr` looks
// for an earlier value, so 65 536 of them through the builder take
// a while; build the messages directly.)
let attrs: Vec<AttributeMessage> = (0..65_536)
.map(|i| build_attr_message(&format!("a{i}"), &AttrValue::I64(i)))
.collect();
let err = build_dense_attrs(&attrs, 0, true)
.err()
.unwrap()
.to_string();
assert!(err.contains("65536 attributes on one object"), "{err}");
assert!(build_dense_attrs(&attrs[1..], 0, true).is_ok());
assert!(build_dense_attrs(&attrs, 0, false).is_ok());
let mut fw = FileWriter::new();
let ds = fw.create_dataset("x");
ds.with_i32_data(&[1]).track_order(true);
for i in 0..20 {
ds.set_attr(&format!("a{i}"), AttrValue::I64(i));
}
assert!(fw.finish().is_ok());
}
#[test]
fn dense_attrs_root_group_self_roundtrip() {
let mut fw = FileWriter::new();