From 193a5f8a8267337c8534bb1f6d1b55a575fb3bce Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:17:37 -0500 Subject: [PATCH] writer: track attribute creation order with track_order h5py's track_order=True orders attributes as well as links; the writer tracked links only. A tracking object's header now sets the attribute creation order tracked/indexed flags and carries per-message creation orders, an Attribute Info message holds the next order (inline too), and dense storage gets a type-9 creation-order index. The file default applies to datasets, with DatasetBuilder::track_order per dataset; more than 65 535 attributes on a tracking object is an error (libhdf5's counter is 2 bytes). The reader lists such attributes in creation order. h5py lists them in order (inline, dense, 20 000 on one dataset) and keeps numbering in r+ mode, including its inline-to-dense move. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 16 + crates/clawhdf5-format/src/attribute.rs | 35 +- crates/clawhdf5-format/src/file_writer.rs | 360 ++++++++++++++---- .../src/object_header_writer.rs | 65 +++- crates/clawhdf5-format/src/type_builders.rs | 23 +- crates/clawhdf5-tools/tests/h5rs_interop.rs | 6 + crates/clawhdf5/src/writer.rs | 8 +- .../clawhdf5/tests/writer_groups_interop.rs | 91 +++++ docs/known-issues.md | 12 +- 9 files changed, 515 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 448d2a0..dbecdc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ ## Unreleased ### Writer: large dense indexes (2026-09-26) +- **`track_order` orders attributes too, as h5py's `track_order=True` + does.** It tracked link creation order only, so h5py listed a tracked + object's attributes by name. A tracking object's header now has the + attribute creation order tracked and indexed flags, an Attribute Info + message with the next creation order (also for inline attributes), and a + creation order on each inline attribute message; dense attribute storage + gets a creation-order index (B-tree type 9). `FileWriter::track_order` / + `FileBuilder::track_order` now apply to datasets' attributes as well, and + `DatasetBuilder::track_order` sets it per dataset. Groups and datasets + that track order are written differently from before; others are + unchanged. More than 65 535 attributes on a tracking object is an error + (libhdf5's creation order counter is 2 bytes). The reader + (`attribute::extract_attributes*`) lists a tracking object's attributes + in creation order. Test `track_order_lists_attributes_in_creation_order` + (h5py lists, reads and extends them in "r+" mode, including libhdf5's + move from inline to dense storage). - **No more 65 535-record limit on the writer's v2 B-trees.** Dense link storage (name index and creation-order index), dense attribute storage and the chunk index of datasets with more than one unlimited dimension diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index 11bde3e..6c4e9ae 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -450,6 +450,8 @@ fn extract_attributes_with( on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result, FormatError> { let mut attrs = Vec::new(); + // Each attribute's creation order, where the file records one. + let mut orders: Vec = Vec::new(); // Collect compact attributes (inline in OH) for msg in &header.messages { @@ -479,7 +481,10 @@ fn extract_attributes_with( }; let attr = attr.and_then(|a| check_in_header(a, header)); match attr { - Ok(attr) => attrs.push(attr), + Ok(attr) => { + attrs.push(attr); + orders.push(msg.creation_order.map_or(0, u32::from)); + } Err(e) => on_error(e)?, } } @@ -487,20 +492,30 @@ fn extract_attributes_with( // Check for dense attributes via AttributeInfo message let attr_info = find_attribute_info(header, offset_size)?; - if let Some(info) = attr_info + if let Some(info) = &attr_info && let Some(fh_addr) = info.fractal_heap_address { extract_dense_attributes( file_data, - &info, + info, fh_addr, offset_size, length_size, &mut attrs, + &mut orders, on_error, )?; } + // An object that tracks attribute creation order lists its attributes + // in that order (h5py's `track_order=True`), as libhdf5 does; otherwise + // they come in storage order. + if attr_info.is_some_and(|i| i.max_creation_index.is_some()) { + let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect(); + paired.sort_by_key(|(o, _)| *o); + attrs = paired.into_iter().map(|(_, a)| a).collect(); + } + Ok(attrs) } @@ -518,7 +533,9 @@ fn find_attribute_info( Ok(None) } -/// Extract attributes from dense storage (fractal heap + B-tree v2). +/// Extract attributes from dense storage (fractal heap + B-tree v2), and +/// each one's creation order into `orders`. +#[allow(clippy::too_many_arguments)] fn extract_dense_attributes( file_data: &[u8], attr_info: &AttributeInfoMessage, @@ -526,6 +543,7 @@ fn extract_dense_attributes( offset_size: u8, length_size: u8, attrs: &mut Vec, + orders: &mut Vec, on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result<(), FormatError> { // Parse fractal heap @@ -561,7 +579,14 @@ fn extract_dense_attributes( AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size) }); match attr { - Ok(attr) => attrs.push(attr), + Ok(attr) => { + attrs.push(attr); + let order = record + .data + .get(id_len + 1..id_len + 5) + .map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])); + orders.push(order); + } Err(e) => on_error(e)?, } } diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index a9b3af5..2568582 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -75,14 +75,57 @@ const DENSE_LINK_THRESHOLD: usize = 8; // ---- OH builders ---- +/// An object's attributes as its header stores them: inline Attribute +/// messages, or (`dense`) the Attribute Info message of dense storage; with +/// `track_order`, their creation order tracked and indexed. +#[derive(Clone, Copy)] +pub(crate) struct AttrStorage<'a> { + pub(crate) attrs: &'a [AttributeMessage], + pub(crate) dense: Option<&'a DenseAttrBlob>, + pub(crate) track_order: bool, +} + +impl AttrStorage<'_> { + /// Add the attribute messages to the header being built. Tracking + /// creation order, as libhdf5 does it: the header's flags say so, an + /// Attribute Info message is written even for inline attributes (it + /// holds the next creation order), and each inline attribute's message + /// carries its creation order. + fn add_to(&self, w: &mut ObjectHeaderWriter) { + if self.track_order { + w.track_attr_order(); + } + if let Some(blob) = self.dense { + w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); + return; + } + if self.track_order { + w.add_message( + MessageType::AttributeInfo, + serialize_attribute_info( + u64::MAX, + u64::MAX, + Some((self.attrs.len() as u16, u64::MAX)), + ), + ); + } + for (i, attr) in self.attrs.iter().enumerate() { + w.add_message_with_order( + MessageType::Attribute, + attr.serialize(LENGTH_SIZE), + i as u16, + ); + } + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn build_chunked_dataset_oh( dt: &Datatype, ds: &Dataspace, layout_message: &[u8], pipeline_message: Option<&[u8]>, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -94,13 +137,7 @@ pub(crate) fn build_chunked_dataset_oh( if let Some(pm) = pipeline_message { w.add_message(MessageType::FilterPipeline, pm.to_vec()); } - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -111,8 +148,7 @@ pub(crate) fn build_dataset_oh( ds: &Dataspace, data_addr: u64, data_size: u64, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -132,13 +168,7 @@ pub(crate) fn build_dataset_oh( dl.extend_from_slice(&data_addr.to_le_bytes()); dl.extend_from_slice(&data_size.to_le_bytes()); w.add_message(MessageType::DataLayout, dl); - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -148,8 +178,7 @@ pub(crate) fn build_compact_dataset_oh( dt: &Datatype, ds: &Dataspace, data: &[u8], - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -164,13 +193,7 @@ pub(crate) fn build_compact_dataset_oh( dl.extend_from_slice(&(data.len() as u16).to_le_bytes()); dl.extend_from_slice(data); w.add_message(MessageType::DataLayout, dl); - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -182,8 +205,7 @@ pub(crate) fn build_group_oh( links: &[LinkMessage], link_info: &[u8], dense_links: bool, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, refcount: u32, ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); @@ -198,13 +220,7 @@ pub(crate) fn build_group_oh( w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE)); } } - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -891,11 +907,31 @@ fn write_frhp(p: WriteFrhp) -> Vec { frhp } +/// libhdf5 numbers the attributes of an object that tracks their creation +/// order with a 2-byte counter. +fn check_tracked_attr_count(track_order: bool, n: usize) -> Result<(), FormatError> { + if track_order && n > usize::from(u16::MAX) { + return Err(FormatError::SerializationError(format!( + "{n} attributes on one object with creation order tracked: libhdf5 \ + numbers at most {} (set fewer, or turn off track_order)", + u16::MAX + ))); + } + Ok(()) +} + /// Build dense attribute storage for a set of attributes. +/// +/// With `track_order` the Attribute Info message tracks creation order (an +/// attribute's creation order is its position in `attrs`) and a type-9 +/// creation-order index follows the name index, as libhdf5 writes for h5py's +/// `track_order=True`. libhdf5 numbers at most 65 535 attributes. pub(crate) fn build_dense_attrs( attrs: &[AttributeMessage], base_address: u64, + track_order: bool, ) -> Result { + check_tracked_attr_count(track_order, attrs.len())?; // Dense attrs use v3 attribute messages (adds character set encoding byte). let serialized: Vec> = attrs.iter().map(|a| a.serialize_v3(LENGTH_SIZE)).collect(); @@ -936,7 +972,30 @@ pub(crate) fn build_dense_attrs( let mut blob = heap.blob; blob.extend_from_slice(&dense_v2_btree(8, record_size, &records, bthd_addr)?); - let attr_info = serialize_attribute_info(frhp_addr, bthd_addr); + let order = if track_order { + // Type 9 records: heap ID, message flags, creation order (the key). + let records: Vec> = heap_ids + .iter() + .enumerate() + .map(|(i, heap_id)| { + let mut rec = heap_id.clone(); + rec.push(0); // msg_flags + rec.extend_from_slice(&(i as u32).to_le_bytes()); + rec + }) + .collect(); + let corder_addr = base_address + blob.len() as u64; + blob.extend_from_slice(&dense_v2_btree( + 9, + heap_id_length + 1 + 4, + &records, + corder_addr, + )?); + Some((attrs.len() as u16, corder_addr)) + } else { + None + }; + let attr_info = serialize_attribute_info(frhp_addr, bthd_addr, order); Ok(DenseAttrBlob { attr_info_message: attr_info, @@ -1139,12 +1198,25 @@ fn encode_managed_id(offset: u64, length: u64, max_heap_size: u16, id_length: u1 id } -fn serialize_attribute_info(fh_addr: u64, btree_name_addr: u64) -> Vec { +/// Serialize an Attribute Info message (version 0). `order` — the next +/// creation order to assign and the creation-order index's address — is +/// present when creation order is tracked and indexed. +fn serialize_attribute_info( + fh_addr: u64, + btree_name_addr: u64, + order: Option<(u16, u64)>, +) -> Vec { let mut data = Vec::new(); data.push(0); // version - data.push(0x00); // flags + data.push(if order.is_some() { 0x03 } else { 0x00 }); // flags: tracked, indexed + if let Some((next, _)) = order { + data.extend_from_slice(&next.to_le_bytes()); + } data.extend_from_slice(&fh_addr.to_le_bytes()); data.extend_from_slice(&btree_name_addr.to_le_bytes()); + if let Some((_, corder_addr)) = order { + data.extend_from_slice(&corder_addr.to_le_bytes()); + } data } @@ -1219,8 +1291,7 @@ pub(crate) fn build_vds_dataset_oh( dt: &Datatype, ds: &Dataspace, global_heap_addr: u64, - attrs: &[AttributeMessage], - dense_blob: Option<&DenseAttrBlob>, + attrs: AttrStorage<'_>, fill_message: &[u8], refcount: u32, ) -> Result, FormatError> { @@ -1235,13 +1306,7 @@ pub(crate) fn build_vds_dataset_oh( dl.extend_from_slice(&global_heap_addr.to_le_bytes()); dl.extend_from_slice(&1u32.to_le_bytes()); // object index 1 in the collection w.add_message(MessageType::DataLayout, dl); - if let Some(blob) = dense_blob { - w.add_message(MessageType::AttributeInfo, blob.attr_info_message.clone()); - } else { - for attr in attrs { - w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); - } - } + attrs.add_to(&mut w); add_refcount(&mut w, refcount); w.serialize() } @@ -1276,7 +1341,8 @@ fn write_undef_offset(buf: &mut Vec, offset_size: u8) { pub struct FileWriter { /// The root group's contents (its name is unused). root: GroupBuilder, - /// Default for groups that do not call [`GroupBuilder::track_order`]. + /// Default for groups and datasets that do not set their own + /// `track_order`. track_order: bool, /// Global alignment threshold: datasets with raw data >= this many bytes /// will have their data aligned to `alignment_bytes`. @@ -1311,11 +1377,18 @@ struct DsFlat { virtual_sources: Option>, /// Number of hard links to the dataset. refcount: u32, + /// Track (and index) attribute creation order. + track_order: bool, } /// Convert a DatasetBuilder into a DsFlat, handling VDS (which does not /// require a `data` field). -fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result { +fn flatten_ds( + db: DatasetBuilder, + refcount: u32, + default_track_order: bool, +) -> Result { + let track_order = db.track_order.unwrap_or(default_track_order); let dt = db.datatype.ok_or(FormatError::DatasetMissingData)?; let shape = db.shape.ok_or(FormatError::DatasetMissingShape)?; let is_vds = db.virtual_sources.is_some(); @@ -1365,6 +1438,7 @@ fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result alignment: db.alignment, virtual_sources: db.virtual_sources, refcount, + track_order, }) } @@ -1421,10 +1495,12 @@ impl FileWriter { self } - /// Track (and index) link creation order in every group that does not - /// set its own [`GroupBuilder::track_order`], the root included — as - /// h5py's `track_order=True`: libhdf5 then lists members in the order - /// they were added. Off by default (members are listed by name). + /// Track (and index) creation order — of links and attributes in every + /// group that does not set its own [`GroupBuilder::track_order`], the + /// root included, and of attributes on every dataset that does not set + /// its own [`DatasetBuilder::track_order`] — as h5py's + /// `track_order=True`: libhdf5 then lists members and attributes in the + /// order they were added. Off by default (they are listed by name). pub fn track_order(&mut self, track: bool) -> &mut Self { self.track_order = track; self @@ -1495,7 +1571,7 @@ impl FileWriter { let all_ds: Vec = tree .datasets .into_iter() - .map(|(db, refcount)| flatten_ds(db, refcount)) + .map(|(db, refcount)| flatten_ds(db, refcount, self.track_order)) .collect::>()?; let groups: Vec = tree .groups @@ -1512,6 +1588,15 @@ impl FileWriter { }) .collect(); + // Refuse up front what dense storage would refuse after the work. + let tracked = groups + .iter() + .map(|g| (g.track_order, g.attrs.len())) + .chain(all_ds.iter().map(|d| (d.track_order, d.attrs.len()))); + for (track, n) in tracked { + check_tracked_attr_count(track, n)?; + } + // Every datatype must have an on-disk encoding before anything is laid // out: `Datatype::serialize` itself cannot report a failure. let group_attrs = groups.iter().flat_map(|g| &g.attrs); @@ -1566,7 +1651,7 @@ impl FileWriter { .map(|(gi, g)| { let dummy_links = g.link_messages(&[], &[]); let attr_blob = group_dense[gi] - .then(|| build_dense_attrs(&g.attrs, 0)) + .then(|| build_dense_attrs(&g.attrs, 0, g.track_order)) .transpose()?; let li = if group_links_dense[gi] { serialize_link_info( @@ -1582,8 +1667,11 @@ impl FileWriter { &dummy_links, &li, group_links_dense[gi], - &g.attrs, - attr_blob.as_ref(), + AttrStorage { + attrs: &g.attrs, + dense: attr_blob.as_ref(), + track_order: g.track_order, + }, g.refcount, ) .map(|oh| oh.len()) @@ -1602,7 +1690,7 @@ impl FileWriter { let mut dummy_cursor = 0u64; for (i, d) in all_ds.iter().enumerate() { let dense_blob = ds_dense[i] - .then(|| build_dense_attrs(&d.attrs, 0)) + .then(|| build_dense_attrs(&d.attrs, 0, d.track_order)) .transpose()?; if is_vds[i] { // VDS: dummy OH with address 0 to get the OH size. The global @@ -1611,8 +1699,11 @@ impl FileWriter { &d.dt, &d.ds, 0, // dummy address - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1651,8 +1742,11 @@ impl FileWriter { &d.ds, &result.layout_message, result.pipeline_message.as_deref(), - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1666,8 +1760,11 @@ impl FileWriter { &d.dt, &d.ds, &d.raw, - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1682,8 +1779,11 @@ impl FileWriter { &d.ds, 0, d.raw.len() as u64, - &d.attrs, - dense_blob.as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: dense_blob.as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1727,7 +1827,7 @@ impl FileWriter { group_link_blob_addrs.push(None); } if group_dense[gi] { - let blob = build_dense_attrs(&g.attrs, cursor2 as u64)?; + let blob = build_dense_attrs(&g.attrs, cursor2 as u64, g.track_order)?; cursor2 += blob.blob.len(); group_dense_blobs.push(Some(blob)); } else { @@ -1744,7 +1844,8 @@ impl FileWriter { let addr = cursor2 as u64; cursor2 += sz; if ds_dense[i] { - let blob = build_dense_attrs(&all_ds[i].attrs, cursor2 as u64)?; + let blob = + build_dense_attrs(&all_ds[i].attrs, cursor2 as u64, all_ds[i].track_order)?; cursor2 += blob.blob.len(); ds_dense_blobs.push(Some(blob)); } else { @@ -1768,8 +1869,11 @@ impl FileWriter { &d.dt, &d.ds, heap_addr, - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1796,8 +1900,11 @@ impl FileWriter { &d.ds, &result.layout_message, result.pipeline_message.as_deref(), - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1812,8 +1919,11 @@ impl FileWriter { &d.dt, &d.ds, &d.raw, - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1838,8 +1948,11 @@ impl FileWriter { &d.ds, cursor2 as u64, d.raw.len() as u64, - &d.attrs, - ds_dense_blobs[i].as_ref(), + AttrStorage { + attrs: &d.attrs, + dense: ds_dense_blobs[i].as_ref(), + track_order: d.track_order, + }, &d.fill_message, d.refcount, )?; @@ -1907,8 +2020,11 @@ impl FileWriter { &links, &li, link_blob.is_some(), - &g.attrs, - group_dense_blobs[gi].as_ref(), + AttrStorage { + attrs: &g.attrs, + dense: group_dense_blobs[gi].as_ref(), + track_order: g.track_order, + }, g.refcount, )?; debug_assert_eq!(oh.len(), group_oh_sizes[gi]); @@ -2139,6 +2255,92 @@ mod tests { assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0, 3.0]); } + /// Attribute names of the object at `path`, in the order the reader + /// lists them. + fn attr_names(bytes: &[u8], path: &str) -> Vec { + let sig = signature::find_signature(bytes).unwrap(); + let sb = Superblock::parse(bytes, sig).unwrap(); + let addr = if path == "/" { + sb.root_group_address + } else { + resolve_path_any(bytes, &sb, path).unwrap() + }; + let hdr = + ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap(); + crate::attribute::extract_attributes_full(bytes, &hdr, sb.offset_size, sb.length_size) + .unwrap() + .into_iter() + .map(|a| a.name) + .collect() + } + + #[test] + fn tracked_attributes_are_read_in_creation_order() { + let set = |names: &[String]| -> Vec<(String, AttrValue)> { + names + .iter() + .enumerate() + .map(|(i, n)| (n.clone(), AttrValue::I64(i as i64))) + .collect() + }; + let compact: Vec = ["zeta", "alpha", "mid"].map(String::from).to_vec(); + let dense: Vec = (0..30).rev().map(|i| format!("a{i:02}")).collect(); + let mut fw = FileWriter::new(); + fw.track_order(true); + for (n, v) in set(&compact) { + fw.set_root_attr(&n, v); + } + let ds = fw.create_dataset("dense"); + ds.with_i32_data(&[1]); + for (n, v) in set(&dense) { + ds.set_attr(&n, v); + } + let ds = fw.create_dataset("untracked"); + ds.with_i32_data(&[1]).track_order(false); + for (n, v) in set(&dense) { + ds.set_attr(&n, v); + } + let mut g = fw.create_group("g"); + g.track_order(false); + for (n, v) in set(&compact) { + g.set_attr(&n, v); + } + fw.add_group(g.finish()); + let bytes = fw.finish().unwrap(); + assert_eq!(attr_names(&bytes, "/"), compact); + assert_eq!(attr_names(&bytes, "dense"), dense); + // Without tracking: storage order (inline: as added; dense: hash). + assert_eq!(attr_names(&bytes, "g"), compact); + let mut by_hash = dense.clone(); + by_hash.sort_by_key(|n| crate::checksum::jenkins_lookup3(n.as_bytes())); + assert_eq!(attr_names(&bytes, "untracked"), by_hash); + } + + #[test] + fn too_many_tracked_attributes_is_an_error() { + // libhdf5 numbers at most 65 535 attributes on an object that + // tracks their creation order (a 2-byte field). (`set_attr` looks + // for an earlier value, so 65 536 of them through the builder take + // a while; build the messages directly.) + let attrs: Vec = (0..65_536) + .map(|i| build_attr_message(&format!("a{i}"), &AttrValue::I64(i))) + .collect(); + let err = build_dense_attrs(&attrs, 0, true) + .err() + .unwrap() + .to_string(); + assert!(err.contains("65536 attributes on one object"), "{err}"); + assert!(build_dense_attrs(&attrs[1..], 0, true).is_ok()); + assert!(build_dense_attrs(&attrs, 0, false).is_ok()); + let mut fw = FileWriter::new(); + let ds = fw.create_dataset("x"); + ds.with_i32_data(&[1]).track_order(true); + for i in 0..20 { + ds.set_attr(&format!("a{i}"), AttrValue::I64(i)); + } + assert!(fw.finish().is_ok()); + } + #[test] fn dense_attrs_root_group_self_roundtrip() { let mut fw = FileWriter::new(); diff --git a/crates/clawhdf5-format/src/object_header_writer.rs b/crates/clawhdf5-format/src/object_header_writer.rs index 8d52ad4..bbb3c0a 100644 --- a/crates/clawhdf5-format/src/object_header_writer.rs +++ b/crates/clawhdf5-format/src/object_header_writer.rs @@ -12,9 +12,16 @@ use crate::message_type::MessageType; /// its size truncated to 16 bits produced files libhdf5 refuses. pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize; +/// Object header flags: attribute creation order tracked (each message +/// then carries a 2-byte creation order) and indexed. +const OHDR_ATTR_CRT_ORDER_TRACKED: u8 = 0x04; +const OHDR_ATTR_CRT_ORDER_INDEXED: u8 = 0x08; + /// Writer for v2 object headers with proper checksums. pub struct ObjectHeaderWriter { - messages: Vec<(MessageType, Vec, u8)>, // (type, data, msg_flags) + messages: Vec<(MessageType, Vec, u8, u16)>, // (type, data, msg_flags, creation order) + /// Attribute creation order tracked and indexed. + attr_order: bool, } impl ObjectHeaderWriter { @@ -22,17 +29,33 @@ impl ObjectHeaderWriter { pub fn new() -> Self { Self { messages: Vec::new(), + attr_order: false, } } + /// Track and index attribute creation order, as libhdf5 does for an + /// object created with `H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED` + /// (h5py's `track_order=True`): the header's flags say so, and every + /// message carries a creation order (an attribute's own; 0 for the + /// others). libhdf5 reads the setting back from these flags. + pub fn track_attr_order(&mut self) { + self.attr_order = true; + } + /// Add a message to the header with default flags (0). pub fn add_message(&mut self, msg_type: MessageType, data: Vec) { - self.messages.push((msg_type, data, 0)); + self.messages.push((msg_type, data, 0, 0)); } /// Add a message with specific flags. pub fn add_message_with_flags(&mut self, msg_type: MessageType, data: Vec, flags: u8) { - self.messages.push((msg_type, data, flags)); + self.messages.push((msg_type, data, flags, 0)); + } + + /// Add a message with its creation order, which is written only when + /// attribute creation order is tracked ([`Self::track_attr_order`]). + pub fn add_message_with_order(&mut self, msg_type: MessageType, data: Vec, order: u16) { + self.messages.push((msg_type, data, 0, order)); } /// Serialize the complete v2 object header (OHDR + messages + checksum). @@ -41,10 +64,10 @@ impl ObjectHeaderWriter { /// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would /// need dense attribute storage), rather than writing a corrupt header. pub fn serialize(&self) -> Result, FormatError> { - if let Some((msg_type, data, _)) = self + if let Some((msg_type, data, _, _)) = self .messages .iter() - .find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE) + .find(|(_, data, _, _)| data.len() > MAX_MESSAGE_SIZE) { return Err(FormatError::SerializationError(format!( "{msg_type:?} message is {} bytes; an object header message holds at most \ @@ -52,11 +75,13 @@ impl ObjectHeaderWriter { data.len() ))); } - // Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data + // Calculate total message bytes: each message has type(1) + size(2) + + // flags(1) [+ creation order(2)] + data + let msg_header = if self.attr_order { 6 } else { 4 }; let msg_bytes_total: usize = self .messages .iter() - .map(|(_, data, _)| 4 + data.len()) + .map(|(_, data, _, _)| msg_header + data.len()) .sum(); // Determine chunk size field width based on msg_bytes_total @@ -68,6 +93,12 @@ impl ObjectHeaderWriter { (0x02u8, 4) }; + let flags = if self.attr_order { + flags | OHDR_ATTR_CRT_ORDER_TRACKED | OHDR_ATTR_CRT_ORDER_INDEXED + } else { + flags + }; + let mut buf = Vec::new(); // OHDR signature @@ -85,7 +116,7 @@ impl ObjectHeaderWriter { } // Messages - for (msg_type, data, msg_flags) in &self.messages { + for (msg_type, data, msg_flags, order) in &self.messages { let type_id = msg_type.to_u16(); assert!( type_id <= 255, @@ -94,6 +125,9 @@ impl ObjectHeaderWriter { buf.push(type_id as u8); // type (1 byte in v2) buf.extend_from_slice(&(data.len() as u16).to_le_bytes()); // size (2 bytes) buf.push(*msg_flags); // flags + if self.attr_order { + buf.extend_from_slice(&order.to_le_bytes()); // creation order + } buf.extend_from_slice(data); } @@ -193,6 +227,21 @@ mod tests { assert_eq!(hdr.messages.len(), 0); } + #[test] + fn tracked_attribute_order_is_in_the_flags_and_every_message() { + let mut writer = ObjectHeaderWriter::new(); + writer.track_attr_order(); + writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]); + writer.add_message_with_order(MessageType::Attribute, vec![5, 6], 7); + let bytes = writer.serialize().unwrap(); + assert_eq!(bytes[5] & 0x0C, 0x0C); + let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap(); + assert_eq!(hdr.messages.len(), 2); + assert_eq!(hdr.messages[0].creation_order, Some(0)); + assert_eq!(hdr.messages[1].creation_order, Some(7)); + assert_eq!(hdr.messages[1].data, vec![5, 6]); + } + #[test] fn two_messages_roundtrip() { let mut writer = ObjectHeaderWriter::new(); diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index ce43f64..3019daa 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -503,6 +503,9 @@ pub struct DatasetBuilder { /// `data` field is ignored; instead the global heap blob is built from /// these mappings and a VDS layout message is emitted. pub(crate) virtual_sources: Option>, + /// Track (and index) attribute creation order; `None` follows the + /// file's default (`FileWriter::track_order`). + pub(crate) track_order: Option, #[cfg(feature = "provenance")] pub(crate) provenance: Option, } @@ -522,11 +525,22 @@ impl DatasetBuilder { compact: false, alignment: 0, virtual_sources: None, + track_order: None, #[cfg(feature = "provenance")] provenance: None, } } + /// Track the creation order of this dataset's attributes, and index it, + /// as h5py's `create_dataset(..., track_order=True)` does: libhdf5 (and + /// h5py) then list the attributes in the order they were set rather + /// than by name. libhdf5 numbers at most 65 535 attributes on an object + /// that tracks their order; more is an error when the file is written. + pub fn track_order(&mut self, track: bool) -> &mut Self { + self.track_order = Some(track); + self + } + pub fn with_f64_data(&mut self, data: &[f64]) -> &mut Self { self.datatype = Some(make_f64_type()); let mut b = Vec::with_capacity(data.len() * 8); @@ -986,10 +1000,11 @@ impl GroupBuilder { self.attrs.push((name.to_string(), value)); } - /// Track the creation order of this group's links, and index it, as - /// h5py's `track_order=True` does: libhdf5 (and h5py) then list the - /// group's members in the order they were added rather than by name. - /// Applies to links only, not to attributes. + /// Track the creation order of this group's links and attributes, and + /// index it, as h5py's `track_order=True` does: libhdf5 (and h5py) then + /// list the group's members, and its attributes, in the order they were + /// added rather than by name. libhdf5 numbers at most 65 535 attributes + /// on an object that tracks their order. pub fn track_order(&mut self, track: bool) -> &mut Self { self.track_order = Some(track); self diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 32f6fb2..f7b0f74 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -940,11 +940,17 @@ fn write_nested_links(dir: &Path) -> Vec { g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]); } g.add_hard_link("back", "/a/b/c"); + // Attribute creation order tracked too: dense, with a type-9 index. + for i in (0..12).rev() { + g.set_attr(&format!("attr{i:02}"), AttrValue::I64(i)); + } b.add_group(g.finish()); let mut g = b.create_group("compact_ordered"); g.track_order(true); g.create_dataset("z").with_i32_data(&[1]); g.create_dataset("a").with_i32_data(&[2]); + g.set_attr("zz", AttrValue::I64(1)); + g.set_attr("aa", AttrValue::I64(2)); b.add_group(g.finish()); let nested = dir.join("nested.h5"); b.write(&nested).unwrap(); diff --git a/crates/clawhdf5/src/writer.rs b/crates/clawhdf5/src/writer.rs index bfa9393..ea48bc9 100644 --- a/crates/clawhdf5/src/writer.rs +++ b/crates/clawhdf5/src/writer.rs @@ -88,9 +88,11 @@ impl FileBuilder { self } - /// Track link creation order in every group that does not set its own - /// (`GroupBuilder::track_order`), as h5py's `track_order=True`: libhdf5 - /// then lists members in the order they were added. + /// Track the creation order of links and attributes in every group, and + /// of attributes on every dataset, that does not set its own + /// (`GroupBuilder::track_order`, `DatasetBuilder::track_order`), as + /// h5py's `track_order=True`: libhdf5 then lists members and attributes + /// in the order they were added. pub fn track_order(&mut self, track: bool) -> &mut Self { self.writer.track_order(track); self diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index d16567c..777ec07 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -531,6 +531,97 @@ fn ten_thousand_links_in_one_group() { ); } +#[test] +fn track_order_lists_attributes_in_creation_order() { + skip_if_no_python!(); + // h5py's track_order=True orders an object's attributes as well as a + // group's links; the writer tracked links only, so h5py listed the + // attributes by name. Now the object header's flags say attribute + // creation order is tracked and indexed, an Attribute Info message + // holds the next order, inline attributes carry theirs, and dense + // storage gets a creation-order index (B-tree type 9). + let dir = tempfile::tempdir().unwrap(); + let small = ["zeta", "alpha", "mid"]; + let mut b = FileBuilder::new(); + b.track_order(true); // the root, and every group and dataset by default + for (i, n) in small.iter().enumerate() { + b.set_attr(n, AttrValue::I64(i as i64)); + } + let mut g = b.create_group("g"); // dense: 30 attributes + for i in (0..30).rev() { + g.set_attr(&format!("a{i:02}"), AttrValue::I64(i)); + } + b.add_group(g.finish()); + let d = b.create_dataset("d"); + d.with_i32_data(&[1]); + for (i, n) in small.iter().enumerate() { + d.set_attr(n, AttrValue::I64(i as i64)); + } + // 20 000 attributes: a one-leaf creation-order index of 20 000 records. + let big = b.create_dataset("big"); + big.with_i32_data(&[2]); + for i in (0..20_000).rev() { + big.set_attr(&format!("b{i:05}"), AttrValue::I64(i)); + } + let plain = b.create_dataset("plain"); + plain.with_i32_data(&[3]).track_order(false); + for (i, n) in small.iter().enumerate() { + plain.set_attr(n, AttrValue::I64(i as i64)); + } + let path = write(&dir, "attr_order.h5", b); + + let out = h5py( + &path, + "def order(o):\n\ + \x20 return o.id.get_create_plist().get_attr_creation_order()\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 big = list(f['big'].attrs)\n\ + \x20 print(json.dumps([list(f.attrs), [int(v) for v in f.attrs.values()],\n\ + \x20 list(f['g'].attrs)[:3], len(f['g'].attrs), list(f['d'].attrs),\n\ + \x20 big[:2], big == ['b%05d' % i for i in range(19999, -1, -1)],\n\ + \x20 int(f['big'].attrs['b00007']), list(f['plain'].attrs),\n\ + \x20 [order(f['/']), order(f['g']), order(f['d']), order(f['plain'])]]))", + ); + assert_eq!( + out, + r#"[["zeta", "alpha", "mid"], [0, 1, 2], ["a29", "a28", "a27"], 30, ["zeta", "alpha", "mid"], ["b19999", "b19998"], true, 7, ["alpha", "mid", "zeta"], [3, 3, 3, 0]]"# + ); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("big").unwrap().attrs().unwrap().len(), 20_000); + assert!(matches!( + f.group("g").unwrap().attrs().unwrap()["a07"], + AttrValue::I64(7) + )); + drop(f); + + // libhdf5 continues the numbering: new attributes come last, also when + // it moves the inline ones of `d` to dense storage. + let out = h5py( + &path, + "with h5py.File(path, 'r+') as f:\n\ + \x20 f.attrs['new'] = 9\n\ + \x20 del f.attrs['alpha']\n\ + \x20 f['g'].attrs['new'] = 9\n\ + \x20 del f['g'].attrs['a15']\n\ + \x20 for i in range(8):\n\ + \x20 f['d'].attrs['x%d' % i] = i\n\ + \x20 f['big'].attrs['new'] = 9\n\ + \x20 for i in range(0, 20000, 2):\n\ + \x20 del f['big'].attrs['b%05d' % i]\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 big = list(f['big'].attrs)\n\ + \x20 print(json.dumps([list(f.attrs), list(f['g'].attrs)[-2:], len(f['g'].attrs),\n\ + \x20 list(f['d'].attrs)[:4], len(f['d'].attrs),\n\ + \x20 big == ['b%05d' % i for i in range(19999, -1, -2)] + ['new']]))", + ); + assert_eq!( + out, + r#"[["zeta", "mid", "new"], ["a00", "new"], 30, ["zeta", "alpha", "mid", "x0"], 11, true]"# + ); + h5dump_ok(&path); +} + #[test] fn track_order_lists_members_in_creation_order() { skip_if_no_python!(); diff --git a/docs/known-issues.md b/docs/known-issues.md index 865102b..fca0df6 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -264,8 +264,16 @@ fill-value item that did is fixed). external links at any depth and optional creation-order tracking; h5py, h5dump and `h5rs check --data` read them (`crates/clawhdf5/tests/writer_groups_interop.rs`, - `crates/clawhdf5-tools/tests/h5rs_interop.rs`). Still missing: - attribute creation order is not tracked. + `crates/clawhdf5-tools/tests/h5rs_interop.rs`). ~~Still missing: + attribute creation order is not tracked.~~ **Fixed 2026-09-26:** + `track_order` (file default, `GroupBuilder`, and the new + `DatasetBuilder::track_order`) tracks and indexes attribute creation + order as h5py's `track_order=True` does; h5py lists the attributes in + the order they were set, inline and dense (20 000 on one dataset), and + keeps numbering them in "r+" mode (tank, + `cargo test -p clawhdf5 --test writer_groups_interop + track_order_lists_attributes_in_creation_order`). libhdf5 numbers at + most 65 535 attributes on such an object, so more is an error. - ~~A group with more than 65 535 links, or an object with more than 65 535 dense attributes, is an error (the index is one B-tree leaf).~~ **Fixed 2026-09-26:** the dense indexes are v2 B-trees of any depth