quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

writer: track attribute creation order with track_order

Browse Source 
h5py's track_order=True orders attributes as well as links; the writer
tracked links only. A tracking object's header now sets the attribute
creation order tracked/indexed flags and carries per-message creation
orders, an Attribute Info message holds the next order (inline too),
and dense storage gets a type-9 creation-order index. The file default
applies to datasets, with DatasetBuilder::track_order per dataset; more
than 65 535 attributes on a tracking object is an error (libhdf5's
counter is 2 bytes). The reader lists such attributes in creation
order.

h5py lists them in order (inline, dense, 20 000 on one dataset) and
keeps numbering in r+ mode, including its inline-to-dense move.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 10:17:37 -05:00
co-authored by Claude Opus 5.5
parent d63c76e7ab
commit 193a5f8a82
9 changed files with 515 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5/src/writer.rs
+5 -3
View File
@@ -88,9 +88,11 @@ impl FileBuilder {
self
}
/// Track link creation order in every group that does not set its own
/// (`GroupBuilder::track_order`), as h5py's `track_order=True`: libhdf5
/// then lists members in the order they were added.
/// Track the creation order of links and attributes in every group, and
/// of attributes on every dataset, that does not set its own
/// (`GroupBuilder::track_order`, `DatasetBuilder::track_order`), as
/// h5py's `track_order=True`: libhdf5 then lists members and attributes
/// in the order they were added.
pub fn track_order(&mut self, track: bool) -> &mut Self {
self.writer.track_order(track);
self
crates/clawhdf5/tests/writer_groups_interop.rs
+91
View File
@@ -531,6 +531,97 @@ fn ten_thousand_links_in_one_group() {
);
}
#[test]
fn track_order_lists_attributes_in_creation_order() {
skip_if_no_python!();
// h5py's track_order=True orders an object's attributes as well as a
// group's links; the writer tracked links only, so h5py listed the
// attributes by name. Now the object header's flags say attribute
// creation order is tracked and indexed, an Attribute Info message
// holds the next order, inline attributes carry theirs, and dense
// storage gets a creation-order index (B-tree type 9).
let dir = tempfile::tempdir().unwrap();
let small = ["zeta", "alpha", "mid"];
let mut b = FileBuilder::new();
b.track_order(true); // the root, and every group and dataset by default
for (i, n) in small.iter().enumerate() {
b.set_attr(n, AttrValue::I64(i as i64));
}
let mut g = b.create_group("g"); // dense: 30 attributes
for i in (0..30).rev() {
g.set_attr(&format!("a{i:02}"), AttrValue::I64(i));
}
b.add_group(g.finish());
let d = b.create_dataset("d");
d.with_i32_data(&[1]);
for (i, n) in small.iter().enumerate() {
d.set_attr(n, AttrValue::I64(i as i64));
}
// 20 000 attributes: a one-leaf creation-order index of 20 000 records.
let big = b.create_dataset("big");
big.with_i32_data(&[2]);
for i in (0..20_000).rev() {
big.set_attr(&format!("b{i:05}"), AttrValue::I64(i));
}
let plain = b.create_dataset("plain");
plain.with_i32_data(&[3]).track_order(false);
for (i, n) in small.iter().enumerate() {
plain.set_attr(n, AttrValue::I64(i as i64));
}
let path = write(&dir, "attr_order.h5", b);
let out = h5py(
&path,
"def order(o):\n\
\x20 return o.id.get_create_plist().get_attr_creation_order()\n\
with h5py.File(path, 'r') as f:\n\
\x20 big = list(f['big'].attrs)\n\
\x20 print(json.dumps([list(f.attrs), [int(v) for v in f.attrs.values()],\n\
\x20 list(f['g'].attrs)[:3], len(f['g'].attrs), list(f['d'].attrs),\n\
\x20 big[:2], big == ['b%05d' % i for i in range(19999, -1, -1)],\n\
\x20 int(f['big'].attrs['b00007']), list(f['plain'].attrs),\n\
\x20 [order(f['/']), order(f['g']), order(f['d']), order(f['plain'])]]))",
);
assert_eq!(
out,
r#"[["zeta", "alpha", "mid"], [0, 1, 2], ["a29", "a28", "a27"], 30, ["zeta", "alpha", "mid"], ["b19999", "b19998"], true, 7, ["alpha", "mid", "zeta"], [3, 3, 3, 0]]"#
);
h5dump_ok(&path);
let f = File::open(&path).unwrap();
assert_eq!(f.dataset("big").unwrap().attrs().unwrap().len(), 20_000);
assert!(matches!(
f.group("g").unwrap().attrs().unwrap()["a07"],
AttrValue::I64(7)
));
drop(f);
// libhdf5 continues the numbering: new attributes come last, also when
// it moves the inline ones of `d` to dense storage.
let out = h5py(
&path,
"with h5py.File(path, 'r+') as f:\n\
\x20 f.attrs['new'] = 9\n\
\x20 del f.attrs['alpha']\n\
\x20 f['g'].attrs['new'] = 9\n\
\x20 del f['g'].attrs['a15']\n\
\x20 for i in range(8):\n\
\x20 f['d'].attrs['x%d' % i] = i\n\
\x20 f['big'].attrs['new'] = 9\n\
\x20 for i in range(0, 20000, 2):\n\
\x20 del f['big'].attrs['b%05d' % i]\n\
with h5py.File(path, 'r') as f:\n\
\x20 big = list(f['big'].attrs)\n\
\x20 print(json.dumps([list(f.attrs), list(f['g'].attrs)[-2:], len(f['g'].attrs),\n\
\x20 list(f['d'].attrs)[:4], len(f['d'].attrs),\n\
\x20 big == ['b%05d' % i for i in range(19999, -1, -2)] + ['new']]))",
);
assert_eq!(
out,
r#"[["zeta", "mid", "new"], ["a00", "new"], 30, ["zeta", "alpha", "mid", "x0"], 11, true]"#
);
h5dump_ok(&path);
}
#[test]
fn track_order_lists_members_in_creation_order() {
skip_if_no_python!();