feat(format): nested groups, soft/hard/external links and creation order in the writer

FileWriter wrote the root group plus one level of groups, and refused
path-like names. The writer now flattens its builders into a group tree
(writer_tree.rs) before layout:

- A name may be a path ("a/b/x", "/a/b/x" at the root); missing
  intermediate groups are created as h5py does, and GroupBuilder gains
  create_group/add_group so builders nest to any depth. A group added at a
  path that already holds a group is merged into it (require_group);
  any other repeated name, an empty or "." component, or an absolute path
  below the root is an error.
- add_soft_link, add_hard_link and add_external_link on FileWriter,
  FileBuilder and GroupBuilder. Hard-link targets are resolved to objects
  at finish (through other hard links; a missing target, a soft link on the
  way or a cycle of paths is an error). Objects with several hard links get
  an Object Reference Count message so libhdf5 can delete one link without
  freeing the object.
- track_order(true) per group, or as the file default, tracks and indexes
  link creation order: Link Info flags and max order, the order in each
  Link message, and a type-6 creation-order B-tree for dense groups.
- A group's link index is one B-tree leaf; more than 65535 links is an
  error.

Groups are laid out depth-first from the root, datasets group by group,
and untracked groups keep writing datasets, then groups, then other links:
files with one level of groups are byte-identical to before.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:33:46 -05:00
co-authored by Claude Opus 5.5
parent 8cbbef3fae
commit d102c06306
11 changed files with 1694 additions and 433 deletions
@@ -773,3 +773,88 @@ fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
assert_eq!(code(&h5rs(&["ls"])), 2);
assert_eq!(code(&h5rs(&["--help"])), 0);
}
// ---------------------------------------------------------------------------
// files clawhdf5 writes: nested groups and links
// ---------------------------------------------------------------------------
/// Nested groups (4 levels, by builders and by path names), soft, hard and
/// external links, creation-order tracking, and dense link and attribute
/// storage, as `FileBuilder` writes them.
fn write_nested_links(dir: &Path) -> Vec<String> {
use clawhdf5::{AttrValue, FileBuilder};
let mut b = FileBuilder::new();
b.set_attr("title", AttrValue::String("links".into()));
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0]);
b.create_dataset("a/b/c/d/leaf")
.with_i32_data(&[4, 5])
.set_attr("depth", AttrValue::I64(5));
b.add_soft_link("soft", "/x/y");
b.add_soft_link("dangling", "/nowhere");
b.add_hard_link("alias", "/x/y");
b.add_external_link("ext", "other.h5", "/data");
let mut g = b.create_group("a/b");
g.set_attr("merged", AttrValue::I64(1));
for i in 0..10 {
g.set_attr(&format!("attr{i}"), AttrValue::F64(i as f64));
}
b.add_group(g.finish());
let mut g = b.create_group("ordered");
g.track_order(true);
for i in (0..40).rev() {
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
}
g.add_hard_link("back", "/a/b/c");
b.add_group(g.finish());
let mut g = b.create_group("compact_ordered");
g.track_order(true);
g.create_dataset("z").with_i32_data(&[1]);
g.create_dataset("a").with_i32_data(&[2]);
b.add_group(g.finish());
let nested = dir.join("nested.h5");
b.write(&nested).unwrap();
let mut b = FileBuilder::new();
let mut g = b.create_group("many");
for i in 0..10_000 {
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
}
b.add_group(g.finish());
let many = dir.join("many.h5");
b.write(&many).unwrap();
[nested, many]
.iter()
.map(|p| p.to_string_lossy().into_owned())
.collect()
}
#[test]
fn check_and_dump_files_with_nested_groups_and_links() {
let dir = tempfile::tempdir().unwrap();
let files = write_nested_links(dir.path());
// `--data` reads every dataset by path, and a lookup in a dense group
// scans all of its links: 10 000 datasets take minutes in a debug
// build, so the big file is checked structurally only (and not dumped).
for (p, data) in [(&files[0], true), (&files[1], false)] {
let args: &[&str] = if data {
&["check", "--data", p]
} else {
&["check", p]
};
let o = h5rs(args);
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
}
if missing(tool_available("h5dump"), "h5dump") {
return;
}
for p in &files[..1] {
let name = Path::new(p).file_name().unwrap().to_string_lossy();
let ours = h5rs(&["dump", p]);
assert!(ours.status.success(), "{p}: {ours:?}");
let reference = run("h5dump", &[p]);
assert!(reference.status.success(), "h5dump {p}: {reference:?}");
let r = stdout(&reference).replacen(p.as_str(), &name, 1);
assert_eq!(stdout(&ours), r, "{name}");
}
}