feat(format): nested groups, soft/hard/external links and creation order in the writer
FileWriter wrote the root group plus one level of groups, and refused
path-like names. The writer now flattens its builders into a group tree
(writer_tree.rs) before layout:
- A name may be a path ("a/b/x", "/a/b/x" at the root); missing
intermediate groups are created as h5py does, and GroupBuilder gains
create_group/add_group so builders nest to any depth. A group added at a
path that already holds a group is merged into it (require_group);
any other repeated name, an empty or "." component, or an absolute path
below the root is an error.
- add_soft_link, add_hard_link and add_external_link on FileWriter,
FileBuilder and GroupBuilder. Hard-link targets are resolved to objects
at finish (through other hard links; a missing target, a soft link on the
way or a cycle of paths is an error). Objects with several hard links get
an Object Reference Count message so libhdf5 can delete one link without
freeing the object.
- track_order(true) per group, or as the file default, tracks and indexes
link creation order: Link Info flags and max order, the order in each
Link message, and a type-6 creation-order B-tree for dense groups.
- A group's link index is one B-tree leaf; more than 65535 links is an
error.
Groups are laid out depth-first from the root, datasets group by group,
and untracked groups keep writing datasets, then groups, then other links:
files with one level of groups are byte-identical to before.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -773,3 +773,88 @@ fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
|
||||
assert_eq!(code(&h5rs(&["ls"])), 2);
|
||||
assert_eq!(code(&h5rs(&["--help"])), 0);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// files clawhdf5 writes: nested groups and links
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Nested groups (4 levels, by builders and by path names), soft, hard and
|
||||
/// external links, creation-order tracking, and dense link and attribute
|
||||
/// storage, as `FileBuilder` writes them.
|
||||
fn write_nested_links(dir: &Path) -> Vec<String> {
|
||||
use clawhdf5::{AttrValue, FileBuilder};
|
||||
let mut b = FileBuilder::new();
|
||||
b.set_attr("title", AttrValue::String("links".into()));
|
||||
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0]);
|
||||
b.create_dataset("a/b/c/d/leaf")
|
||||
.with_i32_data(&[4, 5])
|
||||
.set_attr("depth", AttrValue::I64(5));
|
||||
b.add_soft_link("soft", "/x/y");
|
||||
b.add_soft_link("dangling", "/nowhere");
|
||||
b.add_hard_link("alias", "/x/y");
|
||||
b.add_external_link("ext", "other.h5", "/data");
|
||||
let mut g = b.create_group("a/b");
|
||||
g.set_attr("merged", AttrValue::I64(1));
|
||||
for i in 0..10 {
|
||||
g.set_attr(&format!("attr{i}"), AttrValue::F64(i as f64));
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("ordered");
|
||||
g.track_order(true);
|
||||
for i in (0..40).rev() {
|
||||
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
|
||||
}
|
||||
g.add_hard_link("back", "/a/b/c");
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("compact_ordered");
|
||||
g.track_order(true);
|
||||
g.create_dataset("z").with_i32_data(&[1]);
|
||||
g.create_dataset("a").with_i32_data(&[2]);
|
||||
b.add_group(g.finish());
|
||||
let nested = dir.join("nested.h5");
|
||||
b.write(&nested).unwrap();
|
||||
|
||||
let mut b = FileBuilder::new();
|
||||
let mut g = b.create_group("many");
|
||||
for i in 0..10_000 {
|
||||
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let many = dir.join("many.h5");
|
||||
b.write(&many).unwrap();
|
||||
[nested, many]
|
||||
.iter()
|
||||
.map(|p| p.to_string_lossy().into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_and_dump_files_with_nested_groups_and_links() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let files = write_nested_links(dir.path());
|
||||
// `--data` reads every dataset by path, and a lookup in a dense group
|
||||
// scans all of its links: 10 000 datasets take minutes in a debug
|
||||
// build, so the big file is checked structurally only (and not dumped).
|
||||
for (p, data) in [(&files[0], true), (&files[1], false)] {
|
||||
let args: &[&str] = if data {
|
||||
&["check", "--data", p]
|
||||
} else {
|
||||
&["check", p]
|
||||
};
|
||||
let o = h5rs(args);
|
||||
assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o));
|
||||
assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o));
|
||||
}
|
||||
if missing(tool_available("h5dump"), "h5dump") {
|
||||
return;
|
||||
}
|
||||
for p in &files[..1] {
|
||||
let name = Path::new(p).file_name().unwrap().to_string_lossy();
|
||||
let ours = h5rs(&["dump", p]);
|
||||
assert!(ours.status.success(), "{p}: {ours:?}");
|
||||
let reference = run("h5dump", &[p]);
|
||||
assert!(reference.status.success(), "h5dump {p}: {reference:?}");
|
||||
let r = stdout(&reference).replacen(p.as_str(), &name, 1);
|
||||
assert_eq!(stdout(&ours), r, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user