quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(format): nested groups, soft/hard/external links and creation order in the writer

Browse Source 
FileWriter wrote the root group plus one level of groups, and refused
path-like names. The writer now flattens its builders into a group tree
(writer_tree.rs) before layout:

- A name may be a path ("a/b/x", "/a/b/x" at the root); missing
  intermediate groups are created as h5py does, and GroupBuilder gains
  create_group/add_group so builders nest to any depth. A group added at a
  path that already holds a group is merged into it (require_group);
  any other repeated name, an empty or "." component, or an absolute path
  below the root is an error.
- add_soft_link, add_hard_link and add_external_link on FileWriter,
  FileBuilder and GroupBuilder. Hard-link targets are resolved to objects
  at finish (through other hard links; a missing target, a soft link on the
  way or a cycle of paths is an error). Objects with several hard links get
  an Object Reference Count message so libhdf5 can delete one link without
  freeing the object.
- track_order(true) per group, or as the file default, tracks and indexes
  link creation order: Link Info flags and max order, the order in each
  Link message, and a type-6 creation-order B-tree for dense groups.
- A group's link index is one B-tree leaf; more than 65535 links is an
  error.

Groups are laid out depth-first from the root, datasets group by group,
and untracked groups keep writing datasets, then groups, then other links:
files with one level of groups are byte-identical to before.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:33:46 -05:00
co-authored by Claude Opus 5.5
parent 8cbbef3fae
commit d102c06306
11 changed files with 1694 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5/src/writer.rs
+40 -3
View File
@@ -42,14 +42,17 @@ impl FileBuilder {
}
}
/// Create a dataset at the root level. Returns a mutable reference to
/// a `DatasetBuilder` for configuring data, shape, and attributes.
/// Create a dataset. Returns a mutable reference to a `DatasetBuilder`
/// for configuring data, shape, and attributes. `name` may be a path
/// (`"a/b/x"`): missing intermediate groups are created, as in h5py.
pub fn create_dataset(&mut self, name: &str) -> &mut FormatDatasetBuilder {
self.writer.create_dataset(name)
}
/// Create a group builder. Call `.finish()` on the returned builder
/// to complete it, then pass to `add_group()`.
/// to complete it, then pass to `add_group()`. `name` may be a path;
/// groups nest to any depth (see `GroupBuilder::add_group`), and a group
/// added at a path that already holds a group is merged into it.
pub fn create_group(&mut self, name: &str) -> FormatGroupBuilder {
self.writer.create_group(name)
}
@@ -59,6 +62,40 @@ impl FileBuilder {
self.writer.add_group(group);
}
/// Add a soft link `name` to the path `target`, like h5py's
/// `f[name] = h5py.SoftLink(target)`. The target need not exist.
pub fn add_soft_link(&mut self, name: &str, target: &str) -> &mut Self {
self.writer.add_soft_link(name, target);
self
}
/// Add another hard link `name` to the object at `target`, like h5py's
/// `f[name] = f[target]`. The target must be written in this file.
pub fn add_hard_link(&mut self, name: &str, target: &str) -> &mut Self {
self.writer.add_hard_link(name, target);
self
}
/// Add an external link `name` to `target_path` in `target_file`.
pub fn add_external_link(
&mut self,
name: &str,
target_file: &str,
target_path: &str,
) -> &mut Self {
self.writer
.add_external_link(name, target_file, target_path);
self
}
/// Track link creation order in every group that does not set its own
/// (`GroupBuilder::track_order`), as h5py's `track_order=True`: libhdf5
/// then lists members in the order they were added.
pub fn track_order(&mut self, track: bool) -> &mut Self {
self.writer.track_order(track);
self
}
/// Set an attribute on the root group.
pub fn set_attr(&mut self, name: &str, value: AttrValue) {
self.writer.set_root_attr(name, value);
crates/clawhdf5/tests/writer_groups_interop.rs
+459 -1
View File
@@ -6,7 +6,7 @@
use std::process::Command;
use clawhdf5::{File, FileBuilder};
use clawhdf5::{AttrValue, File, FileBuilder, Group};
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
@@ -134,3 +134,461 @@ fn h5py_can_add_links_to_groups_we_wrote() {
);
assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]);
}
// ---- the whole tree, as h5py and as clawhdf5 read it ----
fn fmt_num(x: f64) -> String {
format!("{x:.6}")
}
fn fmt_attr(v: &AttrValue) -> String {
let join = |v: Vec<String>| v.join(",");
match v {
AttrValue::F64(x) => fmt_num(*x),
AttrValue::I64(x) => fmt_num(*x as f64),
AttrValue::U64(x) => fmt_num(*x as f64),
AttrValue::F64Array(a) => join(a.iter().map(|x| fmt_num(*x)).collect()),
AttrValue::I64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
AttrValue::U64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
AttrValue::String(s) => s.clone(),
AttrValue::StringArray(a) => a.join(","),
AttrValue::Raw { .. } => "raw".to_string(),
}
}
fn fmt_attrs(attrs: std::collections::HashMap<String, AttrValue>) -> String {
let mut v: Vec<_> = attrs.into_iter().collect();
v.sort_by(|a, b| a.0.cmp(&b.0));
v.iter()
.map(|(k, a)| format!("{k}={}", fmt_attr(a)))
.collect::<Vec<_>>()
.join(";")
}
fn child_path(path: &str, name: &str) -> String {
if path == "/" {
format!("/{name}")
} else {
format!("{path}/{name}")
}
}
/// Every group and dataset reachable from `g` (following hard and soft
/// links; the tree must be acyclic), one line each: path, kind, attributes
/// and (datasets) values.
fn walk(g: &Group<'_>, path: &str, out: &mut Vec<String>) {
out.push(format!("{path}|group|{}", fmt_attrs(g.attrs().unwrap())));
let mut names: Vec<(String, bool)> = g
.datasets()
.unwrap()
.into_iter()
.map(|n| (n, false))
.chain(g.groups().unwrap().into_iter().map(|n| (n, true)))
.collect();
names.sort();
for (name, is_group) in names {
let p = child_path(path, &name);
if is_group {
walk(&g.group(&name).unwrap(), &p, out);
} else {
let ds = g.dataset(&name).unwrap();
let values: Vec<String> = ds.read_f64().unwrap().into_iter().map(fmt_num).collect();
out.push(format!(
"{p}|dataset|{}|{}",
fmt_attrs(ds.attrs().unwrap()),
values.join(",")
));
}
}
}
fn clawhdf5_tree(path: &str) -> String {
let f = File::open(path).unwrap();
let mut out = Vec::new();
walk(&f.root(), "/", &mut out);
out.join("\n")
}
/// The same listing as [`walk`], from h5py. External and dangling soft
/// links are skipped, as clawhdf5's group listings skip them.
const H5PY_WALK: &str = r#"
def fmt(v):
if isinstance(v, bytes): return v.decode()
if isinstance(v, str): return v
a = np.asarray(v)
if a.dtype.kind in 'SUO':
return ','.join(x.decode() if isinstance(x, bytes) else str(x) for x in a.ravel())
if a.ndim == 0: return '%.6f' % float(a)
return ','.join('%.6f' % float(x) for x in a.ravel())
def attrs(o): return ';'.join(f'{k}={fmt(o.attrs[k])}' for k in sorted(o.attrs))
out = []
def walk(g, path):
out.append(f'{path}|group|{attrs(g)}')
for k in sorted(g.keys()):
if isinstance(g.get(k, getlink=True), h5py.ExternalLink): continue
o = g.get(k)
if o is None: continue
p = '/' + k if path == '/' else path + '/' + k
if isinstance(o, h5py.Group): walk(o, p)
else:
vals = ','.join('%.6f' % float(x) for x in np.asarray(o[()]).ravel())
out.append(f'{p}|dataset|{attrs(o)}|{vals}')
with h5py.File(path, 'r') as f:
walk(f, '/')
print('\n'.join(out))
"#;
fn h5py_tree(path: &str) -> String {
h5py(path, H5PY_WALK)
}
/// A four-level tree with attributes on every object: nested builders,
/// path names (with intermediate groups made on the way) and a group added
/// twice (merged), with dense attribute storage at one level and dense link
/// storage at another.
fn nested_builder() -> FileBuilder {
let mut b = FileBuilder::new();
b.set_attr("title", AttrValue::String("nested".into()));
let mut l1 = b.create_group("l1");
l1.set_attr("depth", AttrValue::I64(1));
l1.create_dataset("d1")
.with_f64_data(&[1.0, 1.5])
.set_attr("unit", AttrValue::String("m".into()));
let mut l2 = l1.create_group("l2");
l2.set_attr("depth", AttrValue::I64(2));
l2.create_dataset("d2").with_i32_data(&[2, 3, 4]);
let mut l3 = l2.create_group("l3");
for i in 0..10 {
l3.set_attr(&format!("a{i}"), AttrValue::F64(i as f64 / 4.0)); // dense
}
for i in 0..12 {
l3.create_dataset(&format!("x{i:02}")) // dense links
.with_i64_data(&[i, -i])
.set_attr("i", AttrValue::I64(i));
}
let mut l4 = l3.create_group("l4");
l4.set_attr("depth", AttrValue::I64(4));
l4.create_dataset("leaf")
.with_f64_data(&[4.0, 4.25, 4.5])
.set_attr(
"tags",
AttrValue::StringArray(vec!["a".into(), "bc".into()]),
);
l3.add_group(l4.finish());
l2.add_group(l3.finish());
l1.add_group(l2.finish());
b.add_group(l1.finish());
// Path names: /p, /p/q and /p/q/r are made on the way to the dataset.
b.create_dataset("p/q/r/s")
.with_f64_data(&[7.0])
.set_attr("deep", AttrValue::I64(4));
// A group at an existing path is merged into it.
let mut pq = b.create_group("p/q");
pq.set_attr("merged", AttrValue::I64(1));
pq.create_dataset("t").with_i32_data(&[8]);
b.add_group(pq.finish());
let mut l1b = b.create_group("l1/l2/l3/l4/l5");
l1b.set_attr("depth", AttrValue::I64(5));
b.add_group(l1b.finish());
b
}
const NESTED_TREE: &str = "\
/|group|title=nested
/l1|group|depth=1.000000
/l1/d1|dataset|unit=m|1.000000,1.500000
/l1/l2|group|depth=2.000000
/l1/l2/d2|dataset||2.000000,3.000000,4.000000";
#[test]
fn nested_groups_read_the_same_in_h5py_and_clawhdf5() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let path = write(&dir, "nested.h5", nested_builder());
let ours = clawhdf5_tree(&path);
let theirs = h5py_tree(&path);
assert_eq!(ours, theirs);
assert!(ours.starts_with(NESTED_TREE), "{ours}");
for line in [
"/l1/l2/l3|group|a0=0.000000;a1=0.250000;a2=0.500000;a3=0.750000;a4=1.000000;\
a5=1.250000;a6=1.500000;a7=1.750000;a8=2.000000;a9=2.250000",
"/l1/l2/l3/l4|group|depth=4.000000",
"/l1/l2/l3/l4/l5|group|depth=5.000000",
"/l1/l2/l3/l4/leaf|dataset|tags=a,bc|4.000000,4.250000,4.500000",
"/l1/l2/l3/x11|dataset|i=11.000000|11.000000,-11.000000",
"/p|group|",
"/p/q|group|merged=1.000000",
"/p/q/r/s|dataset|deep=4.000000|7.000000",
"/p/q/t|dataset||8.000000",
] {
assert!(
ours.lines().any(|l| l == line),
"missing {line:?} in\n{ours}"
);
}
assert_eq!(ours.lines().count(), 26, "{ours}");
let dump = h5dump_ok(&path);
if !dump.is_empty() {
assert!(dump.contains("GROUP \"l5\""), "{dump}");
assert!(dump.contains("DATASET \"leaf\""), "{dump}");
}
}
#[test]
fn soft_hard_and_external_links() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let mut other = FileBuilder::new();
other.create_dataset("data").with_i32_data(&[42, 43]);
write(&dir, "other.h5", other);
let mut b = FileBuilder::new();
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0, 3.0]);
b.create_dataset("x/z").with_i32_data(&[9]);
b.add_soft_link("soft_abs", "/x/y");
b.add_soft_link("dangling", "/nowhere");
b.add_hard_link("alias", "/x/y");
b.add_hard_link("x_again", "x");
b.add_external_link("ext", "other.h5", "/data");
let mut g = b.create_group("a/b/c");
g.add_soft_link("rel", "sib"); // relative to /a/b/c
g.create_dataset("sib").with_i32_data(&[5]);
g.add_hard_link("deep_alias", "/x_again/z"); // through a hard link
g.add_soft_link("to_group", "/x");
b.add_group(g.finish());
let path = write(&dir, "links.h5", b);
let out = h5py(
&path,
"import os\nos.chdir(os.path.dirname(path))\n\
with h5py.File(path, 'r') as f:\n\
\x20 def kind(g, k):\n\
\x20 l = g.get(k, getlink=True)\n\
\x20 if isinstance(l, h5py.SoftLink): return 'soft:' + l.path\n\
\x20 if isinstance(l, h5py.ExternalLink): return 'ext:' + l.filename + ':' + l.path\n\
\x20 return 'hard'\n\
\x20 print(json.dumps({\n\
\x20 'root': {k: kind(f, k) for k in f},\n\
\x20 'abc': {k: kind(f['a/b/c'], k) for k in f['a/b/c']},\n\
\x20 'same': [f['alias'].id == f['x/y'].id, f['x_again'].id == f['x'].id,\n\
\x20 f['a/b/c/deep_alias'].id == f['x/z'].id],\n\
\x20 'rc': [h5py.h5o.get_info(f['x/y'].id).rc, h5py.h5o.get_info(f['x'].id).rc,\n\
\x20 h5py.h5o.get_info(f['x/z'].id).rc, h5py.h5o.get_info(f['a'].id).rc],\n\
\x20 'vals': [f['soft_abs'][()].tolist(), f['ext'][()].tolist(),\n\
\x20 f['a/b/c/rel'][()].tolist(), sorted(f['a/b/c/to_group'])],\n\
\x20 'dangling': f.get('dangling') is None,\n\
\x20 }, sort_keys=True))",
);
assert_eq!(
out,
r#"{"abc": {"deep_alias": "hard", "rel": "soft:sib", "sib": "hard", "to_group": "soft:/x"}, "dangling": true, "rc": [2, 2, 2, 1], "root": {"a": "hard", "alias": "hard", "dangling": "soft:/nowhere", "ext": "ext:other.h5:/data", "soft_abs": "soft:/x/y", "x": "hard", "x_again": "hard"}, "same": [true, true, true], "vals": [[1.0, 2.0, 3.0], [42, 43], [5], ["y", "z"]]}"#
);
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
h5dump_ok(&path);
let f = File::open(&path).unwrap();
assert_eq!(
f.dataset("alias").unwrap().read_f64().unwrap(),
[1.0, 2.0, 3.0]
);
assert_eq!(
f.dataset("soft_abs").unwrap().read_f64().unwrap(),
[1.0, 2.0, 3.0]
);
assert_eq!(f.dataset("a/b/c/rel").unwrap().read_i32().unwrap(), [5]);
assert_eq!(
f.dataset("a/b/c/deep_alias").unwrap().read_i32().unwrap(),
[9]
);
assert_eq!(
f.dataset("x_again/y").unwrap().read_f64().unwrap(),
[1.0, 2.0, 3.0]
);
drop(f);
// The reference counts let libhdf5 delete one of two hard links and
// keep the object; with a count of 1 it would free an object still
// linked from elsewhere.
let out = h5py(
&path,
"with h5py.File(path, 'r+') as f:\n\
\x20 del f['alias']\n\
\x20 del f['x_again']\n\
\x20 f.create_dataset('filler', data=np.arange(1000))\n\
with h5py.File(path, 'r') as f:\n\
\x20 print(json.dumps([f['x/y'][()].tolist(), sorted(f['x']), h5py.h5o.get_info(f['x/y'].id).rc]))",
);
assert_eq!(out, r#"[[1.0, 2.0, 3.0], ["y", "z"], 1]"#);
h5dump_ok(&path);
}
#[test]
fn a_hard_link_can_make_a_cycle() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let mut b = FileBuilder::new();
let mut g = b.create_group("g");
g.create_dataset("v").with_i32_data(&[1]);
g.add_hard_link("up", "/");
g.add_hard_link("me", ".");
b.add_group(g.finish());
let path = write(&dir, "cycle.h5", b);
let out = h5py(
&path,
"with h5py.File(path, 'r') as f:\n\
\x20 print(json.dumps([sorted(f['g/up/g']), f['g/up/g/me/me/v'][()].tolist(),\n\
\x20 h5py.h5o.get_info(f.id).rc, h5py.h5o.get_info(f['g'].id).rc]))",
);
assert_eq!(out, r#"[["me", "up", "v"], [1], 2, 2]"#);
h5dump_ok(&path);
let f = File::open(&path).unwrap();
assert_eq!(f.dataset("g/up/g/me/v").unwrap().read_i32().unwrap(), [1]);
}
#[test]
fn bad_links_are_errors() {
for setup in [
|b: &mut FileBuilder| {
b.add_hard_link("h", "/missing");
},
|b: &mut FileBuilder| {
b.create_dataset("x").with_i32_data(&[1]);
b.add_soft_link("s", "/x");
b.add_hard_link("h", "/s"); // through a soft link
},
|b: &mut FileBuilder| {
b.add_hard_link("h1", "/h2");
b.add_hard_link("h2", "/h1");
},
|b: &mut FileBuilder| {
b.create_dataset("x").with_i32_data(&[1]);
b.add_hard_link("h", "/x/y"); // a dataset is not a group
},
|b: &mut FileBuilder| {
b.add_soft_link("s", "");
},
|b: &mut FileBuilder| {
b.add_external_link("e", "", "/x");
},
|b: &mut FileBuilder| {
b.create_dataset("x").with_i32_data(&[1]);
b.add_soft_link("x", "/y"); // name taken
},
] {
let mut b = FileBuilder::new();
setup(&mut b);
assert!(b.finish().is_err());
}
}
#[test]
fn ten_thousand_links_in_one_group() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let mut b = FileBuilder::new();
let mut g = b.create_group("many");
for i in 0..10_000 {
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
}
g.set_attr("n", AttrValue::I64(10_000));
b.add_group(g.finish());
// The same in creation order, added in reverse name order, with soft
// links among them.
let mut g = b.create_group("ordered");
g.track_order(true);
for i in (0..10_000).rev() {
if i % 1000 == 0 {
g.add_soft_link(&format!("s{i:05}"), &format!("/many/d{i:05}"));
}
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
}
b.add_group(g.finish());
let path = write(&dir, "many.h5", b);
let out = h5py(
&path,
"with h5py.File(path, 'r') as f:\n\
\x20 m, o = f['many'], f['ordered']\n\
\x20 names = list(m)\n\
\x20 onames = list(o)\n\
\x20 print(json.dumps([len(names), names == sorted(names), names[:2], int(m.attrs['n']),\n\
\x20 [int(m['d%05d' % i][0]) for i in (0, 1, 4096, 9999)],\n\
\x20 len(onames), onames[:3], onames[-2:], int(o['s05000'][0]),\n\
\x20 o.id.get_create_plist().get_link_creation_order()]))",
);
assert_eq!(
out,
r#"[10000, true, ["d00000", "d00001"], 10000, [0, 1, 4096, 9999], 10010, ["d09999", "d09998", "d09997"], ["s00000", "d00000"], 5000, 3]"#
);
h5dump_ok(&path);
let f = File::open(&path).unwrap();
let g = f.group("many").unwrap();
assert_eq!(g.datasets().unwrap().len(), 10_000);
assert_eq!(g.dataset("d09999").unwrap().read_i32().unwrap(), [9999]);
assert_eq!(
f.dataset("ordered/s05000").unwrap().read_i32().unwrap(),
[5000]
);
}
#[test]
fn more_links_than_one_index_leaf_holds_is_an_error() {
let mut b = FileBuilder::new();
for i in 0..70_000 {
b.add_soft_link(&format!("s{i}"), "/x");
}
let err = b.finish().unwrap_err().to_string();
assert!(err.contains("at most 65535 links"), "{err}");
}
#[test]
fn track_order_lists_members_in_creation_order() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let names = ["zeta", "alpha", "mid", "beta"];
let mut b = FileBuilder::new();
b.track_order(true); // the root and every group without its own setting
for n in names {
b.create_dataset(n).with_i32_data(&[1]);
}
let mut g = b.create_group("by_name");
g.track_order(false);
for n in names {
g.create_dataset(n).with_i32_data(&[2]);
}
b.add_group(g.finish());
let mut g = b.create_group("dense");
for i in (0..20).rev() {
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
}
g.add_soft_link("soft", "/zeta");
b.add_group(g.finish());
b.create_dataset("made/on/the/way").with_i32_data(&[3]);
let path = write(&dir, "order.h5", b);
let out = h5py(
&path,
"with h5py.File(path, 'r') as f:\n\
\x20 print(json.dumps([list(f), list(f['by_name']), list(f['dense'])[:3],\n\
\x20 list(f['dense'])[-2:], list(f['made/on'])]))",
);
assert_eq!(
out,
r#"[["zeta", "alpha", "mid", "beta", "by_name", "dense", "made"], ["alpha", "beta", "mid", "zeta"], ["n19", "n18", "n17"], ["n00", "soft"], ["the"]]"#
);
h5dump_ok(&path);
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
// libhdf5 keeps the order when it adds to (and converts) these groups.
let out = h5py(
&path,
"with h5py.File(path, 'r+') as f:\n\
\x20 f['aaa'] = np.arange(2)\n\
\x20 f['dense']['aaa'] = np.arange(2)\n\
\x20 del f['dense/n10']\n\
with h5py.File(path, 'r') as f:\n\
\x20 print(json.dumps([list(f)[-1], list(f['dense'])[-2:], len(f['dense'])]))",
);
assert_eq!(out, r#"["aaa", ["soft", "aaa"], 21]"#);
h5dump_ok(&path);
}