osobhandClaude Opus 5.5
8bcae3c78e
b0a1e4f fixed this for group and root attributes only. Setting a dataset
attribute twice still wrote two attribute messages with one name, and h5py
read back the first value: set_attr("a", 1) then set_attr("a", 2) read as
1, and list(attrs) was ["a", "a"]. DatasetBuilder::set_attr now replaces
the earlier value, compact or dense. Likewise, a hand-set attribute named
like a provenance attribute (_provenance_sha256, ...) is replaced by the
computed one instead of being written next to it and read first.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
941 lines
34 KiB
Rust
941 lines
34 KiB
Rust
//! Groups and links written by `FileBuilder`, read back by h5py (libhdf5)
|
|
//! and h5dump, and by clawhdf5 itself.
|
|
//!
|
|
//! Skipped when python3 with h5py is unavailable, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::{AttrValue, File, FileBuilder, Group};
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn h5dump_available() -> bool {
|
|
Command::new("h5dump")
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
fn run_python(script: &str) -> String {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python");
|
|
if !output.status.success() {
|
|
panic!(
|
|
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&output.stdout),
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
}
|
|
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
|
}
|
|
|
|
/// Run `body` under h5py with `path` bound to the file's path.
|
|
fn h5py(path: &str, body: &str) -> String {
|
|
run_python(&format!(
|
|
"import h5py, numpy as np, json\npath = r'{path}'\n{body}"
|
|
))
|
|
}
|
|
|
|
fn write(dir: &tempfile::TempDir, name: &str, b: FileBuilder) -> String {
|
|
let path = dir.path().join(name).display().to_string();
|
|
b.write(&path).unwrap();
|
|
path
|
|
}
|
|
|
|
/// h5dump must read the whole file without an error.
|
|
fn h5dump_ok(path: &str) -> String {
|
|
if !h5dump_available() {
|
|
assert!(!interop_required(), "h5dump is not available");
|
|
return String::new();
|
|
}
|
|
let o = Command::new("h5dump").arg(path).output().unwrap();
|
|
let out = String::from_utf8_lossy(&o.stdout).to_string();
|
|
assert!(
|
|
o.status.success(),
|
|
"h5dump failed:\n{out}{}",
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
out
|
|
}
|
|
|
|
// ---- libhdf5 can modify the groups we write ----
|
|
|
|
#[test]
|
|
fn h5py_can_add_links_to_groups_we_wrote() {
|
|
skip_if_no_python!();
|
|
// Measured before the fix: h5py in "r+" mode could not add a link to any
|
|
// group we wrote ("Unable to create link (message type not found)"):
|
|
// libhdf5 reads a group's Group Info message before inserting a link, and
|
|
// the writer wrote none.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("x").with_f64_data(&[1.0, 2.0]);
|
|
let mut g = b.create_group("small");
|
|
g.create_dataset("a").with_i32_data(&[1]);
|
|
b.add_group(g.finish());
|
|
let mut g = b.create_group("big"); // dense link storage
|
|
for i in 0..20 {
|
|
g.create_dataset(&format!("d{i:02}")).with_i32_data(&[i]);
|
|
}
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "modify.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r+') as f:\n\
|
|
\x20 f['alias'] = f['x']\n\
|
|
\x20 f['small']['new'] = np.arange(3)\n\
|
|
\x20 f['big']['new'] = np.arange(4)\n\
|
|
\x20 f.create_group('added/deeper')\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([sorted(f), sorted(f['small']), len(f['big']),\n\
|
|
\x20 f['alias'][()].tolist(), f['big/new'][()].tolist(), f['big/d07'][()].tolist()]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"[["added", "alias", "big", "small", "x"], ["a", "new"], 21, [1.0, 2.0], [0, 1, 2, 3], [7]]"#
|
|
);
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(
|
|
f.dataset("big/new").unwrap().read_i64().unwrap(),
|
|
[0, 1, 2, 3]
|
|
);
|
|
assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]);
|
|
}
|
|
|
|
// ---- the whole tree, as h5py and as clawhdf5 read it ----
|
|
|
|
fn fmt_num(x: f64) -> String {
|
|
format!("{x:.6}")
|
|
}
|
|
|
|
fn fmt_attr(v: &AttrValue) -> String {
|
|
let join = |v: Vec<String>| v.join(",");
|
|
match v {
|
|
AttrValue::F64(x) => fmt_num(*x),
|
|
AttrValue::I64(x) => fmt_num(*x as f64),
|
|
AttrValue::U64(x) => fmt_num(*x as f64),
|
|
AttrValue::F64Array(a) => join(a.iter().map(|x| fmt_num(*x)).collect()),
|
|
AttrValue::I64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
|
|
AttrValue::U64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
|
|
AttrValue::String(s) => s.clone(),
|
|
AttrValue::StringArray(a) => a.join(","),
|
|
AttrValue::Raw { .. } => "raw".to_string(),
|
|
}
|
|
}
|
|
|
|
fn fmt_attrs(attrs: std::collections::HashMap<String, AttrValue>) -> String {
|
|
let mut v: Vec<_> = attrs.into_iter().collect();
|
|
v.sort_by(|a, b| a.0.cmp(&b.0));
|
|
v.iter()
|
|
.map(|(k, a)| format!("{k}={}", fmt_attr(a)))
|
|
.collect::<Vec<_>>()
|
|
.join(";")
|
|
}
|
|
|
|
fn child_path(path: &str, name: &str) -> String {
|
|
if path == "/" {
|
|
format!("/{name}")
|
|
} else {
|
|
format!("{path}/{name}")
|
|
}
|
|
}
|
|
|
|
/// Every group and dataset reachable from `g` (following hard and soft
|
|
/// links; the tree must be acyclic), one line each: path, kind, attributes
|
|
/// and (datasets) values.
|
|
fn walk(g: &Group<'_>, path: &str, out: &mut Vec<String>) {
|
|
out.push(format!("{path}|group|{}", fmt_attrs(g.attrs().unwrap())));
|
|
let mut names: Vec<(String, bool)> = g
|
|
.datasets()
|
|
.unwrap()
|
|
.into_iter()
|
|
.map(|n| (n, false))
|
|
.chain(g.groups().unwrap().into_iter().map(|n| (n, true)))
|
|
.collect();
|
|
names.sort();
|
|
for (name, is_group) in names {
|
|
let p = child_path(path, &name);
|
|
if is_group {
|
|
walk(&g.group(&name).unwrap(), &p, out);
|
|
} else {
|
|
let ds = g.dataset(&name).unwrap();
|
|
let values: Vec<String> = ds.read_f64().unwrap().into_iter().map(fmt_num).collect();
|
|
out.push(format!(
|
|
"{p}|dataset|{}|{}",
|
|
fmt_attrs(ds.attrs().unwrap()),
|
|
values.join(",")
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
fn clawhdf5_tree(path: &str) -> String {
|
|
let f = File::open(path).unwrap();
|
|
let mut out = Vec::new();
|
|
walk(&f.root(), "/", &mut out);
|
|
out.join("\n")
|
|
}
|
|
|
|
/// The same listing as [`walk`], from h5py. External and dangling soft
|
|
/// links are skipped, as clawhdf5's group listings skip them.
|
|
const H5PY_WALK: &str = r#"
|
|
def fmt(v):
|
|
if isinstance(v, bytes): return v.decode()
|
|
if isinstance(v, str): return v
|
|
a = np.asarray(v)
|
|
if a.dtype.kind in 'SUO':
|
|
return ','.join(x.decode() if isinstance(x, bytes) else str(x) for x in a.ravel())
|
|
if a.ndim == 0: return '%.6f' % float(a)
|
|
return ','.join('%.6f' % float(x) for x in a.ravel())
|
|
def attrs(o): return ';'.join(f'{k}={fmt(o.attrs[k])}' for k in sorted(o.attrs))
|
|
out = []
|
|
def walk(g, path):
|
|
out.append(f'{path}|group|{attrs(g)}')
|
|
for k in sorted(g.keys()):
|
|
if isinstance(g.get(k, getlink=True), h5py.ExternalLink): continue
|
|
o = g.get(k)
|
|
if o is None: continue
|
|
p = '/' + k if path == '/' else path + '/' + k
|
|
if isinstance(o, h5py.Group): walk(o, p)
|
|
else:
|
|
vals = ','.join('%.6f' % float(x) for x in np.asarray(o[()]).ravel())
|
|
out.append(f'{p}|dataset|{attrs(o)}|{vals}')
|
|
with h5py.File(path, 'r') as f:
|
|
walk(f, '/')
|
|
print('\n'.join(out))
|
|
"#;
|
|
|
|
fn h5py_tree(path: &str) -> String {
|
|
h5py(path, H5PY_WALK)
|
|
}
|
|
|
|
/// A four-level tree with attributes on every object: nested builders,
|
|
/// path names (with intermediate groups made on the way) and a group added
|
|
/// twice (merged), with dense attribute storage at one level and dense link
|
|
/// storage at another.
|
|
fn nested_builder() -> FileBuilder {
|
|
let mut b = FileBuilder::new();
|
|
b.set_attr("title", AttrValue::String("nested".into()));
|
|
let mut l1 = b.create_group("l1");
|
|
l1.set_attr("depth", AttrValue::I64(1));
|
|
l1.create_dataset("d1")
|
|
.with_f64_data(&[1.0, 1.5])
|
|
.set_attr("unit", AttrValue::String("m".into()));
|
|
let mut l2 = l1.create_group("l2");
|
|
l2.set_attr("depth", AttrValue::I64(2));
|
|
l2.create_dataset("d2").with_i32_data(&[2, 3, 4]);
|
|
let mut l3 = l2.create_group("l3");
|
|
for i in 0..10 {
|
|
l3.set_attr(&format!("a{i}"), AttrValue::F64(i as f64 / 4.0)); // dense
|
|
}
|
|
for i in 0..12 {
|
|
l3.create_dataset(&format!("x{i:02}")) // dense links
|
|
.with_i64_data(&[i, -i])
|
|
.set_attr("i", AttrValue::I64(i));
|
|
}
|
|
let mut l4 = l3.create_group("l4");
|
|
l4.set_attr("depth", AttrValue::I64(4));
|
|
l4.create_dataset("leaf")
|
|
.with_f64_data(&[4.0, 4.25, 4.5])
|
|
.set_attr(
|
|
"tags",
|
|
AttrValue::StringArray(vec!["a".into(), "bc".into()]),
|
|
);
|
|
l3.add_group(l4.finish());
|
|
l2.add_group(l3.finish());
|
|
l1.add_group(l2.finish());
|
|
b.add_group(l1.finish());
|
|
// Path names: /p, /p/q and /p/q/r are made on the way to the dataset.
|
|
b.create_dataset("p/q/r/s")
|
|
.with_f64_data(&[7.0])
|
|
.set_attr("deep", AttrValue::I64(4));
|
|
// A group at an existing path is merged into it.
|
|
let mut pq = b.create_group("p/q");
|
|
pq.set_attr("merged", AttrValue::I64(1));
|
|
pq.create_dataset("t").with_i32_data(&[8]);
|
|
b.add_group(pq.finish());
|
|
let mut l1b = b.create_group("l1/l2/l3/l4/l5");
|
|
l1b.set_attr("depth", AttrValue::I64(5));
|
|
b.add_group(l1b.finish());
|
|
b
|
|
}
|
|
|
|
const NESTED_TREE: &str = "\
|
|
/|group|title=nested
|
|
/l1|group|depth=1.000000
|
|
/l1/d1|dataset|unit=m|1.000000,1.500000
|
|
/l1/l2|group|depth=2.000000
|
|
/l1/l2/d2|dataset||2.000000,3.000000,4.000000";
|
|
|
|
#[test]
|
|
fn nested_groups_read_the_same_in_h5py_and_clawhdf5() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = write(&dir, "nested.h5", nested_builder());
|
|
let ours = clawhdf5_tree(&path);
|
|
let theirs = h5py_tree(&path);
|
|
assert_eq!(ours, theirs);
|
|
assert!(ours.starts_with(NESTED_TREE), "{ours}");
|
|
for line in [
|
|
"/l1/l2/l3|group|a0=0.000000;a1=0.250000;a2=0.500000;a3=0.750000;a4=1.000000;\
|
|
a5=1.250000;a6=1.500000;a7=1.750000;a8=2.000000;a9=2.250000",
|
|
"/l1/l2/l3/l4|group|depth=4.000000",
|
|
"/l1/l2/l3/l4/l5|group|depth=5.000000",
|
|
"/l1/l2/l3/l4/leaf|dataset|tags=a,bc|4.000000,4.250000,4.500000",
|
|
"/l1/l2/l3/x11|dataset|i=11.000000|11.000000,-11.000000",
|
|
"/p|group|",
|
|
"/p/q|group|merged=1.000000",
|
|
"/p/q/r/s|dataset|deep=4.000000|7.000000",
|
|
"/p/q/t|dataset||8.000000",
|
|
] {
|
|
assert!(
|
|
ours.lines().any(|l| l == line),
|
|
"missing {line:?} in\n{ours}"
|
|
);
|
|
}
|
|
assert_eq!(ours.lines().count(), 26, "{ours}");
|
|
let dump = h5dump_ok(&path);
|
|
if !dump.is_empty() {
|
|
assert!(dump.contains("GROUP \"l5\""), "{dump}");
|
|
assert!(dump.contains("DATASET \"leaf\""), "{dump}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn soft_hard_and_external_links() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut other = FileBuilder::new();
|
|
other.create_dataset("data").with_i32_data(&[42, 43]);
|
|
write(&dir, "other.h5", other);
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0, 3.0]);
|
|
b.create_dataset("x/z").with_i32_data(&[9]);
|
|
b.add_soft_link("soft_abs", "/x/y");
|
|
b.add_soft_link("dangling", "/nowhere");
|
|
b.add_hard_link("alias", "/x/y");
|
|
b.add_hard_link("x_again", "x");
|
|
b.add_external_link("ext", "other.h5", "/data");
|
|
let mut g = b.create_group("a/b/c");
|
|
g.add_soft_link("rel", "sib"); // relative to /a/b/c
|
|
g.create_dataset("sib").with_i32_data(&[5]);
|
|
g.add_hard_link("deep_alias", "/x_again/z"); // through a hard link
|
|
g.add_soft_link("to_group", "/x");
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "links.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"import os\nos.chdir(os.path.dirname(path))\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 def kind(g, k):\n\
|
|
\x20 l = g.get(k, getlink=True)\n\
|
|
\x20 if isinstance(l, h5py.SoftLink): return 'soft:' + l.path\n\
|
|
\x20 if isinstance(l, h5py.ExternalLink): return 'ext:' + l.filename + ':' + l.path\n\
|
|
\x20 return 'hard'\n\
|
|
\x20 print(json.dumps({\n\
|
|
\x20 'root': {k: kind(f, k) for k in f},\n\
|
|
\x20 'abc': {k: kind(f['a/b/c'], k) for k in f['a/b/c']},\n\
|
|
\x20 'same': [f['alias'].id == f['x/y'].id, f['x_again'].id == f['x'].id,\n\
|
|
\x20 f['a/b/c/deep_alias'].id == f['x/z'].id],\n\
|
|
\x20 'rc': [h5py.h5o.get_info(f['x/y'].id).rc, h5py.h5o.get_info(f['x'].id).rc,\n\
|
|
\x20 h5py.h5o.get_info(f['x/z'].id).rc, h5py.h5o.get_info(f['a'].id).rc],\n\
|
|
\x20 'vals': [f['soft_abs'][()].tolist(), f['ext'][()].tolist(),\n\
|
|
\x20 f['a/b/c/rel'][()].tolist(), sorted(f['a/b/c/to_group'])],\n\
|
|
\x20 'dangling': f.get('dangling') is None,\n\
|
|
\x20 }, sort_keys=True))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"{"abc": {"deep_alias": "hard", "rel": "soft:sib", "sib": "hard", "to_group": "soft:/x"}, "dangling": true, "rc": [2, 2, 2, 1], "root": {"a": "hard", "alias": "hard", "dangling": "soft:/nowhere", "ext": "ext:other.h5:/data", "soft_abs": "soft:/x/y", "x": "hard", "x_again": "hard"}, "same": [true, true, true], "vals": [[1.0, 2.0, 3.0], [42, 43], [5], ["y", "z"]]}"#
|
|
);
|
|
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
|
|
h5dump_ok(&path);
|
|
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(
|
|
f.dataset("alias").unwrap().read_f64().unwrap(),
|
|
[1.0, 2.0, 3.0]
|
|
);
|
|
assert_eq!(
|
|
f.dataset("soft_abs").unwrap().read_f64().unwrap(),
|
|
[1.0, 2.0, 3.0]
|
|
);
|
|
assert_eq!(f.dataset("a/b/c/rel").unwrap().read_i32().unwrap(), [5]);
|
|
assert_eq!(
|
|
f.dataset("a/b/c/deep_alias").unwrap().read_i32().unwrap(),
|
|
[9]
|
|
);
|
|
assert_eq!(
|
|
f.dataset("x_again/y").unwrap().read_f64().unwrap(),
|
|
[1.0, 2.0, 3.0]
|
|
);
|
|
drop(f);
|
|
|
|
// The reference counts let libhdf5 delete one of two hard links and
|
|
// keep the object; with a count of 1 it would free an object still
|
|
// linked from elsewhere.
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r+') as f:\n\
|
|
\x20 del f['alias']\n\
|
|
\x20 del f['x_again']\n\
|
|
\x20 f.create_dataset('filler', data=np.arange(1000))\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([f['x/y'][()].tolist(), sorted(f['x']), h5py.h5o.get_info(f['x/y'].id).rc]))",
|
|
);
|
|
assert_eq!(out, r#"[[1.0, 2.0, 3.0], ["y", "z"], 1]"#);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
#[test]
|
|
fn a_hard_link_can_make_a_cycle() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("g");
|
|
g.create_dataset("v").with_i32_data(&[1]);
|
|
g.add_hard_link("up", "/");
|
|
g.add_hard_link("me", ".");
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "cycle.h5", b);
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([sorted(f['g/up/g']), f['g/up/g/me/me/v'][()].tolist(),\n\
|
|
\x20 h5py.h5o.get_info(f.id).rc, h5py.h5o.get_info(f['g'].id).rc]))",
|
|
);
|
|
assert_eq!(out, r#"[["me", "up", "v"], [1], 2, 2]"#);
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(f.dataset("g/up/g/me/v").unwrap().read_i32().unwrap(), [1]);
|
|
}
|
|
|
|
#[test]
|
|
fn bad_links_are_errors() {
|
|
for setup in [
|
|
|b: &mut FileBuilder| {
|
|
b.add_hard_link("h", "/missing");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.create_dataset("x").with_i32_data(&[1]);
|
|
b.add_soft_link("s", "/x");
|
|
b.add_hard_link("h", "/s"); // through a soft link
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.add_hard_link("h1", "/h2");
|
|
b.add_hard_link("h2", "/h1");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.create_dataset("x").with_i32_data(&[1]);
|
|
b.add_hard_link("h", "/x/y"); // a dataset is not a group
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.add_soft_link("s", "");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.add_external_link("e", "", "/x");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.create_dataset("x").with_i32_data(&[1]);
|
|
b.add_soft_link("x", "/y"); // name taken
|
|
},
|
|
] {
|
|
let mut b = FileBuilder::new();
|
|
setup(&mut b);
|
|
assert!(b.finish().is_err());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ten_thousand_links_in_one_group() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("many");
|
|
for i in 0..10_000 {
|
|
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
|
}
|
|
g.set_attr("n", AttrValue::I64(10_000));
|
|
b.add_group(g.finish());
|
|
// The same in creation order, added in reverse name order, with soft
|
|
// links among them.
|
|
let mut g = b.create_group("ordered");
|
|
g.track_order(true);
|
|
for i in (0..10_000).rev() {
|
|
if i % 1000 == 0 {
|
|
g.add_soft_link(&format!("s{i:05}"), &format!("/many/d{i:05}"));
|
|
}
|
|
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
|
}
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "many.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 m, o = f['many'], f['ordered']\n\
|
|
\x20 names = list(m)\n\
|
|
\x20 onames = list(o)\n\
|
|
\x20 print(json.dumps([len(names), names == sorted(names), names[:2], int(m.attrs['n']),\n\
|
|
\x20 [int(m['d%05d' % i][0]) for i in (0, 1, 4096, 9999)],\n\
|
|
\x20 len(onames), onames[:3], onames[-2:], int(o['s05000'][0]),\n\
|
|
\x20 o.id.get_create_plist().get_link_creation_order()]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"[10000, true, ["d00000", "d00001"], 10000, [0, 1, 4096, 9999], 10010, ["d09999", "d09998", "d09997"], ["s00000", "d00000"], 5000, 3]"#
|
|
);
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
let g = f.group("many").unwrap();
|
|
assert_eq!(g.datasets().unwrap().len(), 10_000);
|
|
assert_eq!(g.dataset("d09999").unwrap().read_i32().unwrap(), [9999]);
|
|
assert_eq!(
|
|
f.dataset("ordered/s05000").unwrap().read_i32().unwrap(),
|
|
[5000]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn more_links_than_one_index_leaf_holds_is_an_error() {
|
|
let mut b = FileBuilder::new();
|
|
for i in 0..70_000 {
|
|
b.add_soft_link(&format!("s{i}"), "/x");
|
|
}
|
|
let err = b.finish().unwrap_err().to_string();
|
|
assert!(
|
|
err.contains("70000 links in one group: at most 65535"),
|
|
"{err}"
|
|
);
|
|
// Dense attributes have the same one-leaf index. Their count used to
|
|
// be written modulo 65 536.
|
|
let mut b = FileBuilder::new();
|
|
let x = b.create_dataset("x");
|
|
x.with_i32_data(&[1]);
|
|
for i in 0..70_000 {
|
|
x.set_attr(&format!("a{i}"), AttrValue::I64(i));
|
|
}
|
|
let err = b.finish().unwrap_err().to_string();
|
|
assert!(
|
|
err.contains("70000 attributes on one object: at most 65535"),
|
|
"{err}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn track_order_lists_members_in_creation_order() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let names = ["zeta", "alpha", "mid", "beta"];
|
|
let mut b = FileBuilder::new();
|
|
b.track_order(true); // the root and every group without its own setting
|
|
for n in names {
|
|
b.create_dataset(n).with_i32_data(&[1]);
|
|
}
|
|
let mut g = b.create_group("by_name");
|
|
g.track_order(false);
|
|
for n in names {
|
|
g.create_dataset(n).with_i32_data(&[2]);
|
|
}
|
|
b.add_group(g.finish());
|
|
let mut g = b.create_group("dense");
|
|
for i in (0..20).rev() {
|
|
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
|
|
}
|
|
g.add_soft_link("soft", "/zeta");
|
|
b.add_group(g.finish());
|
|
b.create_dataset("made/on/the/way").with_i32_data(&[3]);
|
|
let path = write(&dir, "order.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([list(f), list(f['by_name']), list(f['dense'])[:3],\n\
|
|
\x20 list(f['dense'])[-2:], list(f['made/on'])]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"[["zeta", "alpha", "mid", "beta", "by_name", "dense", "made"], ["alpha", "beta", "mid", "zeta"], ["n19", "n18", "n17"], ["n00", "soft"], ["the"]]"#
|
|
);
|
|
h5dump_ok(&path);
|
|
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
|
|
|
|
// libhdf5 keeps the order when it adds to (and converts) these groups.
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r+') as f:\n\
|
|
\x20 f['aaa'] = np.arange(2)\n\
|
|
\x20 f['dense']['aaa'] = np.arange(2)\n\
|
|
\x20 del f['dense/n10']\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([list(f)[-1], list(f['dense'])[-2:], len(f['dense'])]))",
|
|
);
|
|
assert_eq!(out, r#"["aaa", ["soft", "aaa"], 21]"#);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
#[test]
|
|
fn non_ascii_names_are_utf8() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("größe/wert").with_i32_data(&[1]);
|
|
let path = write(&dir, "utf8.h5", b);
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 l = f.id.links.get_info('größe'.encode())\n\
|
|
\x20 print(json.dumps([list(f), list(f['größe']), l.cset], ensure_ascii=False))",
|
|
);
|
|
assert_eq!(out, r#"[["größe"], ["wert"], 1]"#);
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(f.dataset("größe/wert").unwrap().read_i32().unwrap(), [1]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_group_attribute_set_again_takes_the_new_value() {
|
|
skip_if_no_python!();
|
|
// Setting a group attribute twice wrote two attribute messages with one
|
|
// name. Now the later value replaces the earlier, as `attrs[name] = v`
|
|
// does in h5py — also across a group merged from two builders.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
b.set_attr("v", AttrValue::I64(1));
|
|
b.set_attr("v", AttrValue::I64(2));
|
|
let mut g = b.create_group("g");
|
|
g.set_attr("w", AttrValue::I64(1));
|
|
b.add_group(g.finish());
|
|
let mut g = b.create_group("g");
|
|
g.set_attr("w", AttrValue::String("two".into()));
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "attrs.h5", b);
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([list(f.attrs), int(f.attrs['v']), list(f['g'].attrs),\n\
|
|
\x20 f['g'].attrs['w'].decode()]))",
|
|
);
|
|
assert_eq!(out, r#"[["v"], 2, ["w"], "two"]"#);
|
|
let f = File::open(&path).unwrap();
|
|
assert!(matches!(f.root().attrs().unwrap()["v"], AttrValue::I64(2)));
|
|
}
|
|
|
|
// ---- big dense storage: child indirect blocks in the fractal heap ----
|
|
|
|
/// A name `len` bytes long, unique per `i`.
|
|
fn long_name(i: usize, len: usize) -> String {
|
|
let n = format!("link_{i:06}_");
|
|
format!("{n}{}", "x".repeat(len - n.len()))
|
|
}
|
|
|
|
#[test]
|
|
fn dense_links_past_the_direct_blocks_of_the_root() {
|
|
skip_if_no_python!();
|
|
// A dense group's links live in a fractal heap whose root indirect
|
|
// block holds direct blocks up to 64 KiB: 512 KiB of link messages.
|
|
// Rows past that are child indirect blocks. The writer used to write
|
|
// them as direct blocks, which libhdf5 cannot read ("incorrect metadata
|
|
// checksum"), from about 17 000 links with 20-byte names.
|
|
// `g` crosses the first boundary (0.6 MB of links); `deep` has 65 535
|
|
// links of about 110 bytes (7 MB), so its heap reaches the child indirect
|
|
// blocks that hold indirect blocks themselves.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("x").with_i32_data(&[7]);
|
|
let mut g = b.create_group("g");
|
|
for i in 0..20_000 {
|
|
g.create_dataset(&format!("dataset_number_{i:06}"))
|
|
.with_i32_data(&[i]);
|
|
}
|
|
b.add_group(g.finish());
|
|
let mut g = b.create_group("deep");
|
|
g.track_order(true);
|
|
for i in 0..usize::from(u16::MAX) {
|
|
g.add_hard_link(&long_name(i, 100), "/x");
|
|
}
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "big_links.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 g, d = f['g'], f['deep']\n\
|
|
\x20 names = list(g)\n\
|
|
\x20 dn = list(d)\n\
|
|
\x20 print(json.dumps([len(names), names[-1], int(g[names[-1]][0]),\n\
|
|
\x20 sum(int(g[n][0]) for n in names), len(dn), dn[0][:12], dn[-1][:12],\n\
|
|
\x20 int(d[dn[-1]][0]), h5py.h5o.get_info(f['x'].id).rc]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"[20000, "dataset_number_019999", 19999, 199990000, 65535, "link_000000_", "link_065534_", 7, 65536]"#
|
|
);
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
let g = f.group("g").unwrap();
|
|
assert_eq!(g.datasets().unwrap().len(), 20_000);
|
|
assert_eq!(
|
|
g.dataset("dataset_number_019999")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap(),
|
|
[19999]
|
|
);
|
|
let d = f.group("deep").unwrap();
|
|
assert_eq!(d.datasets().unwrap().len(), usize::from(u16::MAX));
|
|
assert_eq!(
|
|
d.dataset(&long_name(65_534, 100))
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap(),
|
|
[7]
|
|
);
|
|
|
|
// libhdf5 can add to and delete from the heap. It could not when the
|
|
// header's block allocation offset was 0: its next block overwrote the
|
|
// first ("bad version number for message"). Adding to `deep` also
|
|
// needs its index's leaf node to have room for at most 65 535 records:
|
|
// a bigger node made libhdf5 overflow the leaf's 2-byte record count
|
|
// (a crash, or "unknown link class" when listing).
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r+') as f:\n\
|
|
\x20 f['g']['zz_new'] = np.arange(3)\n\
|
|
\x20 f['deep']['zz_new'] = np.arange(4)\n\
|
|
\x20 del f['g/dataset_number_000005']\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]), len(f['deep']),\n\
|
|
\x20 list(f['deep'])[-1], int(f['g/dataset_number_019998'][0])]))",
|
|
);
|
|
assert_eq!(out, r#"[20000, 2, 65536, "zz_new", 19998]"#);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
#[test]
|
|
fn dense_attributes_past_the_direct_blocks_of_the_root() {
|
|
skip_if_no_python!();
|
|
// Dense attributes share the heap writer. 150 attributes of up to 56 KB
|
|
// (8 MB) need child indirect blocks, and a big attribute after small
|
|
// ones must skip the small blocks rather than overrun one.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
let ds = b.create_dataset("x");
|
|
ds.with_i32_data(&[1]);
|
|
for i in 0..150usize {
|
|
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
|
|
let v: Vec<f64> = (0..len).map(|k| (i * 100_000 + k) as f64).collect();
|
|
ds.set_attr(&format!("a{i:03}"), AttrValue::F64Array(v));
|
|
}
|
|
let path = write(&dir, "big_attrs.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 a = f['x'].attrs\n\
|
|
\x20 ok = all(np.array_equal(a['a%03d' % i],\n\
|
|
\x20 np.arange(7000 if i % 3 == 0 else 1 + i) + i * 100000) for i in range(150))\n\
|
|
\x20 print(json.dumps([len(a), ok]))",
|
|
);
|
|
assert_eq!(out, "[150, true]");
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
let attrs = f.dataset("x").unwrap().attrs().unwrap();
|
|
assert_eq!(attrs.len(), 150);
|
|
for i in [0usize, 1, 147, 149] {
|
|
let len = if i % 3 == 0 { 7_000 } else { 1 + i };
|
|
let want: Vec<f64> = (0..len).map(|k| (i * 100_000 + k) as f64).collect();
|
|
match &attrs[&format!("a{i:03}")] {
|
|
AttrValue::F64Array(v) => assert_eq!(*v, want, "a{i:03}"),
|
|
other => panic!("a{i:03}: {other:?}"),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_link_too_big_for_dense_storage_is_an_error() {
|
|
// A link message must fit one fractal heap direct block (64 KiB less
|
|
// its header); the writer has no huge-object path. It used to be
|
|
// written anyway, cut off, and libhdf5 could not list the group.
|
|
let mut b = FileBuilder::new();
|
|
for i in 0..10 {
|
|
b.create_dataset(&format!("d{i}")).with_i32_data(&[i]);
|
|
}
|
|
b.add_soft_link("s", &"/y".repeat(40_000));
|
|
let err = b.finish().unwrap_err().to_string();
|
|
assert!(err.contains("fractal heap object holds at most"), "{err}");
|
|
// The same for a dense attribute.
|
|
let mut b = FileBuilder::new();
|
|
let x = b.create_dataset("x");
|
|
x.with_i32_data(&[1]);
|
|
for i in 0..9 {
|
|
x.set_attr(&format!("a{i}"), AttrValue::I64(i));
|
|
}
|
|
x.set_attr("big", AttrValue::F64Array(vec![0.5; 9_000]));
|
|
let err = b.finish().unwrap_err().to_string();
|
|
assert!(err.contains("fractal heap object holds at most"), "{err}");
|
|
|
|
// Just under the limit is fine, and libhdf5 reads it back.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
for i in 0..10 {
|
|
b.create_dataset(&format!("d{i}")).with_i32_data(&[i]);
|
|
}
|
|
let target = format!("/{}", "y".repeat(65_000));
|
|
b.add_soft_link("s", &target);
|
|
let path = write(&dir, "long_soft.h5", b);
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([len(f), len(f.get('s', getlink=True).path)]))",
|
|
);
|
|
assert_eq!(out, "[11, 65001]");
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
#[test]
|
|
fn chained_hard_links_resolve_in_linear_time() {
|
|
skip_if_no_python!();
|
|
// Each link's target goes through the previous link twice. Resolving
|
|
// them without remembering resolved links doubled the work per link:
|
|
// 26 links took 46 s in a debug build, so 60 would never finish.
|
|
fn chain(reverse: bool) -> FileBuilder {
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("g");
|
|
g.create_dataset("v").with_i32_data(&[5]);
|
|
b.add_group(g.finish());
|
|
let mut order: Vec<usize> = (0..60).collect();
|
|
if reverse {
|
|
order.reverse();
|
|
}
|
|
for i in order {
|
|
if i == 0 {
|
|
b.add_hard_link("g/s0", "/g");
|
|
} else {
|
|
b.add_hard_link(&format!("g/s{i}"), &format!("/g/s{}/s{}", i - 1, i - 1));
|
|
}
|
|
}
|
|
b
|
|
}
|
|
let (tx, rx) = std::sync::mpsc::channel();
|
|
std::thread::spawn(move || {
|
|
let bytes = [false, true].map(|r| chain(r).finish().unwrap());
|
|
tx.send(bytes).unwrap();
|
|
});
|
|
let [forward, reverse] = rx
|
|
.recv_timeout(std::time::Duration::from_secs(60))
|
|
.expect("resolving 60 chained hard links took over a minute");
|
|
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for (name, bytes) in [("forward.h5", forward), ("reverse.h5", reverse)] {
|
|
let path = dir.path().join(name).display().to_string();
|
|
std::fs::write(&path, bytes).unwrap();
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([h5py.h5o.get_info(f['g'].id).rc, len(f['g']),\n\
|
|
\x20 int(f['g/s59/s30/s0/v'][0]), f['g/s59'] == f['g']]))",
|
|
);
|
|
assert_eq!(out, "[61, 61, 5, true]", "{name}");
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(f.dataset("g/s59/s0/v").unwrap().read_i32().unwrap(), [5]);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_dataset_attribute_set_again_takes_the_new_value() {
|
|
skip_if_no_python!();
|
|
// Setting a dataset attribute twice wrote two attribute messages with
|
|
// one name, and h5py read back the first value. Also with dense
|
|
// attribute storage (more than 8).
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("x")
|
|
.with_f64_data(&[1.0])
|
|
.set_attr("a", AttrValue::I64(1))
|
|
.set_attr("a", AttrValue::I64(2));
|
|
let d = b.create_dataset("dense");
|
|
d.with_i32_data(&[1]);
|
|
for i in 0..12 {
|
|
d.set_attr(&format!("k{i:02}"), AttrValue::I64(i));
|
|
}
|
|
d.set_attr("k03", AttrValue::String("three".into()));
|
|
let path = write(&dir, "ds_attrs.h5", b);
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 a, d = f['x'].attrs, f['dense'].attrs\n\
|
|
\x20 print(json.dumps([list(a), int(a['a']), len(d), d['k03'].decode(), int(d['k04'])]))",
|
|
);
|
|
assert_eq!(out, r#"[["a"], 2, 12, "three", 4]"#);
|
|
let f = File::open(&path).unwrap();
|
|
assert!(matches!(
|
|
f.dataset("x").unwrap().attrs().unwrap()["a"],
|
|
AttrValue::I64(2)
|
|
));
|
|
}
|
|
|
|
#[cfg(feature = "provenance")]
|
|
#[test]
|
|
fn provenance_attributes_replace_ones_set_by_hand() {
|
|
skip_if_no_python!();
|
|
// A hand-set attribute with a provenance attribute's name was written
|
|
// next to the computed one, and h5py read the hand-set value.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("p")
|
|
.with_i32_data(&[1, 2])
|
|
.with_provenance("me", "2026-09-26T00:00:00Z", None)
|
|
.set_attr("_provenance_sha256", AttrValue::String("forged".into()));
|
|
let path = write(&dir, "prov.h5", b);
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 a = f['p'].attrs\n\
|
|
\x20 h = a['_provenance_sha256']\n\
|
|
\x20 h = h.decode() if isinstance(h, bytes) else h\n\
|
|
\x20 print(json.dumps([list(a).count('_provenance_sha256'), h != 'forged']))",
|
|
);
|
|
assert_eq!(out, "[1, true]");
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(
|
|
f.dataset("p").unwrap().verify_provenance().unwrap(),
|
|
clawhdf5_format::provenance::VerifyResult::Ok
|
|
);
|
|
}
|