osobhandClaude Opus 5.5
d102c06306
FileWriter wrote the root group plus one level of groups, and refused
path-like names. The writer now flattens its builders into a group tree
(writer_tree.rs) before layout:
- A name may be a path ("a/b/x", "/a/b/x" at the root); missing
intermediate groups are created as h5py does, and GroupBuilder gains
create_group/add_group so builders nest to any depth. A group added at a
path that already holds a group is merged into it (require_group);
any other repeated name, an empty or "." component, or an absolute path
below the root is an error.
- add_soft_link, add_hard_link and add_external_link on FileWriter,
FileBuilder and GroupBuilder. Hard-link targets are resolved to objects
at finish (through other hard links; a missing target, a soft link on the
way or a cycle of paths is an error). Objects with several hard links get
an Object Reference Count message so libhdf5 can delete one link without
freeing the object.
- track_order(true) per group, or as the file default, tracks and indexes
link creation order: Link Info flags and max order, the order in each
Link message, and a type-6 creation-order B-tree for dense groups.
- A group's link index is one B-tree leaf; more than 65535 links is an
error.
Groups are laid out depth-first from the root, datasets group by group,
and untracked groups keep writing datasets, then groups, then other links:
files with one level of groups are byte-identical to before.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
595 lines
21 KiB
Rust
595 lines
21 KiB
Rust
//! Groups and links written by `FileBuilder`, read back by h5py (libhdf5)
|
|
//! and h5dump, and by clawhdf5 itself.
|
|
//!
|
|
//! Skipped when python3 with h5py is unavailable, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::{AttrValue, File, FileBuilder, Group};
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn h5dump_available() -> bool {
|
|
Command::new("h5dump")
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
fn run_python(script: &str) -> String {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python");
|
|
if !output.status.success() {
|
|
panic!(
|
|
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&output.stdout),
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
}
|
|
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
|
}
|
|
|
|
/// Run `body` under h5py with `path` bound to the file's path.
|
|
fn h5py(path: &str, body: &str) -> String {
|
|
run_python(&format!(
|
|
"import h5py, numpy as np, json\npath = r'{path}'\n{body}"
|
|
))
|
|
}
|
|
|
|
fn write(dir: &tempfile::TempDir, name: &str, b: FileBuilder) -> String {
|
|
let path = dir.path().join(name).display().to_string();
|
|
b.write(&path).unwrap();
|
|
path
|
|
}
|
|
|
|
/// h5dump must read the whole file without an error.
|
|
fn h5dump_ok(path: &str) -> String {
|
|
if !h5dump_available() {
|
|
assert!(!interop_required(), "h5dump is not available");
|
|
return String::new();
|
|
}
|
|
let o = Command::new("h5dump").arg(path).output().unwrap();
|
|
let out = String::from_utf8_lossy(&o.stdout).to_string();
|
|
assert!(
|
|
o.status.success(),
|
|
"h5dump failed:\n{out}{}",
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
out
|
|
}
|
|
|
|
// ---- libhdf5 can modify the groups we write ----
|
|
|
|
#[test]
|
|
fn h5py_can_add_links_to_groups_we_wrote() {
|
|
skip_if_no_python!();
|
|
// Measured before the fix: h5py in "r+" mode could not add a link to any
|
|
// group we wrote ("Unable to create link (message type not found)"):
|
|
// libhdf5 reads a group's Group Info message before inserting a link, and
|
|
// the writer wrote none.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("x").with_f64_data(&[1.0, 2.0]);
|
|
let mut g = b.create_group("small");
|
|
g.create_dataset("a").with_i32_data(&[1]);
|
|
b.add_group(g.finish());
|
|
let mut g = b.create_group("big"); // dense link storage
|
|
for i in 0..20 {
|
|
g.create_dataset(&format!("d{i:02}")).with_i32_data(&[i]);
|
|
}
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "modify.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r+') as f:\n\
|
|
\x20 f['alias'] = f['x']\n\
|
|
\x20 f['small']['new'] = np.arange(3)\n\
|
|
\x20 f['big']['new'] = np.arange(4)\n\
|
|
\x20 f.create_group('added/deeper')\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([sorted(f), sorted(f['small']), len(f['big']),\n\
|
|
\x20 f['alias'][()].tolist(), f['big/new'][()].tolist(), f['big/d07'][()].tolist()]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"[["added", "alias", "big", "small", "x"], ["a", "new"], 21, [1.0, 2.0], [0, 1, 2, 3], [7]]"#
|
|
);
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(
|
|
f.dataset("big/new").unwrap().read_i64().unwrap(),
|
|
[0, 1, 2, 3]
|
|
);
|
|
assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]);
|
|
}
|
|
|
|
// ---- the whole tree, as h5py and as clawhdf5 read it ----
|
|
|
|
fn fmt_num(x: f64) -> String {
|
|
format!("{x:.6}")
|
|
}
|
|
|
|
fn fmt_attr(v: &AttrValue) -> String {
|
|
let join = |v: Vec<String>| v.join(",");
|
|
match v {
|
|
AttrValue::F64(x) => fmt_num(*x),
|
|
AttrValue::I64(x) => fmt_num(*x as f64),
|
|
AttrValue::U64(x) => fmt_num(*x as f64),
|
|
AttrValue::F64Array(a) => join(a.iter().map(|x| fmt_num(*x)).collect()),
|
|
AttrValue::I64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
|
|
AttrValue::U64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()),
|
|
AttrValue::String(s) => s.clone(),
|
|
AttrValue::StringArray(a) => a.join(","),
|
|
AttrValue::Raw { .. } => "raw".to_string(),
|
|
}
|
|
}
|
|
|
|
fn fmt_attrs(attrs: std::collections::HashMap<String, AttrValue>) -> String {
|
|
let mut v: Vec<_> = attrs.into_iter().collect();
|
|
v.sort_by(|a, b| a.0.cmp(&b.0));
|
|
v.iter()
|
|
.map(|(k, a)| format!("{k}={}", fmt_attr(a)))
|
|
.collect::<Vec<_>>()
|
|
.join(";")
|
|
}
|
|
|
|
fn child_path(path: &str, name: &str) -> String {
|
|
if path == "/" {
|
|
format!("/{name}")
|
|
} else {
|
|
format!("{path}/{name}")
|
|
}
|
|
}
|
|
|
|
/// Every group and dataset reachable from `g` (following hard and soft
|
|
/// links; the tree must be acyclic), one line each: path, kind, attributes
|
|
/// and (datasets) values.
|
|
fn walk(g: &Group<'_>, path: &str, out: &mut Vec<String>) {
|
|
out.push(format!("{path}|group|{}", fmt_attrs(g.attrs().unwrap())));
|
|
let mut names: Vec<(String, bool)> = g
|
|
.datasets()
|
|
.unwrap()
|
|
.into_iter()
|
|
.map(|n| (n, false))
|
|
.chain(g.groups().unwrap().into_iter().map(|n| (n, true)))
|
|
.collect();
|
|
names.sort();
|
|
for (name, is_group) in names {
|
|
let p = child_path(path, &name);
|
|
if is_group {
|
|
walk(&g.group(&name).unwrap(), &p, out);
|
|
} else {
|
|
let ds = g.dataset(&name).unwrap();
|
|
let values: Vec<String> = ds.read_f64().unwrap().into_iter().map(fmt_num).collect();
|
|
out.push(format!(
|
|
"{p}|dataset|{}|{}",
|
|
fmt_attrs(ds.attrs().unwrap()),
|
|
values.join(",")
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
fn clawhdf5_tree(path: &str) -> String {
|
|
let f = File::open(path).unwrap();
|
|
let mut out = Vec::new();
|
|
walk(&f.root(), "/", &mut out);
|
|
out.join("\n")
|
|
}
|
|
|
|
/// The same listing as [`walk`], from h5py. External and dangling soft
|
|
/// links are skipped, as clawhdf5's group listings skip them.
|
|
const H5PY_WALK: &str = r#"
|
|
def fmt(v):
|
|
if isinstance(v, bytes): return v.decode()
|
|
if isinstance(v, str): return v
|
|
a = np.asarray(v)
|
|
if a.dtype.kind in 'SUO':
|
|
return ','.join(x.decode() if isinstance(x, bytes) else str(x) for x in a.ravel())
|
|
if a.ndim == 0: return '%.6f' % float(a)
|
|
return ','.join('%.6f' % float(x) for x in a.ravel())
|
|
def attrs(o): return ';'.join(f'{k}={fmt(o.attrs[k])}' for k in sorted(o.attrs))
|
|
out = []
|
|
def walk(g, path):
|
|
out.append(f'{path}|group|{attrs(g)}')
|
|
for k in sorted(g.keys()):
|
|
if isinstance(g.get(k, getlink=True), h5py.ExternalLink): continue
|
|
o = g.get(k)
|
|
if o is None: continue
|
|
p = '/' + k if path == '/' else path + '/' + k
|
|
if isinstance(o, h5py.Group): walk(o, p)
|
|
else:
|
|
vals = ','.join('%.6f' % float(x) for x in np.asarray(o[()]).ravel())
|
|
out.append(f'{p}|dataset|{attrs(o)}|{vals}')
|
|
with h5py.File(path, 'r') as f:
|
|
walk(f, '/')
|
|
print('\n'.join(out))
|
|
"#;
|
|
|
|
fn h5py_tree(path: &str) -> String {
|
|
h5py(path, H5PY_WALK)
|
|
}
|
|
|
|
/// A four-level tree with attributes on every object: nested builders,
|
|
/// path names (with intermediate groups made on the way) and a group added
|
|
/// twice (merged), with dense attribute storage at one level and dense link
|
|
/// storage at another.
|
|
fn nested_builder() -> FileBuilder {
|
|
let mut b = FileBuilder::new();
|
|
b.set_attr("title", AttrValue::String("nested".into()));
|
|
let mut l1 = b.create_group("l1");
|
|
l1.set_attr("depth", AttrValue::I64(1));
|
|
l1.create_dataset("d1")
|
|
.with_f64_data(&[1.0, 1.5])
|
|
.set_attr("unit", AttrValue::String("m".into()));
|
|
let mut l2 = l1.create_group("l2");
|
|
l2.set_attr("depth", AttrValue::I64(2));
|
|
l2.create_dataset("d2").with_i32_data(&[2, 3, 4]);
|
|
let mut l3 = l2.create_group("l3");
|
|
for i in 0..10 {
|
|
l3.set_attr(&format!("a{i}"), AttrValue::F64(i as f64 / 4.0)); // dense
|
|
}
|
|
for i in 0..12 {
|
|
l3.create_dataset(&format!("x{i:02}")) // dense links
|
|
.with_i64_data(&[i, -i])
|
|
.set_attr("i", AttrValue::I64(i));
|
|
}
|
|
let mut l4 = l3.create_group("l4");
|
|
l4.set_attr("depth", AttrValue::I64(4));
|
|
l4.create_dataset("leaf")
|
|
.with_f64_data(&[4.0, 4.25, 4.5])
|
|
.set_attr(
|
|
"tags",
|
|
AttrValue::StringArray(vec!["a".into(), "bc".into()]),
|
|
);
|
|
l3.add_group(l4.finish());
|
|
l2.add_group(l3.finish());
|
|
l1.add_group(l2.finish());
|
|
b.add_group(l1.finish());
|
|
// Path names: /p, /p/q and /p/q/r are made on the way to the dataset.
|
|
b.create_dataset("p/q/r/s")
|
|
.with_f64_data(&[7.0])
|
|
.set_attr("deep", AttrValue::I64(4));
|
|
// A group at an existing path is merged into it.
|
|
let mut pq = b.create_group("p/q");
|
|
pq.set_attr("merged", AttrValue::I64(1));
|
|
pq.create_dataset("t").with_i32_data(&[8]);
|
|
b.add_group(pq.finish());
|
|
let mut l1b = b.create_group("l1/l2/l3/l4/l5");
|
|
l1b.set_attr("depth", AttrValue::I64(5));
|
|
b.add_group(l1b.finish());
|
|
b
|
|
}
|
|
|
|
const NESTED_TREE: &str = "\
|
|
/|group|title=nested
|
|
/l1|group|depth=1.000000
|
|
/l1/d1|dataset|unit=m|1.000000,1.500000
|
|
/l1/l2|group|depth=2.000000
|
|
/l1/l2/d2|dataset||2.000000,3.000000,4.000000";
|
|
|
|
#[test]
|
|
fn nested_groups_read_the_same_in_h5py_and_clawhdf5() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = write(&dir, "nested.h5", nested_builder());
|
|
let ours = clawhdf5_tree(&path);
|
|
let theirs = h5py_tree(&path);
|
|
assert_eq!(ours, theirs);
|
|
assert!(ours.starts_with(NESTED_TREE), "{ours}");
|
|
for line in [
|
|
"/l1/l2/l3|group|a0=0.000000;a1=0.250000;a2=0.500000;a3=0.750000;a4=1.000000;\
|
|
a5=1.250000;a6=1.500000;a7=1.750000;a8=2.000000;a9=2.250000",
|
|
"/l1/l2/l3/l4|group|depth=4.000000",
|
|
"/l1/l2/l3/l4/l5|group|depth=5.000000",
|
|
"/l1/l2/l3/l4/leaf|dataset|tags=a,bc|4.000000,4.250000,4.500000",
|
|
"/l1/l2/l3/x11|dataset|i=11.000000|11.000000,-11.000000",
|
|
"/p|group|",
|
|
"/p/q|group|merged=1.000000",
|
|
"/p/q/r/s|dataset|deep=4.000000|7.000000",
|
|
"/p/q/t|dataset||8.000000",
|
|
] {
|
|
assert!(
|
|
ours.lines().any(|l| l == line),
|
|
"missing {line:?} in\n{ours}"
|
|
);
|
|
}
|
|
assert_eq!(ours.lines().count(), 26, "{ours}");
|
|
let dump = h5dump_ok(&path);
|
|
if !dump.is_empty() {
|
|
assert!(dump.contains("GROUP \"l5\""), "{dump}");
|
|
assert!(dump.contains("DATASET \"leaf\""), "{dump}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn soft_hard_and_external_links() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut other = FileBuilder::new();
|
|
other.create_dataset("data").with_i32_data(&[42, 43]);
|
|
write(&dir, "other.h5", other);
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("x/y").with_f64_data(&[1.0, 2.0, 3.0]);
|
|
b.create_dataset("x/z").with_i32_data(&[9]);
|
|
b.add_soft_link("soft_abs", "/x/y");
|
|
b.add_soft_link("dangling", "/nowhere");
|
|
b.add_hard_link("alias", "/x/y");
|
|
b.add_hard_link("x_again", "x");
|
|
b.add_external_link("ext", "other.h5", "/data");
|
|
let mut g = b.create_group("a/b/c");
|
|
g.add_soft_link("rel", "sib"); // relative to /a/b/c
|
|
g.create_dataset("sib").with_i32_data(&[5]);
|
|
g.add_hard_link("deep_alias", "/x_again/z"); // through a hard link
|
|
g.add_soft_link("to_group", "/x");
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "links.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"import os\nos.chdir(os.path.dirname(path))\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 def kind(g, k):\n\
|
|
\x20 l = g.get(k, getlink=True)\n\
|
|
\x20 if isinstance(l, h5py.SoftLink): return 'soft:' + l.path\n\
|
|
\x20 if isinstance(l, h5py.ExternalLink): return 'ext:' + l.filename + ':' + l.path\n\
|
|
\x20 return 'hard'\n\
|
|
\x20 print(json.dumps({\n\
|
|
\x20 'root': {k: kind(f, k) for k in f},\n\
|
|
\x20 'abc': {k: kind(f['a/b/c'], k) for k in f['a/b/c']},\n\
|
|
\x20 'same': [f['alias'].id == f['x/y'].id, f['x_again'].id == f['x'].id,\n\
|
|
\x20 f['a/b/c/deep_alias'].id == f['x/z'].id],\n\
|
|
\x20 'rc': [h5py.h5o.get_info(f['x/y'].id).rc, h5py.h5o.get_info(f['x'].id).rc,\n\
|
|
\x20 h5py.h5o.get_info(f['x/z'].id).rc, h5py.h5o.get_info(f['a'].id).rc],\n\
|
|
\x20 'vals': [f['soft_abs'][()].tolist(), f['ext'][()].tolist(),\n\
|
|
\x20 f['a/b/c/rel'][()].tolist(), sorted(f['a/b/c/to_group'])],\n\
|
|
\x20 'dangling': f.get('dangling') is None,\n\
|
|
\x20 }, sort_keys=True))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"{"abc": {"deep_alias": "hard", "rel": "soft:sib", "sib": "hard", "to_group": "soft:/x"}, "dangling": true, "rc": [2, 2, 2, 1], "root": {"a": "hard", "alias": "hard", "dangling": "soft:/nowhere", "ext": "ext:other.h5:/data", "soft_abs": "soft:/x/y", "x": "hard", "x_again": "hard"}, "same": [true, true, true], "vals": [[1.0, 2.0, 3.0], [42, 43], [5], ["y", "z"]]}"#
|
|
);
|
|
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
|
|
h5dump_ok(&path);
|
|
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(
|
|
f.dataset("alias").unwrap().read_f64().unwrap(),
|
|
[1.0, 2.0, 3.0]
|
|
);
|
|
assert_eq!(
|
|
f.dataset("soft_abs").unwrap().read_f64().unwrap(),
|
|
[1.0, 2.0, 3.0]
|
|
);
|
|
assert_eq!(f.dataset("a/b/c/rel").unwrap().read_i32().unwrap(), [5]);
|
|
assert_eq!(
|
|
f.dataset("a/b/c/deep_alias").unwrap().read_i32().unwrap(),
|
|
[9]
|
|
);
|
|
assert_eq!(
|
|
f.dataset("x_again/y").unwrap().read_f64().unwrap(),
|
|
[1.0, 2.0, 3.0]
|
|
);
|
|
drop(f);
|
|
|
|
// The reference counts let libhdf5 delete one of two hard links and
|
|
// keep the object; with a count of 1 it would free an object still
|
|
// linked from elsewhere.
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r+') as f:\n\
|
|
\x20 del f['alias']\n\
|
|
\x20 del f['x_again']\n\
|
|
\x20 f.create_dataset('filler', data=np.arange(1000))\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([f['x/y'][()].tolist(), sorted(f['x']), h5py.h5o.get_info(f['x/y'].id).rc]))",
|
|
);
|
|
assert_eq!(out, r#"[[1.0, 2.0, 3.0], ["y", "z"], 1]"#);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
#[test]
|
|
fn a_hard_link_can_make_a_cycle() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("g");
|
|
g.create_dataset("v").with_i32_data(&[1]);
|
|
g.add_hard_link("up", "/");
|
|
g.add_hard_link("me", ".");
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "cycle.h5", b);
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([sorted(f['g/up/g']), f['g/up/g/me/me/v'][()].tolist(),\n\
|
|
\x20 h5py.h5o.get_info(f.id).rc, h5py.h5o.get_info(f['g'].id).rc]))",
|
|
);
|
|
assert_eq!(out, r#"[["me", "up", "v"], [1], 2, 2]"#);
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
assert_eq!(f.dataset("g/up/g/me/v").unwrap().read_i32().unwrap(), [1]);
|
|
}
|
|
|
|
#[test]
|
|
fn bad_links_are_errors() {
|
|
for setup in [
|
|
|b: &mut FileBuilder| {
|
|
b.add_hard_link("h", "/missing");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.create_dataset("x").with_i32_data(&[1]);
|
|
b.add_soft_link("s", "/x");
|
|
b.add_hard_link("h", "/s"); // through a soft link
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.add_hard_link("h1", "/h2");
|
|
b.add_hard_link("h2", "/h1");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.create_dataset("x").with_i32_data(&[1]);
|
|
b.add_hard_link("h", "/x/y"); // a dataset is not a group
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.add_soft_link("s", "");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.add_external_link("e", "", "/x");
|
|
},
|
|
|b: &mut FileBuilder| {
|
|
b.create_dataset("x").with_i32_data(&[1]);
|
|
b.add_soft_link("x", "/y"); // name taken
|
|
},
|
|
] {
|
|
let mut b = FileBuilder::new();
|
|
setup(&mut b);
|
|
assert!(b.finish().is_err());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ten_thousand_links_in_one_group() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("many");
|
|
for i in 0..10_000 {
|
|
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
|
}
|
|
g.set_attr("n", AttrValue::I64(10_000));
|
|
b.add_group(g.finish());
|
|
// The same in creation order, added in reverse name order, with soft
|
|
// links among them.
|
|
let mut g = b.create_group("ordered");
|
|
g.track_order(true);
|
|
for i in (0..10_000).rev() {
|
|
if i % 1000 == 0 {
|
|
g.add_soft_link(&format!("s{i:05}"), &format!("/many/d{i:05}"));
|
|
}
|
|
g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]);
|
|
}
|
|
b.add_group(g.finish());
|
|
let path = write(&dir, "many.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 m, o = f['many'], f['ordered']\n\
|
|
\x20 names = list(m)\n\
|
|
\x20 onames = list(o)\n\
|
|
\x20 print(json.dumps([len(names), names == sorted(names), names[:2], int(m.attrs['n']),\n\
|
|
\x20 [int(m['d%05d' % i][0]) for i in (0, 1, 4096, 9999)],\n\
|
|
\x20 len(onames), onames[:3], onames[-2:], int(o['s05000'][0]),\n\
|
|
\x20 o.id.get_create_plist().get_link_creation_order()]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"[10000, true, ["d00000", "d00001"], 10000, [0, 1, 4096, 9999], 10010, ["d09999", "d09998", "d09997"], ["s00000", "d00000"], 5000, 3]"#
|
|
);
|
|
h5dump_ok(&path);
|
|
let f = File::open(&path).unwrap();
|
|
let g = f.group("many").unwrap();
|
|
assert_eq!(g.datasets().unwrap().len(), 10_000);
|
|
assert_eq!(g.dataset("d09999").unwrap().read_i32().unwrap(), [9999]);
|
|
assert_eq!(
|
|
f.dataset("ordered/s05000").unwrap().read_i32().unwrap(),
|
|
[5000]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn more_links_than_one_index_leaf_holds_is_an_error() {
|
|
let mut b = FileBuilder::new();
|
|
for i in 0..70_000 {
|
|
b.add_soft_link(&format!("s{i}"), "/x");
|
|
}
|
|
let err = b.finish().unwrap_err().to_string();
|
|
assert!(err.contains("at most 65535 links"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn track_order_lists_members_in_creation_order() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let names = ["zeta", "alpha", "mid", "beta"];
|
|
let mut b = FileBuilder::new();
|
|
b.track_order(true); // the root and every group without its own setting
|
|
for n in names {
|
|
b.create_dataset(n).with_i32_data(&[1]);
|
|
}
|
|
let mut g = b.create_group("by_name");
|
|
g.track_order(false);
|
|
for n in names {
|
|
g.create_dataset(n).with_i32_data(&[2]);
|
|
}
|
|
b.add_group(g.finish());
|
|
let mut g = b.create_group("dense");
|
|
for i in (0..20).rev() {
|
|
g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]);
|
|
}
|
|
g.add_soft_link("soft", "/zeta");
|
|
b.add_group(g.finish());
|
|
b.create_dataset("made/on/the/way").with_i32_data(&[3]);
|
|
let path = write(&dir, "order.h5", b);
|
|
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([list(f), list(f['by_name']), list(f['dense'])[:3],\n\
|
|
\x20 list(f['dense'])[-2:], list(f['made/on'])]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
r#"[["zeta", "alpha", "mid", "beta", "by_name", "dense", "made"], ["alpha", "beta", "mid", "zeta"], ["n19", "n18", "n17"], ["n00", "soft"], ["the"]]"#
|
|
);
|
|
h5dump_ok(&path);
|
|
assert_eq!(clawhdf5_tree(&path), h5py_tree(&path));
|
|
|
|
// libhdf5 keeps the order when it adds to (and converts) these groups.
|
|
let out = h5py(
|
|
&path,
|
|
"with h5py.File(path, 'r+') as f:\n\
|
|
\x20 f['aaa'] = np.arange(2)\n\
|
|
\x20 f['dense']['aaa'] = np.arange(2)\n\
|
|
\x20 del f['dense/n10']\n\
|
|
with h5py.File(path, 'r') as f:\n\
|
|
\x20 print(json.dumps([list(f)[-1], list(f['dense'])[-2:], len(f['dense'])]))",
|
|
);
|
|
assert_eq!(out, r#"["aaa", ["soft", "aaa"], 21]"#);
|
|
h5dump_ok(&path);
|
|
}
|