From 8cbbef3fae6a11286aa096f47f0168ddfcd96463 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:08:31 -0500 Subject: [PATCH] fix(format): write a Group Info message in every group libhdf5 reads a group's Group Info message before it inserts a link, and FileWriter wrote none, so h5py in "r+" mode could not add a link to any group we wrote: "Unable to create link (message type not found)". Each group header now carries a version 0 Group Info message with the default link-phase thresholds, as libhdf5 writes for a new group. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++ crates/clawhdf5-format/src/file_writer.rs | 7 + .../clawhdf5/tests/writer_groups_interop.rs | 136 ++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 crates/clawhdf5/tests/writer_groups_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ef4bd2f..cc1010f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### Writer: groups and links (2026-09-26) +- **libhdf5 could not add links to groups we wrote.** h5py in `"r+"` mode + failed with "Unable to create link (message type not found)" on every + group `FileWriter` wrote: libhdf5 reads a group's Group Info message before + inserting a link, and none was written. Every group now carries one + (version 0, default thresholds: 6 more bytes per group header, so files + are not byte-identical to earlier versions). Regression test: + `crates/clawhdf5/tests/writer_groups_interop.rs`. + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index dbf59f2..36c3fbb 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -196,6 +196,13 @@ pub(crate) fn build_group_oh( li.extend_from_slice(&u64::MAX.to_le_bytes()); // fractal heap addr = UNDEF li.extend_from_slice(&u64::MAX.to_le_bytes()); // btree name index addr = UNDEF w.add_message(MessageType::LinkInfo, li); + } + // Group Info (version 0, default link-phase thresholds, no estimates). + // Readers don't need it, but libhdf5 reads it before inserting a link: + // without one, adding a link to a group we wrote (h5py in "r+" mode) + // failed with "message type not found". + w.add_message(MessageType::GroupInfo, vec![0, 0]); + if dense_link_info.is_none() { for link in links { w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE)); } diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs new file mode 100644 index 0000000..9ea3053 --- /dev/null +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -0,0 +1,136 @@ +//! Groups and links written by `FileBuilder`, read back by h5py (libhdf5) +//! and h5dump, and by clawhdf5 itself. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{File, FileBuilder}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn h5dump_available() -> bool { + Command::new("h5dump") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Run `body` under h5py with `path` bound to the file's path. +fn h5py(path: &str, body: &str) -> String { + run_python(&format!( + "import h5py, numpy as np, json\npath = r'{path}'\n{body}" + )) +} + +fn write(dir: &tempfile::TempDir, name: &str, b: FileBuilder) -> String { + let path = dir.path().join(name).display().to_string(); + b.write(&path).unwrap(); + path +} + +/// h5dump must read the whole file without an error. +fn h5dump_ok(path: &str) -> String { + if !h5dump_available() { + assert!(!interop_required(), "h5dump is not available"); + return String::new(); + } + let o = Command::new("h5dump").arg(path).output().unwrap(); + let out = String::from_utf8_lossy(&o.stdout).to_string(); + assert!( + o.status.success(), + "h5dump failed:\n{out}{}", + String::from_utf8_lossy(&o.stderr) + ); + out +} + +// ---- libhdf5 can modify the groups we write ---- + +#[test] +fn h5py_can_add_links_to_groups_we_wrote() { + skip_if_no_python!(); + // Measured before the fix: h5py in "r+" mode could not add a link to any + // group we wrote ("Unable to create link (message type not found)"): + // libhdf5 reads a group's Group Info message before inserting a link, and + // the writer wrote none. + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + b.create_dataset("x").with_f64_data(&[1.0, 2.0]); + let mut g = b.create_group("small"); + g.create_dataset("a").with_i32_data(&[1]); + b.add_group(g.finish()); + let mut g = b.create_group("big"); // dense link storage + for i in 0..20 { + g.create_dataset(&format!("d{i:02}")).with_i32_data(&[i]); + } + b.add_group(g.finish()); + let path = write(&dir, "modify.h5", b); + + let out = h5py( + &path, + "with h5py.File(path, 'r+') as f:\n\ + \x20 f['alias'] = f['x']\n\ + \x20 f['small']['new'] = np.arange(3)\n\ + \x20 f['big']['new'] = np.arange(4)\n\ + \x20 f.create_group('added/deeper')\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([sorted(f), sorted(f['small']), len(f['big']),\n\ + \x20 f['alias'][()].tolist(), f['big/new'][()].tolist(), f['big/d07'][()].tolist()]))", + ); + assert_eq!( + out, + r#"[["added", "alias", "big", "small", "x"], ["a", "new"], 21, [1.0, 2.0], [0, 1, 2, 3], [7]]"# + ); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("big/new").unwrap().read_i64().unwrap(), + [0, 1, 2, 3] + ); + assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]); +}