fix(format): write a Group Info message in every group
libhdf5 reads a group's Group Info message before it inserts a link, and FileWriter wrote none, so h5py in "r+" mode could not add a link to any group we wrote: "Unable to create link (message type not found)". Each group header now carries a version 0 Group Info message with the default link-phase thresholds, as libhdf5 writes for a new group. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -2,6 +2,15 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Writer: groups and links (2026-09-26)
|
||||
- **libhdf5 could not add links to groups we wrote.** h5py in `"r+"` mode
|
||||
failed with "Unable to create link (message type not found)" on every
|
||||
group `FileWriter` wrote: libhdf5 reads a group's Group Info message before
|
||||
inserting a link, and none was written. Every group now carries one
|
||||
(version 0, default thresholds: 6 more bytes per group header, so files
|
||||
are not byte-identical to earlier versions). Regression test:
|
||||
`crates/clawhdf5/tests/writer_groups_interop.rs`.
|
||||
|
||||
### Plugin filters (2026-09-26)
|
||||
- **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files
|
||||
written by h5py with `compression="lzf"`, or with hdf5plugin's
|
||||
|
||||
@@ -196,6 +196,13 @@ pub(crate) fn build_group_oh(
|
||||
li.extend_from_slice(&u64::MAX.to_le_bytes()); // fractal heap addr = UNDEF
|
||||
li.extend_from_slice(&u64::MAX.to_le_bytes()); // btree name index addr = UNDEF
|
||||
w.add_message(MessageType::LinkInfo, li);
|
||||
}
|
||||
// Group Info (version 0, default link-phase thresholds, no estimates).
|
||||
// Readers don't need it, but libhdf5 reads it before inserting a link:
|
||||
// without one, adding a link to a group we wrote (h5py in "r+" mode)
|
||||
// failed with "message type not found".
|
||||
w.add_message(MessageType::GroupInfo, vec![0, 0]);
|
||||
if dense_link_info.is_none() {
|
||||
for link in links {
|
||||
w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
//! Groups and links written by `FileBuilder`, read back by h5py (libhdf5)
|
||||
//! and h5dump, and by clawhdf5 itself.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::{File, FileBuilder};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn h5dump_available() -> bool {
|
||||
Command::new("h5dump")
|
||||
.arg("--version")
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn run_python(script: &str) -> String {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// Run `body` under h5py with `path` bound to the file's path.
|
||||
fn h5py(path: &str, body: &str) -> String {
|
||||
run_python(&format!(
|
||||
"import h5py, numpy as np, json\npath = r'{path}'\n{body}"
|
||||
))
|
||||
}
|
||||
|
||||
fn write(dir: &tempfile::TempDir, name: &str, b: FileBuilder) -> String {
|
||||
let path = dir.path().join(name).display().to_string();
|
||||
b.write(&path).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
/// h5dump must read the whole file without an error.
|
||||
fn h5dump_ok(path: &str) -> String {
|
||||
if !h5dump_available() {
|
||||
assert!(!interop_required(), "h5dump is not available");
|
||||
return String::new();
|
||||
}
|
||||
let o = Command::new("h5dump").arg(path).output().unwrap();
|
||||
let out = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
assert!(
|
||||
o.status.success(),
|
||||
"h5dump failed:\n{out}{}",
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
// ---- libhdf5 can modify the groups we write ----
|
||||
|
||||
#[test]
|
||||
fn h5py_can_add_links_to_groups_we_wrote() {
|
||||
skip_if_no_python!();
|
||||
// Measured before the fix: h5py in "r+" mode could not add a link to any
|
||||
// group we wrote ("Unable to create link (message type not found)"):
|
||||
// libhdf5 reads a group's Group Info message before inserting a link, and
|
||||
// the writer wrote none.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("x").with_f64_data(&[1.0, 2.0]);
|
||||
let mut g = b.create_group("small");
|
||||
g.create_dataset("a").with_i32_data(&[1]);
|
||||
b.add_group(g.finish());
|
||||
let mut g = b.create_group("big"); // dense link storage
|
||||
for i in 0..20 {
|
||||
g.create_dataset(&format!("d{i:02}")).with_i32_data(&[i]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
let path = write(&dir, "modify.h5", b);
|
||||
|
||||
let out = h5py(
|
||||
&path,
|
||||
"with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 f['alias'] = f['x']\n\
|
||||
\x20 f['small']['new'] = np.arange(3)\n\
|
||||
\x20 f['big']['new'] = np.arange(4)\n\
|
||||
\x20 f.create_group('added/deeper')\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 print(json.dumps([sorted(f), sorted(f['small']), len(f['big']),\n\
|
||||
\x20 f['alias'][()].tolist(), f['big/new'][()].tolist(), f['big/d07'][()].tolist()]))",
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"[["added", "alias", "big", "small", "x"], ["a", "new"], 21, [1.0, 2.0], [0, 1, 2, 3], [7]]"#
|
||||
);
|
||||
h5dump_ok(&path);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
f.dataset("big/new").unwrap().read_i64().unwrap(),
|
||||
[0, 1, 2, 3]
|
||||
);
|
||||
assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]);
|
||||
}
|
||||
Reference in New Issue
Block a user