From 17fa783dced4c982cb51ee102c7d9a99b658bb37 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:08:45 -0500 Subject: [PATCH] fix(format): resolve SOHM-shared messages on every path A message shared through the file's SOHM heap (H5Pset_shared_mesg_index) is referenced by heap ID, which needs the SOHM table from the superblock extension. Only message_data_with_sohm (used for fill values) loaded it; resolve_shared_message passed no table, so a SOHM-shared datatype, dataspace, filter pipeline or attribute failed with "invalid shared message version: 2" and the dataset or attribute could not be read. resolve_shared_message now loads the table when the reference carries a heap ID. Found while making attrs() tolerant: SOHM attributes turned from an error into missing keys in the audit read matrix. With this fix all 36 SOHM cases there match h5py (datasets, fill values and attributes, every shareable message type, libver earliest and latest). Regression test: sohm_shared_messages_resolve (h5py writes files sharing each message type on its own and all of them; values and attributes checked). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 + crates/clawhdf5-format/src/shared_message.rs | 10 +- crates/clawhdf5/tests/sohm_interop.rs | 141 +++++++++++++++++++ 3 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 crates/clawhdf5/tests/sohm_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ce688d1..dc2dfa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -345,6 +345,13 @@ header or B-tree) still fails the call. `clawhdf5-format` gains `attribute::extract_attributes_tolerant`; `extract_attributes_full` stays strict. +- `clawhdf5-format` reader — files with shared object header messages + (SOHM, `H5Pset_shared_mesg_index`): a datatype, dataspace, filter pipeline + or attribute stored in the file's SOHM heap failed with "invalid shared + message version: 2" — only shared fill values loaded the SOHM table — so + such files' datasets and attributes could not be read. + `shared_message::resolve_shared_message` now loads the table when a + reference needs it (36 cases of the audit's read matrix). - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 33334fa..fc7842d 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -529,7 +529,8 @@ pub fn message_data<'a>( /// /// For type 1/3 (shared in another object header), reads the target object header /// and finds the message of the specified type. -/// For type 2 (SOHM), uses the fractal heap from the SOHM table. +/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table, +/// loaded from the superblock extension on demand. pub fn resolve_shared_message( file_data: &[u8], shared_ref: &SharedMessageRef, @@ -537,13 +538,18 @@ pub fn resolve_shared_message( offset_size: u8, length_size: u8, ) -> Result, FormatError> { + let table = if shared_ref.heap_id.is_some() { + load_sohm_table(file_data, offset_size, length_size)? + } else { + None + }; resolve_shared_message_with_sohm( file_data, shared_ref, target_msg_type, offset_size, length_size, - None, + table.as_ref(), ) } diff --git a/crates/clawhdf5/tests/sohm_interop.rs b/crates/clawhdf5/tests/sohm_interop.rs new file mode 100644 index 0000000..a259313 --- /dev/null +++ b/crates/clawhdf5/tests/sohm_interop.rs @@ -0,0 +1,141 @@ +//! Files with shared object header messages (SOHM: datatypes, dataspaces, +//! filter pipelines and attributes stored once in a file-wide heap and +//! referenced by heap ID), written by libhdf5 through h5py. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{AttrValue, File}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// h5py has no binding for the SOHM property-list calls, so they go through +/// the libhdf5 that h5py bundles. `None` when that library is not found. +fn sohm_file(path: &str, libver: &str, mesg_types: u32) -> Option<()> { + let out = run_python(&format!( + "import ctypes, glob, os, h5py, numpy as np\n\ + libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))\n\ + if not libs:\n\ + \x20 print('nolib'); raise SystemExit\n\ + lib = ctypes.CDLL(libs[0])\n\ + lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\ + lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]\n\ + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)\n\ + assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0\n\ + assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, {mesg_types}, 1) >= 0\n\ + fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)\n\ + low = h5py.h5f.LIBVER_EARLIEST if '{libver}' == 'earliest' else h5py.h5f.LIBVER_LATEST\n\ + fapl.set_libver_bounds(low, h5py.h5f.LIBVER_LATEST)\n\ + fid = h5py.h5f.create(r'{path}'.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)\n\ + with h5py.File(fid) as f:\n\ + \x20 for i in range(4):\n\ + \x20 ds = f.create_dataset('d%d' % i, shape=(50,), dtype=' = (0..20).map(|v| f64::from(v) + 2.0).collect(); + expected.resize(50, -9.0); + assert_eq!( + d2.read_f64().unwrap_or_else(|e| panic!("{case}: {e}")), + expected, + "{case}" + ); + let (attrs, errors) = d2.attrs_with_errors().unwrap(); + assert!(errors.is_empty(), "{case}: {errors:?}"); + let shared: Vec = (0..10).map(f64::from).collect(); + assert!( + matches!(&attrs["shared_attr"], AttrValue::F64Array(v) if *v == shared), + "{case}: {:?}", + attrs.get("shared_attr") + ); + assert!( + matches!(&attrs["units"], AttrValue::String(s) if s == "m/s"), + "{case}: {:?}", + attrs.get("units") + ); + assert_eq!( + f.dataset("contig").unwrap().read_i32().unwrap(), + [0, 2, 4, 6, 8, 10, 12], + "{case}" + ); + } + } +}