diff --git a/CHANGELOG.md b/CHANGELOG.md index ce688d1..dc2dfa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -345,6 +345,13 @@ header or B-tree) still fails the call. `clawhdf5-format` gains `attribute::extract_attributes_tolerant`; `extract_attributes_full` stays strict. +- `clawhdf5-format` reader — files with shared object header messages + (SOHM, `H5Pset_shared_mesg_index`): a datatype, dataspace, filter pipeline + or attribute stored in the file's SOHM heap failed with "invalid shared + message version: 2" — only shared fill values loaded the SOHM table — so + such files' datasets and attributes could not be read. + `shared_message::resolve_shared_message` now loads the table when a + reference needs it (36 cases of the audit's read matrix). - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 33334fa..fc7842d 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -529,7 +529,8 @@ pub fn message_data<'a>( /// /// For type 1/3 (shared in another object header), reads the target object header /// and finds the message of the specified type. -/// For type 2 (SOHM), uses the fractal heap from the SOHM table. +/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table, +/// loaded from the superblock extension on demand. pub fn resolve_shared_message( file_data: &[u8], shared_ref: &SharedMessageRef, @@ -537,13 +538,18 @@ pub fn resolve_shared_message( offset_size: u8, length_size: u8, ) -> Result, FormatError> { + let table = if shared_ref.heap_id.is_some() { + load_sohm_table(file_data, offset_size, length_size)? + } else { + None + }; resolve_shared_message_with_sohm( file_data, shared_ref, target_msg_type, offset_size, length_size, - None, + table.as_ref(), ) } diff --git a/crates/clawhdf5/tests/sohm_interop.rs b/crates/clawhdf5/tests/sohm_interop.rs new file mode 100644 index 0000000..a259313 --- /dev/null +++ b/crates/clawhdf5/tests/sohm_interop.rs @@ -0,0 +1,141 @@ +//! Files with shared object header messages (SOHM: datatypes, dataspaces, +//! filter pipelines and attributes stored once in a file-wide heap and +//! referenced by heap ID), written by libhdf5 through h5py. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{AttrValue, File}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// h5py has no binding for the SOHM property-list calls, so they go through +/// the libhdf5 that h5py bundles. `None` when that library is not found. +fn sohm_file(path: &str, libver: &str, mesg_types: u32) -> Option<()> { + let out = run_python(&format!( + "import ctypes, glob, os, h5py, numpy as np\n\ + libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))\n\ + if not libs:\n\ + \x20 print('nolib'); raise SystemExit\n\ + lib = ctypes.CDLL(libs[0])\n\ + lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\ + lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]\n\ + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)\n\ + assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0\n\ + assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, {mesg_types}, 1) >= 0\n\ + fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)\n\ + low = h5py.h5f.LIBVER_EARLIEST if '{libver}' == 'earliest' else h5py.h5f.LIBVER_LATEST\n\ + fapl.set_libver_bounds(low, h5py.h5f.LIBVER_LATEST)\n\ + fid = h5py.h5f.create(r'{path}'.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)\n\ + with h5py.File(fid) as f:\n\ + \x20 for i in range(4):\n\ + \x20 ds = f.create_dataset('d%d' % i, shape=(50,), dtype=' = (0..20).map(|v| f64::from(v) + 2.0).collect(); + expected.resize(50, -9.0); + assert_eq!( + d2.read_f64().unwrap_or_else(|e| panic!("{case}: {e}")), + expected, + "{case}" + ); + let (attrs, errors) = d2.attrs_with_errors().unwrap(); + assert!(errors.is_empty(), "{case}: {errors:?}"); + let shared: Vec = (0..10).map(f64::from).collect(); + assert!( + matches!(&attrs["shared_attr"], AttrValue::F64Array(v) if *v == shared), + "{case}: {:?}", + attrs.get("shared_attr") + ); + assert!( + matches!(&attrs["units"], AttrValue::String(s) if s == "m/s"), + "{case}: {:?}", + attrs.get("units") + ); + assert_eq!( + f.dataset("contig").unwrap().read_i32().unwrap(), + [0, 2, 4, 6, 8, 10, 12], + "{case}" + ); + } + } +}