CHANGELOG.md

@@ -345,6 +345,13 @@
   header or B-tree) still fails the call. `clawhdf5-format` gains
   `attribute::extract_attributes_tolerant`; `extract_attributes_full` stays
   strict.
+- `clawhdf5-format` reader — files with shared object header messages
+  (SOHM, `H5Pset_shared_mesg_index`): a datatype, dataspace, filter pipeline
+  or attribute stored in the file's SOHM heap failed with "invalid shared
+  message version: 2" — only shared fill values loaded the SOHM table — so
+  such files' datasets and attributes could not be read.
+  `shared_message::resolve_shared_message` now loads the table when a
+  reference needs it (36 cases of the audit's read matrix).
 - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:**
   - Extensible Array (one unlimited dimension): chunks from index 244 on were
     written but never indexed and read as 0, by libhdf5 and by us.

crates/clawhdf5-format/src/shared_message.rs

@@ -529,7 +529,8 @@ pub fn message_data<'a>(
 ///
 /// For type 1/3 (shared in another object header), reads the target object header
 /// and finds the message of the specified type.
-/// For type 2 (SOHM), uses the fractal heap from the SOHM table.
+/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table,
+/// loaded from the superblock extension on demand.
 pub fn resolve_shared_message(
     file_data: &[u8],
     shared_ref: &SharedMessageRef,
@@ -537,13 +538,18 @@ pub fn resolve_shared_message(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<u8>, FormatError> {
+    let table = if shared_ref.heap_id.is_some() {
+        load_sohm_table(file_data, offset_size, length_size)?
+    } else {
+        None
+    };
     resolve_shared_message_with_sohm(
         file_data,
         shared_ref,
         target_msg_type,
         offset_size,
         length_size,
-        None,
+        table.as_ref(),
     )
 }
 

crates/clawhdf5/tests/sohm_interop.rs

@@ -0,0 +1,141 @@
+//! Files with shared object header messages (SOHM: datatypes, dataspaces,
+//! filter pipelines and attributes stored once in a file-wide heap and
+//! referenced by heap ID), written by libhdf5 through h5py.
+//!
+//! Skipped when python3 with h5py is unavailable, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::process::Command;
+
+use clawhdf5::{AttrValue, File};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+fn run_python(script: &str) -> String {
+    let output = Command::new(python())
+        .args(["-c", script])
+        .output()
+        .expect("failed to run python");
+    if !output.status.success() {
+        panic!(
+            "Python script failed:\nSTDOUT: {}\nSTDERR: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+    String::from_utf8_lossy(&output.stdout).trim().to_string()
+}
+
+/// h5py has no binding for the SOHM property-list calls, so they go through
+/// the libhdf5 that h5py bundles. `None` when that library is not found.
+fn sohm_file(path: &str, libver: &str, mesg_types: u32) -> Option<()> {
+    let out = run_python(&format!(
+        "import ctypes, glob, os, h5py, numpy as np\n\
+         libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))\n\
+         if not libs:\n\
+         \x20   print('nolib'); raise SystemExit\n\
+         lib = ctypes.CDLL(libs[0])\n\
+         lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\
+         lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]\n\
+         fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)\n\
+         assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0\n\
+         assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, {mesg_types}, 1) >= 0\n\
+         fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)\n\
+         low = h5py.h5f.LIBVER_EARLIEST if '{libver}' == 'earliest' else h5py.h5f.LIBVER_LATEST\n\
+         fapl.set_libver_bounds(low, h5py.h5f.LIBVER_LATEST)\n\
+         fid = h5py.h5f.create(r'{path}'.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)\n\
+         with h5py.File(fid) as f:\n\
+         \x20   for i in range(4):\n\
+         \x20       ds = f.create_dataset('d%d' % i, shape=(50,), dtype='<f8', chunks=(10,), compression='gzip', fillvalue=-9.0)\n\
+         \x20       ds[0:20] = np.arange(20.0) + i\n\
+         \x20       ds.attrs['shared_attr'] = np.arange(10.0)\n\
+         \x20       ds.attrs['units'] = 'm/s'\n\
+         \x20   f.create_dataset('contig', data=np.arange(7, dtype='<i4') * 2)\n\
+         print('ok')\n"
+    ));
+    (out == "ok").then_some(())
+}
+
+/// Every message type libhdf5 can share (`H5O_SHMESG_ALL_FLAG`), and each on
+/// its own.
+const MESG_TYPES: [(u32, &str); 6] = [
+    (0x182A, "all"),
+    (0x02, "dataspace"),
+    (0x08, "datatype"),
+    (0x20, "fill value"),
+    (0x800, "filter pipeline"),
+    (0x1000, "attribute"),
+];
+
+/// A message shared through the SOHM heap was only resolved on the one path
+/// that loaded the SOHM table itself (shared fill values); datatypes,
+/// dataspaces, filter pipelines and attributes stored there failed with
+/// "invalid shared message version: 2", so such files' datasets could not be
+/// read at all.
+#[test]
+fn sohm_shared_messages_resolve() {
+    if !python_available() {
+        assert!(
+            !interop_required(),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+        );
+        eprintln!("SKIP: python3 with h5py not available");
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    for libver in ["earliest", "latest"] {
+        for (flags, what) in MESG_TYPES {
+            let path = dir.path().join("sohm.h5").display().to_string();
+            if sohm_file(&path, libver, flags).is_none() {
+                assert!(
+                    !interop_required(),
+                    "CLAWHDF5_REQUIRE_INTEROP=1 but h5py's bundled libhdf5 was not found"
+                );
+                eprintln!("SKIP: h5py's bundled libhdf5 not found");
+                return;
+            }
+            let case = format!("{what}, libver {libver}");
+            let f = File::open(&path).unwrap();
+            let d2 = f.dataset("d2").unwrap_or_else(|e| panic!("{case}: {e}"));
+            let mut expected: Vec<f64> = (0..20).map(|v| f64::from(v) + 2.0).collect();
+            expected.resize(50, -9.0);
+            assert_eq!(
+                d2.read_f64().unwrap_or_else(|e| panic!("{case}: {e}")),
+                expected,
+                "{case}"
+            );
+            let (attrs, errors) = d2.attrs_with_errors().unwrap();
+            assert!(errors.is_empty(), "{case}: {errors:?}");
+            let shared: Vec<f64> = (0..10).map(f64::from).collect();
+            assert!(
+                matches!(&attrs["shared_attr"], AttrValue::F64Array(v) if *v == shared),
+                "{case}: {:?}",
+                attrs.get("shared_attr")
+            );
+            assert!(
+                matches!(&attrs["units"], AttrValue::String(s) if s == "m/s"),
+                "{case}: {:?}",
+                attrs.get("units")
+            );
+            assert_eq!(
+                f.dataset("contig").unwrap().read_i32().unwrap(),
+                [0, 2, 4, 6, 8, 10, 12],
+                "{case}"
+            );
+        }
+    }
+}