fix(format): resolve shared fill value messages instead of zero-filling

dataset_fill_value treated a shared Fill Value message as "no fill value", so unwritten storage of a dataset whose fill value lives in the file's shared-message (SOHM) heap read as zeros rather than its fill value. libhdf5 shares fill values whenever the file has a SOHM index for them. - fill_value::dataset_fill_value_in follows the reference (another object header, or the SOHM heap); read_full_with_fill and the facade's selection read use it. - dataset_fill_value, which has no file to follow a reference into, now returns UnresolvedSharedMessage for a shared message instead of None. - shared_message::load_sohm_table / message_data_with_sohm load the SOHM table from the superblock extension on demand. - parse_sohm_table skipped each index's leading version byte, reading every field one byte off; SOHM references could never resolve. Fixture shared_fill_value.h5 (HDF5 2.0, gen_shared_fill.py): sohm_b read [0,1,2,3,0,0,0,0] and now reads [0,1,2,3,-7,-7,-7,-7], as h5py does. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-25 21:18:12 -05:00
parent 57e938c438
commit 7c1968a34a
7 changed files with 268 additions and 12 deletions
@@ -0,0 +1,49 @@
+"""Generate shared_fill_value.h5: datasets whose Fill Value message is
+*shared*, in the two ways libhdf5 can share one.
+
+- /sohm_a, /sohm_b: the file has a shared-object-header-message (SOHM) index
+  for fill values, so libhdf5 stores the fill value (-7, int32) in the SOHM
+  heap and /sohm_b's header holds only a reference to it. Chunked, with only
+  the first chunk written, so the rest reads as the fill value.
+- /unwritten_a, /unwritten_b: the same, never written: no storage at all,
+  read entirely as the fill value.
+
+h5py has no API for SOHM indexes, so the file creation property list is
+configured by calling the libhdf5 bundled in the h5py wheel through ctypes.
+Written with h5py 3.16.0 / HDF5 2.0.0. Re-run only to regenerate:
+
+    python gen_shared_fill.py shared_fill_value.h5
+"""
+import ctypes
+import glob
+import os
+import sys
+
+import h5py
+import numpy as np
+
+libdir = os.path.join(os.path.dirname(os.path.dirname(h5py.__file__)), "h5py.libs")
+libs = [p for p in glob.glob(os.path.join(libdir, "libhdf5*.so*")) if "_hl" not in os.path.basename(p)]
+lib = ctypes.CDLL(libs[0])
+lib.H5open()
+
+H5O_SHMESG_FILL_FLAG = 1 << 0x0005
+
+fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)
+lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]
+lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]
+assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0
+assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, H5O_SHMESG_FILL_FLAG, 0) >= 0
+
+fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
+fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
+fid = h5py.h5f.create(sys.argv[1].encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)
+with h5py.File(fid) as f:
+    # Chunked, with only the first chunk written: the rest reads as fill.
+    # libhdf5 keeps the first copy of a message in its own header; the second
+    # identical one (the `_b` datasets) is the SOHM reference.
+    for name in ("sohm_a", "sohm_b"):
+        d = f.create_dataset(name, shape=(8,), chunks=(4,), dtype="<i4", fillvalue=-7)
+        d[:4] = np.arange(4)
+    for name in ("unwritten_a", "unwritten_b"):
+        f.create_dataset(name, shape=(3,), dtype="<i4", fillvalue=-7)
@@ -597,3 +597,50 @@ fn unknown_message_flags_follow_libhdf5_on_tbogus() {
        }
    }
 }
+
+// ---- 8. shared fill value messages ----
+
+#[test]
+fn shared_fill_value_is_resolved_not_zero() {
+    // gen_shared_fill.py: HDF5 2.0 with a SOHM index for fill values, so each
+    // dataset's fill value message is a reference into the SOHM heap. It
+    // used to be read as "no fill value" (zeros) instead of -7.
+    let bytes = include_bytes!("fixtures/shared_fill_value.h5");
+    for (name, shared) in [
+        ("sohm_a", false),
+        ("sohm_b", true),
+        ("unwritten_a", false),
+        ("unwritten_b", true),
+    ] {
+        let (sb, oh) = header_at(bytes, name);
+        let msg = oh
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::FillValue)
+            .unwrap();
+        assert_eq!(
+            clawhdf5_format::shared_message::is_shared(msg.flags),
+            shared,
+            "{name}: fixture layout"
+        );
+        if shared {
+            // Without the file the reference cannot be followed: an error,
+            // never a silent default.
+            assert_eq!(
+                clawhdf5_format::fill_value::dataset_fill_value(&oh.messages),
+                Err(clawhdf5_format::error::FormatError::UnresolvedSharedMessage)
+            );
+        }
+        assert_eq!(
+            clawhdf5_format::fill_value::dataset_fill_value_in(
+                bytes,
+                &oh.messages,
+                sb.offset_size,
+                sb.length_size
+            )
+            .unwrap(),
+            Some((-7i32).to_le_bytes().to_vec()),
+            "{name}"
+        );
+    }
+}