fix(format): resolve SOHM-shared messages on every path
A message shared through the file's SOHM heap (H5Pset_shared_mesg_index) is referenced by heap ID, which needs the SOHM table from the superblock extension. Only message_data_with_sohm (used for fill values) loaded it; resolve_shared_message passed no table, so a SOHM-shared datatype, dataspace, filter pipeline or attribute failed with "invalid shared message version: 2" and the dataset or attribute could not be read. resolve_shared_message now loads the table when the reference carries a heap ID. Found while making attrs() tolerant: SOHM attributes turned from an error into missing keys in the audit read matrix. With this fix all 36 SOHM cases there match h5py (datasets, fill values and attributes, every shareable message type, libver earliest and latest). Regression test: sohm_shared_messages_resolve (h5py writes files sharing each message type on its own and all of them; values and attributes checked). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -529,7 +529,8 @@ pub fn message_data<'a>(
|
||||
///
|
||||
/// For type 1/3 (shared in another object header), reads the target object header
|
||||
/// and finds the message of the specified type.
|
||||
/// For type 2 (SOHM), uses the fractal heap from the SOHM table.
|
||||
/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table,
|
||||
/// loaded from the superblock extension on demand.
|
||||
pub fn resolve_shared_message(
|
||||
file_data: &[u8],
|
||||
shared_ref: &SharedMessageRef,
|
||||
@@ -537,13 +538,18 @@ pub fn resolve_shared_message(
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let table = if shared_ref.heap_id.is_some() {
|
||||
load_sohm_table(file_data, offset_size, length_size)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
resolve_shared_message_with_sohm(
|
||||
file_data,
|
||||
shared_ref,
|
||||
target_msg_type,
|
||||
offset_size,
|
||||
length_size,
|
||||
None,
|
||||
table.as_ref(),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Files with shared object header messages (SOHM: datatypes, dataspaces,
|
||||
//! filter pipelines and attributes stored once in a file-wide heap and
|
||||
//! referenced by heap ID), written by libhdf5 through h5py.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::{AttrValue, File};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn run_python(script: &str) -> String {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// h5py has no binding for the SOHM property-list calls, so they go through
|
||||
/// the libhdf5 that h5py bundles. `None` when that library is not found.
|
||||
fn sohm_file(path: &str, libver: &str, mesg_types: u32) -> Option<()> {
|
||||
let out = run_python(&format!(
|
||||
"import ctypes, glob, os, h5py, numpy as np\n\
|
||||
libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))\n\
|
||||
if not libs:\n\
|
||||
\x20 print('nolib'); raise SystemExit\n\
|
||||
lib = ctypes.CDLL(libs[0])\n\
|
||||
lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\
|
||||
lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]\n\
|
||||
fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)\n\
|
||||
assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0\n\
|
||||
assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, {mesg_types}, 1) >= 0\n\
|
||||
fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)\n\
|
||||
low = h5py.h5f.LIBVER_EARLIEST if '{libver}' == 'earliest' else h5py.h5f.LIBVER_LATEST\n\
|
||||
fapl.set_libver_bounds(low, h5py.h5f.LIBVER_LATEST)\n\
|
||||
fid = h5py.h5f.create(r'{path}'.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)\n\
|
||||
with h5py.File(fid) as f:\n\
|
||||
\x20 for i in range(4):\n\
|
||||
\x20 ds = f.create_dataset('d%d' % i, shape=(50,), dtype='<f8', chunks=(10,), compression='gzip', fillvalue=-9.0)\n\
|
||||
\x20 ds[0:20] = np.arange(20.0) + i\n\
|
||||
\x20 ds.attrs['shared_attr'] = np.arange(10.0)\n\
|
||||
\x20 ds.attrs['units'] = 'm/s'\n\
|
||||
\x20 f.create_dataset('contig', data=np.arange(7, dtype='<i4') * 2)\n\
|
||||
print('ok')\n"
|
||||
));
|
||||
(out == "ok").then_some(())
|
||||
}
|
||||
|
||||
/// Every message type libhdf5 can share (`H5O_SHMESG_ALL_FLAG`), and each on
|
||||
/// its own.
|
||||
const MESG_TYPES: [(u32, &str); 6] = [
|
||||
(0x182A, "all"),
|
||||
(0x02, "dataspace"),
|
||||
(0x08, "datatype"),
|
||||
(0x20, "fill value"),
|
||||
(0x800, "filter pipeline"),
|
||||
(0x1000, "attribute"),
|
||||
];
|
||||
|
||||
/// A message shared through the SOHM heap was only resolved on the one path
|
||||
/// that loaded the SOHM table itself (shared fill values); datatypes,
|
||||
/// dataspaces, filter pipelines and attributes stored there failed with
|
||||
/// "invalid shared message version: 2", so such files' datasets could not be
|
||||
/// read at all.
|
||||
#[test]
|
||||
fn sohm_shared_messages_resolve() {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for libver in ["earliest", "latest"] {
|
||||
for (flags, what) in MESG_TYPES {
|
||||
let path = dir.path().join("sohm.h5").display().to_string();
|
||||
if sohm_file(&path, libver, flags).is_none() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py's bundled libhdf5 was not found"
|
||||
);
|
||||
eprintln!("SKIP: h5py's bundled libhdf5 not found");
|
||||
return;
|
||||
}
|
||||
let case = format!("{what}, libver {libver}");
|
||||
let f = File::open(&path).unwrap();
|
||||
let d2 = f.dataset("d2").unwrap_or_else(|e| panic!("{case}: {e}"));
|
||||
let mut expected: Vec<f64> = (0..20).map(|v| f64::from(v) + 2.0).collect();
|
||||
expected.resize(50, -9.0);
|
||||
assert_eq!(
|
||||
d2.read_f64().unwrap_or_else(|e| panic!("{case}: {e}")),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
let (attrs, errors) = d2.attrs_with_errors().unwrap();
|
||||
assert!(errors.is_empty(), "{case}: {errors:?}");
|
||||
let shared: Vec<f64> = (0..10).map(f64::from).collect();
|
||||
assert!(
|
||||
matches!(&attrs["shared_attr"], AttrValue::F64Array(v) if *v == shared),
|
||||
"{case}: {:?}",
|
||||
attrs.get("shared_attr")
|
||||
);
|
||||
assert!(
|
||||
matches!(&attrs["units"], AttrValue::String(s) if s == "m/s"),
|
||||
"{case}: {:?}",
|
||||
attrs.get("units")
|
||||
);
|
||||
assert_eq!(
|
||||
f.dataset("contig").unwrap().read_i32().unwrap(),
|
||||
[0, 2, 4, 6, 8, 10, 12],
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user