feat(format): decode hyperslab selection versions 1 and 2 in VDS mappings
libhdf5 serializes a VDS hyperslab as version 1 (irregular, 4-byte block corners) for the default format bounds, and as version 2 (regular, 8-byte) for unlimited selections in the 1.10 format. Only version 3 was accepted, so every h5py VDS written with default libver failed with "only version-3 hyperslab selections are supported" (5 libhdf5 test files in the sweep). Decode all three versions following H5S__hyper_deserialize, including irregular hyperslabs (a union of blocks, enumerated in row-major order as libhdf5 iterates them) and the all-ones "unlimited" count/block marker. SerializedSelection exposes the raw form for unlimited-mapping support. Test: vds_interop::vds_version1_irregular_hyperslab_selections compares default-libver h5py VDS reads (contiguous, strided and 2-D block mappings) with libhdf5's values. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,156 @@
|
||||
//! Virtual Dataset (VDS) reads checked against libhdf5 (through h5py).
|
||||
//!
|
||||
//! Each test has h5py build virtual datasets and their source files in a temp
|
||||
//! directory, record what libhdf5 reads back (shape and values) next to them,
|
||||
//! and then compares that with what clawhdf5 reads from the same files.
|
||||
//!
|
||||
//! Skipped when python3 or h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
|
||||
/// The Python interpreter to drive interop checks with (`CLAWHDF5_PYTHON`
|
||||
/// lets these run against a virtualenv holding h5py).
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency
|
||||
/// is a test failure instead of a silent skip.
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py, numpy"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Prelude for every generator script: `expect(file, dset, tag)` records what
|
||||
/// libhdf5 reads for `file:dset` as `<tag>.expect` (shape line, values line).
|
||||
const PRELUDE: &str = r#"
|
||||
import h5py, numpy as np
|
||||
def expect(fn, dset, tag):
|
||||
with h5py.File(fn, "r") as f:
|
||||
d = f[dset]
|
||||
a = d[...]
|
||||
with open(tag + ".expect", "w") as out:
|
||||
out.write(" ".join(str(n) for n in d.shape) + "\n")
|
||||
out.write(" ".join(repr(float(v)) for v in a.ravel()) + "\n")
|
||||
"#;
|
||||
|
||||
/// Run `body` (after [`PRELUDE`]) with `dir` as the working directory, so
|
||||
/// relative source file names land next to the virtual file.
|
||||
fn generate(dir: &Path, body: &str) {
|
||||
let script = format!("{PRELUDE}\n{body}");
|
||||
let out = Command::new(python())
|
||||
.args(["-c", &script])
|
||||
.current_dir(dir)
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"generator failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&out.stdout),
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
/// What libhdf5 read for `tag`: (shape, values as f64).
|
||||
fn expected(dir: &Path, tag: &str) -> (Vec<u64>, Vec<f64>) {
|
||||
let text = std::fs::read_to_string(dir.join(format!("{tag}.expect"))).unwrap();
|
||||
let mut lines = text.lines();
|
||||
let parse_line = |l: Option<&str>| -> Vec<String> {
|
||||
l.unwrap_or("")
|
||||
.split_whitespace()
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
};
|
||||
let shape = parse_line(lines.next())
|
||||
.iter()
|
||||
.map(|s| s.parse().unwrap())
|
||||
.collect();
|
||||
let values = parse_line(lines.next())
|
||||
.iter()
|
||||
.map(|s| s.parse().unwrap())
|
||||
.collect();
|
||||
(shape, values)
|
||||
}
|
||||
|
||||
/// Assert clawhdf5 reads `file:dset` exactly as libhdf5 did for `tag`.
|
||||
fn assert_matches_libhdf5(dir: &Path, file: &str, dset: &str, tag: &str) {
|
||||
let (shape, values) = expected(dir, tag);
|
||||
let f = File::open(dir.join(file)).unwrap();
|
||||
let ds = f.dataset(dset).unwrap();
|
||||
assert_eq!(
|
||||
ds.shape().unwrap(),
|
||||
shape,
|
||||
"{tag}: shape differs from libhdf5"
|
||||
);
|
||||
let got = ds
|
||||
.read_f64()
|
||||
.unwrap_or_else(|e| panic!("{tag}: read failed: {e}"));
|
||||
assert_eq!(got.len(), values.len(), "{tag}: element count differs");
|
||||
for (i, (g, e)) in got.iter().zip(&values).enumerate() {
|
||||
assert!(
|
||||
g == e || (g.is_nan() && e.is_nan()),
|
||||
"{tag}: element {i} is {g}, libhdf5 reads {e}\n ours: {got:?}\n libhdf5: {values:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Selection encodings
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Files written with the default (earliest) format bounds serialize every
|
||||
/// VDS hyperslab as a version-1 *irregular* selection (4-byte block corners),
|
||||
/// and a strided selection as many blocks. These were refused outright.
|
||||
#[test]
|
||||
fn vds_version1_irregular_hyperslab_selections() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
generate(
|
||||
dir.path(),
|
||||
r#"
|
||||
with h5py.File("src.h5", "w") as s:
|
||||
s.create_dataset("a", data=np.arange(12.0))
|
||||
s.create_dataset("m", data=np.arange(20.0).reshape(4, 5))
|
||||
with h5py.File("v1.h5", "w") as f: # default libver: hyperslab version 1
|
||||
f.create_dataset("local", data=np.arange(10.0) * -1)
|
||||
lay = h5py.VirtualLayout(shape=(12,), dtype="f8")
|
||||
lay[0:4] = h5py.VirtualSource(".", "local", shape=(10,))[2:6]
|
||||
lay[4:10] = h5py.VirtualSource("src.h5", "a", shape=(12,))[::2]
|
||||
lay[10:12] = h5py.VirtualSource("src.h5", "a", shape=(12,))[10:12]
|
||||
f.create_virtual_dataset("strided", lay)
|
||||
lay = h5py.VirtualLayout(shape=(4, 6), dtype="f8")
|
||||
lay[:, 0:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 3:5]
|
||||
lay[:, 2:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 0:2]
|
||||
lay[:, 3:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 2:4]
|
||||
f.create_virtual_dataset("grid", lay)
|
||||
expect("v1.h5", "strided", "strided")
|
||||
expect("v1.h5", "grid", "grid")
|
||||
"#,
|
||||
);
|
||||
assert_matches_libhdf5(dir.path(), "v1.h5", "strided", "strided");
|
||||
assert_matches_libhdf5(dir.path(), "v1.h5", "grid", "grid");
|
||||
}
|
||||
Reference in New Issue
Block a user