fix(format): convert float data read as integers instead of returning bit patterns
read_i32/read_i64/read_u64 on a floating-point dataset reinterpreted the IEEE bits (1.5 read as i64 was 4609434218613702656). Convert like libhdf5's hard conversions instead: truncate toward zero and saturate at the target range; NaN reads as 0. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -0,0 +1,129 @@
|
||||
//! Numeric conversions on read, checked against h5py/libhdf5.
|
||||
//!
|
||||
//! h5py writes each file and prints what libhdf5 converts the data to
|
||||
//! (`Dataset.astype`); the typed readers must return the same values.
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py, numpy"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Run `script` (which writes the file at `path`) and return its stdout as
|
||||
/// `key -> values`, one `key v1 v2 ...` line per key.
|
||||
fn run_python(script: &str) -> HashMap<String, Vec<String>> {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"python failed:\n{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&output.stdout)
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let mut words = line.split_whitespace().map(str::to_string);
|
||||
Some((words.next()?, words.collect()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse<T: std::str::FromStr>(values: &[String]) -> Vec<T>
|
||||
where
|
||||
T::Err: std::fmt::Debug,
|
||||
{
|
||||
values.iter().map(|v| v.parse().unwrap()).collect()
|
||||
}
|
||||
|
||||
/// Python prelude: `emit(key, array)` prints one line of integers.
|
||||
const PRELUDE: &str = r#"
|
||||
import h5py, numpy as np
|
||||
def emit(key, arr):
|
||||
print(key, *[int(v) for v in np.asarray(arr).ravel()])
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn float_dataset_read_as_integers_converts_like_libhdf5() {
|
||||
// read_i32/read_i64/read_u64 on a float dataset used to return the raw
|
||||
// IEEE bit patterns (1.5 read as i64 was 4609434218613702656).
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("float_to_int.h5");
|
||||
let script = format!(
|
||||
r#"{PRELUDE}
|
||||
vals = [1.5, -2.75, 3e9, 1e300, -1e300, -0.5, 0.0, 7.99, np.inf, -np.inf, 1e19, -1e19]
|
||||
with h5py.File("{path}", "w") as f:
|
||||
for name, dt in (("f8", "<f8"), ("f8be", ">f8"), ("f4", "<f4")):
|
||||
f.create_dataset(name, data=np.array(vals).astype(dt))
|
||||
# libhdf5's half conversions are not saturating (an infinite half becomes
|
||||
# INT_MIN whatever its sign, a negative one wraps as u64), so the half
|
||||
# case stays finite and its u64 read is checked separately below.
|
||||
f.create_dataset("f2", data=np.array([1.5, -2.75, -0.5, 0.0, 7.99, 65504, -65504], "<f2"))
|
||||
with h5py.File("{path}", "r") as f:
|
||||
for name in ("f8", "f8be", "f4", "f2"):
|
||||
d = f[name]
|
||||
emit(name + ":i32", d.astype("<i4")[()])
|
||||
emit(name + ":i64", d.astype("<i8")[()])
|
||||
emit(name + ":u64", d.astype("<u8")[()])
|
||||
"#,
|
||||
path = path.display()
|
||||
);
|
||||
let expected = run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
for name in ["f8", "f8be", "f4", "f2"] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
assert_eq!(
|
||||
ds.read_i32().unwrap(),
|
||||
parse::<i32>(&expected[&format!("{name}:i32")]),
|
||||
"{name} as i32"
|
||||
);
|
||||
assert_eq!(
|
||||
ds.read_i64().unwrap(),
|
||||
parse::<i64>(&expected[&format!("{name}:i64")]),
|
||||
"{name} as i64"
|
||||
);
|
||||
if name != "f2" {
|
||||
assert_eq!(
|
||||
ds.read_u64().unwrap(),
|
||||
parse::<u64>(&expected[&format!("{name}:u64")]),
|
||||
"{name} as u64"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Negative values saturate at 0 rather than wrapping.
|
||||
let f2 = file.dataset("f2").unwrap();
|
||||
assert_eq!(f2.read_u64().unwrap(), vec![1, 0, 0, 0, 7, 65504, 0]);
|
||||
}
|
||||
Reference in New Issue
Block a user